def sync_stream(config: Dict, state: Dict, table_spec: Dict, stream: Dict) -> int:
"""
Sync the stream
:param config: Connection and stream config
:param state: current state
:param table_spec: table specs
:param stream: stream
:return: count of streamed records
"""
table_name = table_spec['table_name']
modified_since = utils.strptime_with_tz(get_bookmark(state, table_name, 'modified_since') or
config['start_date'])
LOGGER.info('Syncing table "%s".', table_name)
LOGGER.info('Getting files modified since %s.', modified_since)
s3_files = s3.get_input_files_for_table(
config, table_spec, modified_since)
records_streamed = 0
# We sort here so that tracking the modified_since bookmark makes
# sense. This means that we can't sync s3 buckets that are larger than
# we can sort in memory which is suboptimal. If we could bookmark
# based on anything else then we could just sync files as we see them.
for s3_file in sorted(s3_files, key=lambda item: item['last_modified']):
records_streamed += sync_table_file(
config, s3_file['key'], table_spec, stream)
state = write_bookmark(state, table_name, 'modified_since', s3_file['last_modified'].isoformat())
write_state(state)
LOGGER.info('Wrote %s records for table "%s".', records_streamed, table_name)
return records_streamed
def sync_stream(config, state, table_spec, stream):
table_name = table_spec['table_name']
modified_since = utils.strptime_with_tz(
singer.get_bookmark(state, table_name, 'modified_since')
or config['start_date'])
LOGGER.info('Syncing table "%s".', table_name)
LOGGER.info('Getting files modified since %s.', modified_since)
s3_files = s3.get_input_files_for_table(config, table_spec, modified_since)
LOGGER.info('Found %s files to be synced.', len(s3_files))
records_streamed = 0
if not s3_files:
return records_streamed
for s3_file in s3_files:
records_streamed += sync_table_file(config, s3_file['key'], table_spec,
stream)
state = singer.write_bookmark(state, table_name, 'modified_since',
s3_file['last_modified'].isoformat())
singer.write_state(state)
LOGGER.info('Wrote %s records for table "%s".', records_streamed,
table_name)
return records_streamed
def get_sampled_schema_for_table(config, table_spec):
logger.info('Sampling records to determine table schema.')
s3_files = s3.get_input_files_for_table(config, table_spec)
samples = s3.sample_files(config, table_spec, s3_files)
metadata_schema = {
'_s3_source_bucket': {
'type': 'string'
},
'_s3_source_file': {
'type': 'string'
},
'_s3_source_lineno': {
'type': 'integer'
},
}
data_schema = conversion.generate_schema(samples)
return {
'type': 'object',
'properties': merge_dicts(data_schema, metadata_schema)
}
def sync_stream(config, state, table_spec, stream):
table_name = table_spec['table_name']
modified_since = singer_utils.strptime_with_tz(singer.get_bookmark(state, table_name, 'modified_since') or
config['start_date'])
LOGGER.info('Syncing table "%s".', table_name)
LOGGER.info('Getting files modified since %s.', modified_since)
s3_files = s3.get_input_files_for_table(
config, table_spec, modified_since)
records_streamed = 0
# We sort here so that tracking the modified_since bookmark makes
# sense. This means that we can't sync s3 buckets that are larger than
# we can sort in memory which is suboptimal. If we could bookmark
# based on anything else then we could just sync files as we see them.
for s3_file in sorted(s3_files, key=lambda item: item['last_modified']):
records_streamed += sync_table_file(
config, s3_file['key'], table_spec, stream)
state = singer.write_bookmark(state, table_name, 'modified_since', s3_file['last_modified'].isoformat())
singer.write_state(state)
if s3.skipped_files_count:
LOGGER.warn("%s files got skipped during the last sync.",s3.skipped_files_count)
LOGGER.info('Wrote %s records for table "%s".', records_streamed, table_name)
return records_streamed
def sync_stream(config, state, table_spec, stream):
table_name = table_spec["table_name"]
modified_since = utils.strptime_with_tz(
singer.get_bookmark(state, table_name, "modified_since")
or config["start_date"])
LOGGER.info('Syncing table "%s".', table_name)
LOGGER.info("Getting files modified since %s.", modified_since)
s3_files = s3.get_input_files_for_table(config, table_spec, modified_since)
records_streamed = 0
# We sort here so that tracking the modified_since bookmark makes
# sense. This means that we can't sync s3 buckets that are larger than
# we can sort in memory which is suboptimal. If we could bookmark
# based on anything else then we could just sync files as we see them.
for s3_file in sorted(s3_files, key=lambda item: item["last_modified"]):
records_streamed += sync_table_file(config, s3_file["key"], table_spec,
stream, s3_file["last_modified"])
state = singer.write_bookmark(state, table_name, "modified_since",
s3_file["last_modified"].isoformat())
singer.write_state(state)
LOGGER.info('Wrote %s records for table "%s".', records_streamed,
table_name)
return records_streamed
def sync_stream(config, state, table_spec, stream):
table_name = table_spec['table_name']
bookmark = singer.get_bookmark(state, table_name, 'modified_since')
modified_since = utils.strptime_with_tz(bookmark or '1990-01-01T00:00:00Z')
LOGGER.info('Syncing table "%s".', table_name)
LOGGER.info('Getting files modified since %s.', modified_since)
s3_files = s3.get_input_files_for_table(config, table_spec, modified_since)
records_streamed = 0
# Original implementation sorted by 'modified_since' so that the modified_since bookmark makes
# sense. We sort by 'key' because we import multiple part files generated from Spark where the
# names are incremental order.
# This means that we can't sync s3 buckets that are larger than
# we can sort in memory which is suboptimal. If we could bookmark
# based on anything else then we could just sync files as we see them.
for s3_file in sorted(s3_files, key=lambda item: item['key']):
records_streamed += sync_table_file(config, s3_file['key'], table_spec,
stream)
state = singer.write_bookmark(state, table_name, 'modified_since',
s3_file['last_modified'].isoformat())
singer.write_state(state)
LOGGER.info('Wrote %s records for table "%s".', records_streamed,
table_name)
return records_streamed
def sync_table(config, state, table_spec):
table_name = table_spec['name']
modified_since = dateutil.parser.parse(
state.get(table_name, {}).get('modified_since') or
config['start_date'])
logger.info('Syncing table "{}".'.format(table_name))
logger.info('Getting files modified since {}.'.format(modified_since))
s3_files = s3.get_input_files_for_table(
config, table_spec, modified_since)
logger.info('Found {} files to be synced.'
.format(len(s3_files)))
if not s3_files:
return state
inferred_schema = get_sampled_schema_for_table(config, table_spec)
override_schema = {'properties': table_spec.get('schema_overrides', {})}
schema = merge_dicts(
inferred_schema,
override_schema)
singer.write_schema(
table_name,
schema,
key_properties=table_spec['key_properties'])
records_streamed = 0
for s3_file in s3_files:
records_streamed += sync_table_file(
config, s3_file['key'], table_spec, schema)
state[table_name] = {
'modified_since': s3_file['last_modified'].isoformat()
}
singer.write_state(state)
logger.info('Wrote {} records for table "{}".'
.format(records_streamed, table_name))
return state