def do_sync(config: Dict, catalog: Dict, state: Dict) -> None:
"""
Syncs every selected stream in the catalog and updates the state
:param config: connection and streams information
:param catalog: Streams catalog
:param state: current state
:return: Nothing
"""
LOGGER.info('Starting sync.')
for stream in catalog['streams']:
stream_name = stream['tap_stream_id']
mdata = metadata.to_map(stream['metadata'])
table_spec = next(s for s in config['tables']
if s['table_name'] == stream_name)
if not stream_is_selected(mdata):
LOGGER.info("%s: Skipping - not selected", stream_name)
continue
singer.write_state(state)
key_properties = metadata.get(mdata, (), 'table-key-properties')
singer.write_schema(stream_name, stream['schema'], key_properties)
LOGGER.info("%s: Starting sync", stream_name)
counter_value = sync_stream(config, state, table_spec, stream)
LOGGER.info("%s: Completed sync (%s rows)", stream_name, counter_value)
LOGGER.info('Done syncing.')
def do_sync(config, catalog, state):
LOGGER.info('Starting sync.')
for stream in catalog['streams']:
stream_name = stream['tap_stream_id']
mdata = metadata.to_map(stream['metadata'])
table_spec = next(s for s in config['tables']
if s['table_name'] == stream_name)
singer.write_state(state)
key_properties = metadata.get(mdata, (), 'table-key-properties')
singer.write_schema(stream_name, stream['schema'], key_properties)
LOGGER.info("%s: Starting sync", stream_name)
counter_value = sync_stream(config, state, table_spec, stream)
LOGGER.info("%s: Completed sync (%s rows)", stream_name, counter_value)
LOGGER.info('Done syncing.')
def do_sync(config, catalog, state):
LOGGER.info('Starting sync.')
for stream in catalog.get_selected_streams(state):
stream_name = stream.tap_stream_id
mdata = metadata.to_map(stream.metadata)
table_spec = next(s for s in config['tables'] if s['table_name'] == stream_name)
if not stream_is_selected(mdata):
LOGGER.info("%s: Skipping - not selected", stream_name)
continue
singer.write_state(state)
key_properties = metadata.get(mdata, (), 'table-key-properties')
singer.write_schema(stream_name, stream.schema.to_dict(), key_properties)
LOGGER.info("%s: Starting sync", stream_name)
counter_value = sync_stream(config, state, table_spec, stream)
LOGGER.info("%s: Completed sync (%s rows)", stream_name, counter_value)
LOGGER.info('Done syncing.')
