def test_empty_state(self):
empty_state = {}
# Case with no value to fall back on
self.assertIsNone(bookmarks.get_currently_syncing(empty_state))
# Case with a given default
self.assertEqual(
bookmarks.get_currently_syncing(empty_state, 'default_value'),
'default_value')
def validate_state(config, catalog, state):
for stream in catalog["streams"]:
for mdata in stream['metadata']:
if mdata['breadcrumb'] == [] and mdata['metadata'].get('selected') != True:
# If a stream is deselected while it's the current stream, unset the
# current stream.
if stream["tap_stream_id"] == get_currently_syncing(state):
set_currently_syncing(state, None)
break
replication_key = determine_replication_key(stream['tap_stream_id'])
if not replication_key:
continue
# If there's no bookmark for a stream (new integration, newly selected,
# reset, etc) we need to use the default start date from the config.
bookmark = get_bookmark(state,
stream["tap_stream_id"],
replication_key)
if bookmark is None:
state = write_bookmark(state,
stream["tap_stream_id"],
replication_key,
config["start_date"])
singer.write_state(state)
return state
def sync(client, catalog, config, state):
starting_stream = bookmarks.get_currently_syncing(state)
if starting_stream:
singer.log_info("Resuming sync from %s", starting_stream)
else:
singer.log_info("Starting sync")
for stream in catalog["streams"]:
# Skip unselected streams.
mdata = metadata.to_map(stream['metadata'])
if not metadata.get(mdata, (), 'selected'):
singer.log_info("%s: not selected", stream["tap_stream_id"])
continue
# Skip streams that have already be synced when resuming.
if starting_stream and stream["tap_stream_id"] != starting_stream:
singer.log_info("%s: already synced", stream["tap_stream_id"])
continue
singer.log_info("%s: starting sync", stream["tap_stream_id"])
# Now that we've started, there's no more "starting stream". Set
# the current stream to resume on next run.
starting_stream = None
state = bookmarks.set_currently_syncing(state, stream["tap_stream_id"])
singer.write_state(state)
# Sync stream based on type.
if stream["tap_stream_id"] == "activity_types":
state, record_count = sync_activity_types(client, state, stream)
elif stream["tap_stream_id"] == "leads":
state, record_count = sync_leads(client, state, stream, config)
elif stream["tap_stream_id"].startswith("activities_"):
state, record_count = sync_activities(client, state, stream,
config)
elif stream["tap_stream_id"] in ["campaigns", "lists"]:
state, record_count = sync_paginated(client, state, stream)
elif stream["tap_stream_id"] == "programs":
state, record_count = sync_programs(client, state, stream)
else:
raise Exception("Stream %s not implemented" %
stream["tap_stream_id"])
# Emit metric for record count.
counter = singer.metrics.record_counter(stream["tap_stream_id"])
counter.value = record_count
counter._pop() # pylint: disable=protected-access
# Unset current stream.
state = bookmarks.set_currently_syncing(state, None)
singer.write_state(state)
singer.log_info("%s: finished sync", stream["tap_stream_id"])
# If Corona is not supported, log a warning near the end of the tap
# log with instructions on how to get Corona supported.
singer.log_info("Finished sync.")
if not client.use_corona:
singer.log_warning(NO_CORONA_WARNING)
def test_non_empty_state(self):
stream_id_1 = 'customers'
bookmark_key_1 = 'datetime'
bookmark_val_1 = 123456789
offset_val = 'fizzy water'
non_empty_state = {
'bookmarks': {
stream_id_1: {
bookmark_key_1: bookmark_val_1,
'offset': offset_val
}
},
'currently_syncing': stream_id_1
}
# Case with no value to fall back on
self.assertEqual(bookmarks.get_currently_syncing(non_empty_state),
stream_id_1)
# Case with a given default
self.assertEqual(
bookmarks.get_currently_syncing(non_empty_state, 'default_value'),
stream_id_1)