def test_empty_state(self): empty_state = {} # Case with no value to fall back on self.assertIsNone(bookmarks.get_currently_syncing(empty_state)) # Case with a given default self.assertEqual( bookmarks.get_currently_syncing(empty_state, 'default_value'), 'default_value')
def validate_state(config, catalog, state): for stream in catalog["streams"]: for mdata in stream['metadata']: if mdata['breadcrumb'] == [] and mdata['metadata'].get('selected') != True: # If a stream is deselected while it's the current stream, unset the # current stream. if stream["tap_stream_id"] == get_currently_syncing(state): set_currently_syncing(state, None) break replication_key = determine_replication_key(stream['tap_stream_id']) if not replication_key: continue # If there's no bookmark for a stream (new integration, newly selected, # reset, etc) we need to use the default start date from the config. bookmark = get_bookmark(state, stream["tap_stream_id"], replication_key) if bookmark is None: state = write_bookmark(state, stream["tap_stream_id"], replication_key, config["start_date"]) singer.write_state(state) return state
def sync(client, catalog, config, state): starting_stream = bookmarks.get_currently_syncing(state) if starting_stream: singer.log_info("Resuming sync from %s", starting_stream) else: singer.log_info("Starting sync") for stream in catalog["streams"]: # Skip unselected streams. mdata = metadata.to_map(stream['metadata']) if not metadata.get(mdata, (), 'selected'): singer.log_info("%s: not selected", stream["tap_stream_id"]) continue # Skip streams that have already be synced when resuming. if starting_stream and stream["tap_stream_id"] != starting_stream: singer.log_info("%s: already synced", stream["tap_stream_id"]) continue singer.log_info("%s: starting sync", stream["tap_stream_id"]) # Now that we've started, there's no more "starting stream". Set # the current stream to resume on next run. starting_stream = None state = bookmarks.set_currently_syncing(state, stream["tap_stream_id"]) singer.write_state(state) # Sync stream based on type. if stream["tap_stream_id"] == "activity_types": state, record_count = sync_activity_types(client, state, stream) elif stream["tap_stream_id"] == "leads": state, record_count = sync_leads(client, state, stream, config) elif stream["tap_stream_id"].startswith("activities_"): state, record_count = sync_activities(client, state, stream, config) elif stream["tap_stream_id"] in ["campaigns", "lists"]: state, record_count = sync_paginated(client, state, stream) elif stream["tap_stream_id"] == "programs": state, record_count = sync_programs(client, state, stream) else: raise Exception("Stream %s not implemented" % stream["tap_stream_id"]) # Emit metric for record count. counter = singer.metrics.record_counter(stream["tap_stream_id"]) counter.value = record_count counter._pop() # pylint: disable=protected-access # Unset current stream. state = bookmarks.set_currently_syncing(state, None) singer.write_state(state) singer.log_info("%s: finished sync", stream["tap_stream_id"]) # If Corona is not supported, log a warning near the end of the tap # log with instructions on how to get Corona supported. singer.log_info("Finished sync.") if not client.use_corona: singer.log_warning(NO_CORONA_WARNING)
def test_non_empty_state(self): stream_id_1 = 'customers' bookmark_key_1 = 'datetime' bookmark_val_1 = 123456789 offset_val = 'fizzy water' non_empty_state = { 'bookmarks': { stream_id_1: { bookmark_key_1: bookmark_val_1, 'offset': offset_val } }, 'currently_syncing': stream_id_1 } # Case with no value to fall back on self.assertEqual(bookmarks.get_currently_syncing(non_empty_state), stream_id_1) # Case with a given default self.assertEqual( bookmarks.get_currently_syncing(non_empty_state, 'default_value'), stream_id_1)