def do_sync_logical_replication(conn_config, stream, state, desired_columns, md_map): if get_bookmark(state, stream.tap_stream_id, 'lsn'): LOGGER.info("Stream %s is using logical replication. end lsn %s", stream.tap_stream_id, logical_replication.fetch_current_lsn(conn_config)) logical_replication.add_automatic_properties(stream) send_schema_message(stream, ['lsn']) state = logical_replication.sync_table(conn_config, stream, state, desired_columns, md_map) else: #start off with full-table replication end_lsn = logical_replication.fetch_current_lsn(conn_config) LOGGER.info( "Stream %s is using logical replication. performing initial full table sync", stream.tap_stream_id) send_schema_message(stream, []) state = full_table.sync_table(conn_config, stream, state, desired_columns, md_map) state = singer.write_bookmark(state, stream.tap_stream_id, 'xmin', None) #once we are done with full table, write the lsn to the state state = singer.write_bookmark(state, stream.tap_stream_id, 'lsn', end_lsn) return state
def do_sync_logical_replication(conn_config, stream, state, desired_columns, md_map): LOGGER.info("Stream %s is using logical replication", stream.tap_stream_id) if get_bookmark(state, stream.tap_stream_id, 'xmin') and get_bookmark( state, stream.tap_stream_id, 'lsn'): #finishing previously interrupted full-table (first stage of logical replication) LOGGER.info( "Initial stage of full table sync was interrupted. resuming...") send_schema_message(stream, []) state = full_table.sync_table(conn_config, stream, state, desired_columns, md_map) state = singer.write_bookmark(state, stream.tap_stream_id, 'xmin', None) state = singer.write_bookmark(state, stream.tap_stream_id, 'initial_logical_replication_complete', False) #inconsistent state elif get_bookmark( state, stream.tap_stream_id, 'xmin') and not get_bookmark(state, stream.tap_stream_id, 'lsn'): raise Exception( "Xmin found(%s) in state implying full-table replication but no lsn is present" ) elif not get_bookmark(state, stream.tap_stream_id, 'xmin') and not get_bookmark( state, stream.tap_stream_id, 'lsn'): #initial full-table phase of logical replication end_lsn = logical_replication.fetch_current_lsn(conn_config) LOGGER.info("Performing initial full table sync") state = singer.write_bookmark(state, stream.tap_stream_id, 'lsn', end_lsn) send_schema_message(stream, []) state = full_table.sync_table(conn_config, stream, state, desired_columns, md_map) state = singer.write_bookmark(state, stream.tap_stream_id, 'xmin', None) state = singer.write_bookmark(state, stream.tap_stream_id, 'initial_logical_replication_complete', False) elif not get_bookmark(state, stream.tap_stream_id, 'xmin') and get_bookmark( state, stream.tap_stream_id, 'lsn'): #initial stage of logical replication(full-table) has been completed. moving onto pure logical replication LOGGER.info("Pure Logical Replication upto lsn %s", logical_replication.fetch_current_lsn(conn_config)) logical_replication.add_automatic_properties(stream) send_schema_message(stream, ['lsn']) state = logical_replication.sync_table(conn_config, stream, state, desired_columns, md_map) return state
def do_sync(conn_config, catalog, default_replication_method, state): currently_syncing = singer.get_currently_syncing(state) streams = list(filter(is_selected_via_metadata, catalog['streams'])) streams.sort(key=lambda s: s['tap_stream_id']) LOGGER.info("Selected streams: %s ", list(map(lambda s: s['tap_stream_id'], streams))) if any_logical_streams(streams, default_replication_method): LOGGER.info("Use of logical replication requires fetching an lsn...") end_lsn = logical_replication.fetch_current_lsn(conn_config) LOGGER.info("End LSN: %s ", end_lsn) else: end_lsn = None sync_method_lookup, traditional_streams, logical_streams = sync_method_for_streams(streams, state, default_replication_method) if currently_syncing: LOGGER.info("found currently_syncing: %s", currently_syncing) currently_syncing_stream = list(filter(lambda s: s['tap_stream_id'] == currently_syncing, traditional_streams)) if currently_syncing_stream is None: LOGGER.warning("unable to locate currently_syncing(%s) amongst selected traditional streams(%s). will ignore", currently_syncing, list(map(lambda s: s['tap_stream_id'], traditional_streams))) other_streams = list(filter(lambda s: s['tap_stream_id'] != currently_syncing, traditional_streams)) traditional_streams = currently_syncing_stream + other_streams else: LOGGER.info("No currently_syncing found") for stream in traditional_streams: state = sync_traditional_stream(conn_config, stream, state, sync_method_lookup[stream['tap_stream_id']], end_lsn) logical_streams.sort(key=lambda s: metadata.to_map(s['metadata']).get(()).get('database-name')) for dbname, streams in itertools.groupby(logical_streams, lambda s: metadata.to_map(s['metadata']).get(()).get('database-name')): conn_config['dbname'] = dbname state = sync_logical_streams(conn_config, list(streams), state, end_lsn) return state
def do_sync(conn_config, catalog, default_replication_method, state, state_file=None): """ Orchestrates sync of all streams """ currently_syncing = singer.get_currently_syncing(state) streams = list(filter(is_selected_via_metadata, catalog['streams'])) streams.sort(key=lambda s: s['tap_stream_id']) LOGGER.info("Selected streams: %s ", [s['tap_stream_id'] for s in streams]) if any_logical_streams(streams, default_replication_method): # Use of logical replication requires fetching an lsn end_lsn = logical_replication.fetch_current_lsn(conn_config) LOGGER.debug("end_lsn = %s ", end_lsn) else: end_lsn = None refresh_streams_schema(conn_config, streams) sync_method_lookup, traditional_streams, logical_streams = \ sync_method_for_streams(streams, state, default_replication_method) if currently_syncing: LOGGER.debug("Found currently_syncing: %s", currently_syncing) currently_syncing_stream = list( filter(lambda s: s['tap_stream_id'] == currently_syncing, traditional_streams)) if not currently_syncing_stream: LOGGER.warning( "unable to locate currently_syncing(%s) amongst selected traditional streams(%s). " "Will ignore", currently_syncing, {s['tap_stream_id'] for s in traditional_streams}) other_streams = list( filter(lambda s: s['tap_stream_id'] != currently_syncing, traditional_streams)) traditional_streams = currently_syncing_stream + other_streams else: LOGGER.info("No streams marked as currently_syncing in state file") for stream in traditional_streams: state = sync_traditional_stream( conn_config, stream, state, sync_method_lookup[stream['tap_stream_id']], end_lsn) logical_streams.sort(key=lambda s: metadata.to_map(s['metadata']).get( ()).get('database-name')) for dbname, streams in itertools.groupby( logical_streams, lambda s: metadata.to_map(s['metadata']).get( ()).get('database-name')): conn_config['dbname'] = dbname state = sync_logical_streams(conn_config, list(streams), state, end_lsn, state_file) return state
def sync_logical_streams(conn_config, logical_streams, state): if logical_streams: LOGGER.info("Pure Logical Replication upto lsn %s for (%s)", logical_replication.fetch_current_lsn(conn_config), list(map(lambda s: s['tap_stream_id'], logical_streams))) logical_streams = list(map(logical_replication.add_automatic_properties, logical_streams)) state = logical_replication.sync_tables(conn_config, logical_streams, state) return state
def sync_traditional_stream(conn_config, stream, state, sync_method): LOGGER.info("Beginning sync of stream(%s) with sync method(%s)", stream['tap_stream_id'], sync_method) md_map = metadata.to_map(stream['metadata']) conn_config['dbname'] = md_map.get(()).get('database-name') desired_columns = [ c for c in stream['schema']['properties'].keys() if should_sync_column(md_map, c) ] desired_columns.sort() if len(desired_columns) == 0: LOGGER.warning( 'There are no columns selected for stream %s, skipping it', stream['tap_stream_id']) return state register_type_adapters(conn_config) if sync_method == 'full': state = singer.set_currently_syncing(state, stream['tap_stream_id']) state = do_sync_full_table(conn_config, stream, state, desired_columns, md_map) elif sync_method == 'incremental': state = singer.set_currently_syncing(state, stream['tap_stream_id']) state = do_sync_incremental(conn_config, stream, state, desired_columns, md_map) elif sync_method == 'logical_initial': state = singer.set_currently_syncing(state, stream['tap_stream_id']) end_lsn = logical_replication.fetch_current_lsn(conn_config) LOGGER.info("Performing initial full table sync") state = singer.write_bookmark(state, stream['tap_stream_id'], 'lsn', end_lsn) sync_common.send_schema_message(stream, []) state = full_table.sync_table(conn_config, stream, state, desired_columns, md_map) state = singer.write_bookmark(state, stream['tap_stream_id'], 'xmin', None) elif sync_method == 'logical_initial_interrupted': state = singer.set_currently_syncing(state, stream['tap_stream_id']) LOGGER.info( "Initial stage of full table sync was interrupted. resuming...") sync_common.send_schema_message(stream, []) state = full_table.sync_table(conn_config, stream, state, desired_columns, md_map) else: raise Exception("unknown sync method {} for stream {}".format( sync_method, stream['tap_stream_id'])) state = singer.set_currently_syncing(state, None) singer.write_message(singer.StateMessage(value=copy.deepcopy(state))) return state