def sync_traditional_stream(conn_config, stream, state, sync_method, end_lsn): """ Sync INCREMENTAL and FULL_TABLE streams """ LOGGER.info("Beginning sync of stream(%s) with sync method(%s)", stream['tap_stream_id'], sync_method) md_map = metadata.to_map(stream['metadata']) conn_config['dbname'] = md_map.get(()).get('database-name') desired_columns = [ c for c in stream['schema']['properties'].keys() if sync_common.should_sync_column(md_map, c) ] desired_columns.sort() if len(desired_columns) == 0: LOGGER.warning( 'There are no columns selected for stream %s, skipping it', stream['tap_stream_id']) return state register_type_adapters(conn_config) if sync_method == 'full': state = singer.set_currently_syncing(state, stream['tap_stream_id']) state = do_sync_full_table(conn_config, stream, state, desired_columns, md_map) elif sync_method == 'incremental': state = singer.set_currently_syncing(state, stream['tap_stream_id']) state = do_sync_incremental(conn_config, stream, state, desired_columns, md_map) elif sync_method == 'logical_initial': state = singer.set_currently_syncing(state, stream['tap_stream_id']) LOGGER.info("Performing initial full table sync") state = singer.write_bookmark(state, stream['tap_stream_id'], 'lsn', end_lsn) sync_common.send_schema_message(stream, []) state = full_table.sync_table(conn_config, stream, state, desired_columns, md_map) state = singer.write_bookmark(state, stream['tap_stream_id'], 'xmin', None) elif sync_method == 'logical_initial_interrupted': state = singer.set_currently_syncing(state, stream['tap_stream_id']) LOGGER.info( "Initial stage of full table sync was interrupted. resuming...") sync_common.send_schema_message(stream, []) state = full_table.sync_table(conn_config, stream, state, desired_columns, md_map) else: raise Exception( f"unknown sync method {sync_method} for stream {stream['tap_stream_id']}" ) state = singer.set_currently_syncing(state, None) singer.write_message(singer.StateMessage(value=copy.deepcopy(state))) return state
def do_sync_logical_replication(conn_config, stream, state, desired_columns, md_map): LOGGER.info("Stream %s is using logical replication", stream.tap_stream_id) if get_bookmark(state, stream.tap_stream_id, 'xmin') and get_bookmark( state, stream.tap_stream_id, 'lsn'): #finishing previously interrupted full-table (first stage of logical replication) LOGGER.info( "Initial stage of full table sync was interrupted. resuming...") send_schema_message(stream, []) state = full_table.sync_table(conn_config, stream, state, desired_columns, md_map) state = singer.write_bookmark(state, stream.tap_stream_id, 'xmin', None) state = singer.write_bookmark(state, stream.tap_stream_id, 'initial_logical_replication_complete', False) #inconsistent state elif get_bookmark( state, stream.tap_stream_id, 'xmin') and not get_bookmark(state, stream.tap_stream_id, 'lsn'): raise Exception( "Xmin found(%s) in state implying full-table replication but no lsn is present" ) elif not get_bookmark(state, stream.tap_stream_id, 'xmin') and not get_bookmark( state, stream.tap_stream_id, 'lsn'): #initial full-table phase of logical replication end_lsn = logical_replication.fetch_current_lsn(conn_config) LOGGER.info("Performing initial full table sync") state = singer.write_bookmark(state, stream.tap_stream_id, 'lsn', end_lsn) send_schema_message(stream, []) state = full_table.sync_table(conn_config, stream, state, desired_columns, md_map) state = singer.write_bookmark(state, stream.tap_stream_id, 'xmin', None) state = singer.write_bookmark(state, stream.tap_stream_id, 'initial_logical_replication_complete', False) elif not get_bookmark(state, stream.tap_stream_id, 'xmin') and get_bookmark( state, stream.tap_stream_id, 'lsn'): #initial stage of logical replication(full-table) has been completed. moving onto pure logical replication LOGGER.info("Pure Logical Replication upto lsn %s", logical_replication.fetch_current_lsn(conn_config)) logical_replication.add_automatic_properties(stream) send_schema_message(stream, ['lsn']) state = logical_replication.sync_table(conn_config, stream, state, desired_columns, md_map) return state
def sync_traditional_stream(conn_config, stream, state, sync_method, end_lsn): LOGGER.info("Beginning sync of stream(%s) with sync method(%s)", stream["tap_stream_id"], sync_method) md_map = metadata.to_map(stream["metadata"]) conn_config["dbname"] = md_map.get(()).get("database-name") desired_columns = [ c for c in stream["schema"]["properties"].keys() if sync_common.should_sync_column(md_map, c) ] desired_columns.sort() if len(desired_columns) == 0: LOGGER.warning( "There are no columns selected for stream %s, skipping it", stream["tap_stream_id"]) return state register_type_adapters(conn_config) if sync_method == "full": state = singer.set_currently_syncing(state, stream["tap_stream_id"]) state = do_sync_full_table(conn_config, stream, state, desired_columns, md_map) elif sync_method == "incremental": state = singer.set_currently_syncing(state, stream["tap_stream_id"]) state = do_sync_incremental(conn_config, stream, state, desired_columns, md_map) elif sync_method == "logical_initial": state = singer.set_currently_syncing(state, stream["tap_stream_id"]) LOGGER.info("Performing initial full table sync") state = singer.write_bookmark(state, stream["tap_stream_id"], "lsn", end_lsn) sync_common.send_schema_message(stream, []) state = full_table.sync_table(conn_config, stream, state, desired_columns, md_map) state = singer.write_bookmark(state, stream["tap_stream_id"], "xmin", None) elif sync_method == "logical_initial_interrupted": state = singer.set_currently_syncing(state, stream["tap_stream_id"]) LOGGER.info( "Initial stage of full table sync was interrupted. resuming...") sync_common.send_schema_message(stream, []) state = full_table.sync_table(conn_config, stream, state, desired_columns, md_map) else: raise Exception("unknown sync method {} for stream {}".format( sync_method, stream["tap_stream_id"])) state = singer.set_currently_syncing(state, None) singer.write_message(singer.StateMessage(value=copy.deepcopy(state))) return state
def do_sync_logical_replication(conn_config, stream, state, desired_columns, md_map): if get_bookmark(state, stream.tap_stream_id, 'lsn'): LOGGER.info("Stream %s is using logical replication. end lsn %s", stream.tap_stream_id, logical_replication.fetch_current_lsn(conn_config)) logical_replication.add_automatic_properties(stream) send_schema_message(stream, ['lsn']) state = logical_replication.sync_table(conn_config, stream, state, desired_columns, md_map) else: #start off with full-table replication end_lsn = logical_replication.fetch_current_lsn(conn_config) LOGGER.info( "Stream %s is using logical replication. performing initial full table sync", stream.tap_stream_id) send_schema_message(stream, []) state = full_table.sync_table(conn_config, stream, state, desired_columns, md_map) state = singer.write_bookmark(state, stream.tap_stream_id, 'xmin', None) #once we are done with full table, write the lsn to the state state = singer.write_bookmark(state, stream.tap_stream_id, 'lsn', end_lsn) return state
def do_sync_full_table(conn_config, stream, state, desired_columns, md_map): LOGGER.info("Stream %s is using full_table replication", stream['tap_stream_id']) sync_common.send_schema_message(stream, []) if md_map.get((), {}).get('is-view'): state = full_table.sync_view(conn_config, stream, state, desired_columns, md_map) else: state = full_table.sync_table(conn_config, stream, state, desired_columns, md_map) return state