def sync_traditional_stream(conn_config, stream, state, sync_method, end_lsn): LOGGER.info("Beginning sync of stream(%s) with sync method(%s)", stream["tap_stream_id"], sync_method) md_map = metadata.to_map(stream["metadata"]) conn_config["dbname"] = md_map.get(()).get("database-name") desired_columns = [ c for c in stream["schema"]["properties"].keys() if sync_common.should_sync_column(md_map, c) ] desired_columns.sort() if len(desired_columns) == 0: LOGGER.warning( "There are no columns selected for stream %s, skipping it", stream["tap_stream_id"]) return state register_type_adapters(conn_config) if sync_method == "full": state = singer.set_currently_syncing(state, stream["tap_stream_id"]) state = do_sync_full_table(conn_config, stream, state, desired_columns, md_map) elif sync_method == "incremental": state = singer.set_currently_syncing(state, stream["tap_stream_id"]) state = do_sync_incremental(conn_config, stream, state, desired_columns, md_map) elif sync_method == "logical_initial": state = singer.set_currently_syncing(state, stream["tap_stream_id"]) LOGGER.info("Performing initial full table sync") state = singer.write_bookmark(state, stream["tap_stream_id"], "lsn", end_lsn) sync_common.send_schema_message(stream, []) state = full_table.sync_table(conn_config, stream, state, desired_columns, md_map) state = singer.write_bookmark(state, stream["tap_stream_id"], "xmin", None) elif sync_method == "logical_initial_interrupted": state = singer.set_currently_syncing(state, stream["tap_stream_id"]) LOGGER.info( "Initial stage of full table sync was interrupted. resuming...") sync_common.send_schema_message(stream, []) state = full_table.sync_table(conn_config, stream, state, desired_columns, md_map) else: raise Exception("unknown sync method {} for stream {}".format( sync_method, stream["tap_stream_id"])) state = singer.set_currently_syncing(state, None) singer.write_message(singer.StateMessage(value=copy.deepcopy(state))) return state
def sync_method_for_streams(streams, state, default_replication_method): lookup = {} traditional_steams = [] logical_streams = [] for stream in streams: stream_metadata = metadata.to_map(stream["metadata"]) replication_method = stream_metadata.get( (), {}).get("replication-method", default_replication_method) replication_key = stream_metadata.get((), {}).get("replication-key") state = clear_state_on_replication_change(state, stream["tap_stream_id"], replication_key, replication_method) if replication_method not in set( ["LOG_BASED", "FULL_TABLE", "INCREMENTAL"]): raise Exception( "Unrecognized replication_method {} for stream {}".format( replication_method, stream["tap_stream_id"])) md_map = metadata.to_map(stream["metadata"]) desired_columns = [ c for c in stream["schema"]["properties"].keys() if sync_common.should_sync_column(md_map, c) ] desired_columns.sort() if len(desired_columns) == 0: LOGGER.warning( "There are no columns selected for stream %s, skipping it", stream["tap_stream_id"]) continue if replication_method == "LOG_BASED" and stream_metadata.get( (), {}).get("is-view"): raise Exception( "Logical Replication is NOT supported for views." "Please change the replication method for {}".format( stream["tap_stream_id"])) if replication_method == "FULL_TABLE": lookup[stream["tap_stream_id"]] = "full" traditional_steams.append(stream) elif replication_method == "INCREMENTAL": lookup[stream["tap_stream_id"]] = "incremental" traditional_steams.append(stream) elif get_bookmark(state, stream["tap_stream_id"], "xmin") and get_bookmark( state, stream["tap_stream_id"], "lsn"): # finishing previously interrupted full-table (first stage of logical replication) lookup[stream["tap_stream_id"]] = "logical_initial_interrupted" traditional_steams.append(stream) # inconsistent state elif get_bookmark( state, stream["tap_stream_id"], "xmin") and not get_bookmark( state, stream["tap_stream_id"], "lsn"): raise Exception( "Xmin found(%s) in state implying full-table replication but no lsn is present" ) elif not get_bookmark(state, stream["tap_stream_id"], "xmin") and not get_bookmark( state, stream["tap_stream_id"], "lsn"): # initial full-table phase of logical replication lookup[stream["tap_stream_id"]] = "logical_initial" traditional_steams.append(stream) else: # no xmin but we have an lsn # initial stage of logical replication(full-table) has been completed. # Moving onto pure logical replication lookup[stream["tap_stream_id"]] = "pure_logical" logical_streams.append(stream) return lookup, traditional_steams, logical_streams