def log_engine(mysql_conn, catalog_entry):
is_view = common.get_is_view(catalog_entry)
database_name = common.get_database_name(catalog_entry)
if is_view:
LOGGER.info("Beginning sync for view %s.%s", database_name, catalog_entry.table)
else:
with connect_with_backoff(mysql_conn) as open_conn:
with open_conn.cursor() as cur:
cur.execute(
"""
SELECT engine
FROM information_schema.tables
WHERE table_schema = %s
AND table_name = %s
""",
(database_name, catalog_entry.table),
)
row = cur.fetchone()
if row:
LOGGER.info(
"Beginning sync for %s table %s.%s",
row[0],
database_name,
catalog_entry.table,
)
def get_non_binlog_streams(mysql_conn, catalog, config, state):
"""
Returns the Catalog of data we're going to sync for all SELECT-based
streams (i.e. INCREMENTAL, FULL_TABLE, and LOG_BASED that require a historical
sync). LOG_BASED streams that require a historical sync are inferred from lack
of any state.
Using the Catalog provided from the input file, this function will return a
Catalog representing exactly which tables and columns that will be emitted
by SELECT-based syncs. This is achieved by comparing the input Catalog to a
freshly discovered Catalog to determine the resulting Catalog.
The resulting Catalog will include the following any streams marked as
"selected" that currently exist in the database. Columns marked as "selected"
and those labeled "automatic" (e.g. primary keys and replication keys) will be
included. Streams will be prioritized in the following order:
1. currently_syncing if it is SELECT-based
2. any streams that do not have state
3. any streams that do not have a replication method of LOG_BASED
"""
discovered = discover_catalog(mysql_conn, config.get('filter_dbs'))
# Filter catalog to include only selected streams
selected_streams = list(filter(common.stream_is_selected, catalog.streams))
streams_with_state = []
streams_without_state = []
for stream in selected_streams:
stream_metadata = metadata.to_map(stream.metadata)
replication_method = stream_metadata.get((), {}).get('replication-method')
stream_state = state.get('bookmarks', {}).get(stream.tap_stream_id)
if not stream_state:
if replication_method == 'LOG_BASED':
LOGGER.info("LOG_BASED stream %s requires full historical sync", stream.tap_stream_id)
streams_without_state.append(stream)
elif stream_state and replication_method == 'LOG_BASED' and binlog_stream_requires_historical(stream, state):
is_view = common.get_is_view(stream)
if is_view:
raise Exception(
f"Unable to replicate stream({stream.stream}) with binlog because it is a view.")
LOGGER.info("LOG_BASED stream %s will resume its historical sync", stream.tap_stream_id)
streams_with_state.append(stream)
elif stream_state and replication_method != 'LOG_BASED':
streams_with_state.append(stream)
# If the state says we were in the middle of processing a stream, skip
# to that stream. Then process streams without prior state and finally
# move onto streams with state (i.e. have been synced in the past)
currently_syncing = singer.get_currently_syncing(state)
# prioritize streams that have not been processed
ordered_streams = streams_without_state + streams_with_state
if currently_syncing:
currently_syncing_stream = list(filter(
lambda s: s.tap_stream_id == currently_syncing and is_valid_currently_syncing_stream(s, state),
streams_with_state))
non_currently_syncing_streams = list(filter(lambda s: s.tap_stream_id != currently_syncing, ordered_streams))
streams_to_sync = currently_syncing_stream + non_currently_syncing_streams
else:
# prioritize streams that have not been processed
streams_to_sync = ordered_streams
return resolve_catalog(discovered, streams_to_sync)
def do_sync_historical_binlog(mysql_conn, catalog_entry, state, columns, use_gtid: bool, engine: str):
binlog.verify_binlog_config(mysql_conn)
if use_gtid and engine == MYSQL_ENGINE:
binlog.verify_gtid_config(mysql_conn)
is_view = common.get_is_view(catalog_entry)
if is_view:
raise Exception(f"Unable to replicate stream({catalog_entry.stream}) with binlog because it is a view.")
log_file = singer.get_bookmark(state,
catalog_entry.tap_stream_id,
'log_file')
log_pos = singer.get_bookmark(state,
catalog_entry.tap_stream_id,
'log_pos')
gtid = None
if use_gtid:
gtid = singer.get_bookmark(state,
catalog_entry.tap_stream_id,
'gtid')
max_pk_values = singer.get_bookmark(state,
catalog_entry.tap_stream_id,
'max_pk_values')
write_schema_message(catalog_entry)
stream_version = common.get_stream_version(catalog_entry.tap_stream_id, state)
if max_pk_values and ((use_gtid and gtid) or (log_file and log_pos)):
LOGGER.info("Resuming initial full table sync for LOG_BASED stream %s", catalog_entry.tap_stream_id)
full_table.sync_table(mysql_conn, catalog_entry, state, columns, stream_version)
else:
LOGGER.info("Performing initial full table sync for LOG_BASED stream %s", catalog_entry.tap_stream_id)
state = singer.write_bookmark(state,
catalog_entry.tap_stream_id,
'initial_binlog_complete',
False)
current_log_file, current_log_pos = binlog.fetch_current_log_file_and_pos(mysql_conn)
current_gtid = None
if use_gtid:
current_gtid = binlog.fetch_current_gtid_pos(mysql_conn, engine)
state = singer.write_bookmark(state,
catalog_entry.tap_stream_id,
'version',
stream_version)
if full_table.pks_are_auto_incrementing(mysql_conn, catalog_entry):
# We must save log_file, log_pos, gtid across FULL_TABLE syncs when using
# an incrementing PK
state = singer.write_bookmark(state,
catalog_entry.tap_stream_id,
'log_file',
current_log_file)
state = singer.write_bookmark(state,
catalog_entry.tap_stream_id,
'log_pos',
current_log_pos)
if current_gtid:
state = singer.write_bookmark(state,
catalog_entry.tap_stream_id,
'gtid',
current_gtid)
full_table.sync_table(mysql_conn, catalog_entry, state, columns, stream_version)
else:
full_table.sync_table(mysql_conn, catalog_entry, state, columns, stream_version)
state = singer.write_bookmark(state,
catalog_entry.tap_stream_id,
'log_file',
current_log_file)
state = singer.write_bookmark(state,
catalog_entry.tap_stream_id,
'log_pos',
current_log_pos)
if current_gtid:
state = singer.write_bookmark(state,
catalog_entry.tap_stream_id,
'gtid',
current_gtid)
def do_sync_historical_binlog(mysql_conn, config, catalog_entry, state, columns):
binlog.verify_binlog_config(mysql_conn)
is_view = common.get_is_view(catalog_entry)
key_properties = common.get_key_properties(catalog_entry)
if is_view:
raise Exception("Unable to replicate stream({}) with binlog because it is a view.".format(catalog_entry.stream))
log_file = singer.get_bookmark(state,
catalog_entry.tap_stream_id,
'log_file')
log_pos = singer.get_bookmark(state,
catalog_entry.tap_stream_id,
'log_pos')
max_pk_values = singer.get_bookmark(state,
catalog_entry.tap_stream_id,
'max_pk_values')
last_pk_fetched = singer.get_bookmark(state,
catalog_entry.tap_stream_id,
'last_pk_fetched')
write_schema_message(catalog_entry)
stream_version = common.get_stream_version(catalog_entry.tap_stream_id, state)
if log_file and log_pos and max_pk_values:
LOGGER.info("Resuming initial full table sync for LOG_BASED stream %s", catalog_entry.tap_stream_id)
full_table.sync_table(mysql_conn, catalog_entry, state, columns, stream_version)
else:
LOGGER.info("Performing initial full table sync for LOG_BASED stream %s", catalog_entry.tap_stream_id)
state = singer.write_bookmark(state,
catalog_entry.tap_stream_id,
'initial_binlog_complete',
False)
current_log_file, current_log_pos = binlog.fetch_current_log_file_and_pos(mysql_conn)
state = singer.write_bookmark(state,
catalog_entry.tap_stream_id,
'version',
stream_version)
if full_table.sync_is_resumable(mysql_conn, catalog_entry):
# We must save log_file and log_pos across FULL_TABLE syncs when performing
# a resumable full table sync
state = singer.write_bookmark(state,
catalog_entry.tap_stream_id,
'log_file',
current_log_file)
state = singer.write_bookmark(state,
catalog_entry.tap_stream_id,
'log_pos',
current_log_pos)
full_table.sync_table(mysql_conn, catalog_entry, state, columns, stream_version)
else:
full_table.sync_table(mysql_conn, catalog_entry, state, columns, stream_version)
state = singer.write_bookmark(state,
catalog_entry.tap_stream_id,
'log_file',
current_log_file)
state = singer.write_bookmark(state,
catalog_entry.tap_stream_id,
'log_pos',
current_log_pos)
def do_sync_historical_binlog(mysql_conn, catalog_entry, state, columns):
binlog.verify_binlog_config(mysql_conn)
is_view = common.get_is_view(catalog_entry)
if is_view:
raise Exception(
f"Unable to replicate stream({catalog_entry.stream}) with binlog because it is a view."
)
log_file = singer.get_bookmark(state, catalog_entry.tap_stream_id, "log_file")
log_pos = singer.get_bookmark(state, catalog_entry.tap_stream_id, "log_pos")
max_pk_values = singer.get_bookmark(
state, catalog_entry.tap_stream_id, "max_pk_values"
)
write_schema_message(catalog_entry)
stream_version = common.get_stream_version(catalog_entry.tap_stream_id, state)
if log_file and log_pos and max_pk_values:
LOGGER.info(
"Resuming initial full table sync for LOG_BASED stream %s",
catalog_entry.tap_stream_id,
)
full_table.sync_table(mysql_conn, catalog_entry, state, columns, stream_version)
else:
LOGGER.info(
"Performing initial full table sync for LOG_BASED stream %s",
catalog_entry.tap_stream_id,
)
state = singer.write_bookmark(
state, catalog_entry.tap_stream_id, "initial_binlog_complete", False
)
current_log_file, current_log_pos = binlog.fetch_current_log_file_and_pos(
mysql_conn
)
state = singer.write_bookmark(
state, catalog_entry.tap_stream_id, "version", stream_version
)
if full_table.pks_are_auto_incrementing(mysql_conn, catalog_entry):
# We must save log_file and log_pos across FULL_TABLE syncs when using
# an incrementing PK
state = singer.write_bookmark(
state, catalog_entry.tap_stream_id, "log_file", current_log_file
)
state = singer.write_bookmark(
state, catalog_entry.tap_stream_id, "log_pos", current_log_pos
)
full_table.sync_table(
mysql_conn, catalog_entry, state, columns, stream_version
)
else:
full_table.sync_table(
mysql_conn, catalog_entry, state, columns, stream_version
)
state = singer.write_bookmark(
state, catalog_entry.tap_stream_id, "log_file", current_log_file
)
state = singer.write_bookmark(
state, catalog_entry.tap_stream_id, "log_pos", current_log_pos
)