def validate_dolt_as_source(db_conn, db_table, get_db_target_writer, dolt_repo, dolt_table):
"""
Verifies that given a Dolt repository that has a a series of updates applied to it (defined in the fixture
create_dolt_test_data_commits) that after syncing at each of the commits, the Dolt repository and the target
relational database, so far MySQL or Postgres server instance contain the same data. Tests creates, updates, and
deletes.
:param db_conn:
:param db_table:
:param get_db_target_writer:
:param dolt_repo:
:param dolt_table:
:return:
"""
target_writer = get_db_target_writer(db_conn, True)
table_mapping = {str(dolt_table.name): str(db_table.name)}
commits = list(dolt_repo.log().keys())
commits_to_check = [commits[0], commits[1], commits[2], commits[3], commits[4]]
commits_to_check.reverse()
for i, commit in enumerate(commits_to_check):
logger.info('Syncing from Dolt at commit {}, {} of {}'.format(commit, i, len(commits_to_check)))
table_reader = get_dolt_table_reader_diffs(commit)
sync_from_dolt(get_dolt_source_reader(dolt_repo, table_reader), target_writer, table_mapping)
db_data = get_data_for_comparison(db_conn)
_, dolt_data = get_dolt_table_reader(commit)(str(dolt_table.name), dolt_repo)
assert assert_rows_equal(db_data, list(dolt_data))
def assertion_helper(commit: str, expected_diff: List[dict]):
"""
Validates that both the HEAD of the current branch of the Dolt repo match MySQL, and that the difIfs created by
the write match what is expected.
"""
_, dolt_data = get_dolt_table_reader(commit)(str(dolt_table.name), dssc)
db_table_metadata = get_table_metadata(db_engine, str(db_table.name))
db_data = get_db_table_reader()(db_engine, db_table_metadata)
assert_rows_equal(list(dolt_data), db_data, datetime_strict=datetime_strict)
_, dolt_diff_data = get_dolt_table_reader_diffs(commit)(str(dolt_table.name), dssc)
assert_rows_equal(expected_diff, list(dolt_diff_data), datetime_strict=datetime_strict)
def test_postgres_to_dolt_array_types(postgres_with_table_with_arrays,
db_with_table_with_arrays):
postgres_engine, postgres_table = postgres_with_table_with_arrays
dssc, dolt_table = db_with_table_with_arrays
get_postgres_target_writer(postgres_engine)({
str(postgres_table.name): ([], TEST_DATA_WITH_ARRAYS)
})
source_reader = get_source_reader(postgres_engine, get_table_reader())
target_writer = get_dolt_target_writer(dssc, commit=True)
sync_to_dolt(source_reader, target_writer,
{str(postgres_table.name): str(dolt_table.name)})
latest_commit = list(dssc.dolt.log().keys())[0]
_, dolt_data = get_dolt_table_reader(latest_commit)(str(dolt_table.name),
dssc)
db_table_metadata = get_table_metadata(postgres_engine,
str(postgres_table.name))
db_data = get_table_reader()(postgres_engine, db_table_metadata)
clean_dolt_data = deserialize_longtext(dolt_data)
assert_rows_equal(clean_dolt_data, db_data, lambda dic: dic['id'])
def validate_dolt_as_target(db_engine: Engine,
db_table: Table,
get_db_source_reader,
get_db_target_writer,
get_db_table_reader,
dssc,
dolt_table,
datetime_strict: bool = True):
"""
Validates syncing from a relational database, so far MySQL and Postgres, to Dolt. Work by making a series of writes
to the relational database (running in a Docker container provided by a fixture), executing a sync, and then
validating the HEAD of master of the Dolt repo has the expected values. It also validates that the Dolt history is
correct after every write. Finally validates that deletes flow through to Dolt.
:param db_engine:
:param db_table:
:param get_db_source_reader:
:param get_db_target_writer:
:param get_db_table_reader:
:param dssc:
:param dolt_table:
:param datetime_strict:
:return:
"""
def sync_to_dolt_helper():
source_reader = get_db_source_reader(db_engine, get_db_table_reader())
target_writer = get_dolt_target_writer(dssc, commit=True)
sync_to_dolt(source_reader, target_writer, {str(db_table.name): str(dolt_table.name)})
def assertion_helper(commit: str, expected_diff: List[dict]):
"""
Validates that both the HEAD of the current branch of the Dolt repo match MySQL, and that the difIfs created by
the write match what is expected.
"""
_, dolt_data = get_dolt_table_reader(commit)(str(dolt_table.name), dssc)
db_table_metadata = get_table_metadata(db_engine, str(db_table.name))
db_data = get_db_table_reader()(db_engine, db_table_metadata)
assert_rows_equal(list(dolt_data), db_data, datetime_strict=datetime_strict)
_, dolt_diff_data = get_dolt_table_reader_diffs(commit)(str(dolt_table.name), dssc)
assert_rows_equal(expected_diff, list(dolt_diff_data), datetime_strict=datetime_strict)
update_sequence = [
TEST_DATA_INITIAL,
TEST_DATA_APPEND_MULTIPLE_ROWS,
TEST_DATA_APPEND_SINGLE_ROW,
TEST_DATA_UPDATE_SINGLE_ROW
]
for update_data in update_sequence:
get_db_target_writer(db_engine)({str(db_table.name): ([], update_data)})
sync_to_dolt_helper()
latest_commit = list(dssc.log().keys())[0]
assertion_helper(latest_commit, update_data)
with db_engine.connect() as conn:
conn.execute(db_table.delete().where(db_table.c.first_name == 'Novak'))
sync_to_dolt_helper()
latest_commit = list(dssc.log().keys())[0]
_, dolt_data = get_dolt_table_reader(latest_commit)(str(dolt_table.name), dssc)
db_data = get_db_table_reader()(db_engine, db_table)
assert_rows_equal(list(dolt_data), db_data, datetime_strict=datetime_strict)
dropped_pks, _ = get_dolt_table_reader_diffs(latest_commit)(str(dolt_table.name), dssc)
assert dropped_pks == [{'first_name': 'Novak', 'last_name': 'Djokovic'}]