def validate_drop_primary_keys(engine: Engine, table: Table):
"""
Verify that dropping a primary key from using drop_primary_keys leaves a relational database in the required state.
:param engine:
:param table:
:return:
"""
with engine.connect() as conn:
conn.execute(table.insert(), TEST_DATA_APPEND_MULTIPLE_ROWS)
pks_to_drop = [{'first_name': 'Stefanos', 'last_name': 'Tsitsipas'}]
drop_primary_keys(engine, table, pks_to_drop)
result = get_data_for_comparison(engine)
assert_rows_equal(TEST_DATA_APPEND_MULTIPLE_ROWS_WITH_DELETE, result)
def assertion_helper(commit: str, expected_diff: List[dict]):
"""
Validates that both the HEAD of the current branch of the Dolt repo match MySQL, and that the diffs created by
the write match what is expected.
"""
_, dolt_data = get_dolt_table_reader(commit)(str(dolt_table.name),
dolt_repo)
db_table_metadata = get_table_metadata(db_engine, str(db_table.name))
db_data = get_db_table_reader()(db_engine, db_table_metadata)
assert_rows_equal(list(dolt_data), db_data)
_, dolt_diff_data = get_dolt_table_reader_diffs(commit)(str(
dolt_table.name), dolt_repo)
assert_rows_equal(expected_diff, list(dolt_diff_data))
def validate_dolt_as_source(db_conn, db_table, get_db_target_writer, dolt_repo,
dolt_table):
"""
Verifies that given a Dolt repository that has a a series of updates applied to it (defined in the fixture
create_dolt_test_data_commits) that after syncing at each of the commits, the Dolt repository and the target
relational database, so far MySQL or Postgres server instance contain the same data. Tests creates, updates, and
deletes.
:param db_conn:
:param db_table:
:param get_db_target_writer:
:param dolt_repo:
:param dolt_table:
:return:
"""
target_writer = get_db_target_writer(db_conn, True)
table_mapping = {str(dolt_table.name): str(db_table.name)}
commits = list(dolt_repo.log().keys())
commits_to_check = [
commits[0], commits[1], commits[2], commits[3], commits[4]
]
commits_to_check.reverse()
for i, commit in enumerate(commits_to_check):
logger.info('Syncing from Dolt at commit {}, {} of {}'.format(
commit, i, len(commits_to_check)))
table_reader = get_dolt_table_reader_diffs(commit)
sync_from_dolt(get_dolt_source_reader(dolt_repo, table_reader),
target_writer, table_mapping)
db_data = get_data_for_comparison(db_conn)
_, dolt_data = get_dolt_table_reader(commit)(str(dolt_table.name),
dolt_repo)
assert assert_rows_equal(db_data, list(dolt_data))
def _test_dolt_table_reader_helper(repo: Dolt,
table: Table,
build_table_reader: Callable[[str], Callable[[str, Dolt], DoltTableUpdate]],
get_expected: Callable[[int], Tuple[List[dict], List[dict]]]):
commits = list(repo.log().keys())
update_to_commit = {
FIRST_UPDATE: commits[4],
SECOND_UPDATE: commits[3],
THIRD_UPDATE: commits[2],
FOURTH_UPDATE: commits[1],
FIFTH_UPDATE: commits[0]
}
for update_num, commit in update_to_commit.items():
logger.info('comparison for commit/update_num {}/{}'.format(commit, update_num))
dropped_pks, dolt_data = build_table_reader(commit)(str(table.name), repo)
expected_dropped_pks, expected_data = get_expected(update_num)
assert expected_dropped_pks == dropped_pks
assert_rows_equal(expected_data, list(dolt_data))
def test_get_target_writer(repo_with_table):
"""
When writing to Dolt from a relational database we want to replicate the state of the database at each commit, since
the database itself stores no history, we must read the entire dataset each time, and delete appropriate PKs
:param repo_with_table:
:return:
"""
repo, dolt_table = repo_with_table
update_sequence = [
TEST_DATA_INITIAL, TEST_DATA_INITIAL + TEST_DATA_APPEND_MULTIPLE_ROWS,
TEST_DATA_APPEND_MULTIPLE_ROWS + TEST_DATA_APPEND_SINGLE_ROW,
TEST_DATA_APPEND_MULTIPLE_ROWS + TEST_DATA_UPDATE_SINGLE_ROW
]
for i, update in enumerate(update_sequence):
logger.info('Making {} of {} updates and validating'.format(
i, len(update_sequence)))
get_target_writer(repo, commit=True)({str(dolt_table.name): update})
result = _dolt_table_read_helper(repo, str(dolt_table.name))
assert_rows_equal(update, result)
def test_postgres_to_dolt_array_types(postgres_with_table_with_arrays,
repo_with_table_with_arrays):
postgres_engine, postgres_table = postgres_with_table_with_arrays
dolt_repo, dolt_table = repo_with_table_with_arrays
get_postgres_target_writer(postgres_engine)({
str(postgres_table.name): ([], TEST_DATA_WITH_ARRAYS)
})
source_reader = get_source_reader(postgres_engine, get_table_reader())
target_writer = get_dolt_target_writer(dolt_repo, commit=True)
sync_to_dolt(source_reader, target_writer,
{str(postgres_table.name): str(dolt_table.name)})
latest_commit = list(dolt_repo.log().keys())[0]
_, dolt_data = get_dolt_table_reader(latest_commit)(str(dolt_table.name),
dolt_repo)
db_table_metadata = get_table_metadata(postgres_engine,
str(postgres_table.name))
db_data = get_table_reader()(postgres_engine, db_table_metadata)
clean_dolt_data = deserialize_longtext(dolt_data)
assert_rows_equal(clean_dolt_data, db_data, lambda dic: dic['id'])
def validate_dolt_as_target(db_engine: Engine,
db_table: Table,
get_db_source_reader,
get_db_target_writer,
get_db_table_reader,
dolt_repo,
dolt_table,
datetime_strict: bool = True):
"""
Validates syncing from a relational database, so far MySQL and Postgres, to Dolt. Work by making a series of writes
to the relational database (running in a Docker container provided by a fixture), executing a sync, and then
validating the HEAD of master of the Dolt repo has the expected values. It also validates that the Dolt history is
correct after every write. Finally validates that deletes flow through to Dolt.
:param db_engine:
:param db_table:
:param get_db_source_reader:
:param get_db_target_writer:
:param get_db_table_reader:
:param dolt_repo:
:param dolt_table:
:param datetime_strict:
:return:
"""
def sync_to_dolt_helper():
source_reader = get_db_source_reader(db_engine, get_db_table_reader())
target_writer = get_dolt_target_writer(dolt_repo, commit=True)
sync_to_dolt(source_reader, target_writer,
{str(db_table.name): str(dolt_table.name)})
def assertion_helper(commit: str, expected_diff: List[dict]):
"""
Validates that both the HEAD of the current branch of the Dolt repo match MySQL, and that the diffs created by
the write match what is expected.
"""
_, dolt_data = get_dolt_table_reader(commit)(str(dolt_table.name),
dolt_repo)
db_table_metadata = get_table_metadata(db_engine, str(db_table.name))
db_data = get_db_table_reader()(db_engine, db_table_metadata)
assert_rows_equal(list(dolt_data),
db_data,
datetime_strict=datetime_strict)
_, dolt_diff_data = get_dolt_table_reader_diffs(commit)(str(
dolt_table.name), dolt_repo)
assert_rows_equal(expected_diff,
list(dolt_diff_data),
datetime_strict=datetime_strict)
update_sequence = [
TEST_DATA_INITIAL, TEST_DATA_APPEND_MULTIPLE_ROWS,
TEST_DATA_APPEND_SINGLE_ROW, TEST_DATA_UPDATE_SINGLE_ROW
]
for update_data in update_sequence:
get_db_target_writer(db_engine)({
str(db_table.name): ([], update_data)
})
sync_to_dolt_helper()
latest_commit = list(dolt_repo.log().keys())[0]
assertion_helper(latest_commit, update_data)
with db_engine.connect() as conn:
conn.execute(db_table.delete().where(db_table.c.first_name == 'Novak'))
sync_to_dolt_helper()
latest_commit = list(dolt_repo.log().keys())[0]
_, dolt_data = get_dolt_table_reader(latest_commit)(str(dolt_table.name),
dolt_repo)
db_data = get_db_table_reader()(db_engine, db_table)
assert_rows_equal(list(dolt_data),
db_data,
datetime_strict=datetime_strict)
dropped_pks, _ = get_dolt_table_reader_diffs(latest_commit)(str(
dolt_table.name), dolt_repo)
assert dropped_pks == [{'first_name': 'Novak', 'last_name': 'Djokovic'}]
def _write_and_diff_helper(data: List[dict], update_num: int):
get_target_writer(engine, True)({str(table.name): ([], data)})
result = get_data_for_comparison(engine)
_, expected_data = get_expected_data(update_num)
assert_rows_equal(expected_data, result)