def delta_add_check_and_disable_option_common(db_conn, tmpdir, mocked_config,
logger, importer_name,
historic_tbl_name):
"""Common code to verify delta add check and CLI option to disable check.
1) Verify that import fails if the check is enabled and the record to add is already in db.
2) Verify '--disable-delta-adds-check' CLI option and verify that, if disabled, the import succeeds.
"""
imei_norm_one = '12345678901243'
imei_norm_three = '32345678901234'
imei_norm_pk = 'imei_norm'
if importer_name == 'golden_list':
imei_norm_one = imeis_md5_hashing_uuid(imei_norm_one,
convert_to_uuid=True)
imei_norm_three = imeis_md5_hashing_uuid(imei_norm_three,
convert_to_uuid=True)
imei_norm_pk = 'hashed_imei_norm'
# populate historic_tbl
imei_norm_to_insert_list = [imei_norm_one, imei_norm_three]
# write into csv
csv_imei_change_type_tuples = [('12345678901243', 'add')]
# Test part 1)
historic_table_insert_params_from_dict(db_conn, importer_name,
imei_norm_to_insert_list)
valid_zip_import_data_file_path = write_import_csv(
tmpdir, importer_name, csv_imei_change_type_tuples)
runner = CliRunner()
result = runner.invoke(
dirbs_import_cli,
[importer_name, '--delta', valid_zip_import_data_file_path],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 1
assert 'Failed add delta validation check. Cannot add item that is already in db. ' \
'Failing rows: {imei_norm_pk}: {imei_norm}'.format(imei_norm_pk=imei_norm_pk,
imei_norm=imei_norm_one) \
in logger_stream_contents(logger)
# historic_table rows
count, res = fetch_tbl_rows(historic_tbl_name, db_conn)
# only 2 rows that were inserted manually at the beginning of the test
assert count == 2
# Test part 2)
result = runner.invoke(dirbs_import_cli, [
importer_name, '--delta', '--disable-delta-adds-check',
valid_zip_import_data_file_path
],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
# historic_table rows
count, res = fetch_tbl_rows(historic_tbl_name, db_conn)
# only 2 rows that were inserted manually at the beginning of the test
assert count == 2
def delta_add_same_entries_common(db_conn, tmpdir, mocked_config,
importer_name, historic_tbl_name):
"""Common code to verify that is possible to add in delta list.
An info message is generated and only the first row gets inserted into historic table.
"""
# write into csv
csv_imei_change_type_tuples = [('12345678901243', 'add'),
('12345678901243', 'add')]
valid_zip_import_data_file_path = write_import_csv(
tmpdir, importer_name, csv_imei_change_type_tuples)
runner = CliRunner()
result = runner.invoke(
dirbs_import_cli,
[importer_name, '--delta', valid_zip_import_data_file_path],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
count, res = fetch_tbl_rows(historic_tbl_name, db_conn)
assert count == 1
imei_one = '12345678901243'
imei_norm_pk_name = 'imei_norm'
if importer_name == 'golden_list':
imei_one = imeis_md5_hashing_uuid(imei_one, convert_to_uuid=True)
imei_norm_pk_name = 'hashed_imei_norm'
imei_norm_set = {getattr(r, imei_norm_pk_name) for r in res}
assert imei_norm_set == {imei_one}
def delta_remove_check_and_disable_option_common(db_conn, historic_tbl_name, tmpdir, mocked_config, logger,
importer_name):
"""Common code to verify delta remove check and CLI option to disable it.
1) Verify that import fails if the check is enabled and the record to remove is not in db.
2) Verify '--disable-delta-removes-check' CLI option and verify that, if disabled, the import succeeds.
"""
# write into csv
csv_imei_change_type_tuples = [('12345678901243', 'remove')]
# Test part 1)
valid_zip_import_data_file_path = write_import_csv(tmpdir, importer_name, csv_imei_change_type_tuples)
runner = CliRunner()
result = runner.invoke(dirbs_import_cli, [importer_name, '--delta', valid_zip_import_data_file_path],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 1
imei_norm_expected = '12345678901243'
imei_norm_pk = 'imei_norm'
if importer_name == 'golden_list':
imei_norm_expected = imeis_md5_hashing_uuid(imei_norm_expected, convert_to_uuid=True)
imei_norm_pk = 'hashed_imei_norm'
assert 'Failed remove delta validation check. Cannot remove records not in db. ' \
'Failing rows: {imei_norm_pk}: {imei_norm_expected}'.format(imei_norm_pk=imei_norm_pk,
imei_norm_expected=imei_norm_expected) \
in logger_stream_contents(logger)
# Test part 2)
result = runner.invoke(dirbs_import_cli, [importer_name, '--delta', '--disable-delta-removes-check',
valid_zip_import_data_file_path], obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
# historic_table rows
count, res = fetch_tbl_rows(historic_tbl_name, db_conn)
assert count == 0
def row_count_stats_common(postgres, db_conn, tmpdir, mocked_config, logger,
importer_name, historic_tbl_name):
"""Test Depot not available yet. Verify output stats for CLI import command."""
# Part 1) populate import_table and verify before import row count
# Part 2) import file containing one duplicate (same row) and verify that staging_row_count value includes
# duplicates and import_table_new_row_count doesn't.
imei_norm_one = '12345678901243'
imei_norm_two = '22345678901243'
imei_norm_three = '32345678901234'
imei_norm_four = '42345678901243'
imei_norm_five = '52345678901243'
imei_norm_pk = 'imei_norm'
if importer_name == 'golden_list':
imei_norm_one = imeis_md5_hashing_uuid(imei_norm_one,
convert_to_uuid=True)
imei_norm_two = imeis_md5_hashing_uuid(imei_norm_two,
convert_to_uuid=True)
imei_norm_three = imeis_md5_hashing_uuid(imei_norm_three,
convert_to_uuid=True)
imei_norm_four = imeis_md5_hashing_uuid(imei_norm_four,
convert_to_uuid=True)
imei_norm_five = imeis_md5_hashing_uuid(imei_norm_five,
convert_to_uuid=True)
imei_norm_pk = 'hashed_imei_norm'
# populate historic_tbl
imei_norm_to_insert_list = [imei_norm_one, imei_norm_three]
# write into csv
csv_imei_change_type_tuples = [('12345678901243', 'remove'),
('22345678901243', 'add'),
('22345678901243', 'add'),
('42345678901243', 'add'),
('52345678901243', 'add')]
# Test part 1)
historic_table_insert_params_from_dict(db_conn, importer_name,
imei_norm_to_insert_list)
valid_zip_import_data_file_path = write_import_csv(
tmpdir, importer_name, csv_imei_change_type_tuples)
runner = CliRunner()
result = runner.invoke(
dirbs_import_cli,
[importer_name, '--delta', valid_zip_import_data_file_path],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
count, res = fetch_tbl_rows(historic_tbl_name, db_conn)
assert count == 5
imei_norm_set = {getattr(r, imei_norm_pk) for r in res}
assert imei_norm_set == {
imei_norm_one, imei_norm_two, imei_norm_three, imei_norm_four,
imei_norm_five
}
# Test Part 1)
assert 'Rows in table prior to import: 2' in logger_stream_contents(logger)
# Test Part 2) - self.staging_row_count
assert 'Rows supplied in delta input file: 5' in logger_stream_contents(
logger)
# Test Part 2) - import_table_new_row_count=rows_before + rows_inserted - rows_deleted
assert 'Rows in table after import: 4 (3 new, 0 updated, 1 removed)' in logger_stream_contents(
logger)
def delta_list_import_common(db_conn, mocked_config, tmpdir, importer_name,
historic_tbl_name, logger):
"""Common code to test delta list import.
1) Verify import failure without using delta option because delta file pre-validation fails due to change_type col
2) Verify import succeeds using delta option
2a) Verify that all record added have end_date None
2b) Verify that record with imei_norm '52345678901234' has been removed and end_date set to not None.
"""
imei_one = '12345678901234'
imei_two = '22345678901234'
imei_five = '52345678901234'
imei_norm_pk_name = 'imei_norm'
if importer_name == 'golden_list':
imei_one = imeis_md5_hashing_uuid(imei_one, convert_to_uuid=True)
imei_two = imeis_md5_hashing_uuid(imei_two, convert_to_uuid=True)
imei_five = imeis_md5_hashing_uuid(imei_five, convert_to_uuid=True)
imei_norm_pk_name = 'hashed_imei_norm'
# populate historic_tbl
imei_norm_to_insert_list = [imei_five]
# write into csv
csv_imei_change_type_tuples = [('52345678901234', 'remove'),
('12345678901234', 'add'),
('22345678901234', 'add')]
# populate table
historic_table_insert_params_from_dict(db_conn, importer_name,
imei_norm_to_insert_list)
# write csv
valid_zip_import_data_file_path = write_import_csv(
tmpdir, importer_name, csv_imei_change_type_tuples)
runner = CliRunner()
# Test part 1) Verify import failure without delta option
result = runner.invoke(dirbs_import_cli,
[importer_name, valid_zip_import_data_file_path],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 1
assert 'Pre-validation failed: b\'Error: ' \
'Metadata header, cannot find the column headers - change_type' in logger_stream_contents(logger)
# Test part 2) Verify import success using delta option
result = runner.invoke(dirbs_import_cli, [
importer_name, '--delta', '--disable-delta-adds-check',
valid_zip_import_data_file_path
],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
count, res = fetch_tbl_rows(historic_tbl_name, db_conn)
assert count == 3
expected_output = {imei_one, imei_two, imei_five}
imei_norm_set = {getattr(r, imei_norm_pk_name) for r in res}
assert imei_norm_set == expected_output
# Test part 2a
assert all([(r.end_date is None) for r in res
if getattr(r, imei_norm_pk_name) not in imei_five])
# Test part 2b
assert all([(r.end_date is not None) for r in res
if getattr(r, imei_norm_pk_name) in imei_five])
def full_list_import_common(tmpdir, db_conn, mocked_config, importer_name, historic_tbl_name, delta_import=False):
"""Common code to test full list import.
Import '52345678901234', '32345678901234' in db, with start_date '20160420' and end_date null.
Import a full list containing '12345678901234', '22345678901234', '32345678901234'.
1) Verify that 52345678901234 is removed(set end_date to not job date) because is only in db.
2) Verify that 32345678901234 is ignored and remains the same (start_date '20160420') because is in both
db and list.
3) Verify that 12345678901234 and 22345678901234 are added with start date different from '20160420'
and end date null because are in current list and not in db.
"""
imei_one = '12345678901234'
imei_two = '22345678901234'
imei_three = '32345678901234'
imei_five = '52345678901234'
imei_norm_pk_name = 'imei_norm'
if importer_name == 'golden_list':
imei_one = imeis_md5_hashing_uuid(imei_one, convert_to_uuid=True)
imei_two = imeis_md5_hashing_uuid(imei_two, convert_to_uuid=True)
imei_three = imeis_md5_hashing_uuid(imei_three, convert_to_uuid=True)
imei_five = imeis_md5_hashing_uuid(imei_five, convert_to_uuid=True)
imei_norm_pk_name = 'hashed_imei_norm'
# populate historic_tbl
imei_norm_to_insert_list = [imei_five,
imei_three]
# write into csv
csv_imei_change_type_tuples = [('12345678901234', None), ('22345678901234', None),
('32345678901234', None)]
# populate historic table
historic_table_insert_params_from_dict(db_conn, importer_name, imei_norm_to_insert_list)
# write csv input file
valid_zip_import_data_file_path = write_import_csv(tmpdir, importer_name, csv_imei_change_type_tuples,
delta_import=delta_import)
runner = CliRunner()
result = runner.invoke(dirbs_import_cli, [importer_name, valid_zip_import_data_file_path],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
count, res = fetch_tbl_rows(historic_tbl_name, db_conn)
assert count == 4
expected_output = {imei_one,
imei_two,
imei_three,
imei_five}
imei_norm_set = {getattr(r, imei_norm_pk_name) for r in res}
assert imei_norm_set == expected_output
date_time_before_import = datetime(2016, 4, 20)
# Test part 1
assert all([(r.end_date is not None) for r in res if getattr(r, imei_norm_pk_name) in imei_five])
# Test part 2 and 3
assert all([(r.end_date is None) for r in res if getattr(r, imei_norm_pk_name) in (imei_one, imei_two,
imei_three)])
assert all([(r.start_date != date_time_before_import) for r in res if getattr(r, imei_norm_pk_name)
in (imei_one, imei_two)])
assert all([(r.start_date == date_time_before_import) for r in res if getattr(r, imei_norm_pk_name)
not in (imei_one, imei_two)])
def test_delta_update_check_and_disable_option(db_conn, tmpdir, mocked_config,
logger):
"""Test Depot not available yet. Verify delta update check and CLI option to disable it.
1) Verify that import fails if the check is enabled and the record to remove is not in db.
2) Verify '--disable-delta-updates-check' CLI option and verify that, if disabled, the import succeeds.
3) Verify that per each row updated a new row is added (and the existing row end_date is not null anymore)
to historic_stolen_list table.
"""
# Test part 1)
csv_imei_change_type_tuples = [('22345678901234', 'update')]
valid_zip_import_data_file_path = write_import_csv(
tmpdir, importer_name, csv_imei_change_type_tuples)
runner = CliRunner()
result = runner.invoke(dirbs_import_cli, [
'stolen_list', '--delta', '--disable-delta-removes-check',
valid_zip_import_data_file_path
],
obj={'APP_CONFIG': mocked_config},
catch_exceptions=False)
assert result.exit_code == 1
assert 'Failed update delta validation check. ' \
'Cannot update records not in db. Failing rows: imei_norm: 22345678901234' in logger_stream_contents(logger)
# Test part 2)
result = runner.invoke(dirbs_import_cli, [
'stolen_list', '--delta', '--disable-delta-updates-check',
valid_zip_import_data_file_path
],
catch_exceptions=False,
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
# Test part 3)
imei_one = '12345678901234'
# populate historic_tbl
imei_norm_to_insert_list = [imei_one]
# write into csv
csv_imei_change_type_tuples = [('12345678901234', 'update')]
# populate table
historic_table_insert_params_from_dict(db_conn, importer_name,
imei_norm_to_insert_list)
# write csv
valid_zip_import_data_file_path = write_import_csv(
tmpdir, importer_name, csv_imei_change_type_tuples)
runner = CliRunner()
result = runner.invoke(
dirbs_import_cli,
[importer_name, '--delta', valid_zip_import_data_file_path],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
count, res = fetch_tbl_rows(historic_tbl_name, db_conn)
assert count == 3
# verifying that there are 3 rows in historic_stolen_list table
# one row for 22345678901234 added in step 1 and two rows for imei_norm 12345678901234 that has been updated.
assert [
r.imei_norm for r in res
if r.end_date is None and r.imei_norm != '22345678901234'
] == ['12345678901234']
assert [
r.imei_norm for r in res
if r.end_date is not None and r.imei_norm != '22345678901234'
] == ['12345678901234']
