def test_fields_normalised(golden_list_importer, logger, db_conn):
"""Test Depot ID not known yet.
Verify that the data was imported hashed into the database after having been normalized per the
following criteria:
If the first 14 characters of the IMEI are digits, the normalised IMEI is the first 14 characters.
If the imei does not start with 14 leading digits, no normalisation is done and we just copy the imei
upper-case value to the imei_norm column.
"""
expect_success(golden_list_importer, 2, db_conn, logger)
first_imei_couple_normalized_uuid = imeis_md5_hashing_uuid(
'64320204327947', convert_to_uuid=True)
second_imei_couple_normalized_uuid = imeis_md5_hashing_uuid(
'14A20204327947', convert_to_uuid=True)
with db_conn.cursor() as cursor:
cursor.execute(
'SELECT DISTINCT hashed_imei_norm FROM golden_list ORDER BY hashed_imei_norm'
)
res = [x.hashed_imei_norm for x in cursor.fetchall()]
assert res == [
first_imei_couple_normalized_uuid,
second_imei_couple_normalized_uuid
]
def row_count_stats_common(postgres, db_conn, tmpdir, mocked_config, logger,
importer_name, historic_tbl_name):
"""Test Depot not available yet. Verify output stats for CLI import command."""
# Part 1) populate import_table and verify before import row count
# Part 2) import file containing one duplicate (same row) and verify that staging_row_count value includes
# duplicates and import_table_new_row_count doesn't.
imei_norm_one = '12345678901243'
imei_norm_two = '22345678901243'
imei_norm_three = '32345678901234'
imei_norm_four = '42345678901243'
imei_norm_five = '52345678901243'
imei_norm_pk = 'imei_norm'
if importer_name == 'golden_list':
imei_norm_one = imeis_md5_hashing_uuid(imei_norm_one,
convert_to_uuid=True)
imei_norm_two = imeis_md5_hashing_uuid(imei_norm_two,
convert_to_uuid=True)
imei_norm_three = imeis_md5_hashing_uuid(imei_norm_three,
convert_to_uuid=True)
imei_norm_four = imeis_md5_hashing_uuid(imei_norm_four,
convert_to_uuid=True)
imei_norm_five = imeis_md5_hashing_uuid(imei_norm_five,
convert_to_uuid=True)
imei_norm_pk = 'hashed_imei_norm'
# populate historic_tbl
imei_norm_to_insert_list = [imei_norm_one, imei_norm_three]
# write into csv
csv_imei_change_type_tuples = [('12345678901243', 'remove'),
('22345678901243', 'add'),
('22345678901243', 'add'),
('42345678901243', 'add'),
('52345678901243', 'add')]
# Test part 1)
historic_table_insert_params_from_dict(db_conn, importer_name,
imei_norm_to_insert_list)
valid_zip_import_data_file_path = write_import_csv(
tmpdir, importer_name, csv_imei_change_type_tuples)
runner = CliRunner()
result = runner.invoke(
dirbs_import_cli,
[importer_name, '--delta', valid_zip_import_data_file_path],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
count, res = fetch_tbl_rows(historic_tbl_name, db_conn)
assert count == 5
imei_norm_set = {getattr(r, imei_norm_pk) for r in res}
assert imei_norm_set == {
imei_norm_one, imei_norm_two, imei_norm_three, imei_norm_four,
imei_norm_five
}
# Test Part 1)
assert 'Rows in table prior to import: 2' in logger_stream_contents(logger)
# Test Part 2) - self.staging_row_count
assert 'Rows supplied in delta input file: 5' in logger_stream_contents(
logger)
# Test Part 2) - import_table_new_row_count=rows_before + rows_inserted - rows_deleted
assert 'Rows in table after import: 4 (3 new, 0 updated, 1 removed)' in logger_stream_contents(
logger)
def delta_list_import_common(db_conn, mocked_config, tmpdir, importer_name,
historic_tbl_name, logger):
"""Common code to test delta list import.
1) Verify import failure without using delta option because delta file pre-validation fails due to change_type col
2) Verify import succeeds using delta option
2a) Verify that all record added have end_date None
2b) Verify that record with imei_norm '52345678901234' has been removed and end_date set to not None.
"""
imei_one = '12345678901234'
imei_two = '22345678901234'
imei_five = '52345678901234'
imei_norm_pk_name = 'imei_norm'
if importer_name == 'golden_list':
imei_one = imeis_md5_hashing_uuid(imei_one, convert_to_uuid=True)
imei_two = imeis_md5_hashing_uuid(imei_two, convert_to_uuid=True)
imei_five = imeis_md5_hashing_uuid(imei_five, convert_to_uuid=True)
imei_norm_pk_name = 'hashed_imei_norm'
# populate historic_tbl
imei_norm_to_insert_list = [imei_five]
# write into csv
csv_imei_change_type_tuples = [('52345678901234', 'remove'),
('12345678901234', 'add'),
('22345678901234', 'add')]
# populate table
historic_table_insert_params_from_dict(db_conn, importer_name,
imei_norm_to_insert_list)
# write csv
valid_zip_import_data_file_path = write_import_csv(
tmpdir, importer_name, csv_imei_change_type_tuples)
runner = CliRunner()
# Test part 1) Verify import failure without delta option
result = runner.invoke(dirbs_import_cli,
[importer_name, valid_zip_import_data_file_path],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 1
assert 'Pre-validation failed: b\'Error: ' \
'Metadata header, cannot find the column headers - change_type' in logger_stream_contents(logger)
# Test part 2) Verify import success using delta option
result = runner.invoke(dirbs_import_cli, [
importer_name, '--delta', '--disable-delta-adds-check',
valid_zip_import_data_file_path
],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
count, res = fetch_tbl_rows(historic_tbl_name, db_conn)
assert count == 3
expected_output = {imei_one, imei_two, imei_five}
imei_norm_set = {getattr(r, imei_norm_pk_name) for r in res}
assert imei_norm_set == expected_output
# Test part 2a
assert all([(r.end_date is None) for r in res
if getattr(r, imei_norm_pk_name) not in imei_five])
# Test part 2b
assert all([(r.end_date is not None) for r in res
if getattr(r, imei_norm_pk_name) in imei_five])
def full_list_import_common(tmpdir, db_conn, mocked_config, importer_name, historic_tbl_name, delta_import=False):
"""Common code to test full list import.
Import '52345678901234', '32345678901234' in db, with start_date '20160420' and end_date null.
Import a full list containing '12345678901234', '22345678901234', '32345678901234'.
1) Verify that 52345678901234 is removed(set end_date to not job date) because is only in db.
2) Verify that 32345678901234 is ignored and remains the same (start_date '20160420') because is in both
db and list.
3) Verify that 12345678901234 and 22345678901234 are added with start date different from '20160420'
and end date null because are in current list and not in db.
"""
imei_one = '12345678901234'
imei_two = '22345678901234'
imei_three = '32345678901234'
imei_five = '52345678901234'
imei_norm_pk_name = 'imei_norm'
if importer_name == 'golden_list':
imei_one = imeis_md5_hashing_uuid(imei_one, convert_to_uuid=True)
imei_two = imeis_md5_hashing_uuid(imei_two, convert_to_uuid=True)
imei_three = imeis_md5_hashing_uuid(imei_three, convert_to_uuid=True)
imei_five = imeis_md5_hashing_uuid(imei_five, convert_to_uuid=True)
imei_norm_pk_name = 'hashed_imei_norm'
# populate historic_tbl
imei_norm_to_insert_list = [imei_five,
imei_three]
# write into csv
csv_imei_change_type_tuples = [('12345678901234', None), ('22345678901234', None),
('32345678901234', None)]
# populate historic table
historic_table_insert_params_from_dict(db_conn, importer_name, imei_norm_to_insert_list)
# write csv input file
valid_zip_import_data_file_path = write_import_csv(tmpdir, importer_name, csv_imei_change_type_tuples,
delta_import=delta_import)
runner = CliRunner()
result = runner.invoke(dirbs_import_cli, [importer_name, valid_zip_import_data_file_path],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
count, res = fetch_tbl_rows(historic_tbl_name, db_conn)
assert count == 4
expected_output = {imei_one,
imei_two,
imei_three,
imei_five}
imei_norm_set = {getattr(r, imei_norm_pk_name) for r in res}
assert imei_norm_set == expected_output
date_time_before_import = datetime(2016, 4, 20)
# Test part 1
assert all([(r.end_date is not None) for r in res if getattr(r, imei_norm_pk_name) in imei_five])
# Test part 2 and 3
assert all([(r.end_date is None) for r in res if getattr(r, imei_norm_pk_name) in (imei_one, imei_two,
imei_three)])
assert all([(r.start_date != date_time_before_import) for r in res if getattr(r, imei_norm_pk_name)
in (imei_one, imei_two)])
assert all([(r.start_date == date_time_before_import) for r in res if getattr(r, imei_norm_pk_name)
not in (imei_one, imei_two)])
def delta_add_check_and_disable_option_common(db_conn, tmpdir, mocked_config,
logger, importer_name,
historic_tbl_name):
"""Common code to verify delta add check and CLI option to disable check.
1) Verify that import fails if the check is enabled and the record to add is already in db.
2) Verify '--disable-delta-adds-check' CLI option and verify that, if disabled, the import succeeds.
"""
imei_norm_one = '12345678901243'
imei_norm_three = '32345678901234'
imei_norm_pk = 'imei_norm'
if importer_name == 'golden_list':
imei_norm_one = imeis_md5_hashing_uuid(imei_norm_one,
convert_to_uuid=True)
imei_norm_three = imeis_md5_hashing_uuid(imei_norm_three,
convert_to_uuid=True)
imei_norm_pk = 'hashed_imei_norm'
# populate historic_tbl
imei_norm_to_insert_list = [imei_norm_one, imei_norm_three]
# write into csv
csv_imei_change_type_tuples = [('12345678901243', 'add')]
# Test part 1)
historic_table_insert_params_from_dict(db_conn, importer_name,
imei_norm_to_insert_list)
valid_zip_import_data_file_path = write_import_csv(
tmpdir, importer_name, csv_imei_change_type_tuples)
runner = CliRunner()
result = runner.invoke(
dirbs_import_cli,
[importer_name, '--delta', valid_zip_import_data_file_path],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 1
assert 'Failed add delta validation check. Cannot add item that is already in db. ' \
'Failing rows: {imei_norm_pk}: {imei_norm}'.format(imei_norm_pk=imei_norm_pk,
imei_norm=imei_norm_one) \
in logger_stream_contents(logger)
# historic_table rows
count, res = fetch_tbl_rows(historic_tbl_name, db_conn)
# only 2 rows that were inserted manually at the beginning of the test
assert count == 2
# Test part 2)
result = runner.invoke(dirbs_import_cli, [
importer_name, '--delta', '--disable-delta-adds-check',
valid_zip_import_data_file_path
],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
# historic_table rows
count, res = fetch_tbl_rows(historic_tbl_name, db_conn)
# only 2 rows that were inserted manually at the beginning of the test
assert count == 2
def test_import_data_hashing_option(golden_list_importer, db_conn,
metadata_db_conn, mocked_config, logger,
tmpdir, mocked_statsd):
"""Test Depot ID not known yet.
Verify option to import either hashed or unhashed data.
"""
# Step 1 import unhashed data
# Step 2 import already hashed data
imei_one_uuid = imeis_md5_hashing_uuid('64220204327947',
convert_to_uuid=True)
imei_two_uuid = imeis_md5_hashing_uuid('12875502464321',
convert_to_uuid=True)
# Step 1
expect_success(golden_list_importer, 2, db_conn, logger)
with db_conn.cursor() as cur:
cur.execute(
'SELECT hashed_imei_norm FROM golden_list ORDER BY hashed_imei_norm'
)
res = {x.hashed_imei_norm for x in cur.fetchall()}
assert len(res) == 2
assert res == {imei_one_uuid, imei_two_uuid}
# Step 2
imei_one_hashed = imeis_md5_hashing_uuid('64220204327947',
convert_to_uuid=False)
imei_two_hashed = imeis_md5_hashing_uuid('12875502464321',
convert_to_uuid=False)
with get_importer(
GoldenListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
GoldenListParams(content='GOLDEN_IMEI\n{0}\n{1}\n'.format(
imei_one_hashed, imei_two_hashed),
prehashed_input_data=True)) as imp:
expect_success(imp, 2, db_conn, logger)
with db_conn.cursor() as cur:
cur.execute(
'SELECT hashed_imei_norm FROM golden_list ORDER BY hashed_imei_norm'
)
res = {x.hashed_imei_norm for x in cur.fetchall()}
assert len(res) == 2
assert res == {imei_one_uuid, imei_two_uuid}
def delta_add_same_entries_common(db_conn, tmpdir, mocked_config,
importer_name, historic_tbl_name):
"""Common code to verify that is possible to add in delta list.
An info message is generated and only the first row gets inserted into historic table.
"""
# write into csv
csv_imei_change_type_tuples = [('12345678901243', 'add'),
('12345678901243', 'add')]
valid_zip_import_data_file_path = write_import_csv(
tmpdir, importer_name, csv_imei_change_type_tuples)
runner = CliRunner()
result = runner.invoke(
dirbs_import_cli,
[importer_name, '--delta', valid_zip_import_data_file_path],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
count, res = fetch_tbl_rows(historic_tbl_name, db_conn)
assert count == 1
imei_one = '12345678901243'
imei_norm_pk_name = 'imei_norm'
if importer_name == 'golden_list':
imei_one = imeis_md5_hashing_uuid(imei_one, convert_to_uuid=True)
imei_norm_pk_name = 'hashed_imei_norm'
imei_norm_set = {getattr(r, imei_norm_pk_name) for r in res}
assert imei_norm_set == {imei_one}
def delta_remove_check_and_disable_option_common(db_conn, historic_tbl_name, tmpdir, mocked_config, logger,
importer_name):
"""Common code to verify delta remove check and CLI option to disable it.
1) Verify that import fails if the check is enabled and the record to remove is not in db.
2) Verify '--disable-delta-removes-check' CLI option and verify that, if disabled, the import succeeds.
"""
# write into csv
csv_imei_change_type_tuples = [('12345678901243', 'remove')]
# Test part 1)
valid_zip_import_data_file_path = write_import_csv(tmpdir, importer_name, csv_imei_change_type_tuples)
runner = CliRunner()
result = runner.invoke(dirbs_import_cli, [importer_name, '--delta', valid_zip_import_data_file_path],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 1
imei_norm_expected = '12345678901243'
imei_norm_pk = 'imei_norm'
if importer_name == 'golden_list':
imei_norm_expected = imeis_md5_hashing_uuid(imei_norm_expected, convert_to_uuid=True)
imei_norm_pk = 'hashed_imei_norm'
assert 'Failed remove delta validation check. Cannot remove records not in db. ' \
'Failing rows: {imei_norm_pk}: {imei_norm_expected}'.format(imei_norm_pk=imei_norm_pk,
imei_norm_expected=imei_norm_expected) \
in logger_stream_contents(logger)
# Test part 2)
result = runner.invoke(dirbs_import_cli, [importer_name, '--delta', '--disable-delta-removes-check',
valid_zip_import_data_file_path], obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
# historic_table rows
count, res = fetch_tbl_rows(historic_tbl_name, db_conn)
assert count == 0
def multiple_changes_check_common(logger, mocked_config, tmpdir, importer_name):
"""Common code to verify that is not possible to add and remove a record at the same time."""
# write into csv
csv_imei_change_type_tuples = [('12345678901234', 'add'), ('12345678901234', 'remove')]
valid_zip_import_data_file_path = write_import_csv(tmpdir, importer_name, csv_imei_change_type_tuples)
runner = CliRunner()
result = runner.invoke(dirbs_import_cli, [importer_name, '--delta', valid_zip_import_data_file_path],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 1
imei_norm_expected = '12345678901234'
imei_norm_pk = 'imei_norm'
if importer_name == 'golden_list':
imei_norm_expected = imeis_md5_hashing_uuid(imei_norm_expected, convert_to_uuid=True)
imei_norm_pk = 'hashed_imei_norm'
assert 'Same record cannot be added or removed at the same time in delta list. ' \
'Failing rows: {imei_norm_pk}: {imei_norm_expected}'.format(imei_norm_pk=imei_norm_pk,
imei_norm_expected=imei_norm_expected) \
in logger_stream_contents(logger)
