def test_delta_file_prevalidation(logger, db_conn, metadata_db_conn,
mocked_config, tmpdir, mocked_statsd,
golden_list_importer):
"""Test Depot not available yet.
Test pre-validation schemas.
"""
# update change-type is allowed only for stolen
expect_failure(
golden_list_importer,
exc_message=
'Pre-validation failed: b\'Error: regex("^(add|remove)$") fails for line: 2, '
'column: change_type, value: "update"\\nFAIL')
# change_type must be lower case
with get_importer(
GoldenListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
GoldenListParams(content='golden_imei,change_type\n'
'12345678901234,ADD',
delta=True)) as imp:
expect_failure(
imp,
exc_message=
'Pre-validation failed: b\'Error: regex("^(add|remove)$") fails for line: '
'1, column: change_type, value: "ADD"\\nFAIL')
def test_repeat_import(golden_list_importer, logger, mocked_statsd, db_conn,
metadata_db_conn, mocked_config, tmpdir):
"""Test Depot ID not known yet.
Verify that valid golden list data can be successfully imported into the database
when repeating the import of the same file.
"""
expect_success(golden_list_importer, 21, db_conn, logger)
with get_importer(
GoldenListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
GoldenListParams(filename='sample_golden_list_v1.csv')) as imp:
expect_success(imp, 21, db_conn, logger)
def test_override_historical_check(golden_list_importer, logger, mocked_statsd,
db_conn, metadata_db_conn, mocked_config,
tmpdir):
"""Test Depot ID not known yet.
Verify that the user can override historical checks when importing Golden List data.
"""
expect_success(golden_list_importer, 20, db_conn, logger)
with get_importer(
GoldenListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
GoldenListParams(filename='sample_golden_list_historicalcheck.csv',
perform_historic_check=False)) as imp:
expect_success(imp, 9, db_conn, logger)
def test_historical_check_empty(golden_list_importer, logger, mocked_statsd,
db_conn, metadata_db_conn, mocked_config,
tmpdir):
"""Test Depot ID not known yet.
Verify that it fails to import an empty file after importing a non empty file.
"""
expect_success(golden_list_importer, 20, db_conn, logger)
with get_importer(
GoldenListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
GoldenListParams(
filename='empty_goldenlist_historical_check.csv')) as imp:
expect_failure(imp, exc_message='Failed import size historic check')
def test_import_data_hashing_option(golden_list_importer, db_conn,
metadata_db_conn, mocked_config, logger,
tmpdir, mocked_statsd):
"""Test Depot ID not known yet.
Verify option to import either hashed or unhashed data.
"""
# Step 1 import unhashed data
# Step 2 import already hashed data
imei_one_uuid = imeis_md5_hashing_uuid('64220204327947',
convert_to_uuid=True)
imei_two_uuid = imeis_md5_hashing_uuid('12875502464321',
convert_to_uuid=True)
# Step 1
expect_success(golden_list_importer, 2, db_conn, logger)
with db_conn.cursor() as cur:
cur.execute(
'SELECT hashed_imei_norm FROM golden_list ORDER BY hashed_imei_norm'
)
res = {x.hashed_imei_norm for x in cur.fetchall()}
assert len(res) == 2
assert res == {imei_one_uuid, imei_two_uuid}
# Step 2
imei_one_hashed = imeis_md5_hashing_uuid('64220204327947',
convert_to_uuid=False)
imei_two_hashed = imeis_md5_hashing_uuid('12875502464321',
convert_to_uuid=False)
with get_importer(
GoldenListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
GoldenListParams(content='GOLDEN_IMEI\n{0}\n{1}\n'.format(
imei_one_hashed, imei_two_hashed),
prehashed_input_data=True)) as imp:
expect_success(imp, 2, db_conn, logger)
with db_conn.cursor() as cur:
cur.execute(
'SELECT hashed_imei_norm FROM golden_list ORDER BY hashed_imei_norm'
)
res = {x.hashed_imei_norm for x in cur.fetchall()}
assert len(res) == 2
assert res == {imei_one_uuid, imei_two_uuid}
def test_malformed_imeis(golden_list_importer, logger, mocked_statsd, db_conn,
metadata_db_conn, mocked_config, tmpdir):
"""Test Depot ID not known yet.
Verify that the Golden List data file is accepted
and imported if the data contains IMEIs with #, and *.
"""
expect_success(golden_list_importer, 20, db_conn, logger)
# attempting to import golden list file containing symbol not allowed '%'.
with get_importer(
GoldenListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
GoldenListParams(
filename='golden_list_hexpound_bad_symbol.csv')) as imp:
expect_failure(imp,
exc_message='regex("^[0-9A-Fa-f\\\\*\\\\#]{1,16}$") '
'fails for line: 1, column: golden_imei, value: '
'"62%222222222222"\\nFAIL')
def test_historical_check_percentage_fails(golden_list_importer, logger,
mocked_statsd, db_conn,
metadata_db_conn, mocked_config,
tmpdir):
"""Test Depot ID not known yet.
Verify that a local golden data containing 9 rows fails to be imported after having imported a 20
rows file because Historical golden list check is greater than 25% drop in import size;.
"""
expect_success(golden_list_importer, 20, db_conn, logger)
with get_importer(
GoldenListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
GoldenListParams(
filename='sample_golden_list_historicalcheck.csv',
import_size_variation_percent=mocked_config.
golden_threshold_config.import_size_variation_percent,
import_size_variation_absolute=mocked_config.
golden_threshold_config.import_size_variation_absolute)
) as imp:
expect_failure(imp, exc_message='Failed import size historic check')
def test_historical_check_percentage_succeeds(golden_list_importer, logger, mocked_statsd,
db_conn, metadata_db_conn, mocked_config, tmpdir):
"""Test Depot ID not known yet.
Verify that a local golden data is successfully imported after having imported two files where the
second file has 80% size of the first one and the threshold value is 75.
"""
expect_success(golden_list_importer, 100, db_conn, logger)
with get_importer(GoldenListImporter,
db_conn,
metadata_db_conn,
mocked_config.db_config,
tmpdir,
logger,
mocked_statsd,
GoldenListParams(filename=data_file_to_test(80, imei_imsi=False,
imei_custom_header='golden_imei'),
import_size_variation_percent=mocked_config.golden_threshold_config.
import_size_variation_percent,
import_size_variation_absolute=mocked_config.golden_threshold_config.
import_size_variation_absolute)) as imp:
expect_success(imp, 80, db_conn, logger)
result_list = [res.hashed_imei_norm for res in cursor]
assert result.exit_code == 0
assert result_list == [
'14d7f294-462f-1847-bb3c-f2f271308684',
'17186a62-8378-e3d2-7f17-0adefd81a1aa',
'5aa286a1-c3e6-cb88-ba86-047fba391f29',
'7b00c461-1cac-79cd-b391-3a03ca5423b9',
'a6728112-3cfc-025a-3b99-5301f1ffe5ba',
'de9aa2e0-8df1-958b-bd0e-3bd9a4482476'
]
@pytest.mark.parametrize(
'golden_list_importer',
[GoldenListParams(filename='golden_list_missingheader.csv')],
indirect=True)
def test_missing_header(golden_list_importer, logger, db_conn):
"""Test Depot ID not known yet.
Verify that the Golden List data is not imported if a header column is missing.
"""
expect_failure(
golden_list_importer,
exc_message='Metadata header, cannot find the column headers - '
'golden_imei, 642222222222222 - .\\nFAIL')
@pytest.mark.parametrize(
'golden_list_importer',
[GoldenListParams(filename='sample_golden_list_normalize.csv')],