def test_normal_sample(pairing_list_importer, logger, db_conn):
"""Test Depot ID 96615/1.
Verify that valid Paring List Data can be successfully imported into the database.
"""
expect_success(pairing_list_importer, 5, db_conn, logger)
def test_case_insensitive_headers(pairing_list_importer, logger, db_conn):
"""Test Depot ID 96773/9.
Verify that the pairing list file is accepted and imported if the headers have mixed cases.
"""
expect_success(pairing_list_importer, 1, db_conn, logger)
def test_duplicate_check_fails(golden_list_importer, logger, db_conn):
"""Test Depot ID not known yet.
Verify that it doesn't fail to import file with same rows.
"""
expect_success(golden_list_importer, 20, db_conn, logger)
def test_duplicate_check_override(golden_list_importer, db_conn, logger):
"""Test Depot ID not known yet.
Verify that it fails to import an empty file after importing a non empty file.
"""
expect_success(golden_list_importer, 20, db_conn, logger)
def test_duplicate_check_fails(registration_list_importer, logger, db_conn):
"""Test Depot ID not known yet.
Verify that it doesn't fail to import sam rows.
"""
expect_success(registration_list_importer, 18, db_conn, logger)
def test_simple_import(golden_list_importer, logger, db_conn):
"""Test Depot ID not known yet.
Verify that valid golden data can be successfully imported into the database.
"""
expect_success(golden_list_importer, 20, db_conn, logger)
def test_matching_normalization(monitoring_list_importer, logger, db_conn):
"""Verify that IMEIs that normalize to the same value are successfully imported into the database."""
expect_success(monitoring_list_importer, 5, db_conn, logger)
def test_prune_classification_state(db_conn, metadata_db_conn, tmpdir, logger, mocked_config,
operator_data_importer, stolen_list_importer, monkeypatch,
gsma_tac_db_importer, postgres, mocked_statsd):
"""Test Depot ID not known yet.
A regulator/partner should be able to run a CLI command to prune classification_state table.
It will remove any classification state data related to obsolete conditions and
data with end_date is earlier than the start of the retention window.
"""
# Step 1:
# import gsma_dump empty non tac and classify for all the conditions
# ['gsma_not_found', 'local_stolen', 'duplicate_mk1', 'malformed_imei', 'not_on_registration_list', ..]
# classification_state_table contains records for cond_name "gsma_not_found". They all have end_date==None
# Step 2 - TEST RETENTION WINDOW:
# CLI prune will delete rows where the end_date is earlier than the start of the retention window.
# retention_months=6
# curr_date = datetime.date(2017, 7, 13)
# Import different gsma_db and classify to have different end date for gsma_not_found records in
# the classification table.
# Step 3 - TEST CONDITIONS NOT EXISTING:
# CLI prune for classification_state will look at the current configured conditions and
# remove any entries corresponding to cond_names that no longer exist in the config.
# Load a new yaml file without stolen_list condition and run the prune CLI command to test.
# -- yaml cond config list:
# ['gsma_not_found', 'malformed_imei', 'not_on_registration_list']
# -- classification_state condition list:
# ['gsma_not_found', 'local_stolen', 'malformed_imei', 'not_on_registration_list']
# Step 1
operator_data_importer.import_data()
stolen_list_importer.import_data()
gsma_tac_db_importer.import_data()
runner = CliRunner()
db_conn.commit()
runner.invoke(dirbs_classify_cli, ['--no-safety-check', '--curr-date', '20170713'],
obj={'APP_CONFIG': mocked_config})
with db_conn.cursor() as cur:
cur.execute('SELECT imei_norm, cond_name, end_date FROM classification_state ORDER BY cond_name, imei_norm')
res_list = cur.fetchall()
assert len(res_list) == 32
assert [(x.imei_norm, x.cond_name, x.end_date) for x in res_list] == \
[('21111111111111', 'gsma_not_found', None),
('21111111111112', 'gsma_not_found', None),
('8888#888622222', 'gsma_not_found', None),
('88888862222209', 'gsma_not_found', None),
('88888888622222', 'gsma_not_found', None),
('12432807272315', 'local_stolen', None),
('12640904324427', 'local_stolen', None),
('12640904372723', 'local_stolen', None),
('12727231272313', 'local_stolen', None),
('12875502464321', 'local_stolen', None),
('12875502572723', 'local_stolen', None),
('12875507272312', 'local_stolen', None),
('12904502843271', 'local_stolen', None),
('12909602432585', 'local_stolen', None),
('12909602872723', 'local_stolen', None),
('12922902206948', 'local_stolen', None),
('12922902243260', 'local_stolen', None),
('12922902432742', 'local_stolen', None),
('12922902432776', 'local_stolen', None),
('12957272313271', 'local_stolen', None),
('17272317272723', 'local_stolen', None),
('56773605727231', 'local_stolen', None),
('64220204327947', 'local_stolen', None),
('64220297727231', 'local_stolen', None),
('72723147267231', 'local_stolen', None),
('72723147267631', 'local_stolen', None),
('8888#888622222', 'malformed_imei', None),
('21111111111111', 'not_on_registration_list', None),
('21111111111112', 'not_on_registration_list', None),
('8888#888622222', 'not_on_registration_list', None),
('88888862222209', 'not_on_registration_list', None),
('88888888622222', 'not_on_registration_list', None)]
# Step 2
# all records have end_date == None. Classify twice to have records with different end_date
# first classification
with get_importer(GSMADataImporter,
db_conn,
metadata_db_conn,
mocked_config.db_config,
tmpdir,
logger,
mocked_statsd,
GSMADataParams(filename='gsma_not_found_anonymized.txt')) as imp:
expect_success(imp, 1, db_conn, logger)
runner.invoke(dirbs_classify_cli, ['--no-safety-check', '--curr-date', '20170713'],
obj={'APP_CONFIG': mocked_config})
# with db_conn.cursor() as cur:
cur.execute("""SELECT imei_norm, cond_name, end_date
FROM classification_state
ORDER BY cond_name, imei_norm""")
res_list = cur.fetchall()
gsma_not_found_list = [(x.imei_norm, x.cond_name, x.end_date) for x in res_list
if x.cond_name == 'gsma_not_found']
assert gsma_not_found_list == [('21111111111111', 'gsma_not_found', None),
('21111111111112', 'gsma_not_found', None),
('8888#888622222', 'gsma_not_found', None),
('88888862222209', 'gsma_not_found', None),
('88888888622222', 'gsma_not_found', datetime.date(2017, 7, 13))]
# second classification
with get_importer(GSMADataImporter,
db_conn,
metadata_db_conn,
mocked_config.db_config,
tmpdir,
logger,
mocked_statsd,
GSMADataParams(filename='prune_classification_state_gsma.txt')) as imp:
expect_success(imp, 1, db_conn, logger)
runner.invoke(dirbs_classify_cli, ['--no-safety-check', '--curr-date', '20160101'],
obj={'APP_CONFIG': mocked_config})
# with db_conn.cursor() as cur:
cur.execute('SELECT imei_norm, cond_name, end_date '
' FROM classification_state '
'ORDER BY cond_name, imei_norm, end_date')
res_list = cur.fetchall()
gsma_not_found_list = [(x.imei_norm, x.cond_name, x.end_date) for x in res_list if
x.cond_name == 'gsma_not_found']
assert gsma_not_found_list == [('21111111111111', 'gsma_not_found', datetime.date(2016, 1, 1)),
('21111111111112', 'gsma_not_found', datetime.date(2016, 1, 1)),
('8888#888622222', 'gsma_not_found', None),
('88888862222209', 'gsma_not_found', None),
('88888888622222', 'gsma_not_found', datetime.date(2017, 7, 13)),
('88888888622222', 'gsma_not_found', None)]
# Step 3
# Expect not to be in classification_state table after prune:
# IMEIs 21111111111111 and 21111111111112 for condition gsma_not found (due to end_date)
# IMEIs for condition stolen_list (due to condition no longer exist)
# this commit is to remove locks from the classification_state table so that
# the table can be dropped inside the prune. The locks were activated by the CLI to classify.
db_conn.commit()
cond_dict_list = [{'label': 'gsma_not_found',
'reason': 'TAC not found in GSMA TAC database',
'grace_period_days': 30,
'blocking': True,
'dimensions': [{'module': 'gsma_not_found'}]
},
{'label': 'malformed_imei',
'reason': 'Invalid characters detected in IMEI',
'grace_period_days': 0,
'blocking': False,
'dimensions': [{'module': 'malformed_imei'}]
},
{'label': 'not_on_registration_list',
'reason': 'IMEI not found on local registration list',
'grace_period_days': 0,
'blocking': True,
'max_allowed_matching_ratio': 1.0,
'dimensions': [{'module': 'not_on_registration_list'}]
}]
monkeypatch.setattr(mocked_config, 'conditions', from_cond_dict_list_to_cond_list(cond_dict_list))
with db_conn.cursor() as cur:
result = runner.invoke(dirbs_prune_cli, ['--curr-date', '20170913',
'classification_state'],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
# ITEMS REMOVED
# [('17272317272723', 'local_stolen', None), ('12909602872723', 'local_stolen', None),
# ('12875502572723', 'local_stolen', None), ('12875507272312', 'local_stolen', None),
# ('64220297727231', 'local_stolen', None), ('12909602432585', 'local_stolen', None),
# ('64220204327947', 'local_stolen', None), ('72723147267631', 'local_stolen', None),
# ('72723147267231', 'local_stolen', None), ('12922902243260', 'local_stolen', None),
# ('12875502464321', 'local_stolen', None), ('12922902432776', 'local_stolen', None),
# ('12957272313271', 'local_stolen', None), ('12640904324427', 'local_stolen', None),
# ('12904502843271', 'local_stolen', None), ('12922902432742', 'local_stolen', None),
# ('12432807272315', 'local_stolen', None), ('12922902206948', 'local_stolen', None),
# ('56773605727231', 'local_stolen', None), ('12727231272313', 'local_stolen', None),
# ('12640904372723', 'local_stolen', None),
# ('21111111111111', 'gsma_not_found', datetime.date(2016, 1, 1)),
# ('21111111111112', 'gsma_not_found', datetime.date(2016, 1, 1))]
cur.execute('SELECT imei_norm, cond_name, end_date '
'FROM classification_state '
'ORDER BY cond_name, imei_norm, end_date')
res_list = cur.fetchall()
pruned_class_state_table = [(x.imei_norm, x.cond_name, x.end_date) for x in res_list]
assert pruned_class_state_table == [('8888#888622222', 'gsma_not_found', None),
('88888862222209', 'gsma_not_found', None),
('88888888622222', 'gsma_not_found', datetime.date(2017, 7, 13)),
('88888888622222', 'gsma_not_found', None),
('8888#888622222', 'malformed_imei', None),
('21111111111111', 'not_on_registration_list', None),
('21111111111112', 'not_on_registration_list', None),
('8888#888622222', 'not_on_registration_list', None),
('88888862222209', 'not_on_registration_list', None),
('88888888622222', 'not_on_registration_list', None)]
def test_simple_import(monitoring_list_importer, logger, db_conn):
"""Verify that the valid monitoring list data can be successfully imported into the db."""
expect_success(monitoring_list_importer, 20, db_conn, logger)
def test_simple_import(barred_list_importer, logger, db_conn):
"""Verify that the valid barred list data can be successfully imported into the db."""
expect_success(barred_list_importer, 20, db_conn, logger)
def test_duplicate_check_fails(stolen_list_importer, logger, db_conn):
"""Test Depot ID not known yet.
Verify that rows with same imei and reporting_date are not considered duplicates.
"""
expect_success(stolen_list_importer, 20, db_conn, logger)
def test_reporting_date_optional(stolen_list_importer, logger, db_conn,
mocked_config, metadata_db_conn, tmpdir,
mocked_statsd):
"""Test Depot ID not known yet.
Verify that valid stolen data can be successfully imported into the database with optional field reporting_date.
Verify in case of two IMEIs with different reporting_date, only the one with min(reporting_date) is imported.
"""
# Basically, the decorator @optional in the stolen_list schema for the field reporting-date, allows whitespaces.
# If the field was text and we imported a row containing a reporting-date whitespace-only,
# the row would have been imported with withespaces into the db.
# In this case the reporting-date field is of type date so,
# when we import a whitespace data of type text into a date type column, we get an Error Type
# valid imei, reporting_date
expect_success(stolen_list_importer, 2, db_conn, logger)
with db_conn, db_conn.cursor() as cur:
cur.execute(
'SELECT imei_norm, reporting_date FROM historic_stolen_list ORDER BY imei_norm'
)
res = cur.fetchall()
assert len(res) == 2
stolen_list_rows = {(r.imei_norm, r.reporting_date) for r in res}
assert stolen_list_rows == {
('12222222222222', datetime.date(datetime(2016, 4, 25))),
('62222222222222', datetime.date(datetime(2016, 4, 26)))
}
# Verify in case of two IMEIs with different reporting_date, duplicate check will raise an exception.
with get_importer(
StolenListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
StolenListParams(perform_historic_check=False,
content='imei,reporting_date,status\n'
'01234567891234,20160401,\n'
'01234567891234,20160402,')) as imp:
expect_failure(
imp,
'Conflicting rows check failed (1 rows with same primary key and conflicting data)'
)
assert 'Found 1 conflicting row(s) with primary key (\'imei_norm\',): (\'01234567891234\',)' \
in logger_stream_contents(logger)
# valid imei, reporting_date whitespace only - error reporting_date type is date not string
with get_importer(
StolenListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
StolenListParams(content='imei,reporting_date,status\n'
'622222222222222, ,\n'
'122222222222223,20160425,')) as imp:
expect_failure(
imp,
exc_message='Pre-validation failed: b\'Error: '
'regex("^(20[0-9]{2}((0[13578]|1[02])31|(01|0[3-9]|1[0-2])(29|30)|'
'(0[1-9]|1[0-2])(0[1-9]|1[0-9]|2[0-8]))|20([02468][048]|[13579][26])0229)?$") '
'fails for line: 1, column: reporting_date, value: " "\\nFAIL')
db_conn.commit()
with db_conn, db_conn.cursor() as cur:
cur.execute('TRUNCATE historic_stolen_list')
# valid imei, reporting_date empty - type null
with get_importer(
StolenListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
StolenListParams(content='imei,reporting_date,status\n'
'722222222222222,,\n'
'122222222222223,20160425,')) as imp:
expect_success(imp, 2, db_conn, logger)
with db_conn, db_conn.cursor() as cur:
cur.execute(
'SELECT imei_norm, reporting_date FROM historic_stolen_list ORDER BY imei_norm'
)
res = cur.fetchall()
assert len(res) == 2
stolen_list_rows = {(r.imei_norm, r.reporting_date) for r in res}
assert stolen_list_rows == {('12222222222222',
datetime.date(datetime(2016, 4, 25))),
('72222222222222', None)}
def test_simple_import(stolen_list_importer, logger, db_conn):
"""Test Depot ID 96591/5.
Verify that valid stolen data can be successfully imported into the database.
"""
expect_success(stolen_list_importer, 20, db_conn, logger)
