def test_historical_check_percentage_succeeds(stolen_list_importer, logger,
mocked_statsd, db_conn,
mocked_config, metadata_db_conn,
tmpdir):
"""Test Depot ID not known yet.
Verify that a local stolen data is successfully imported after having imported two files where the
second file has 80% size of the first one and the threshold value is 75.
"""
expect_success(stolen_list_importer, 100, db_conn, logger)
with get_importer(
StolenListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
StolenListParams(
filename=data_file_to_test(
80,
imei_custom_header='imei,reporting_date,'
'status',
imei_imsi=False),
import_size_variation_percent=mocked_config.
stolen_threshold_config.import_size_variation_percent,
import_size_variation_absolute=mocked_config.
stolen_threshold_config.import_size_variation_absolute)
) as imp:
expect_success(imp, 80, db_conn, logger)
def test_repeat_import(stolen_list_importer, logger, mocked_statsd, db_conn,
metadata_db_conn, mocked_config, tmpdir):
"""Test Depot ID not known yet.
Verify that valid stolen list data can be successfully imported into the database
when repeating the import of the same file.
"""
expect_success(stolen_list_importer, 21, db_conn, logger)
with get_importer(
StolenListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
StolenListParams(filename='sample_stolen_list_v1.csv')) as imp:
expect_success(imp, 21, db_conn, logger)
def test_override_historical_check(stolen_list_importer, logger, mocked_statsd,
db_conn, metadata_db_conn, mocked_config,
tmpdir):
"""Test Depot ID 96594/8.
Verify that the user can override historical checks when importing Stolen List data.
"""
expect_success(stolen_list_importer, 20, db_conn, logger)
with get_importer(
StolenListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
StolenListParams(filename='sample_stolen_list_historicalcheck.csv',
perform_historic_check=False)) as imp:
expect_success(imp, 9, db_conn, logger)
def test_historical_check_empty(stolen_list_importer, logger, mocked_statsd,
db_conn, metadata_db_conn, mocked_config,
tmpdir):
"""Test Depot ID not known yet.
Verify that it fails to import an empty file after importing a non empty file.
"""
expect_success(stolen_list_importer, 20, db_conn, logger)
with get_importer(
StolenListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
StolenListParams(
filename='empty_stolenlist_historical_check.csv')) as imp:
expect_failure(imp, exc_message='Failed import size historic check')
def test_update_existing_record(stolen_list_importer, logger, db_conn,
mocked_config, metadata_db_conn, tmpdir,
mocked_statsd):
"""Test update existing record."""
# populate historic table
expect_success(stolen_list_importer, 2, db_conn, logger)
# Verify that the update consisted in add + remove:
# total rows in db = 4
# two rows are added
# new rows have end_date set to null and new reporting date '20160425'
# existing rows have end_date set to job start time and old reporting date '20160420'
with get_importer(
StolenListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
StolenListParams(content='imei,reporting_date,status\n'
'62222222222222,20160425,\n'
'122222222222223,20160425,')) as imp:
expect_success(imp, 2, db_conn, logger)
with db_conn, db_conn.cursor() as cur:
cur.execute(
'SELECT imei_norm, reporting_date, end_date FROM historic_stolen_list'
)
res = cur.fetchall()
assert len(res) == 4
stolen_list_rows = {(r.imei_norm, r.reporting_date) for r in res}
assert stolen_list_rows == {
('12222222222222', datetime.date(datetime(2016, 4, 25))),
('62222222222222', datetime.date(datetime(2016, 4, 25))),
('12222222222222', datetime.date(datetime(2016, 4, 20))),
('62222222222222', datetime.date(datetime(2016, 4, 20)))
}
# existing rows (with rep_date 20160420) will have end_date not null
assert all([
r.end_date is not None for r in res
if r.reporting_date == datetime.date(datetime(2016, 4, 20))
])
# viceversa added rows (with rep_date 20160425) will have end_date null
assert all([
r.end_date is None for r in res
if r.reporting_date == datetime.date(datetime(2016, 4, 25))
])
def test_malformed_imeis(stolen_list_importer, logger, mocked_statsd, db_conn,
metadata_db_conn, mocked_config, tmpdir):
"""Test Depot ID 95690/4.
Verify that the Stolen List data file is accepted
and imported if the data contains IMEIs with #, and *.
"""
expect_success(stolen_list_importer, 20, db_conn, logger)
# attempting to import stolen list file containing symbol not allowed '%'.
with get_importer(
StolenListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
StolenListParams(
filename='stolen_list_hexpound_bad_symbol.csv')) as imp:
expect_failure(imp,
exc_message='regex("^[0-9A-Fa-f\\\\*\\\\#]{1,16}$") '
'fails for line: 1, column: imei, value: '
'"62%222222222222"\\nFAIL')
def test_historical_check_percentage_fails(stolen_list_importer, logger,
mocked_statsd, db_conn,
mocked_config, metadata_db_conn,
tmpdir):
"""Test Depot ID 96593/7.
Verify that a local stolen data containing 9 rows fails to be imported after having imported a 20
rows file because Historical stolen list check is greater than 25% drop in import size;.
"""
expect_success(stolen_list_importer, 20, db_conn, logger)
with get_importer(
StolenListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
StolenListParams(
filename='sample_stolen_list_historicalcheck.csv',
import_size_variation_percent=mocked_config.
stolen_threshold_config.import_size_variation_percent,
import_size_variation_absolute=mocked_config.
stolen_threshold_config.import_size_variation_absolute)
) as imp:
expect_failure(imp, exc_message='Failed import size historic check')
runner = CliRunner()
result = runner.invoke(dirbs_db_cli, ['upgrade'],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
@pytest.mark.parametrize(
'operator_data_importer, stolen_list_importer, pairing_list_importer, '
'gsma_tac_db_importer, registration_list_importer',
[(OperatorDataParams(
filename=
'testData1-operator-operator1-anonymized_20161101_20161130.csv',
operator='operator1',
perform_unclean_checks=False,
extract=False),
StolenListParams(filename='testData1-sample_stolen_list-anonymized.csv'),
PairListParams(filename='testData1-sample_pairinglist-anonymized.csv'),
GSMADataParams(
filename='testData1-gsmatac_operator4_operator1_anonymized.txt'),
RegistrationListParams(filename='sample_registration_list.csv'))],
indirect=True)
def test_cli_repartition(postgres, mocked_config, db_conn,
operator_data_importer, registration_list_importer,
pairing_list_importer, stolen_list_importer,
gsma_tac_db_importer, tmpdir, logger,
metadata_db_conn, mocked_statsd):
"""Test that the dirbs-db partition script runs without an error."""
import_data(operator_data_importer, 'operator_data', 17, db_conn, logger)
import_data(gsma_tac_db_importer, 'gsma_data', 13, db_conn, logger)
import_data(stolen_list_importer, 'stolen_list', 21, db_conn, logger)
import_data(registration_list_importer, 'registration_list', 20, db_conn,
def test_reporting_date_optional(stolen_list_importer, logger, db_conn,
mocked_config, metadata_db_conn, tmpdir,
mocked_statsd):
"""Test Depot ID not known yet.
Verify that valid stolen data can be successfully imported into the database with optional field reporting_date.
Verify in case of two IMEIs with different reporting_date, only the one with min(reporting_date) is imported.
"""
# Basically, the decorator @optional in the stolen_list schema for the field reporting-date, allows whitespaces.
# If the field was text and we imported a row containing a reporting-date whitespace-only,
# the row would have been imported with withespaces into the db.
# In this case the reporting-date field is of type date so,
# when we import a whitespace data of type text into a date type column, we get an Error Type
# valid imei, reporting_date
expect_success(stolen_list_importer, 2, db_conn, logger)
with db_conn, db_conn.cursor() as cur:
cur.execute(
'SELECT imei_norm, reporting_date FROM historic_stolen_list ORDER BY imei_norm'
)
res = cur.fetchall()
assert len(res) == 2
stolen_list_rows = {(r.imei_norm, r.reporting_date) for r in res}
assert stolen_list_rows == {
('12222222222222', datetime.date(datetime(2016, 4, 25))),
('62222222222222', datetime.date(datetime(2016, 4, 26)))
}
# Verify in case of two IMEIs with different reporting_date, duplicate check will raise an exception.
with get_importer(
StolenListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
StolenListParams(perform_historic_check=False,
content='imei,reporting_date,status\n'
'01234567891234,20160401,\n'
'01234567891234,20160402,')) as imp:
expect_failure(
imp,
'Conflicting rows check failed (1 rows with same primary key and conflicting data)'
)
assert 'Found 1 conflicting row(s) with primary key (\'imei_norm\',): (\'01234567891234\',)' \
in logger_stream_contents(logger)
# valid imei, reporting_date whitespace only - error reporting_date type is date not string
with get_importer(
StolenListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
StolenListParams(content='imei,reporting_date,status\n'
'622222222222222, ,\n'
'122222222222223,20160425,')) as imp:
expect_failure(
imp,
exc_message='Pre-validation failed: b\'Error: '
'regex("^(20[0-9]{2}((0[13578]|1[02])31|(01|0[3-9]|1[0-2])(29|30)|'
'(0[1-9]|1[0-2])(0[1-9]|1[0-9]|2[0-8]))|20([02468][048]|[13579][26])0229)?$") '
'fails for line: 1, column: reporting_date, value: " "\\nFAIL')
db_conn.commit()
with db_conn, db_conn.cursor() as cur:
cur.execute('TRUNCATE historic_stolen_list')
# valid imei, reporting_date empty - type null
with get_importer(
StolenListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
StolenListParams(content='imei,reporting_date,status\n'
'722222222222222,,\n'
'122222222222223,20160425,')) as imp:
expect_success(imp, 2, db_conn, logger)
with db_conn, db_conn.cursor() as cur:
cur.execute(
'SELECT imei_norm, reporting_date FROM historic_stolen_list ORDER BY imei_norm'
)
res = cur.fetchall()
assert len(res) == 2
stolen_list_rows = {(r.imei_norm, r.reporting_date) for r in res}
assert stolen_list_rows == {('12222222222222',
datetime.date(datetime(2016, 4, 25))),
('72222222222222', None)}
with db_conn.cursor() as cursor:
cursor.execute('SELECT imei_norm FROM stolen_list ORDER BY imei_norm')
result_list = [res.imei_norm for res in cursor]
assert result.exit_code == 0
assert result_list == [
'10000000000000', '10000000000001', '10000000000002', '10000000000003',
'10000000000004', '10000000000005'
]
# DIRBS-335
@pytest.mark.parametrize(
'stolen_list_importer',
[StolenListParams(filename='stolen_list_missingheader.csv')],
indirect=True)
def test_missing_header(stolen_list_importer, logger, db_conn):
"""Test Depot ID 96588/2.
Verify that the Stolen List data is not imported if a header column is missing.
"""
expect_failure(
stolen_list_importer,
exc_message='cannot find the column headers - , 642222222222222, imei, '
'reporting_date, status')
# DIRBS-457
@pytest.mark.parametrize(
'stolen_list_importer',