def test_empty_file_no_headers(pairing_list_importer, logger):
"""Test Depot ID 96777/17.
Verify that an empty pairing list file cannot be imported into the database.
"""
expect_failure(pairing_list_importer, exc_message='metadata file is empty but '
'should contain at least a header\\nFAIL')
def test_out_of_order_columns(pairing_list_importer, logger):
"""Test Depot ID 96784/18.
Verify that the pairing list file is rejected and not
imported if it the headers are in the wrong order.
"""
expect_failure(pairing_list_importer)
def test_historic_failure_metrics(mocked_statsd, mocked_config, logger,
gsma_tac_db_importer, tmpdir, db_conn,
metadata_db_conn):
"""Test Depot ID TBD.
Verify that StatsD is sent stats when a threshold validation failure happens during import.
"""
gsma_tac_db_importer.import_data()
# Try a small import
with get_importer(
GSMADataImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
GSMADataParams(
filename='gsma_dump_small_july_2016.txt',
import_size_variation_percent=mocked_config.
gsma_threshold_config.import_size_variation_percent,
import_size_variation_absolute=mocked_config.
gsma_threshold_config.import_size_variation_absolute,
extract=False)) as gsma_small_importer:
expect_failure(gsma_small_importer,
exc_message='Failed import size historic check')
# Expected call is statsd.gauge(key, 1, delta=True)
mocked_statsd.gauge.assert_any_call(
'dirbs.import.gsma_tac.validation_failures.historic_import_size',
1,
delta=True)
def test_delta_file_prevalidation(logger, db_conn, metadata_db_conn,
mocked_config, tmpdir, mocked_statsd,
pairing_list_importer):
"""Test Depot not available yet.
Test pre-validation schemas.
"""
# update change-type is allowed only for stolen
expect_failure(
pairing_list_importer,
exc_message=
'Pre-validation failed: b\'Error: regex("^(add|remove)$") fails for line: 2, '
'column: change_type, value: "update"\\nFAIL')
# change_type must be lower case
with get_importer(
PairingListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
PairListParams(content='imei,imsi,msisdn,change_type\n'
'12345678901234,11111111111111,222222222222222,ADD',
delta=True)) as imp:
expect_failure(
imp,
exc_message=
'Pre-validation failed: b\'Error: regex("^(add|remove)$") fails for line: '
'1, column: change_type, value: "ADD"\\nFAIL')
def test_duplicate_tac_count(gsma_tac_db_importer):
"""Test duplicates."""
# gsma entries: 38245933(2 entries), 38245932(4 entries), 38245931(2 entries)
# expect the duplicates to be 1+3+1=5
expect_failure(gsma_tac_db_importer,
exc_message='Conflicting rows check failed '
'(5 rows with same primary key and conflicting data)')
def test_preprocess_uneven_rows(gsma_tac_db_importer, logger):
"""Test Depot ID 96695/18.
Verify that the gsma data file is rejected and not
imported if inconsistent number of fields per row.
"""
expect_failure(gsma_tac_db_importer, exc_message='Inconsistent number of fields per row')
def test_invalid_uid_val(device_association_list_importer, logger):
"""Verify that the subscribers list data is checked for invalid uid and is not imported."""
expect_failure(
device_association_list_importer,
exc_message='regex("^[0-9A-Za-z\\\\-\\\\\\\\]{1,20}$") fails '
'for line: 1, column: uid, value: "2333)3333&3333%3'
'"\\nFAIL')
def test_preprocess_missing_headers(gsma_tac_db_importer, logger):
"""Test Depot ID 96573/4.
Verify that the gsma data file is rejected and not imported if a header column is missing.
Test file with no extra fields and missing headers.
"""
expect_failure(gsma_tac_db_importer, exc_message='Missing mandatory field')
def test_incorrect_delimiter(gsma_tac_db_importer, logger):
"""Test Depot ID 96572/3.
Verify that the GSMA data file is rejected and not
imported if it is not "|" delimited.
"""
expect_failure(gsma_tac_db_importer, exc_message='Missing mandatory field')
def test_missing_header(golden_list_importer, logger, db_conn):
"""Test Depot ID not known yet.
Verify that the Golden List data is not imported if a header column is missing.
"""
expect_failure(golden_list_importer, exc_message='Metadata header, cannot find the column headers - '
'golden_imei, 642222222222222 - .\\nFAIL')
def test_delta_file_prevalidation(monitoring_list_importer):
"""Test pre-validation schema."""
expect_failure(
monitoring_list_importer,
exc_message=
'regex("^(add|remove)$") fails for line: 1, column: change_type, value: "ADD"'
)
def test_invalid_imsi_char(pairing_list_importer, logger):
"""Test Depot ID 96617/3.
Verify that Pairing List data is checked for invalid
IMSI(s) and is not imported into the database.
"""
expect_failure(pairing_list_importer, exc_message='regex("^[0-9]{1,15}$") fails for line: 1, '
'column: imsi, value: "&*^&*^*(&^"\\nFAIL')
def test_malformed_date(gsma_tac_db_importer, db_conn, logger):
"""Test Depot ID 96575/6.
Verify that the GSMA data file is rejected if
the file contains a malformed date.
"""
expect_failure(gsma_tac_db_importer,
exc_message='fails for line: 3, column: Allocation_Date, value: "Sep-23-2011"\\nFAIL')
def test_invalid_column_data_two(gsma_tac_db_importer, logger, db_conn, tmpdir):
"""Test Depot ID 96570/10.
Verify that GSMA data is pre-checked for invalid column specific information
and is not inserted into the DB.
"""
expect_failure(gsma_tac_db_importer,
exc_message='length(8) fails for line: 1, column: TAC, value: "9113177"\\nFAIL')
def test_invalid_column_data_three(gsma_tac_db_importer, logger, db_conn, tmpdir):
"""Test Depot ID 96570/10.
Verify that GSMA data is pre-checked for invalid column specific information
and is not inserted into the DB.
"""
expect_failure(gsma_tac_db_importer,
exc_message='fails for line: 1, column: Allocation_Date, value: "2011-23-Sep"\\nFAIL')
def test_duplicate_tac_mismatch(gsma_tac_db_importer):
"""Test Depot ID not known yet.
Verify that partial duplicate entries(Same TAC and at least 1 identical column value) in
another row is marked as duplicate and is not imported into the DB.
"""
expect_failure(gsma_tac_db_importer, exc_message='Conflicting rows check failed '
'(1 rows with same primary key and conflicting data)')
def test_historical_check_percentage(device_association_list_importer, logger, mocked_statsd, db_conn,
metadata_db_conn, mocked_config, tmpdir):
"""Verify that the list data is not imported if it fails the historical check."""
expect_success(device_association_list_importer, 5, db_conn, logger)
# size increased, importer succeeds
with get_importer(DeviceAssociationListImporter,
db_conn,
metadata_db_conn,
mocked_config.db_config,
tmpdir,
logger,
mocked_statsd,
DeviceAssociationListParams(filename='sample_association_list_large.csv',
import_size_variation_percent=mocked_config
.associations_threshold_config
.import_size_variation_percent,
import_size_variation_absolute=mocked_config
.associations_threshold_config
.import_size_variation_absolute
)) as imp:
expect_success(imp, 99, db_conn, logger)
# importing file with drop in size greater then 5%
with get_importer(DeviceAssociationListImporter,
db_conn,
metadata_db_conn,
mocked_config.db_config,
tmpdir,
logger,
mocked_statsd,
DeviceAssociationListParams(filename='sample_association_list_large_drop_5_percent_greater.csv',
import_size_variation_percent=mocked_config
.associations_threshold_config
.import_size_variation_percent,
import_size_variation_absolute=mocked_config
.associations_threshold_config
.import_size_variation_absolute
)) as imp:
expect_failure(imp, exc_message='Failed import size historic check')
# importing file with drop in size greater then 5%
with get_importer(DeviceAssociationListImporter,
db_conn,
metadata_db_conn,
mocked_config.db_config,
tmpdir,
logger,
mocked_statsd,
DeviceAssociationListParams(filename='sample_association_list_large_drop_5_percent_less.csv',
import_size_variation_percent=mocked_config
.associations_threshold_config
.import_size_variation_percent,
import_size_variation_absolute=mocked_config
.associations_threshold_config
.import_size_variation_absolute
)) as imp:
expect_success(imp, 95, db_conn, logger)
def test_threshold_failure_metrics(mocked_statsd, logger, operator_data_importer):
"""Test Depot ID TBD.
Verify that StatsD is sent stats when a threshold validation failure happens during import.
"""
expect_failure(operator_data_importer)
# Expected call is statsd.gauge(key, 1, delta=True)
mocked_statsd.gauge.assert_any_call(
'dirbs.import.operator.test_operator.validation_failures.null_imsi', 1, delta=True)
def test_missing_header(stolen_list_importer, logger, db_conn):
"""Test Depot ID 96588/2.
Verify that the Stolen List data is not imported if a header column is missing.
"""
expect_failure(
stolen_list_importer,
exc_message='cannot find the column headers - , 642222222222222, imei, '
'reporting_date, status')
def test_invalid_imei_char(pairing_list_importer, logger):
"""Test Depot ID 96616/2.
Verify that Pairing List data is checked for invalid IMEI(s) and is not
imported into the database.
"""
expect_failure(pairing_list_importer, exc_message='regex("^[0-9A-Fa-f\\\\*\\\\#]{1,16}$") fails '
'for line: 1, column: imei, value: '
'"InvalidIMEI"\\nFAIL')
def test_header_case_insensitivity(registration_list_importer, logger, db_conn):
"""Test Depot ID not known yet.
Verify that the Registration List data is not imported if a header column is missing.
"""
expect_failure(registration_list_importer, exc_message='Metadata header, cannot find the column headers - '
'10000000000000, device_id, model, '
'device_type, model_number, '
'status, brand_name, make, approved_imei, '
'radio_interface - .\\nFAIL')
def test_file_level_failure_metrics(mocked_statsd, logger, operator_data_importer, gsma_tac_db_importer):
"""Test Depot ID TBD.
Verify that StatsD is sent stats when a file-level validation failure happens during import.
"""
gsma_tac_db_importer.import_data()
expect_failure(operator_data_importer)
# Expected call is statsd.gauge(key, 1, delta=True)
mocked_statsd.gauge.assert_any_call(
'dirbs.import.operator.test_operator.validation_failures.leading_zero', 1, delta=True)
def test_empty_imsi(pairing_list_importer):
"""Test Depot not known yet.
Verify that Pairing List data is checked for empty IMSI(s) and is not
imported into the database. This test has been added to verify that imsi column doesn't need to be trimmed because
empty IMSI column will be refused by the validator.
"""
expect_failure(pairing_list_importer, exc_message='Pre-validation failed: b\'Error: '
'regex("^[0-9]{1,15}$") fails for line: 1, column: imsi, '
'value: ""\\nFAIL')
def test_delta_file_prevalidation(stolen_list_importer):
"""Test Depot not available yet.
Test pre-validation schemas.
"""
# change_type must be lower case
expect_failure(
stolen_list_importer,
exc_message=
'regex("^(add|remove|update)$") fails for line: 1, column: change_type, value: "ADD"'
)
def test_historical_check_empty(monitoring_list_importer, logger,
mocked_statsd, db_conn, metadata_db_conn,
mocked_config, tmpdir):
"""Verify that empty file import fails after importing a non empty file."""
expect_success(monitoring_list_importer, 20, db_conn, logger)
with get_importer(
MonitoringListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
MonitoringListParams(
filename='empty_monitoringlist_historical_check.csv')) as imp:
expect_failure(imp, exc_message='Failed import size historic check')
def test_header_case_insensitivity(registration_list_importer, logger,
db_conn):
"""Test Depot ID not known yet.
Verify that the Registration List data is not imported if a header column is missing.
"""
expect_failure(
registration_list_importer,
exc_message='Metadata header, cannot find the column headers - '
'10000000000000, model, status, make, '
'approved_imei - .\\nFAIL')
def test_historical_check_percentage(pairing_list_importer, logger, mocked_statsd, db_conn, metadata_db_conn,
mocked_config, tmpdir):
"""Test Depot ID 96662/5.
Verify that pairing list data is not imported if it fails the historical check.
"""
expect_success(pairing_list_importer, 5, db_conn, logger)
# size increased, the importer succeeds.
with get_importer(PairingListImporter,
db_conn,
metadata_db_conn,
mocked_config.db_config,
tmpdir,
logger,
mocked_statsd,
PairListParams(filename='sample_pairing_list_large.csv',
import_size_variation_percent=mocked_config.pairing_threshold_config.
import_size_variation_percent,
import_size_variation_absolute=mocked_config.pairing_threshold_config.
import_size_variation_absolute)) as imp:
expect_success(imp, 99, db_conn, logger)
# importing file with drop in size greater than 5%
with get_importer(PairingListImporter,
db_conn,
metadata_db_conn,
mocked_config.db_config,
tmpdir,
logger,
mocked_statsd,
PairListParams(filename=data_file_to_test(90, imei_imsi=True),
import_size_variation_percent=mocked_config.pairing_threshold_config.
import_size_variation_percent,
import_size_variation_absolute=mocked_config.pairing_threshold_config.
import_size_variation_absolute)) as imp:
expect_failure(imp, exc_message='Failed import size historic check')
# importing file with drop in size less than 5%
with get_importer(PairingListImporter,
db_conn,
metadata_db_conn,
mocked_config.db_config,
tmpdir,
logger,
mocked_statsd,
PairListParams(filename=data_file_to_test(95, imei_imsi=True),
import_size_variation_percent=mocked_config.pairing_threshold_config.
import_size_variation_percent,
import_size_variation_absolute=mocked_config.pairing_threshold_config.
import_size_variation_absolute)) as imp:
expect_success(imp, 95, db_conn, logger)
def test_historical_check_empty(subscribers_list_importer, logger,
mocked_statsd, db_conn, metadata_db_conn,
mocked_config, tmpdir):
"""Verify that subscribers list data is not imported if it fails historical check."""
expect_success(subscribers_list_importer, 5, db_conn, logger)
# attempting to import empty subscribers list
with get_importer(
SubscribersListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
SubscribersListParams(
filename='sample_subscriberslist_historial_check.csv')) as imp:
expect_failure(imp, exc_message='Failed import size historic check')
def test_historical_check_1000(pairing_list_importer, logger, mocked_statsd,
db_conn, metadata_db_conn, mocked_config,
tmpdir):
"""Test Depot ID not known yet.
Historical pairing list check failed greater than 5% drop in import size 1000000 down to 98000.
"""
expect_success(pairing_list_importer, 1000, db_conn, logger)
with get_importer(
PairingListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
PairListParams(filename=data_file_to_test(
900, imei_imsi_msisdn=True))) as imp:
expect_failure(imp)
def test_historical_check_empty(pairing_list_importer, logger, mocked_statsd,
db_conn, metadata_db_conn, mocked_config,
tmpdir):
"""Test Depot ID 96662/5.
Verify that a the pairing list data is not imported if it fails the historical check.
"""
expect_success(pairing_list_importer, 5, db_conn, logger)
# attempting to import empty pairing list file after having successfully imported 99 rows.
with get_importer(
PairingListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
PairListParams(
filename='sample_pairinglist_historial_check.csv')) as imp:
expect_failure(imp, exc_message='Failed import size historic check')
