def test_historical_check_percentage(pairing_list_importer, logger, mocked_statsd, db_conn, metadata_db_conn,
mocked_config, tmpdir):
"""Test Depot ID 96662/5.
Verify that pairing list data is not imported if it fails the historical check.
"""
expect_success(pairing_list_importer, 5, db_conn, logger)
# size increased, the importer succeeds.
with get_importer(PairingListImporter,
db_conn,
metadata_db_conn,
mocked_config.db_config,
tmpdir,
logger,
mocked_statsd,
PairListParams(filename='sample_pairing_list_large.csv',
import_size_variation_percent=mocked_config.pairing_threshold_config.
import_size_variation_percent,
import_size_variation_absolute=mocked_config.pairing_threshold_config.
import_size_variation_absolute)) as imp:
expect_success(imp, 99, db_conn, logger)
# importing file with drop in size greater than 5%
with get_importer(PairingListImporter,
db_conn,
metadata_db_conn,
mocked_config.db_config,
tmpdir,
logger,
mocked_statsd,
PairListParams(filename=data_file_to_test(90, imei_imsi=True),
import_size_variation_percent=mocked_config.pairing_threshold_config.
import_size_variation_percent,
import_size_variation_absolute=mocked_config.pairing_threshold_config.
import_size_variation_absolute)) as imp:
expect_failure(imp, exc_message='Failed import size historic check')
# importing file with drop in size less than 5%
with get_importer(PairingListImporter,
db_conn,
metadata_db_conn,
mocked_config.db_config,
tmpdir,
logger,
mocked_statsd,
PairListParams(filename=data_file_to_test(95, imei_imsi=True),
import_size_variation_percent=mocked_config.pairing_threshold_config.
import_size_variation_percent,
import_size_variation_absolute=mocked_config.pairing_threshold_config.
import_size_variation_absolute)) as imp:
expect_success(imp, 95, db_conn, logger)
def test_imei_api_pairing_list(per_test_flask_app, per_test_postgres, logger,
mocked_statsd, tmpdir, request, mocked_config,
api_version):
"""Test IMEI API call after pairing list import."""
dsn = per_test_postgres.dsn()
db_config = DBConfig(ignore_env=True, **dsn)
with create_db_connection(db_config) as conn, \
create_db_connection(db_config, autocommit=True) as metadata_conn:
with get_importer(
GSMADataImporter, conn, metadata_conn, db_config, tmpdir,
logger, mocked_statsd,
GSMADataParams(
filename='gsma_dump_small_july_2016.txt')) as imp:
imp.import_data()
with get_importer(
PairingListImporter, conn, metadata_conn, db_config, tmpdir,
logger, mocked_statsd,
PairListParams(content='imei,imsi\n'
'811111013136460,111018001111111\n'
'359000000000000,111015113222222\n'
'357756065985824,111015113333333')) as imp:
imp.import_data()
if api_version == 'v1':
rv = per_test_flask_app.get(
url_for('{0}.imei_api'.format(api_version), imei='21260934000003'))
assert rv.status_code == 200
else: # api version 2.0
rv = per_test_flask_app.get(
url_for('{0}.imei_get_pairings_api'.format(api_version),
imei='21260934000003'))
assert rv.status_code == 200
def test_delta_file_prevalidation(logger, db_conn, metadata_db_conn,
mocked_config, tmpdir, mocked_statsd,
pairing_list_importer):
"""Test Depot not available yet.
Test pre-validation schemas.
"""
# update change-type is allowed only for stolen
expect_failure(
pairing_list_importer,
exc_message=
'Pre-validation failed: b\'Error: regex("^(add|remove)$") fails for line: 2, '
'column: change_type, value: "update"\\nFAIL')
# change_type must be lower case
with get_importer(
PairingListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
PairListParams(content='imei,imsi,msisdn,change_type\n'
'12345678901234,11111111111111,222222222222222,ADD',
delta=True)) as imp:
expect_failure(
imp,
exc_message=
'Pre-validation failed: b\'Error: regex("^(add|remove)$") fails for line: '
'1, column: change_type, value: "ADD"\\nFAIL')
def test_repeat_import(pairing_list_importer, logger, mocked_statsd, db_conn,
metadata_db_conn, mocked_config, tmpdir):
"""Test Depot not known yet. Test same import doesn't affect db."""
expect_success(pairing_list_importer, 5, db_conn, logger)
with get_importer(
PairingListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
PairListParams(filename='sample_pairinglist.csv')) as imp:
expect_success(imp, 5, db_conn, logger)
def test_historical_check_1000(pairing_list_importer, logger, mocked_statsd,
db_conn, metadata_db_conn, mocked_config,
tmpdir):
"""Test Depot ID not known yet.
Historical pairing list check failed greater than 5% drop in import size 1000000 down to 98000.
"""
expect_success(pairing_list_importer, 1000, db_conn, logger)
with get_importer(
PairingListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
PairListParams(filename=data_file_to_test(
900, imei_imsi_msisdn=True))) as imp:
expect_failure(imp)
def test_override_historical_check(pairing_list_importer, logger,
mocked_statsd, db_conn, metadata_db_conn,
mocked_config, tmpdir):
"""Test Depot ID 96661/4.
Verify that the user can override historical checks when importing pairing list data.
"""
expect_success(pairing_list_importer, 5, db_conn, logger)
with get_importer(
PairingListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
PairListParams(filename='sample_pairinglist_historial_check.csv',
perform_historic_check=False)) as imp:
expect_success(imp, 0, db_conn, logger)
def test_historical_check_empty(pairing_list_importer, logger, mocked_statsd,
db_conn, metadata_db_conn, mocked_config,
tmpdir):
"""Test Depot ID 96662/5.
Verify that a the pairing list data is not imported if it fails the historical check.
"""
expect_success(pairing_list_importer, 5, db_conn, logger)
# attempting to import empty pairing list file after having successfully imported 99 rows.
with get_importer(
PairingListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
PairListParams(
filename='sample_pairinglist_historial_check.csv')) as imp:
expect_failure(imp, exc_message='Failed import size historic check')
def test_listgen(per_test_postgres, tmpdir, logger, mocked_statsd, db_user, mocked_config, monkeypatch,
classification_data):
"""Test that the dirbs-listgen instance runs without an error."""
dsn = per_test_postgres.dsn()
db_config = DBConfig(ignore_env=True, **dsn)
with create_db_connection(db_config) as conn, create_db_connection(db_config, autocommit=True) as metadata_conn:
with get_importer(OperatorDataImporter,
conn,
metadata_conn,
db_config,
tmpdir,
logger,
mocked_statsd,
OperatorDataParams(
content='date,imei,imsi,msisdn\n'
'20160203,811111013136460,111018001111111,223338000000\n'
'20160203,359000000000000,111015113222222,223355000000\n'
'20160203,357756065985824,111015113333333,223355111111',
cc=['22', '74'],
mcc_mnc_pairs=[{'mcc': '111', 'mnc': '01'}],
operator='operator1',
extract=False)) as imp:
imp.import_data()
with get_importer(PairingListImporter,
conn,
metadata_conn,
db_config,
tmpdir,
logger,
mocked_statsd,
PairListParams(
content='imei,imsi,msisdn\n'
'811111013136460,111018001111111,234555555555550\n'
'359000000000000,111015113222222,234555555555551\n'
'357756065985824,111015113333333,234555555555552')) as imp:
imp.import_data()
# Now run listgen as requested user
runner = CliRunner()
monkeypatch.setattr(mocked_config.db_config, 'user', db_user)
output_dir = str(tmpdir)
result = runner.invoke(dirbs_listgen_cli, [output_dir], obj={'APP_CONFIG': mocked_config})
if db_user in ['dirbs_poweruser_login', 'dirbs_listgen_user']:
assert result.exit_code == 0
else:
assert result.exit_code != 0
result = runner.invoke(dirbs_db_cli, ['upgrade'],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
@pytest.mark.parametrize(
'operator_data_importer, stolen_list_importer, pairing_list_importer, '
'gsma_tac_db_importer, registration_list_importer',
[(OperatorDataParams(
filename=
'testData1-operator-operator1-anonymized_20161101_20161130.csv',
operator='operator1',
perform_unclean_checks=False,
extract=False),
StolenListParams(filename='testData1-sample_stolen_list-anonymized.csv'),
PairListParams(filename='testData1-sample_pairinglist-anonymized.csv'),
GSMADataParams(
filename='testData1-gsmatac_operator4_operator1_anonymized.txt'),
RegistrationListParams(filename='sample_registration_list.csv'))],
indirect=True)
def test_cli_repartition(postgres, mocked_config, db_conn,
operator_data_importer, registration_list_importer,
pairing_list_importer, stolen_list_importer,
gsma_tac_db_importer, tmpdir, logger,
metadata_db_conn, mocked_statsd):
"""Test that the dirbs-db partition script runs without an error."""
import_data(operator_data_importer, 'operator_data', 17, db_conn, logger)
import_data(gsma_tac_db_importer, 'gsma_data', 13, db_conn, logger)
import_data(stolen_list_importer, 'stolen_list', 21, db_conn, logger)
import_data(registration_list_importer, 'registration_list', 20, db_conn,
logger)
with db_conn.cursor() as cursor:
cursor.execute(
'SELECT imei_norm, imsi FROM pairing_list ORDER BY imei_norm, imsi'
)
res = {(res.imsi, res.imei_norm) for res in cursor.fetchall()}
assert result.exit_code == 0
assert len(res) == 3
assert res == {('11108080805796', '35362602204562'),
('11108951160476', '35412003020863'),
('11108864102404', '35671206060150')}
@pytest.mark.parametrize(
'pairing_list_importer',
[PairListParams(filename='pairing_list_insensitive_header.csv')],
indirect=True)
def test_case_insensitive_headers(pairing_list_importer, logger, db_conn):
"""Test Depot ID 96773/9.
Verify that the pairing list file is accepted and imported if the headers have mixed cases.
"""
expect_success(pairing_list_importer, 1, db_conn, logger)
@pytest.mark.parametrize(
'pairing_list_importer',
[PairListParams(filename='pairing_list_duplicate_record.csv')],
indirect=True)
def test_duplicate_check_override(pairing_list_importer, logger, db_conn):
"""Test Depot ID 96774/10.
"""Test Depot ID not known yet.
Verify IMEI API respects the include_seen_with flag and only returns seen_with data iff include_seen_with is True.
"""
operator_data_importer.import_data()
data = check_output_data(flask_app, '3884773337002638', api_version,
class_block_dupl_bool=True, real_gsma_bool=True)
assert data['seen_with'] == [{'imsi': '11104803062043', 'msisdn': '22300049781840'}]
data = check_output_data(flask_app, '3884773337002638', api_version, include_seen_with_bool=False,
class_block_dupl_bool=True, real_gsma_bool=True)
assert data.get('seen_with', None) is None
@pytest.mark.parametrize('pairing_list_importer',
[PairListParams(content='imei,imsi\n'
'38847733370026,111018001111111\n'
'38847733370026,111015113222222\n'
'38847733370020,111015113333333')],
indirect=True)
@pytest.mark.parametrize('operator_data_importer',
[OperatorDataParams(filename='testData1-operator-operator4-anonymized_20161101_20161130.csv',
extract=False,
perform_unclean_checks=False,
perform_region_checks=False,
perform_home_network_check=False)],
indirect=True)
@pytest.mark.parametrize('classification_data',
['classification_state/imei_api_class_state_v1.csv'],
indirect=True)
def test_paired_with(flask_app, operator_data_importer, pairing_list_importer,
db_conn, tmpdir, logger, classification_data, api_version):
"""Test Depot ID not known yet.