def test_prune(per_test_postgres, tmpdir, logger, mocked_statsd, db_user,
mocked_config, monkeypatch):
"""Test prune works with the poweruser security role."""
dsn = per_test_postgres.dsn()
db_config = DBConfig(ignore_env=True, **dsn)
with create_db_connection(db_config) as conn, create_db_connection(
db_config, autocommit=True) as metadata_conn:
with get_importer(
OperatorDataImporter, conn, metadata_conn, db_config, tmpdir,
logger, mocked_statsd,
OperatorDataParams(
filename=
'testData1-operator-operator4-anonymized_20161101_20161130.csv',
operator='1',
extract=False,
perform_leading_zero_check=False,
mcc_mnc_pairs=[{
'mcc': '111',
'mnc': '04'
}],
perform_unclean_checks=False,
perform_file_daterange_check=False)) as imp:
imp.import_data()
conn.commit()
runner = CliRunner()
monkeypatch.setattr(mocked_config.db_config, 'user', db_user)
result = runner.invoke(dirbs_prune_cli, ['triplets'],
obj={'APP_CONFIG': mocked_config})
if db_user in ['dirbs_poweruser_login']:
assert result.exit_code == 0
else:
assert result.exit_code != 0
def test_unobserved_valid_imeis(flask_app, gsma_tac_db_importer, operator_data_importer, classification_data,
db_conn, metadata_db_conn, mocked_config, tmpdir, logger, mocked_statsd, api_version):
"""Test Depot ID 96544/1.
Verify the IMEI API supports HTTP GET and responds with correct
HTTP Status codes and response body.
"""
gsma_tac_db_importer.import_data()
operator_data_importer.import_data()
with get_importer(OperatorDataImporter,
db_conn,
metadata_db_conn,
mocked_config.db_config,
tmpdir,
logger,
mocked_statsd,
OperatorDataParams(
filename='testData1-operator-operator4-anonymized_20161101_20161130.csv',
extract=False,
operator='operator4',
perform_unclean_checks=False,
perform_region_checks=False,
perform_home_network_check=False)) as new_imp:
new_imp.import_data()
rv = flask_app.get(url_for('{0}.imei_api'.format(api_version), imei='3884773337002633'))
assert rv.status_code == 200
data = json.loads(rv.data.decode('utf-8'))
assert data['imei_norm'] == '38847733370026'
for k, v in data['classification_state']['blocking_conditions'].items():
assert v is False
for k, v in data['classification_state']['informative_conditions'].items():
assert v is False
assert data['realtime_checks']['invalid_imei'] is False
def test_report(per_test_postgres, tmpdir, db_user, logger, mocked_statsd,
mocked_config, monkeypatch):
"""Test catalog works with the security role created based on abstract role."""
dsn = per_test_postgres.dsn()
db_config = DBConfig(ignore_env=True, **dsn)
with create_db_connection(db_config) as conn, create_db_connection(
db_config, autocommit=True) as metadata_conn:
with get_importer(
OperatorDataImporter, conn, metadata_conn, db_config, tmpdir,
logger, mocked_statsd,
OperatorDataParams(
filename=
'testData1-operator-operator1-anonymized_20161101_20161130.csv',
operator='operator1',
perform_unclean_checks=False,
extract=False)) as imp:
imp.import_data()
runner = CliRunner()
output_dir = str(tmpdir)
monkeypatch.setattr(mocked_config.db_config, 'user', db_user)
result = runner.invoke(dirbs_report_cli, [
'standard', '--disable-retention-check', '--disable-data-check', '11',
'2016', output_dir
],
obj={'APP_CONFIG': mocked_config})
if db_user in ['dirbs_poweruser_login', 'dirbs_report_user']:
assert result.exit_code == 0
else:
assert result.exit_code != 0
def test_imei_api(per_test_flask_app, per_test_postgres, logger, mocked_statsd, tmpdir, request, mocked_config,
api_version):
"""Test IMEI API call works with the security role created based on abstract role."""
dsn = per_test_postgres.dsn()
db_config = DBConfig(ignore_env=True, **dsn)
with create_db_connection(db_config) as conn, \
create_db_connection(db_config, autocommit=True) as metadata_conn:
with get_importer(OperatorDataImporter,
conn,
metadata_conn,
db_config,
tmpdir,
logger,
mocked_statsd,
OperatorDataParams(
filename='testData1-operator-operator1-anonymized_20161101_20161130.csv',
operator='operator1',
perform_unclean_checks=False,
extract=False)) as imp:
imp.import_data()
current_user = request.node.callspec.params['per_test_flask_app']
if api_version == 'v1':
if current_user in ['dirbs_poweruser_login', 'dirbs_api_user']:
rv = per_test_flask_app.get(url_for('{0}.imei_api'.format(api_version),
imei='388260336982806', include_seen_with=1))
assert rv.status_code == 200
assert json.loads(rv.data.decode('utf-8'))['seen_with'] == \
[{'imsi': '11101400135251', 'msisdn': '22300825684694'},
{'imsi': '11101400135252', 'msisdn': '22300825684692'}]
assert json.loads(rv.data.decode('utf-8'))['realtime_checks']['ever_observed_on_network'] is True
else:
with pytest.raises(DatabaseRoleCheckException):
per_test_flask_app.get(url_for('{0}.imei_api'.format(api_version),
imei='388260336982806', include_seen_with=1))
else: # api version 2.0
if current_user in ['dirbs_poweruser_login', 'dirbs_api_user']:
rv = per_test_flask_app.get(url_for('{0}.imei_get_subscribers_api'.format(api_version),
imei='388260336982806'))
assert rv.status_code == 200
data = json.loads(rv.data.decode('utf-8'))
assert len(data['subscribers']) != 0
assert data['subscribers'] == [
{
'imsi': '11101400135251',
'last_seen': '2016-11-01',
'msisdn': '22300825684694'
},
{
'imsi': '11101400135252',
'last_seen': '2016-11-02',
'msisdn': '22300825684692'
}]
else:
with pytest.raises(DatabaseRoleCheckException):
per_test_flask_app.get(url_for('{0}.imei_get_subscribers_api'.format(api_version),
imei='388260336982806'))
def test_classify(per_test_postgres, db_user, tmpdir, logger, mocked_statsd, monkeypatch, mocked_config):
"""Test classify works with the security role created based on abstract role."""
dsn = per_test_postgres.dsn()
db_config = DBConfig(ignore_env=True, **dsn)
with create_db_connection(db_config) as conn, create_db_connection(db_config, autocommit=True) as metadata_conn:
with get_importer(OperatorDataImporter,
conn,
metadata_conn,
db_config,
tmpdir,
logger,
mocked_statsd,
OperatorDataParams(
content='date,imei,imsi,msisdn\n'
'20110101,8888#888622222,123456789012345,123456789012345\n'
'20110101,88888888622222,123456789012345,123456789012345\n'
'20110101,8888888862222209,123456789012345,123456789012345\n'
'20110101,88888862222209**,123456789012345,123456789012345',
extract=False,
perform_unclean_checks=False,
perform_region_checks=False,
perform_home_network_check=False,
operator='operator1')) as imp:
imp.import_data()
with get_importer(GSMADataImporter,
conn,
metadata_conn,
db_config,
tmpdir,
logger,
mocked_statsd,
GSMADataParams(filename='gsma_not_found_anonymized.txt')) as imp:
imp.import_data()
with get_importer(RegistrationListImporter,
conn,
metadata_conn,
db_config,
tmpdir,
logger,
mocked_statsd,
RegistrationListParams(filename='sample_registration_list.csv')) as imp:
imp.import_data()
# Run dirbs-classify using db args from the temp postgres instance
runner = CliRunner()
monkeypatch.setattr(mocked_config.db_config, 'user', db_user)
result = runner.invoke(dirbs_classify_cli, ['--no-safety-check'], obj={'APP_CONFIG': mocked_config})
if db_user in ['dirbs_poweruser_login', 'dirbs_classify_user']:
assert result.exit_code == 0
else:
assert result.exit_code != 0
def test_listgen(per_test_postgres, tmpdir, logger, mocked_statsd, db_user, mocked_config, monkeypatch,
classification_data):
"""Test that the dirbs-listgen instance runs without an error."""
dsn = per_test_postgres.dsn()
db_config = DBConfig(ignore_env=True, **dsn)
with create_db_connection(db_config) as conn, create_db_connection(db_config, autocommit=True) as metadata_conn:
with get_importer(OperatorDataImporter,
conn,
metadata_conn,
db_config,
tmpdir,
logger,
mocked_statsd,
OperatorDataParams(
content='date,imei,imsi,msisdn\n'
'20160203,811111013136460,111018001111111,223338000000\n'
'20160203,359000000000000,111015113222222,223355000000\n'
'20160203,357756065985824,111015113333333,223355111111',
cc=['22', '74'],
mcc_mnc_pairs=[{'mcc': '111', 'mnc': '01'}],
operator='operator1',
extract=False)) as imp:
imp.import_data()
with get_importer(PairingListImporter,
conn,
metadata_conn,
db_config,
tmpdir,
logger,
mocked_statsd,
PairListParams(
content='imei,imsi,msisdn\n'
'811111013136460,111018001111111,234555555555550\n'
'359000000000000,111015113222222,234555555555551\n'
'357756065985824,111015113333333,234555555555552')) as imp:
imp.import_data()
# Now run listgen as requested user
runner = CliRunner()
monkeypatch.setattr(mocked_config.db_config, 'user', db_user)
output_dir = str(tmpdir)
result = runner.invoke(dirbs_listgen_cli, [output_dir], obj={'APP_CONFIG': mocked_config})
if db_user in ['dirbs_poweruser_login', 'dirbs_listgen_user']:
assert result.exit_code == 0
else:
assert result.exit_code != 0
# Verify that all CLI command also generates a statsd message
@click.command()
@click.pass_context
@common.unhandled_exception_handler
def test_click_command(ctxt):
"""Test click program."""
raise ValueError('Testing!')
runner = CliRunner()
runner.invoke(test_click_command, obj={'STATSD_CLIENT': mocked_statsd})
mocked_statsd.incr.assert_any_call('dirbs.exceptions.cli.unknown')
@pytest.mark.parametrize('operator_data_importer', [
OperatorDataParams(filename='operator1_invalid_imei_20160701_20160731.csv',
extract=False)
],
indirect=True)
def test_prevalidation_failure_metrics(mocker, mocked_statsd, logger,
operator_data_importer):
"""Test Depot ID TBD.
Verify that StatsD is sent stats when a prevalidation error happens during import.
"""
expect_failure(operator_data_importer)
# Expected call is statsd.gauge(key, 1, delta=True)
mocked_statsd.gauge.assert_any_call(
'dirbs.import.operator.test_operator.validation_failures.prevalidation',
1,
delta=True)
def test_cli_repartition(postgres, mocked_config, db_conn,
operator_data_importer, registration_list_importer,
pairing_list_importer, stolen_list_importer,
gsma_tac_db_importer, tmpdir, logger,
metadata_db_conn, mocked_statsd):
"""Test that the dirbs-db partition script runs without an error."""
import_data(operator_data_importer, 'operator_data', 17, db_conn, logger)
import_data(gsma_tac_db_importer, 'gsma_data', 13, db_conn, logger)
import_data(stolen_list_importer, 'stolen_list', 21, db_conn, logger)
import_data(registration_list_importer, 'registration_list', 20, db_conn,
logger)
import_data(pairing_list_importer, 'pairing_list', 7, db_conn, logger)
# Import second month of operator data to ensure that we have 2 months worth for the same operator
with get_importer(
OperatorDataImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
OperatorDataParams(
content='date,imei,imsi,msisdn\n'
'20161201,64220496727231,123456789012345,123456789012345\n'
'20161201,64220496727232,123456789012345,123456789012345',
operator='operator1',
extract=False,
perform_leading_zero_check=False,
perform_unclean_checks=False,
perform_home_network_check=False,
perform_region_checks=False,
perform_historic_checks=False,
mcc_mnc_pairs=[{
'mcc': '111',
'mnc': '04'
}],
perform_file_daterange_check=False)) as new_imp:
expect_success(new_imp, 19, db_conn, logger)
# Run dirbs-classify and dirbs-listgen to populate some tables prior to re-partition
runner = CliRunner()
output_dir = str(tmpdir)
result = runner.invoke(dirbs_classify_cli, ['--no-safety-check'],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
result = runner.invoke(dirbs_listgen_cli, [output_dir],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
# Make sure that if we run with some invalid phyical shards, that it fails
for num_shards in [-1, 0, 101]:
result = runner.invoke(
dirbs_db_cli,
['repartition', '--num-physical-shards={0:d}'.format(num_shards)],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code != 0
partitioned_tables = [
'classification_state', 'historic_pairing_list',
'historic_registration_list', 'network_imeis',
'monthly_network_triplets_per_mno_operator1_2016_11',
'monthly_network_triplets_country_2016_11', 'blacklist',
'exceptions_lists_operator1', 'notifications_lists_operator1',
'historic_stolen_list'
]
with db_conn, db_conn.cursor() as cursor:
# Manually add one record into the notifications_lists for operator_1 so that the repartitioned table
# is not empty
cursor.execute(
"""INSERT INTO notifications_lists_operator1 (operator_id, imei_norm, imsi, msisdn, block_date,
reasons, start_run_id, end_run_id, delta_reason,
virt_imei_shard)
VALUES ('operator1', '12345678901234', '12345678901234', '1', '20170110',
ARRAY['condition1'], 1125, NULL, 'new', calc_virt_imei_shard('12345678901234'))
""")
# Run dirbs-db repartition to 8 partitions and check that it works
result = runner.invoke(dirbs_db_cli,
['repartition', '--num-physical-shards=8'],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
with db_conn, db_conn.cursor() as cursor:
for base_table in partitioned_tables:
cursor.execute(
sql.SQL('SELECT COUNT(*) FROM {0}').format(
sql.Identifier(base_table)))
tbl_count = cursor.fetchone()[0]
assert tbl_count > 0
cursor.execute(
"""SELECT TABLE_NAME
FROM information_schema.tables
WHERE TABLE_NAME LIKE %s
ORDER BY TABLE_NAME""",
['{0}%'.format(base_table)]),
res = [x.table_name for x in cursor]
assert res == [
'{0}'.format(base_table), '{0}_0_12'.format(base_table),
'{0}_13_25'.format(base_table), '{0}_26_38'.format(base_table),
'{0}_39_51'.format(base_table), '{0}_52_63'.format(base_table),
'{0}_64_75'.format(base_table), '{0}_76_87'.format(base_table),
'{0}_88_99'.format(base_table)
]
# Re-partition back to the default 4 shards so that we do not change state for other tests
result = runner.invoke(dirbs_db_cli,
['repartition', '--num-physical-shards=4'],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
with db_conn, db_conn.cursor() as cursor:
for base_table in partitioned_tables:
cursor.execute(
sql.SQL('SELECT COUNT(*) FROM {0}').format(
sql.Identifier(base_table)))
tbl_count = cursor.fetchone()[0]
assert tbl_count > 0
cursor.execute(
"""SELECT TABLE_NAME
FROM information_schema.tables
WHERE TABLE_NAME LIKE %s
ORDER BY TABLE_NAME""",
['{0}%'.format(base_table)]),
res = [x.table_name for x in cursor]
assert res == [
'{0}'.format(base_table), '{0}_0_24'.format(base_table),
'{0}_25_49'.format(base_table), '{0}_50_74'.format(base_table),
'{0}_75_99'.format(base_table)
]
def test_basic_cli_upgrade(postgres, mocked_config):
"""Test that the dirbs-db upgrade script runs without an error."""
runner = CliRunner()
result = runner.invoke(dirbs_db_cli, ['upgrade'],
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
@pytest.mark.parametrize(
'operator_data_importer, stolen_list_importer, pairing_list_importer, '
'gsma_tac_db_importer, registration_list_importer',
[(OperatorDataParams(
filename=
'testData1-operator-operator1-anonymized_20161101_20161130.csv',
operator='operator1',
perform_unclean_checks=False,
extract=False),
StolenListParams(filename='testData1-sample_stolen_list-anonymized.csv'),
PairListParams(filename='testData1-sample_pairinglist-anonymized.csv'),
GSMADataParams(
filename='testData1-gsmatac_operator4_operator1_anonymized.txt'),
RegistrationListParams(filename='sample_registration_list.csv'))],
indirect=True)
def test_cli_repartition(postgres, mocked_config, db_conn,
operator_data_importer, registration_list_importer,
pairing_list_importer, stolen_list_importer,
gsma_tac_db_importer, tmpdir, logger,
metadata_db_conn, mocked_statsd):
"""Test that the dirbs-db partition script runs without an error."""
import_data(operator_data_importer, 'operator_data', 17, db_conn, logger)
""" MSISDN API should return a 404 status for a zero-length MSISDN """
if api_version == 'v1':
rv = flask_app.get(
url_for('{0}.msisdn_api'.format(api_version), msisdn=''))
assert rv.status_code == 404
else: # msisdn version 2
rv = flask_app.get(
url_for('{0}.msisdn_get_api'.format(api_version), msisdn=''))
assert rv.status_code == 404
@pytest.mark.parametrize('operator_data_importer', [
OperatorDataParams(
filename=
'testData1-operator-operator4-anonymized_20161101_20161130.csv',
extract=False,
perform_unclean_checks=False,
perform_region_checks=False,
perform_home_network_check=False)
],
indirect=True)
@pytest.mark.parametrize('gsma_tac_db_importer', [
GSMADataParams(
filename='testData1-gsmatac_operator1_operator4_anonymized.txt')
],
indirect=True)
@pytest.mark.parametrize('registration_list_importer', [
RegistrationListParams(
filename='registration_list_msisidn_api_test_data.csv')
],
indirect=True)
from dirbs.cli.prune import cli as dirbs_prune_cli
from dirbs.cli.classify import cli as dirbs_classify_cli
from dirbs.importer.gsma_data_importer import GSMADataImporter
from _fixtures import * # noqa: F403, F401
from _helpers import get_importer, expect_success, from_cond_dict_list_to_cond_list
from _importer_params import OperatorDataParams, StolenListParams, GSMADataParams
@pytest.mark.parametrize('operator_data_importer', [
OperatorDataParams(
filename=
'testData1-operator-operator4-anonymized_20161101_20161130.csv',
operator='1',
extract=False,
perform_leading_zero_check=False,
mcc_mnc_pairs=[{
'mcc': '111',
'mnc': '04'
}],
perform_unclean_checks=False,
perform_file_daterange_check=False)
],
indirect=True)
def test_persistent_network_imeis(postgres, db_conn, tmpdir, logger,
operator_data_importer, mocked_config):
"""Test Depot ID 96759/1.
Verify DIRBS core instance maintain a persistent list of all IMEIs that
have ever been seen in operator data. Verify the all-time seen IMEI list does not contain
subscriber information. Verify the all-time seen IMEI list is not be impacted when operator data is pruned.
Verify the all-time seen IMEI list record the date that the IMEI first appeared on a per-operator basis.
data = json.loads(rv.data.decode('utf-8'))
expected_norm = i if len(i) < 14 else i[:14]
assert data['imei_norm'] == expected_norm
@pytest.mark.parametrize('gsma_tac_db_importer',
[GSMADataParams(filename='testData1-gsmatac_operator1_operator4_anonymized.txt',
extract=False)],
indirect=True)
@pytest.mark.parametrize('classification_data',
['classification_state/imei_api_class_state.csv'],
indirect=True)
@pytest.mark.parametrize('operator_data_importer',
[OperatorDataParams(filename='testData1-operator-operator1-anonymized_20161101_20161130.csv',
extract=False,
perform_unclean_checks=False,
perform_region_checks=False,
perform_home_network_check=False)],
indirect=True)
def test_unobserved_valid_imeis(flask_app, gsma_tac_db_importer, operator_data_importer, classification_data,
db_conn, metadata_db_conn, mocked_config, tmpdir, logger, mocked_statsd, api_version):
"""Test Depot ID 96544/1.
Verify the IMEI API supports HTTP GET and responds with correct
HTTP Status codes and response body.
"""
gsma_tac_db_importer.import_data()
operator_data_importer.import_data()
with get_importer(OperatorDataImporter,
db_conn,
metadata_db_conn,