def test_imei_api_registration_list(per_test_flask_app, per_test_postgres, logger, mocked_statsd, tmpdir, request,
mocked_config, api_version):
"""Test IMEI API call after registration list import."""
dsn = per_test_postgres.dsn()
db_config = DBConfig(ignore_env=True, **dsn)
with create_db_connection(db_config) as conn, \
create_db_connection(db_config, autocommit=True) as metadata_conn:
with get_importer(GSMADataImporter,
conn,
metadata_conn,
db_config,
tmpdir,
logger,
mocked_statsd,
GSMADataParams(filename='gsma_dump_small_july_2016.txt')) as imp:
imp.import_data()
with get_importer(RegistrationListImporter,
conn,
metadata_conn,
db_config,
tmpdir,
logger,
mocked_statsd,
RegistrationListParams(content='APPROVED_IMEI,make,model,status,model_number,brand_name,'
'device_type,radio_interface,device_id\n'
'21260934000003,,,,,,,,1')) as imp:
imp.import_data()
if api_version == 'v1':
rv = per_test_flask_app.get(url_for('{0}.imei_api'.format(api_version), imei='21260934000003'))
assert rv.status_code == 200
else: # api version 2.0
rv = per_test_flask_app.get(url_for('{0}.imei_get_api'.format(api_version), imei='21260934000003'))
assert rv.status_code == 200
def test_prune(per_test_postgres, tmpdir, logger, mocked_statsd, db_user, mocked_config, monkeypatch):
"""Test prune works with the poweruser security role."""
dsn = per_test_postgres.dsn()
db_config = DBConfig(ignore_env=True, **dsn)
with create_db_connection(db_config) as conn, create_db_connection(db_config, autocommit=True) as metadata_conn:
with get_importer(OperatorDataImporter,
conn,
metadata_conn,
db_config,
tmpdir,
logger,
mocked_statsd,
OperatorDataParams(
filename='testData1-operator-operator4-anonymized_20161101_20161130.csv',
operator='1',
extract=False,
perform_leading_zero_check=False,
mcc_mnc_pairs=[{'mcc': '111', 'mnc': '04'}],
perform_unclean_checks=False,
perform_file_daterange_check=False)) as imp:
imp.import_data()
conn.commit()
runner = CliRunner()
monkeypatch.setattr(mocked_config.db_config, 'user', db_user)
result = runner.invoke(dirbs_prune_cli, ['triplets'], obj={'APP_CONFIG': mocked_config})
if db_user in ['dirbs_poweruser_login']:
assert result.exit_code == 0
else:
assert result.exit_code != 0
def test_report(per_test_postgres, tmpdir, db_user, logger, mocked_statsd, mocked_config, monkeypatch):
"""Test catalog works with the security role created based on abstract role."""
dsn = per_test_postgres.dsn()
db_config = DBConfig(ignore_env=True, **dsn)
with create_db_connection(db_config) as conn, create_db_connection(db_config, autocommit=True) as metadata_conn:
with get_importer(OperatorDataImporter,
conn,
metadata_conn,
db_config,
tmpdir,
logger,
mocked_statsd,
OperatorDataParams(
filename='testData1-operator-operator1-anonymized_20161101_20161130.csv',
operator='operator1',
perform_unclean_checks=False,
extract=False)) as imp:
imp.import_data()
runner = CliRunner()
output_dir = str(tmpdir)
monkeypatch.setattr(mocked_config.db_config, 'user', db_user)
result = runner.invoke(dirbs_report_cli, ['standard', '--disable-retention-check', '--disable-data-check',
'11', '2016', output_dir], obj={'APP_CONFIG': mocked_config})
if db_user in ['dirbs_poweruser_login', 'dirbs_report_user']:
assert result.exit_code == 0
else:
assert result.exit_code != 0
def test_imei_api_pairing_list(per_test_flask_app, per_test_postgres, logger,
mocked_statsd, tmpdir, request, mocked_config,
api_version):
"""Test IMEI API call after pairing list import."""
dsn = per_test_postgres.dsn()
db_config = DBConfig(ignore_env=True, **dsn)
with create_db_connection(db_config) as conn, \
create_db_connection(db_config, autocommit=True) as metadata_conn:
with get_importer(
GSMADataImporter, conn, metadata_conn, db_config, tmpdir,
logger, mocked_statsd,
GSMADataParams(
filename='gsma_dump_small_july_2016.txt')) as imp:
imp.import_data()
with get_importer(
PairingListImporter, conn, metadata_conn, db_config, tmpdir,
logger, mocked_statsd,
PairListParams(
content='imei,imsi,msisdn\n'
'811111013136460,111018001111111,222222222222222\n'
'359000000000000,111015113222222,222222222222223\n'
'357756065985824,111015113333333,222222222222224')) as imp:
imp.import_data()
if api_version == 'v1':
rv = per_test_flask_app.get(
url_for('{0}.imei_api'.format(api_version), imei='21260934000003'))
assert rv.status_code == 200
else: # api version 2.0
rv = per_test_flask_app.get(
url_for('{0}.imei_get_pairings_api'.format(api_version),
imei='21260934000003'))
assert rv.status_code == 200
def test_imei_api(per_test_flask_app, per_test_postgres, logger, mocked_statsd, tmpdir, request, mocked_config,
api_version):
"""Test IMEI API call works with the security role created based on abstract role."""
dsn = per_test_postgres.dsn()
db_config = DBConfig(ignore_env=True, **dsn)
with create_db_connection(db_config) as conn, \
create_db_connection(db_config, autocommit=True) as metadata_conn:
with get_importer(OperatorDataImporter,
conn,
metadata_conn,
db_config,
tmpdir,
logger,
mocked_statsd,
OperatorDataParams(
filename='testData1-operator-operator1-anonymized_20161101_20161130.csv',
operator='operator1',
perform_unclean_checks=False,
extract=False)) as imp:
imp.import_data()
current_user = request.node.callspec.params['per_test_flask_app']
if api_version == 'v1':
if current_user in ['dirbs_poweruser_login', 'dirbs_api_user']:
rv = per_test_flask_app.get(url_for('{0}.imei_api'.format(api_version),
imei='388260336982806', include_seen_with=1))
assert rv.status_code == 200
assert json.loads(rv.data.decode('utf-8'))['seen_with'] == \
[{'imsi': '11101400135251', 'msisdn': '22300825684694'},
{'imsi': '11101400135252', 'msisdn': '22300825684692'}]
assert json.loads(rv.data.decode('utf-8'))['realtime_checks']['ever_observed_on_network'] is True
else:
with pytest.raises(DatabaseRoleCheckException):
per_test_flask_app.get(url_for('{0}.imei_api'.format(api_version),
imei='388260336982806', include_seen_with=1))
else: # api version 2.0
if current_user in ['dirbs_poweruser_login', 'dirbs_api_user']:
rv = per_test_flask_app.get(url_for('{0}.imei_get_subscribers_api'.format(api_version),
imei='388260336982806'))
assert rv.status_code == 200
data = json.loads(rv.data.decode('utf-8'))
assert len(data['subscribers']) != 0
assert data['subscribers'] == [
{
'imsi': '11101400135251',
'last_seen': '2016-11-01',
'msisdn': '22300825684694'
},
{
'imsi': '11101400135252',
'last_seen': '2016-11-02',
'msisdn': '22300825684692'
}]
else:
with pytest.raises(DatabaseRoleCheckException):
per_test_flask_app.get(url_for('{0}.imei_get_subscribers_api'.format(api_version),
imei='388260336982806'))
def test_classify(per_test_postgres, db_user, tmpdir, logger, mocked_statsd, monkeypatch, mocked_config):
"""Test classify works with the security role created based on abstract role."""
dsn = per_test_postgres.dsn()
db_config = DBConfig(ignore_env=True, **dsn)
with create_db_connection(db_config) as conn, create_db_connection(db_config, autocommit=True) as metadata_conn:
with get_importer(OperatorDataImporter,
conn,
metadata_conn,
db_config,
tmpdir,
logger,
mocked_statsd,
OperatorDataParams(
content='date,imei,imsi,msisdn\n'
'20110101,8888#888622222,123456789012345,123456789012345\n'
'20110101,88888888622222,123456789012345,123456789012345\n'
'20110101,8888888862222209,123456789012345,123456789012345\n'
'20110101,88888862222209**,123456789012345,123456789012345',
extract=False,
perform_unclean_checks=False,
perform_region_checks=False,
perform_home_network_check=False,
operator='operator1')) as imp:
imp.import_data()
with get_importer(GSMADataImporter,
conn,
metadata_conn,
db_config,
tmpdir,
logger,
mocked_statsd,
GSMADataParams(filename='gsma_not_found_anonymized.txt')) as imp:
imp.import_data()
with get_importer(RegistrationListImporter,
conn,
metadata_conn,
db_config,
tmpdir,
logger,
mocked_statsd,
RegistrationListParams(filename='sample_registration_list.csv')) as imp:
imp.import_data()
# Run dirbs-classify using db args from the temp postgres instance
runner = CliRunner()
monkeypatch.setattr(mocked_config.db_config, 'user', db_user)
result = runner.invoke(dirbs_classify_cli, ['--no-safety-check'], obj={'APP_CONFIG': mocked_config})
if db_user in ['dirbs_poweruser_login', 'dirbs_classify_user']:
assert result.exit_code == 0
else:
assert result.exit_code != 0
def test_listgen(per_test_postgres, tmpdir, logger, mocked_statsd, db_user, mocked_config, monkeypatch,
classification_data):
"""Test that the dirbs-listgen instance runs without an error."""
dsn = per_test_postgres.dsn()
db_config = DBConfig(ignore_env=True, **dsn)
with create_db_connection(db_config) as conn, create_db_connection(db_config, autocommit=True) as metadata_conn:
with get_importer(OperatorDataImporter,
conn,
metadata_conn,
db_config,
tmpdir,
logger,
mocked_statsd,
OperatorDataParams(
content='date,imei,imsi,msisdn\n'
'20160203,811111013136460,111018001111111,223338000000\n'
'20160203,359000000000000,111015113222222,223355000000\n'
'20160203,357756065985824,111015113333333,223355111111',
cc=['22', '74'],
mcc_mnc_pairs=[{'mcc': '111', 'mnc': '01'}],
operator='operator1',
extract=False)) as imp:
imp.import_data()
with get_importer(PairingListImporter,
conn,
metadata_conn,
db_config,
tmpdir,
logger,
mocked_statsd,
PairListParams(
content='imei,imsi,msisdn\n'
'811111013136460,111018001111111,234555555555550\n'
'359000000000000,111015113222222,234555555555551\n'
'357756065985824,111015113333333,234555555555552')) as imp:
imp.import_data()
# Now run listgen as requested user
runner = CliRunner()
monkeypatch.setattr(mocked_config.db_config, 'user', db_user)
output_dir = str(tmpdir)
result = runner.invoke(dirbs_listgen_cli, [output_dir], obj={'APP_CONFIG': mocked_config})
if db_user in ['dirbs_poweruser_login', 'dirbs_listgen_user']:
assert result.exit_code == 0
else:
assert result.exit_code != 0
def test_tac_api(per_test_flask_app, per_test_postgres, logger, mocked_statsd,
tmpdir, request, mocked_config, api_version):
"""Test TAC API call works with the security role created based on abstract role."""
dsn = per_test_postgres.dsn()
dsn['user'] = '******'
db_config = DBConfig(ignore_env=True, **dsn)
with create_db_connection(db_config) as conn, \
create_db_connection(db_config, autocommit=True) as metadata_conn:
with get_importer(
GSMADataImporter, conn, metadata_conn, db_config, tmpdir,
logger, mocked_statsd,
GSMADataParams(
filename='sample_gsma_import_list_anonymized.txt')) as imp:
imp.import_data()
current_user = request.node.callspec.params['per_test_flask_app']
if api_version == 'v1':
if current_user in ['dirbs_poweruser_login', 'dirbs_api_user']:
rv = per_test_flask_app.get(
url_for('{0}.tac_api'.format(api_version), tac='01234404'))
assert rv.status_code == 200
results = json.loads(rv.data.decode('utf-8'))
assert results['gsma'] is not None
else:
with pytest.raises(DatabaseRoleCheckException):
per_test_flask_app.get(
url_for('{0}.tac_api'.format(api_version), tac='01234404'))
else: # api version 2.0
if current_user in ['dirbs_poweruser_login', 'dirbs_api_user']:
rv = per_test_flask_app.get(
url_for('{0}.tac_get_api'.format(api_version), tac='01234404'))
data = json.loads(rv.data.decode('utf-8'))
assert data['gsma'] is not None
else:
with pytest.raises(DatabaseRoleCheckException):
per_test_flask_app.get(
url_for('{0}.tac_get_api'.format(api_version),
tac='01234404'))
def __init__(self, *, ignore_env, **yaml_config):
"""Constructor performing common section parsing for config sections."""
self.db_config = DBConfig(ignore_env=ignore_env,
**(yaml_config.get('postgresql', {}) or {}))
self.region_config = RegionConfig(ignore_env=ignore_env,
**(yaml_config.get('region', {})
or {}))
self.log_config = LoggingConfig(ignore_env=ignore_env,
**(yaml_config.get('logging', {})
or {}))
self.import_config = ImporterConfig(ignore_env=ignore_env,
**(yaml_config.get('import', {})
or {}))
self.conditions = [
ConditionConfig(ignore_env=ignore_env, **c)
for c in yaml_config.get('conditions', [])
]
cond_names = [c.label.lower() for c in self.conditions]
# Check that condition names are unique and case-insensitive
dupl_cond_name_found_error_message = 'Duplicate condition names {0} found in config. ' \
'Condition names are case insensitive!'
check_for_duplicates(cond_names, dupl_cond_name_found_error_message)
self.operator_threshold_config = OperatorThresholdConfig(
ignore_env=ignore_env,
**(yaml_config.get('operator_threshold', {}) or {}))
self.gsma_threshold_config = GSMAThresholdConfig(
ignore_env=ignore_env,
**(yaml_config.get('gsma_threshold', {}) or {}))
self.pairing_threshold_config = PairingListThresholdConfig(
ignore_env=ignore_env,
**(yaml_config.get('pairing_list_threshold', {}) or {}))
self.subscribers_threshold_config = SubscribersListThresholdConfig(
ignore_env=ignore_env,
**(yaml_config.get('subscribers_list_threshold', {}) or {}))
self.stolen_threshold_config = StolenListThresholdConfig(
ignore_env=ignore_env,
**(yaml_config.get('stolen_list_threshold', {}) or {}))
self.import_threshold_config = \
RegistrationListThresholdConfig(ignore_env=ignore_env,
**(yaml_config.get('registration_list_threshold', {}) or {}))
self.golden_threshold_config = GoldenListThresholdConfig(
ignore_env=ignore_env,
**(yaml_config.get('golden_list_threshold', {}) or {}))
self.barred_threshold_config = BarredListThresholdConfig(
ignore_env=ignore_env,
**(yaml_config.get('barred_list_threshold', {}) or {}))
self.monitoring_threshold_config = MonitoringListThresholdConfig(
ignore_env=ignore_env,
**(yaml_config.get('monitoring_list_threshold', {}) or {}))
self.barred_tac_threshold_config = BarredTacListThresholdConfig(
ignore_env=ignore_env,
**(yaml_config.get('barred_tac_list_threshold', {}) or {}))
self.associations_threshold_config = DeviceAssociationListThresholdConfig(
ignore_env=ignore_env,
**(yaml_config.get('association_list_threshold', {}) or {}))
self.retention_config = RetentionConfig(
ignore_env=ignore_env,
**(yaml_config.get('data_retention', {}) or {}))
self.listgen_config = ListGenerationConfig(
ignore_env=ignore_env,
**(yaml_config.get('list_generation', {}) or {}))
self.report_config = ReportGenerationConfig(
ignore_env=ignore_env,
**(yaml_config.get('report_generation', {}) or {}))
self.multiprocessing_config = MultiprocessingConfig(
ignore_env=ignore_env,
**(yaml_config.get('multiprocessing', {}) or {}))
self.statsd_config = StatsdConfig(ignore_env=ignore_env,
**(yaml_config.get('statsd', {})
or {}))
self.catalog_config = CatalogConfig(ignore_env=ignore_env,
**(yaml_config.get('catalog', {})
or {}))
self.amnesty_config = AmnestyConfig(ignore_env=ignore_env,
**(yaml_config.get('amnesty', {})
or {}))
self.broker_config = BrokerConfig(ignore_env=ignore_env,
**(yaml_config.get('broker', {})
or {}))
self.operational_config = OperationalConfig(
ignore_env=ignore_env,
**(yaml_config.get('operational', {}) or {}))