def test_imei_api_registration_list(per_test_flask_app, per_test_postgres,
logger, mocked_statsd, tmpdir, request,
mocked_config, api_version):
"""Test IMEI API call after registration list import."""
dsn = per_test_postgres.dsn()
db_config = DBConfig(ignore_env=True, **dsn)
with create_db_connection(db_config) as conn, \
create_db_connection(db_config, autocommit=True) as metadata_conn:
with get_importer(
GSMADataImporter, conn, metadata_conn, db_config, tmpdir,
logger, mocked_statsd,
GSMADataParams(
filename='gsma_dump_small_july_2016.txt')) as imp:
imp.import_data()
with get_importer(
RegistrationListImporter, conn, metadata_conn, db_config,
tmpdir, logger, mocked_statsd,
RegistrationListParams(
content=
'APPROVED_IMEI,make,model,status,model_number,brand_name,'
'device_type,radio_interface,device_id\n'
'21260934000003,,,,,,,,1')) as imp:
imp.import_data()
if api_version == 'v1':
rv = per_test_flask_app.get(
url_for('{0}.imei_api'.format(api_version), imei='21260934000003'))
assert rv.status_code == 200
else: # api version 2.0
rv = per_test_flask_app.get(
url_for('{0}.imei_get_api'.format(api_version),
imei='21260934000003'))
assert rv.status_code == 200
def test_historical_check_percentage_succeeds(registration_list_importer,
logger, mocked_statsd, db_conn,
metadata_db_conn, mocked_config,
tmpdir):
"""Test Depot ID not known yet.
Verify that a local import data is successfully imported after having imported two files where the
second file has 80% size of the first one and the threshold value is 75.
"""
expect_success(registration_list_importer, 100, db_conn, logger)
with get_importer(
RegistrationListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
RegistrationListParams(
filename=data_file_to_test(80,
imei_custom_header='approved_imei,'
'make,model,'
'status,model_number,'
'brand_name,device_type,'
'radio_interface,'
'device_id'),
import_size_variation_percent=mocked_config.
import_threshold_config.import_size_variation_percent,
import_size_variation_absolute=mocked_config.
import_threshold_config.import_size_variation_absolute)
) as imp:
expect_success(imp, 80, db_conn, logger)
def test_delta_file_prevalidation(logger, db_conn, metadata_db_conn,
mocked_config, tmpdir, mocked_statsd,
registration_list_importer):
"""Test Depot not available yet.
Test pre-validation schemas.
"""
# update change-type is allowed only for stolen
expect_failure(
registration_list_importer,
exc_message=
'Pre-validation failed: b\'Error: regex("^(add|remove)$") fails for line: 2, '
'column: change_type, value: "update"\\nFAIL')
# change_type must be lower case
with get_importer(
RegistrationListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
RegistrationListParams(
content='approved_imei,make,model,status,change_type\n'
'12345678901234,,,,ADD',
delta=True)) as imp:
expect_failure(
imp,
exc_message=
'Pre-validation failed: b\'Error: regex("^(add|remove)$") fails for line: '
'1, column: change_type, value: "ADD"\\nFAIL')
def test_classify(per_test_postgres, db_user, tmpdir, logger, mocked_statsd, monkeypatch, mocked_config):
"""Test classify works with the security role created based on abstract role."""
dsn = per_test_postgres.dsn()
db_config = DBConfig(ignore_env=True, **dsn)
with create_db_connection(db_config) as conn, create_db_connection(db_config, autocommit=True) as metadata_conn:
with get_importer(OperatorDataImporter,
conn,
metadata_conn,
db_config,
tmpdir,
logger,
mocked_statsd,
OperatorDataParams(
content='date,imei,imsi,msisdn\n'
'20110101,8888#888622222,123456789012345,123456789012345\n'
'20110101,88888888622222,123456789012345,123456789012345\n'
'20110101,8888888862222209,123456789012345,123456789012345\n'
'20110101,88888862222209**,123456789012345,123456789012345',
extract=False,
perform_unclean_checks=False,
perform_region_checks=False,
perform_home_network_check=False,
operator='operator1')) as imp:
imp.import_data()
with get_importer(GSMADataImporter,
conn,
metadata_conn,
db_config,
tmpdir,
logger,
mocked_statsd,
GSMADataParams(filename='gsma_not_found_anonymized.txt')) as imp:
imp.import_data()
with get_importer(RegistrationListImporter,
conn,
metadata_conn,
db_config,
tmpdir,
logger,
mocked_statsd,
RegistrationListParams(filename='sample_registration_list.csv')) as imp:
imp.import_data()
# Run dirbs-classify using db args from the temp postgres instance
runner = CliRunner()
monkeypatch.setattr(mocked_config.db_config, 'user', db_user)
result = runner.invoke(dirbs_classify_cli, ['--no-safety-check'], obj={'APP_CONFIG': mocked_config})
if db_user in ['dirbs_poweruser_login', 'dirbs_classify_user']:
assert result.exit_code == 0
else:
assert result.exit_code != 0
def test_repeat_import(registration_list_importer, logger, mocked_statsd,
db_conn, metadata_db_conn, mocked_config, tmpdir):
"""Test Depot ID not known yet.
Verify that valid registration_list data can be successfully imported into the database
when repeating the import of the same file.
"""
expect_success(registration_list_importer, 20, db_conn, logger)
with get_importer(
RegistrationListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
RegistrationListParams(
filename='sample_registration_list.csv')) as imp:
expect_success(imp, 20, db_conn, logger)
def test_override_historical_check(registration_list_importer, logger,
mocked_statsd, db_conn, metadata_db_conn,
mocked_config, tmpdir):
"""Test Depot ID not known yet.
Verify that the user can override historical checks when importing registration_list data.
"""
expect_success(registration_list_importer, 20, db_conn, logger)
with get_importer(
RegistrationListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
RegistrationListParams(
filename='sample_registration_list_historicalcheck.csv',
perform_historic_check=False)) as imp:
expect_success(imp, 9, db_conn, logger)
def test_historical_check_empty(registration_list_importer, logger,
mocked_statsd, db_conn, metadata_db_conn,
mocked_config, tmpdir):
"""Test Depot ID not known yet.
Verify that it fails to import an empty file after importing a non empty file.
"""
expect_success(registration_list_importer, 20, db_conn, logger)
with get_importer(
RegistrationListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
RegistrationListParams(
filename='empty_registration_list_historical_check.csv')
) as imp:
expect_failure(imp, exc_message='Failed import size historic check')
def test_malformed_imeis(registration_list_importer, logger, mocked_statsd,
db_conn, metadata_db_conn, mocked_config, tmpdir):
"""Test Depot ID not known yet.
Verify that the registration_list data file is accepted
and imported if the data contains IMEIs with #, and *.
"""
expect_success(registration_list_importer, 20, db_conn, logger)
# attempting to import registration_list file containing symbol not allowed '%'.
with get_importer(
RegistrationListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
RegistrationListParams(
filename='registration_list_hexpound_bad_symbol.csv')) as imp:
expect_failure(imp,
exc_message='regex("^[0-9A-Fa-f\\\\*\\\\#]{1,16}$") '
'fails for line: 1, column: approved_imei, value: '
'"1000000%000000"\\nFAIL')
def test_device_id_required_field(logger, db_conn, metadata_db_conn,
mocked_config, tmpdir, mocked_statsd,
registration_list_importer):
"""Test Depot not available yet.
Verify that the data in device_id is required.
"""
with get_importer(
RegistrationListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
RegistrationListParams(content='approved_imei,make,model,status,'
'model_number,brand_name,device_type,'
'radio_interface,device_id\n'
'12345678901234,,,,,,,,',
delta=False)) as imp:
expect_failure(
imp,
exc_message=
'Pre-validation failed: b\'Error: regex("[a-zA-Z0-9]+") fails for line: '
'1, column: device_id, value: ""\\nFAIL')
def test_delta_file_prevalidation(logger, db_conn, metadata_db_conn,
mocked_config, tmpdir, mocked_statsd,
registration_list_importer):
"""Test Depot not available yet.
Test pre-validation schemas.
"""
# change_type must be lower case
with get_importer(
RegistrationListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
RegistrationListParams(content='approved_imei,make,model,status,'
'model_number,brand_name,device_type,'
'radio_interface,device_id,change_type\n'
'12345678901234,,,,,,,,123,ADD',
delta=True)) as imp:
expect_failure(
imp,
exc_message=
'Pre-validation failed: b\'Error: regex("^(add|remove|update)$") fails for line: '
'1, column: change_type, value: "ADD"\\nFAIL')
def test_historical_check_percentage_fails(registration_list_importer, logger,
mocked_statsd, db_conn,
metadata_db_conn, mocked_config,
tmpdir):
"""Test Depot ID not known yet.
Verify that an import import data containing 9 rows fails to be imported after having imported a 20
rows file because Historical registration_list check is greater than 25% drop in import size;.
"""
expect_success(registration_list_importer, 20, db_conn, logger)
with get_importer(
RegistrationListImporter, db_conn, metadata_db_conn,
mocked_config.db_config, tmpdir, logger, mocked_statsd,
RegistrationListParams(
filename='sample_registration_list_historicalcheck.csv',
import_size_variation_percent=mocked_config.
import_threshold_config.import_size_variation_percent,
import_size_variation_absolute=mocked_config.
import_threshold_config.import_size_variation_absolute)
) as imp:
expect_failure(imp, exc_message='Failed import size historic check')
with db_conn.cursor() as cursor:
cursor.execute(
'SELECT imei_norm FROM registration_list ORDER BY imei_norm')
result_list = [res[0] for res in cursor.fetchall()]
assert result.exit_code == 0
assert result_list == [
'10000000000000', '10000000000001', '10000000000002', '10000000000003',
'10000000000004', '10000000000005'
]
@pytest.mark.parametrize(
'registration_list_importer',
[RegistrationListParams(filename='registration_list_missingheader.csv')],
indirect=True)
def test_header_case_insensitivity(registration_list_importer, logger,
db_conn):
"""Test Depot ID not known yet.
Verify that the Registration List data is not imported if a header column is missing.
"""
expect_failure(
registration_list_importer,
exc_message='Metadata header, cannot find the column headers - '
'10000000000000, device_id, model, '
'device_type, model_number, '
'status, brand_name, make, approved_imei, '
'radio_interface - .\\nFAIL')
@pytest.mark.parametrize(
'operator_data_importer, stolen_list_importer, pairing_list_importer, '
'gsma_tac_db_importer, registration_list_importer',
[(OperatorDataParams(
filename=
'testData1-operator-operator1-anonymized_20161101_20161130.csv',
operator='operator1',
perform_unclean_checks=False,
extract=False),
StolenListParams(filename='testData1-sample_stolen_list-anonymized.csv'),
PairListParams(filename='testData1-sample_pairinglist-anonymized.csv'),
GSMADataParams(
filename='testData1-gsmatac_operator4_operator1_anonymized.txt'),
RegistrationListParams(filename='sample_registration_list.csv'))],
indirect=True)
def test_cli_repartition(postgres, mocked_config, db_conn,
operator_data_importer, registration_list_importer,
pairing_list_importer, stolen_list_importer,
gsma_tac_db_importer, tmpdir, logger,
metadata_db_conn, mocked_statsd):
"""Test that the dirbs-db partition script runs without an error."""
import_data(operator_data_importer, 'operator_data', 17, db_conn, logger)
import_data(gsma_tac_db_importer, 'gsma_data', 13, db_conn, logger)
import_data(stolen_list_importer, 'stolen_list', 21, db_conn, logger)
import_data(registration_list_importer, 'registration_list', 20, db_conn,
logger)
import_data(pairing_list_importer, 'pairing_list', 7, db_conn, logger)
# Import second month of operator data to ensure that we have 2 months worth for the same operator
OperatorDataParams(
filename=
'testData1-operator-operator4-anonymized_20161101_20161130.csv',
extract=False,
perform_unclean_checks=False,
perform_region_checks=False,
perform_home_network_check=False)
],
indirect=True)
@pytest.mark.parametrize('gsma_tac_db_importer', [
GSMADataParams(
filename='testData1-gsmatac_operator1_operator4_anonymized.txt')
],
indirect=True)
@pytest.mark.parametrize('registration_list_importer', [
RegistrationListParams(
filename='registration_list_msisidn_api_test_data.csv')
],
indirect=True)
def test_observed_msisdn_with_registration_list(flask_app,
operator_data_importer,
gsma_tac_db_importer,
registration_list_importer,
db_conn, tmpdir, logger):
"""Test Depot ID not known yet.
Verify MSISDN API (version 2.0) should return IMEI information from GSMA, Network and Device Registration System.
"""
# operator input file contains imei_norm = 38847733370026 with msisdn=22300049781840
# gsma input file contains tac 38847733 with manufacturer = 1d4e632daf5249ba6f4165cca4cb4ff5025ddae6
# registration list contains imei_norm = 38847733370026
operator_data_importer.import_data()
with db_conn.cursor() as cursor:
cursor.execute(
'SELECT imei_norm FROM registration_list ORDER BY imei_norm')
result_list = [res[0] for res in cursor.fetchall()]
assert result.exit_code == 0
assert result_list == [
'10000000000000', '10000000000001', '10000000000002', '10000000000003',
'10000000000004', '10000000000005'
]
@pytest.mark.parametrize(
'registration_list_importer',
[RegistrationListParams(filename='registration_list_missingheader.csv')],
indirect=True)
def test_header_case_insensitivity(registration_list_importer, logger,
db_conn):
"""Test Depot ID not known yet.
Verify that the Registration List data is not imported if a header column is missing.
"""
expect_failure(
registration_list_importer,
exc_message='Metadata header, cannot find the column headers - '
'10000000000000, model, status, make, '
'approved_imei - .\\nFAIL')
@pytest.mark.parametrize('registration_list_importer', [
from _fixtures import * # noqa: F403, F401
from _importer_params import GSMADataParams, OperatorDataParams, RegistrationListParams, PairListParams
from _helpers import get_importer
def check_in_registration_list_helper(imei_list, expect_to_find_in_reg, api_version, flask_app):
"""Helper function to make a request and check in_registration_list value in the response."""
for i in imei_list:
rv = flask_app.get(url_for('{0}.imei_api'.format(api_version), imei=i))
assert rv.status_code == 200
data = json.loads(rv.data.decode('utf-8'))
assert data['realtime_checks']['in_registration_list'] is expect_to_find_in_reg
@pytest.mark.parametrize('registration_list_importer',
[RegistrationListParams(filename='sample_registration_list.csv')],
indirect=True)
@pytest.mark.parametrize('gsma_tac_db_importer',
[GSMADataParams(filename='sample_gsma_import_list_anonymized.txt',
extract=False)],
indirect=True)
def test_check_in_registration_list(flask_app, registration_list_importer, gsma_tac_db_importer,
api_version, monkeypatch, mocked_config):
"""Test Depot not known yet.
Verify that IMEI API response contains a Real-time check for IMEI in registration list.
"""
# APPROVED_IMEI
# 10000000000000
# 10000000000001
# 10000000000002 ....
