def test_normal_sample(pairing_list_importer, logger, db_conn): """Test Depot ID 96615/1. Verify that valid Paring List Data can be successfully imported into the database. """ expect_success(pairing_list_importer, 5, db_conn, logger)
def test_case_insensitive_headers(pairing_list_importer, logger, db_conn): """Test Depot ID 96773/9. Verify that the pairing list file is accepted and imported if the headers have mixed cases. """ expect_success(pairing_list_importer, 1, db_conn, logger)
def test_duplicate_check_fails(golden_list_importer, logger, db_conn): """Test Depot ID not known yet. Verify that it doesn't fail to import file with same rows. """ expect_success(golden_list_importer, 20, db_conn, logger)
def test_duplicate_check_override(golden_list_importer, db_conn, logger): """Test Depot ID not known yet. Verify that it fails to import an empty file after importing a non empty file. """ expect_success(golden_list_importer, 20, db_conn, logger)
def test_duplicate_check_fails(registration_list_importer, logger, db_conn): """Test Depot ID not known yet. Verify that it doesn't fail to import sam rows. """ expect_success(registration_list_importer, 18, db_conn, logger)
def test_simple_import(golden_list_importer, logger, db_conn): """Test Depot ID not known yet. Verify that valid golden data can be successfully imported into the database. """ expect_success(golden_list_importer, 20, db_conn, logger)
def test_matching_normalization(monitoring_list_importer, logger, db_conn): """Verify that IMEIs that normalize to the same value are successfully imported into the database.""" expect_success(monitoring_list_importer, 5, db_conn, logger)
def test_prune_classification_state(db_conn, metadata_db_conn, tmpdir, logger, mocked_config, operator_data_importer, stolen_list_importer, monkeypatch, gsma_tac_db_importer, postgres, mocked_statsd): """Test Depot ID not known yet. A regulator/partner should be able to run a CLI command to prune classification_state table. It will remove any classification state data related to obsolete conditions and data with end_date is earlier than the start of the retention window. """ # Step 1: # import gsma_dump empty non tac and classify for all the conditions # ['gsma_not_found', 'local_stolen', 'duplicate_mk1', 'malformed_imei', 'not_on_registration_list', ..] # classification_state_table contains records for cond_name "gsma_not_found". They all have end_date==None # Step 2 - TEST RETENTION WINDOW: # CLI prune will delete rows where the end_date is earlier than the start of the retention window. # retention_months=6 # curr_date = datetime.date(2017, 7, 13) # Import different gsma_db and classify to have different end date for gsma_not_found records in # the classification table. # Step 3 - TEST CONDITIONS NOT EXISTING: # CLI prune for classification_state will look at the current configured conditions and # remove any entries corresponding to cond_names that no longer exist in the config. # Load a new yaml file without stolen_list condition and run the prune CLI command to test. # -- yaml cond config list: # ['gsma_not_found', 'malformed_imei', 'not_on_registration_list'] # -- classification_state condition list: # ['gsma_not_found', 'local_stolen', 'malformed_imei', 'not_on_registration_list'] # Step 1 operator_data_importer.import_data() stolen_list_importer.import_data() gsma_tac_db_importer.import_data() runner = CliRunner() db_conn.commit() runner.invoke(dirbs_classify_cli, ['--no-safety-check', '--curr-date', '20170713'], obj={'APP_CONFIG': mocked_config}) with db_conn.cursor() as cur: cur.execute('SELECT imei_norm, cond_name, end_date FROM classification_state ORDER BY cond_name, imei_norm') res_list = cur.fetchall() assert len(res_list) == 32 assert [(x.imei_norm, x.cond_name, x.end_date) for x in res_list] == \ [('21111111111111', 'gsma_not_found', None), ('21111111111112', 'gsma_not_found', None), ('8888#888622222', 'gsma_not_found', None), ('88888862222209', 'gsma_not_found', None), ('88888888622222', 'gsma_not_found', None), ('12432807272315', 'local_stolen', None), ('12640904324427', 'local_stolen', None), ('12640904372723', 'local_stolen', None), ('12727231272313', 'local_stolen', None), ('12875502464321', 'local_stolen', None), ('12875502572723', 'local_stolen', None), ('12875507272312', 'local_stolen', None), ('12904502843271', 'local_stolen', None), ('12909602432585', 'local_stolen', None), ('12909602872723', 'local_stolen', None), ('12922902206948', 'local_stolen', None), ('12922902243260', 'local_stolen', None), ('12922902432742', 'local_stolen', None), ('12922902432776', 'local_stolen', None), ('12957272313271', 'local_stolen', None), ('17272317272723', 'local_stolen', None), ('56773605727231', 'local_stolen', None), ('64220204327947', 'local_stolen', None), ('64220297727231', 'local_stolen', None), ('72723147267231', 'local_stolen', None), ('72723147267631', 'local_stolen', None), ('8888#888622222', 'malformed_imei', None), ('21111111111111', 'not_on_registration_list', None), ('21111111111112', 'not_on_registration_list', None), ('8888#888622222', 'not_on_registration_list', None), ('88888862222209', 'not_on_registration_list', None), ('88888888622222', 'not_on_registration_list', None)] # Step 2 # all records have end_date == None. Classify twice to have records with different end_date # first classification with get_importer(GSMADataImporter, db_conn, metadata_db_conn, mocked_config.db_config, tmpdir, logger, mocked_statsd, GSMADataParams(filename='gsma_not_found_anonymized.txt')) as imp: expect_success(imp, 1, db_conn, logger) runner.invoke(dirbs_classify_cli, ['--no-safety-check', '--curr-date', '20170713'], obj={'APP_CONFIG': mocked_config}) # with db_conn.cursor() as cur: cur.execute("""SELECT imei_norm, cond_name, end_date FROM classification_state ORDER BY cond_name, imei_norm""") res_list = cur.fetchall() gsma_not_found_list = [(x.imei_norm, x.cond_name, x.end_date) for x in res_list if x.cond_name == 'gsma_not_found'] assert gsma_not_found_list == [('21111111111111', 'gsma_not_found', None), ('21111111111112', 'gsma_not_found', None), ('8888#888622222', 'gsma_not_found', None), ('88888862222209', 'gsma_not_found', None), ('88888888622222', 'gsma_not_found', datetime.date(2017, 7, 13))] # second classification with get_importer(GSMADataImporter, db_conn, metadata_db_conn, mocked_config.db_config, tmpdir, logger, mocked_statsd, GSMADataParams(filename='prune_classification_state_gsma.txt')) as imp: expect_success(imp, 1, db_conn, logger) runner.invoke(dirbs_classify_cli, ['--no-safety-check', '--curr-date', '20160101'], obj={'APP_CONFIG': mocked_config}) # with db_conn.cursor() as cur: cur.execute('SELECT imei_norm, cond_name, end_date ' ' FROM classification_state ' 'ORDER BY cond_name, imei_norm, end_date') res_list = cur.fetchall() gsma_not_found_list = [(x.imei_norm, x.cond_name, x.end_date) for x in res_list if x.cond_name == 'gsma_not_found'] assert gsma_not_found_list == [('21111111111111', 'gsma_not_found', datetime.date(2016, 1, 1)), ('21111111111112', 'gsma_not_found', datetime.date(2016, 1, 1)), ('8888#888622222', 'gsma_not_found', None), ('88888862222209', 'gsma_not_found', None), ('88888888622222', 'gsma_not_found', datetime.date(2017, 7, 13)), ('88888888622222', 'gsma_not_found', None)] # Step 3 # Expect not to be in classification_state table after prune: # IMEIs 21111111111111 and 21111111111112 for condition gsma_not found (due to end_date) # IMEIs for condition stolen_list (due to condition no longer exist) # this commit is to remove locks from the classification_state table so that # the table can be dropped inside the prune. The locks were activated by the CLI to classify. db_conn.commit() cond_dict_list = [{'label': 'gsma_not_found', 'reason': 'TAC not found in GSMA TAC database', 'grace_period_days': 30, 'blocking': True, 'dimensions': [{'module': 'gsma_not_found'}] }, {'label': 'malformed_imei', 'reason': 'Invalid characters detected in IMEI', 'grace_period_days': 0, 'blocking': False, 'dimensions': [{'module': 'malformed_imei'}] }, {'label': 'not_on_registration_list', 'reason': 'IMEI not found on local registration list', 'grace_period_days': 0, 'blocking': True, 'max_allowed_matching_ratio': 1.0, 'dimensions': [{'module': 'not_on_registration_list'}] }] monkeypatch.setattr(mocked_config, 'conditions', from_cond_dict_list_to_cond_list(cond_dict_list)) with db_conn.cursor() as cur: result = runner.invoke(dirbs_prune_cli, ['--curr-date', '20170913', 'classification_state'], obj={'APP_CONFIG': mocked_config}) assert result.exit_code == 0 # ITEMS REMOVED # [('17272317272723', 'local_stolen', None), ('12909602872723', 'local_stolen', None), # ('12875502572723', 'local_stolen', None), ('12875507272312', 'local_stolen', None), # ('64220297727231', 'local_stolen', None), ('12909602432585', 'local_stolen', None), # ('64220204327947', 'local_stolen', None), ('72723147267631', 'local_stolen', None), # ('72723147267231', 'local_stolen', None), ('12922902243260', 'local_stolen', None), # ('12875502464321', 'local_stolen', None), ('12922902432776', 'local_stolen', None), # ('12957272313271', 'local_stolen', None), ('12640904324427', 'local_stolen', None), # ('12904502843271', 'local_stolen', None), ('12922902432742', 'local_stolen', None), # ('12432807272315', 'local_stolen', None), ('12922902206948', 'local_stolen', None), # ('56773605727231', 'local_stolen', None), ('12727231272313', 'local_stolen', None), # ('12640904372723', 'local_stolen', None), # ('21111111111111', 'gsma_not_found', datetime.date(2016, 1, 1)), # ('21111111111112', 'gsma_not_found', datetime.date(2016, 1, 1))] cur.execute('SELECT imei_norm, cond_name, end_date ' 'FROM classification_state ' 'ORDER BY cond_name, imei_norm, end_date') res_list = cur.fetchall() pruned_class_state_table = [(x.imei_norm, x.cond_name, x.end_date) for x in res_list] assert pruned_class_state_table == [('8888#888622222', 'gsma_not_found', None), ('88888862222209', 'gsma_not_found', None), ('88888888622222', 'gsma_not_found', datetime.date(2017, 7, 13)), ('88888888622222', 'gsma_not_found', None), ('8888#888622222', 'malformed_imei', None), ('21111111111111', 'not_on_registration_list', None), ('21111111111112', 'not_on_registration_list', None), ('8888#888622222', 'not_on_registration_list', None), ('88888862222209', 'not_on_registration_list', None), ('88888888622222', 'not_on_registration_list', None)]
def test_simple_import(monitoring_list_importer, logger, db_conn): """Verify that the valid monitoring list data can be successfully imported into the db.""" expect_success(monitoring_list_importer, 20, db_conn, logger)
def test_simple_import(barred_list_importer, logger, db_conn): """Verify that the valid barred list data can be successfully imported into the db.""" expect_success(barred_list_importer, 20, db_conn, logger)
def test_duplicate_check_fails(stolen_list_importer, logger, db_conn): """Test Depot ID not known yet. Verify that rows with same imei and reporting_date are not considered duplicates. """ expect_success(stolen_list_importer, 20, db_conn, logger)
def test_reporting_date_optional(stolen_list_importer, logger, db_conn, mocked_config, metadata_db_conn, tmpdir, mocked_statsd): """Test Depot ID not known yet. Verify that valid stolen data can be successfully imported into the database with optional field reporting_date. Verify in case of two IMEIs with different reporting_date, only the one with min(reporting_date) is imported. """ # Basically, the decorator @optional in the stolen_list schema for the field reporting-date, allows whitespaces. # If the field was text and we imported a row containing a reporting-date whitespace-only, # the row would have been imported with withespaces into the db. # In this case the reporting-date field is of type date so, # when we import a whitespace data of type text into a date type column, we get an Error Type # valid imei, reporting_date expect_success(stolen_list_importer, 2, db_conn, logger) with db_conn, db_conn.cursor() as cur: cur.execute( 'SELECT imei_norm, reporting_date FROM historic_stolen_list ORDER BY imei_norm' ) res = cur.fetchall() assert len(res) == 2 stolen_list_rows = {(r.imei_norm, r.reporting_date) for r in res} assert stolen_list_rows == { ('12222222222222', datetime.date(datetime(2016, 4, 25))), ('62222222222222', datetime.date(datetime(2016, 4, 26))) } # Verify in case of two IMEIs with different reporting_date, duplicate check will raise an exception. with get_importer( StolenListImporter, db_conn, metadata_db_conn, mocked_config.db_config, tmpdir, logger, mocked_statsd, StolenListParams(perform_historic_check=False, content='imei,reporting_date,status\n' '01234567891234,20160401,\n' '01234567891234,20160402,')) as imp: expect_failure( imp, 'Conflicting rows check failed (1 rows with same primary key and conflicting data)' ) assert 'Found 1 conflicting row(s) with primary key (\'imei_norm\',): (\'01234567891234\',)' \ in logger_stream_contents(logger) # valid imei, reporting_date whitespace only - error reporting_date type is date not string with get_importer( StolenListImporter, db_conn, metadata_db_conn, mocked_config.db_config, tmpdir, logger, mocked_statsd, StolenListParams(content='imei,reporting_date,status\n' '622222222222222, ,\n' '122222222222223,20160425,')) as imp: expect_failure( imp, exc_message='Pre-validation failed: b\'Error: ' 'regex("^(20[0-9]{2}((0[13578]|1[02])31|(01|0[3-9]|1[0-2])(29|30)|' '(0[1-9]|1[0-2])(0[1-9]|1[0-9]|2[0-8]))|20([02468][048]|[13579][26])0229)?$") ' 'fails for line: 1, column: reporting_date, value: " "\\nFAIL') db_conn.commit() with db_conn, db_conn.cursor() as cur: cur.execute('TRUNCATE historic_stolen_list') # valid imei, reporting_date empty - type null with get_importer( StolenListImporter, db_conn, metadata_db_conn, mocked_config.db_config, tmpdir, logger, mocked_statsd, StolenListParams(content='imei,reporting_date,status\n' '722222222222222,,\n' '122222222222223,20160425,')) as imp: expect_success(imp, 2, db_conn, logger) with db_conn, db_conn.cursor() as cur: cur.execute( 'SELECT imei_norm, reporting_date FROM historic_stolen_list ORDER BY imei_norm' ) res = cur.fetchall() assert len(res) == 2 stolen_list_rows = {(r.imei_norm, r.reporting_date) for r in res} assert stolen_list_rows == {('12222222222222', datetime.date(datetime(2016, 4, 25))), ('72222222222222', None)}
def test_simple_import(stolen_list_importer, logger, db_conn): """Test Depot ID 96591/5. Verify that valid stolen data can be successfully imported into the database. """ expect_success(stolen_list_importer, 20, db_conn, logger)