def test_gsma_data_importer(per_test_postgres, tmpdir, db_user, monkeypatch, mocked_config): """Test gsma data import works with the security role created based on abstract role.""" files_to_zip = [ 'unittest_data/gsma/sample_gsma_import_list_anonymized.txt' ] zip_files_to_tmpdir(files_to_zip, tmpdir) zipped_file_path = str( tmpdir.join('sample_gsma_import_list_anonymized.zip')) # Run dirbs-import using db args from the temp postgres instance runner = CliRunner() monkeypatch.setattr(mocked_config.db_config, 'user', db_user) result = runner.invoke(dirbs_import_cli, ['gsma_tac', zipped_file_path], obj={'APP_CONFIG': mocked_config}) if db_user in ['dirbs_poweruser_login', 'dirbs_import_gsma_user']: assert result.exit_code == 0 else: assert result.exit_code != 0 # Re-run to verify data is imported correctly result = runner.invoke(dirbs_import_cli, ['gsma_tac', zipped_file_path], obj={'APP_CONFIG': mocked_config}) if db_user in ['dirbs_poweruser_login', 'dirbs_import_gsma_user']: assert result.exit_code == 0 else: assert result.exit_code != 0
def test_catalog(per_test_postgres, tmpdir, db_user, mocked_config, monkeypatch): """Test catalog works with the security role created based on abstract role.""" files_to_zip = ['unittest_data/operator/operator1_with_rat_info_20160701_20160731.csv'] zip_files_to_tmpdir(files_to_zip, tmpdir) catalog_config_dict = { 'prospectors': [ { 'file_type': 'operator', 'paths': [str(tmpdir.join('operator1_with_rat_info_20160701_20160731.zip'))], 'schema_filename': 'OperatorImportSchema_v2.csvs' } ], 'perform_prevalidation': False } catalog_config = CatalogConfig(ignore_env=True, **catalog_config_dict) monkeypatch.setattr(mocked_config, 'catalog_config', catalog_config) # Run dirbs-catalog using db args from the temp postgres instance runner = CliRunner() monkeypatch.setattr(mocked_config.db_config, 'user', db_user) result = runner.invoke(dirbs_catalog_cli, obj={'APP_CONFIG': mocked_config}) if db_user in ['dirbs_poweruser_login', 'dirbs_catalog_user']: assert result.exit_code == 0 else: assert result.exit_code != 0
def test_operator_data_importer(per_test_postgres, tmpdir, db_user, mocked_config, monkeypatch): """Test operator import works with the security role created based on abstract role.""" files_to_zip = ['unittest_data/operator/Foo_Wireless_20160101_20160331.csv'] zip_files_to_tmpdir(files_to_zip, tmpdir) zipped_file_path = str(tmpdir.join('Foo_Wireless_20160101_20160331.zip')) # Run dirbs-import using db args from the temp postgres instance runner = CliRunner() monkeypatch.setattr(mocked_config.db_config, 'user', db_user) result = runner.invoke(dirbs_import_cli, ['operator', '--disable-clean-check', '--disable-rat-import', '--disable-home-check', '--disable-region-check', 'operator1', zipped_file_path], obj={'APP_CONFIG': mocked_config}) if db_user in ['dirbs_poweruser_login', 'dirbs_import_operator_user']: assert result.exit_code == 0 else: assert result.exit_code != 0 # Re-run to verify data is imported correctly result = runner.invoke(dirbs_import_cli, ['operator', '--disable-clean-check', '--disable-rat-import', '--disable-home-check', '--disable-region-check', 'operator1', zipped_file_path], obj={'APP_CONFIG': mocked_config}) if db_user in ['dirbs_poweruser_login', 'dirbs_import_operator_user']: assert result.exit_code == 0 else: assert result.exit_code != 0
def test_perform_prevalidation_option(postgres, db_conn, tmpdir, monkeypatch, mocked_config): """Test pre-validation is not performed if option is turned off in the config.""" files_to_zip = ['unittest_data/operator/operator1_with_rat_info_20160701_20160731.csv'] zip_files_to_tmpdir(files_to_zip, tmpdir) catalog_config_dict = { 'prospectors': [ { 'file_type': 'operator', 'paths': [str(tmpdir.join('operator1_with_rat_info_20160701_20160731.zip'))], 'schema_filename': 'OperatorImportSchema_v2.csvs' } ], 'perform_prevalidation': False } catalog_config = CatalogConfig(ignore_env=True, **catalog_config_dict) monkeypatch.setattr(mocked_config, 'catalog_config', catalog_config) # Run dirbs-catalog using db args from the temp postgres instance runner = CliRunner() result = runner.invoke(dirbs_catalog_cli, obj={'APP_CONFIG': mocked_config}) assert result.exit_code == 0 # This test basically checks that when pre-validation is disabled, then it is skipped during catalog. # The is_valid_format field would be NULL is that scenario as tested below. The scenario with # pre-validation enabled is implicitly tested in test_all_files_are_harvested test case. with db_conn.cursor() as cursor: cursor.execute('SELECT is_valid_format FROM data_catalog WHERE filename = ' '\'operator1_with_rat_info_20160701_20160731.zip\'') assert cursor.fetchone().is_valid_format is None
def test_file_specified_explicitly_is_cataloged_correctly(postgres, db_conn, tmpdir, mocked_config, monkeypatch): """Test that if file is specified explicitly; it is pre-validated using the correct schema.""" files_to_zip = ['unittest_data/operator/operator1_with_rat_info_20160701_20160731.csv'] zip_files_to_tmpdir(files_to_zip, tmpdir) catalog_config_dict = { 'prospectors': [ { 'file_type': 'operator', 'paths': [str(tmpdir)], 'schema_filename': 'OperatorImportSchema.csvs' }, { 'file_type': 'operator', 'paths': [str(tmpdir.join('operator1_with_rat_info_20160701_20160731.zip'))], 'schema_filename': 'OperatorImportSchema_v2.csvs' } ], 'perform_prevalidation': True } catalog_config = CatalogConfig(ignore_env=True, **catalog_config_dict) monkeypatch.setattr(mocked_config, 'catalog_config', catalog_config) # Run dirbs-catalog using db args from the temp postgres instance runner = CliRunner() result = runner.invoke(dirbs_catalog_cli, obj={'APP_CONFIG': mocked_config}) assert result.exit_code == 0 with db_conn.cursor() as cursor: cursor.execute('SELECT is_valid_format FROM data_catalog WHERE filename = ' '\'operator1_with_rat_info_20160701_20160731.zip\'') assert cursor.fetchone().is_valid_format
def test_all_files_are_harvested(postgres, db_conn, tmpdir, logger, monkeypatch, mocked_config): """Test all input files are correctly harvested and cataloged.""" files_to_zip = [ 'unittest_data/operator/operator1_with_rat_info_20160701_20160731.csv', 'unittest_data/gsma/sample_gsma_import_list_anonymized.txt', 'unittest_data/stolen_list/sample_stolen_list.csv', 'unittest_data/registration_list/sample_registration_list.csv', 'unittest_data/pairing_list/sample_pairinglist.csv', 'unittest_data/golden_list/sample_golden_list.csv' ] zip_files_to_tmpdir(files_to_zip, tmpdir) catalog_config_dict = { 'prospectors': [{ 'file_type': 'operator', 'paths': [ str( tmpdir.join( 'operator1_with_rat_info_20160701_20160731.zip')) ], 'schema_filename': 'OperatorImportSchema_v2.csvs' }, { 'file_type': 'gsma_tac', 'paths': [str(tmpdir.join('sample_gsma_import_list_anonymized.zip'))], 'schema_filename': 'GSMASchema.csvs' }, { 'file_type': 'stolen_list', 'paths': [str(tmpdir.join('sample_stolen_list.zip'))], 'schema_filename': 'StolenListSchema.csvs' }, { 'file_type': 'pairing_list', 'paths': [str(tmpdir.join('sample_pairinglist.zip'))], 'schema_filename': 'PairingListSchema.csvs' }, { 'file_type': 'registration_list', 'paths': [str(tmpdir.join('sample_registration_list.zip'))], 'schema_filename': 'RegistrationListSchema.csvs' }, { 'file_type': 'golden_list', 'paths': [str(tmpdir.join('sample_golden_list.zip'))], 'schema_filename': 'GoldenListSchemaData.csvs' }], 'perform_prevalidation': True } catalog_config = CatalogConfig(ignore_env=True, **catalog_config_dict) monkeypatch.setattr(mocked_config, 'catalog_config', catalog_config) # Run dirbs-catalog using db args from the temp postgres instance runner = CliRunner() # Run dirbs-catalog using db args from the temp postgres instance runner = CliRunner() result = runner.invoke(dirbs_catalog_cli, obj={'APP_CONFIG': mocked_config}) assert result.exit_code == 0 with db_conn.cursor() as cursor: cursor.execute('SELECT * FROM data_catalog') res = [(res.filename, res.file_type, res.compressed_size_bytes, res.is_valid_zip, res.is_valid_format, res.extra_attributes) for res in cursor.fetchall()] assert ('operator1_with_rat_info_20160701_20160731.zip', 'operator', 797, True, True, { 'filename_check': True }) in res assert ('sample_gsma_import_list_anonymized.zip', 'gsma_tac', 1083, True, True, {}) in res assert ('sample_stolen_list.zip', 'stolen_list', 529, True, True, {}) in res assert ('sample_registration_list.zip', 'registration_list', 858, True, True, {}) in res assert ('sample_pairinglist.zip', 'pairing_list', 312, True, True, {}) in res assert ('sample_golden_list.zip', 'golden_list', 474, True, True, {}) in res # Run dirbs-catalog again to verify that no new files are discovered result = runner.invoke(dirbs_catalog_cli, obj={'APP_CONFIG': mocked_config}) assert result.exit_code == 0 assert 'Data catalog is already up-to-date!' in logger_stream_contents( logger)