def test_gsma_data_importer(per_test_postgres, tmpdir, db_user, monkeypatch,
mocked_config):
"""Test gsma data import works with the security role created based on abstract role."""
files_to_zip = [
'unittest_data/gsma/sample_gsma_import_list_anonymized.txt'
]
zip_files_to_tmpdir(files_to_zip, tmpdir)
zipped_file_path = str(
tmpdir.join('sample_gsma_import_list_anonymized.zip'))
# Run dirbs-import using db args from the temp postgres instance
runner = CliRunner()
monkeypatch.setattr(mocked_config.db_config, 'user', db_user)
result = runner.invoke(dirbs_import_cli, ['gsma_tac', zipped_file_path],
obj={'APP_CONFIG': mocked_config})
if db_user in ['dirbs_poweruser_login', 'dirbs_import_gsma_user']:
assert result.exit_code == 0
else:
assert result.exit_code != 0
# Re-run to verify data is imported correctly
result = runner.invoke(dirbs_import_cli, ['gsma_tac', zipped_file_path],
obj={'APP_CONFIG': mocked_config})
if db_user in ['dirbs_poweruser_login', 'dirbs_import_gsma_user']:
assert result.exit_code == 0
else:
assert result.exit_code != 0
def test_catalog(per_test_postgres, tmpdir, db_user, mocked_config, monkeypatch):
"""Test catalog works with the security role created based on abstract role."""
files_to_zip = ['unittest_data/operator/operator1_with_rat_info_20160701_20160731.csv']
zip_files_to_tmpdir(files_to_zip, tmpdir)
catalog_config_dict = {
'prospectors': [
{
'file_type': 'operator',
'paths': [str(tmpdir.join('operator1_with_rat_info_20160701_20160731.zip'))],
'schema_filename': 'OperatorImportSchema_v2.csvs'
}
],
'perform_prevalidation': False
}
catalog_config = CatalogConfig(ignore_env=True, **catalog_config_dict)
monkeypatch.setattr(mocked_config, 'catalog_config', catalog_config)
# Run dirbs-catalog using db args from the temp postgres instance
runner = CliRunner()
monkeypatch.setattr(mocked_config.db_config, 'user', db_user)
result = runner.invoke(dirbs_catalog_cli, obj={'APP_CONFIG': mocked_config})
if db_user in ['dirbs_poweruser_login', 'dirbs_catalog_user']:
assert result.exit_code == 0
else:
assert result.exit_code != 0
def test_operator_data_importer(per_test_postgres, tmpdir, db_user, mocked_config, monkeypatch):
"""Test operator import works with the security role created based on abstract role."""
files_to_zip = ['unittest_data/operator/Foo_Wireless_20160101_20160331.csv']
zip_files_to_tmpdir(files_to_zip, tmpdir)
zipped_file_path = str(tmpdir.join('Foo_Wireless_20160101_20160331.zip'))
# Run dirbs-import using db args from the temp postgres instance
runner = CliRunner()
monkeypatch.setattr(mocked_config.db_config, 'user', db_user)
result = runner.invoke(dirbs_import_cli, ['operator', '--disable-clean-check', '--disable-rat-import',
'--disable-home-check', '--disable-region-check',
'operator1', zipped_file_path],
obj={'APP_CONFIG': mocked_config})
if db_user in ['dirbs_poweruser_login', 'dirbs_import_operator_user']:
assert result.exit_code == 0
else:
assert result.exit_code != 0
# Re-run to verify data is imported correctly
result = runner.invoke(dirbs_import_cli, ['operator', '--disable-clean-check', '--disable-rat-import',
'--disable-home-check', '--disable-region-check',
'operator1', zipped_file_path],
obj={'APP_CONFIG': mocked_config})
if db_user in ['dirbs_poweruser_login', 'dirbs_import_operator_user']:
assert result.exit_code == 0
else:
assert result.exit_code != 0
def test_perform_prevalidation_option(postgres, db_conn, tmpdir, monkeypatch, mocked_config):
"""Test pre-validation is not performed if option is turned off in the config."""
files_to_zip = ['unittest_data/operator/operator1_with_rat_info_20160701_20160731.csv']
zip_files_to_tmpdir(files_to_zip, tmpdir)
catalog_config_dict = {
'prospectors': [
{
'file_type': 'operator',
'paths': [str(tmpdir.join('operator1_with_rat_info_20160701_20160731.zip'))],
'schema_filename': 'OperatorImportSchema_v2.csvs'
}
],
'perform_prevalidation': False
}
catalog_config = CatalogConfig(ignore_env=True, **catalog_config_dict)
monkeypatch.setattr(mocked_config, 'catalog_config', catalog_config)
# Run dirbs-catalog using db args from the temp postgres instance
runner = CliRunner()
result = runner.invoke(dirbs_catalog_cli, obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
# This test basically checks that when pre-validation is disabled, then it is skipped during catalog.
# The is_valid_format field would be NULL is that scenario as tested below. The scenario with
# pre-validation enabled is implicitly tested in test_all_files_are_harvested test case.
with db_conn.cursor() as cursor:
cursor.execute('SELECT is_valid_format FROM data_catalog WHERE filename = '
'\'operator1_with_rat_info_20160701_20160731.zip\'')
assert cursor.fetchone().is_valid_format is None
def test_file_specified_explicitly_is_cataloged_correctly(postgres, db_conn, tmpdir, mocked_config, monkeypatch):
"""Test that if file is specified explicitly; it is pre-validated using the correct schema."""
files_to_zip = ['unittest_data/operator/operator1_with_rat_info_20160701_20160731.csv']
zip_files_to_tmpdir(files_to_zip, tmpdir)
catalog_config_dict = {
'prospectors': [
{
'file_type': 'operator',
'paths': [str(tmpdir)],
'schema_filename': 'OperatorImportSchema.csvs'
},
{
'file_type': 'operator',
'paths': [str(tmpdir.join('operator1_with_rat_info_20160701_20160731.zip'))],
'schema_filename': 'OperatorImportSchema_v2.csvs'
}
],
'perform_prevalidation': True
}
catalog_config = CatalogConfig(ignore_env=True, **catalog_config_dict)
monkeypatch.setattr(mocked_config, 'catalog_config', catalog_config)
# Run dirbs-catalog using db args from the temp postgres instance
runner = CliRunner()
result = runner.invoke(dirbs_catalog_cli, obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
with db_conn.cursor() as cursor:
cursor.execute('SELECT is_valid_format FROM data_catalog WHERE filename = '
'\'operator1_with_rat_info_20160701_20160731.zip\'')
assert cursor.fetchone().is_valid_format
def test_all_files_are_harvested(postgres, db_conn, tmpdir, logger,
monkeypatch, mocked_config):
"""Test all input files are correctly harvested and cataloged."""
files_to_zip = [
'unittest_data/operator/operator1_with_rat_info_20160701_20160731.csv',
'unittest_data/gsma/sample_gsma_import_list_anonymized.txt',
'unittest_data/stolen_list/sample_stolen_list.csv',
'unittest_data/registration_list/sample_registration_list.csv',
'unittest_data/pairing_list/sample_pairinglist.csv',
'unittest_data/golden_list/sample_golden_list.csv'
]
zip_files_to_tmpdir(files_to_zip, tmpdir)
catalog_config_dict = {
'prospectors': [{
'file_type':
'operator',
'paths': [
str(
tmpdir.join(
'operator1_with_rat_info_20160701_20160731.zip'))
],
'schema_filename':
'OperatorImportSchema_v2.csvs'
}, {
'file_type':
'gsma_tac',
'paths':
[str(tmpdir.join('sample_gsma_import_list_anonymized.zip'))],
'schema_filename':
'GSMASchema.csvs'
}, {
'file_type': 'stolen_list',
'paths': [str(tmpdir.join('sample_stolen_list.zip'))],
'schema_filename': 'StolenListSchema.csvs'
}, {
'file_type': 'pairing_list',
'paths': [str(tmpdir.join('sample_pairinglist.zip'))],
'schema_filename': 'PairingListSchema.csvs'
}, {
'file_type':
'registration_list',
'paths': [str(tmpdir.join('sample_registration_list.zip'))],
'schema_filename':
'RegistrationListSchema.csvs'
}, {
'file_type': 'golden_list',
'paths': [str(tmpdir.join('sample_golden_list.zip'))],
'schema_filename': 'GoldenListSchemaData.csvs'
}],
'perform_prevalidation':
True
}
catalog_config = CatalogConfig(ignore_env=True, **catalog_config_dict)
monkeypatch.setattr(mocked_config, 'catalog_config', catalog_config)
# Run dirbs-catalog using db args from the temp postgres instance
runner = CliRunner()
# Run dirbs-catalog using db args from the temp postgres instance
runner = CliRunner()
result = runner.invoke(dirbs_catalog_cli,
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
with db_conn.cursor() as cursor:
cursor.execute('SELECT * FROM data_catalog')
res = [(res.filename, res.file_type, res.compressed_size_bytes,
res.is_valid_zip, res.is_valid_format, res.extra_attributes)
for res in cursor.fetchall()]
assert ('operator1_with_rat_info_20160701_20160731.zip', 'operator',
797, True, True, {
'filename_check': True
}) in res
assert ('sample_gsma_import_list_anonymized.zip', 'gsma_tac', 1083,
True, True, {}) in res
assert ('sample_stolen_list.zip', 'stolen_list', 529, True, True,
{}) in res
assert ('sample_registration_list.zip', 'registration_list', 858, True,
True, {}) in res
assert ('sample_pairinglist.zip', 'pairing_list', 312, True, True,
{}) in res
assert ('sample_golden_list.zip', 'golden_list', 474, True, True,
{}) in res
# Run dirbs-catalog again to verify that no new files are discovered
result = runner.invoke(dirbs_catalog_cli,
obj={'APP_CONFIG': mocked_config})
assert result.exit_code == 0
assert 'Data catalog is already up-to-date!' in logger_stream_contents(
logger)
