def test_builder(file_info_mock, header_mock):
file_info_mock.side_effect = gem_mocks.mock_get_obs_metadata
test_config = mc.Config()
test_config.working_directory = '/test_files'
test_config.proxy_fqn = os.path.join(gem_mocks.TEST_DATA_DIR,
'test_proxy.pem')
test_reader = gemini_metadata.GeminiMetadataReader(Mock(), Mock(), Mock())
test_metadata = gemini_metadata.GeminiMetadataLookup(test_reader)
test_subject = builder.GemObsIDBuilder(test_config, test_reader,
test_metadata)
test_entries = ['S20050825S0143.fits', 'TX20131117_raw.3002.fits']
for test_entry in test_entries:
for task_type in [mc.TaskType.INGEST, mc.TaskType.SCRAPE]:
test_config.task_types = [task_type]
test_result = test_subject.build(test_entry)
assert test_result is not None, f'expect a result'
assert (test_result.file_uri ==
f'{SCHEME}:{COLLECTION}/{test_entry}'), 'wrong file uri'
assert (test_result.prev_uri ==
f'{SCHEME}:{COLLECTION}/{test_result.prev}'
), 'wrong preview uri'
assert (test_result.thumb_uri ==
f'{V_SCHEME}:{COLLECTION}/{test_result.thumb}'
), 'wrong thumb uri'
assert test_result.obs_id is not None, f'expect an obs id'
def test_builder(obs_metadata_mock, tap_client_mock):
obs_metadata_mock.side_effect = gem_mocks.mock_get_obs_metadata
test_config = mc.Config()
test_config.working_directory = '/test_files'
test_config.proxy_fqn = os.path.join(gem_mocks.TEST_DATA_DIR,
'test_proxy.pem')
em.init_global(config=test_config)
test_subject = builder.GemObsIDBuilder(test_config)
test_entry = 'S20050825S0143.fits'
for support in [False, True]:
test_config.features.supports_latest_client = support
test_config.features.use_file_names = True
for task_type in [mc.TaskType.INGEST, mc.TaskType.SCRAPE]:
test_config.task_types = [task_type]
test_result = test_subject.build(test_entry)
assert test_result is not None, \
f'expect a result support {support}'
expected_path = COLLECTION if support else ARCHIVE
assert test_result.file_uri == \
f'{SCHEME}:{expected_path}/{test_entry}', 'wrong file uri'
assert test_result.prev_uri == \
f'{SCHEME}:{expected_path}/{test_result.prev}', \
'wrong preview uri'
expected_scheme = V_SCHEME if support else A_SCHEME
assert test_result.thumb_uri == \
f'{expected_scheme}:{expected_path}/{test_result.thumb}', \
'wrong thumb uri'
test_config.task_types = [mc.TaskType.INGEST]
test_config.features.use_file_names = False
with pytest.raises(mc.CadcException):
test_result = test_subject.build(test_entry)
def _common_init():
config = mc.Config()
config.get_executors()
clients = GemClientCollection(config)
meta_visitors = META_VISITORS
gemini_session = mc.get_endpoint_session()
provenance_finder = gemini_metadata.ProvenanceFinder(
config, clients.query_client, gemini_session)
svofps_session = mc.get_endpoint_session()
filter_cache = svofps.FilterMetadataCache(svofps_session)
clients.gemini_session = gemini_session
clients.svo_session = svofps_session
if config.use_local_files or mc.TaskType.SCRAPE in config.task_types:
metadata_reader = gemini_metadata.GeminiFileMetadataReader(
gemini_session, provenance_finder, filter_cache)
meta_visitors = [
fits2caom2_augmentation,
preview_augmentation,
cleanup_augmentation,
]
elif [mc.TaskType.VISIT] == config.task_types:
metadata_reader = gemini_metadata.GeminiStorageClientReader(
clients.data_client,
gemini_session,
provenance_finder,
filter_cache,
)
else:
metadata_reader = gemini_metadata.GeminiMetadataReader(
gemini_session, provenance_finder, filter_cache)
reader_lookup = gemini_metadata.GeminiMetadataLookup(metadata_reader)
reader_lookup.reader = metadata_reader
name_builder = builder.GemObsIDBuilder(config, metadata_reader,
reader_lookup)
return clients, config, metadata_reader, meta_visitors, name_builder
def _get_lineage(dirname, basename, config):
jpg_file = basename.replace('.fits.header', '.jpg')
name_builder = builder.GemObsIDBuilder(config)
storage_name = name_builder.build(basename.replace('.header', ''))
if os.path.exists(os.path.join(dirname, jpg_file)):
jpg_storage_name = name_builder.build(jpg_file)
jpg = jpg_storage_name.lineage
fits = storage_name.lineage.replace('.header', '')
result = f'{jpg} {fits}'
else:
result = storage_name.lineage.replace('.header', '')
return result
def _run():
"""
Uses a todo file with file names, even though Gemini provides
information about existing data referenced by observation ID.
"""
config = mc.Config()
config.get_executors()
external_metadata.init_global(config=config)
name_builder = builder.GemObsIDBuilder(config)
meta_visitors = _define_meta_visitors(config)
return rc.run_by_todo(config,
name_builder,
chooser=None,
command_name=main_app.APPLICATION,
meta_visitors=meta_visitors)
def test_builder_local(file_info_mock, header_mock, json_mock):
json_mock.side_effect = gem_mocks.mock_retrieve_json
test_reader = gemini_metadata.GeminiFileMetadataReader(
Mock(), Mock(), Mock())
test_metadata = gemini_metadata.GeminiMetadataLookup(test_reader)
test_config = mc.Config()
test_config.data_sources = ['/test_files']
test_config.use_local_files = True
test_entry = '/test_files/S20191214S0301.fits'
test_config.task_types = [mc.TaskType.INGEST]
test_subject = builder.GemObsIDBuilder(test_config, test_reader,
test_metadata)
test_result = test_subject.build(test_entry)
assert test_result is not None, 'expect a result'
assert test_result.file_uri == 'gemini:GEMINI/S20191214S0301.fits', 'file'
assert test_result.prev_uri == 'gemini:GEMINI/S20191214S0301.jpg', 'prev'
assert (
test_result.thumb_uri == 'cadc:GEMINI/S20191214S0301_th.jpg'), 'thumb'
assert (test_result.source_names[0] == '/test_files/S20191214S0301.fits'
), 'wrong source_names'
def test_different_obs_id_cases(json_mock, headers_mock, file_info_mock):
json_mock.side_effect = gem_mocks.mock_retrieve_json
# these are all special cases, where the data label from Gemini is not
# what's used as the observationID value
test_cases = {
'N20100104S0208.fits.header': 'GN-2009B-Q-121-15-001',
'N20200810A0490r.fits': 'N20200810A0490',
'N20200810A0490r': 'N20200810A0490',
'SDCH_20200131_0010.fits': 'GS-CAL20200131-10-0131',
'GN2001BQ013-04': 'GN2001BQ013-04',
}
test_reader = gemini_metadata.GeminiFileMetadataReader(
Mock(), Mock(), Mock())
test_metadata = gemini_metadata.GeminiMetadataLookup(test_reader)
test_config = mc.Config()
test_subject = builder.GemObsIDBuilder(test_config, test_reader,
test_metadata)
for file_name, obs_id in test_cases.items():
test_result = test_subject.build(file_name)
assert test_result is not None, 'expect a result'
assert test_result.obs_id == obs_id, f'got {test_result.obs_id}'
assert file_name.split('.')[0] in test_result.source_names
def _run_test_common(
data_sources,
get_pi_mock,
svofps_mock,
headers_mock,
pf_mock,
json_mock,
file_type_mock,
test_set,
expected_fqn,
):
get_pi_mock.side_effect = mock_get_pi_metadata
svofps_mock.side_effect = mock_get_votable
headers_mock.side_effect = _mock_headers
pf_mock.get.side_effect = mock_get_data_label
json_mock.side_effect = mock_get_obs_metadata
file_type_mock.return_value = 'application/fits'
with TemporaryDirectory() as tmp_dir_name:
test_config = mc.Config()
test_config.task_types = [mc.TaskType.SCRAPE]
test_config.use_local_files = True
test_config.data_sources = data_sources
test_config.working_directory = tmp_dir_name
test_config.proxy_fqn = f'{tmp_dir_name}/test_proxy.pem'
with open(test_config.proxy_fqn, 'w') as f:
f.write('test content')
observation = None
in_fqn = expected_fqn.replace('.expected.', '.in.')
if os.path.exists(in_fqn):
observation = mc.read_obs_from_file(in_fqn)
actual_fqn = expected_fqn.replace('expected', 'actual')
if os.path.exists(actual_fqn):
os.unlink(actual_fqn)
for entry in test_set:
filter_cache = svofps.FilterMetadataCache(svofps_mock)
metadata_reader = gemini_metadata.GeminiFileMetadataReader(
Mock(), pf_mock, filter_cache)
test_metadata = gemini_metadata.GeminiMetadataLookup(
metadata_reader)
test_builder = builder.GemObsIDBuilder(test_config,
metadata_reader,
test_metadata)
storage_name = test_builder.build(entry)
client_mock = Mock()
client_mock.metadata_client = headers_mock
kwargs = {
'storage_name': storage_name,
'metadata_reader': metadata_reader,
'clients': client_mock,
}
logging.getLogger('caom2utils.caom2blueprint', ).setLevel(
logging.INFO)
logging.getLogger('GeminiFits2caom2Visitor').setLevel(logging.INFO)
logging.getLogger('ValueRepairCache').setLevel(logging.INFO)
logging.getLogger('root').setLevel(logging.INFO)
# logging.getLogger('Gmos').setLevel(logging.INFO)
observation = fits2caom2_augmentation.visit(observation, **kwargs)
compare(expected_fqn, actual_fqn, observation)
def test_going_public(
pf_mock,
get_pi_mock,
svofps_mock,
headers_mock,
json_mock,
file_type_mock,
remote_headers_mock,
):
# test case is the data is going public, so the file needs to be
# retrieved from archive.gemini.edu. The mocks are for the case
# where the headers are not yet available from CADC.
test_fid = 'N20150929S0013'
test_f_name = f'{test_fid}.fits'
mock_return_fqn = f'{gem_mocks.TEST_DATA_DIR}/GMOS/{test_f_name}.header'
remote_headers_mock.side_effect = [
ac.make_headers_from_file(mock_return_fqn)
]
expected_fqn = f'{gem_mocks.TEST_DATA_DIR}/GMOS/{test_fid}.expected.xml'
get_pi_mock.side_effect = gem_mocks.mock_get_pi_metadata
svofps_mock.side_effect = gem_mocks.mock_get_votable
headers_mock.side_effect = exceptions.UnexpectedException
pf_mock.get.side_effect = gem_mocks.mock_get_data_label
json_mock.side_effect = gem_mocks.mock_get_obs_metadata
file_type_mock.return_value = 'application/fits'
with TemporaryDirectory() as tmp_dir_name:
test_config = mc.Config()
test_config.task_types = [mc.TaskType.SCRAPE]
test_config.use_local_files = True
test_config.data_sources = [f'{gem_mocks.TEST_DATA_DIR}/broken_files']
test_config.working_directory = tmp_dir_name
test_config.proxy_fqn = f'{tmp_dir_name}/test_proxy.pem'
with open(test_config.proxy_fqn, 'w') as f:
f.write('test content')
in_fqn = expected_fqn.replace('.expected.', '.in.')
observation = mc.read_obs_from_file(in_fqn)
actual_fqn = expected_fqn.replace('expected', 'actual')
if exists(actual_fqn):
unlink(actual_fqn)
filter_cache = svofps.FilterMetadataCache(svofps_mock)
metadata_reader = gemini_metadata.GeminiStorageClientReader(
Mock(), Mock(), pf_mock, filter_cache
)
test_metadata = gemini_metadata.GeminiMetadataLookup(metadata_reader)
test_builder = builder.GemObsIDBuilder(
test_config, metadata_reader, test_metadata
)
storage_name = test_builder.build(mock_return_fqn)
client_mock = Mock()
client_mock.metadata_client = headers_mock
kwargs = {
'storage_name': storage_name,
'metadata_reader': metadata_reader,
'clients': client_mock,
}
ignore = fits2caom2_augmentation.visit(observation, **kwargs)
assert remote_headers_mock.called, 'expect remote header retrieval'
remote_headers_mock.assert_called_with(
'N20150929S0013.fits.header', ANY, ANY
), 'wrong remote header args'