def _common_init():
config = mc.Config()
config.get_executors()
clients = GemClientCollection(config)
meta_visitors = META_VISITORS
gemini_session = mc.get_endpoint_session()
provenance_finder = gemini_metadata.ProvenanceFinder(
config, clients.query_client, gemini_session)
svofps_session = mc.get_endpoint_session()
filter_cache = svofps.FilterMetadataCache(svofps_session)
clients.gemini_session = gemini_session
clients.svo_session = svofps_session
if config.use_local_files or mc.TaskType.SCRAPE in config.task_types:
metadata_reader = gemini_metadata.GeminiFileMetadataReader(
gemini_session, provenance_finder, filter_cache)
meta_visitors = [
fits2caom2_augmentation,
preview_augmentation,
cleanup_augmentation,
]
elif [mc.TaskType.VISIT] == config.task_types:
metadata_reader = gemini_metadata.GeminiStorageClientReader(
clients.data_client,
gemini_session,
provenance_finder,
filter_cache,
)
else:
metadata_reader = gemini_metadata.GeminiMetadataReader(
gemini_session, provenance_finder, filter_cache)
reader_lookup = gemini_metadata.GeminiMetadataLookup(metadata_reader)
reader_lookup.reader = metadata_reader
name_builder = builder.GemObsIDBuilder(config, metadata_reader,
reader_lookup)
return clients, config, metadata_reader, meta_visitors, name_builder
def test_pull_augmentation(http_mock, json_mock, header_mock, file_type_mock):
obs = mc.read_obs_from_file(TEST_OBS_AD_URI_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
original_uri = 'gemini:GEMINI/GN2001BQ013-04.fits'
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
assert (
original_uri
in obs.planes[TEST_PRODUCT_ID].artifacts.keys()), 'initial condition'
test_uri = f'{SCHEME}:{COLLECTION}/{TEST_PRODUCT_ID}.fits'
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
cadc_client_mock = Mock()
cadc_client_mock.return_value.info.return_value = None
clients_mock = Mock()
clients_mock.data_client = cadc_client_mock
json_mock.side_effect = gem_mocks.mock_retrieve_json
filter_cache = svofps.FilterMetadataCache(Mock())
test_reader = gemini_metadata.GeminiFileMetadataReader(
Mock(), Mock(), filter_cache)
test_fqn = f'{gem_mocks.TEST_DATA_DIR}/GMOS/GN2001BQ013-04.fits.header'
test_storage_name = gem_name.GemName(file_name='GN2001BQ013-04.fits')
header_mock.side_effect = gem_mocks._mock_headers
file_type_mock.return_values = 'application/fits'
test_reader.set(test_storage_name)
kwargs = {
'working_directory': TEST_DATA_DIR,
'clients': clients_mock,
'observable': test_observable,
'metadata_reader': test_reader,
'storage_name': test_storage_name,
}
obs = pull_augmentation.visit(obs, **kwargs)
test_url = f'{pull_augmentation.FILE_URL}/{TEST_PRODUCT_ID}.fits'
test_prev = f'{TEST_DATA_DIR}/{TEST_PRODUCT_ID}.fits'
http_mock.assert_called_with(test_url, test_prev), 'mock not called'
assert cadc_client_mock.put.called, 'put mock not called'
cadc_client_mock.put.assert_called_with(
TEST_DATA_DIR, 'gemini:GEMINI/GN2001BQ013-04.fits'), 'wrong put args'
assert obs is not None, 'expect a result'
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'no new artifacts'
try:
ignore = obs.planes[TEST_PRODUCT_ID].artifacts[test_uri]
except KeyError as ke:
# because CAOM does magic
result = obs.planes[TEST_PRODUCT_ID].artifacts[original_uri]
assert result.uri == test_uri, f'wrong uri {result.uri}'
def test_get_filter_metadata(get_vo_mock):
get_vo_mock.side_effect = gem_mocks.mock_get_votable
cache = Mock()
test_subject = svofps.FilterMetadataCache(Mock())
test_result = test_subject.get_filter_metadata(Inst.NIRI, 'filters',
'telescope_name')
assert get_vo_mock.call_count == 2, 'wrong number of calls'
get_vo_mock.assert_called_with(
'http://svo2.cab.inta-csic.es/svo/theory/fps3/fps.php?ID=Gemini/'
'NIRI.filtersw&VERB=0',
ANY,
), 'wrong call args'
assert test_result is None, 'do not expect a result'
# do the same thing again, check that the result has been cached
test_result = test_subject.get_filter_metadata(Inst.NIRI, 'filters',
'telescope_name')
assert get_vo_mock.call_count == 2, 'wrong number of calls'
assert test_result is None, 'do not expect a result this time either'
def _run_test_common(
data_sources,
get_pi_mock,
svofps_mock,
headers_mock,
pf_mock,
json_mock,
file_type_mock,
test_set,
expected_fqn,
):
get_pi_mock.side_effect = mock_get_pi_metadata
svofps_mock.side_effect = mock_get_votable
headers_mock.side_effect = _mock_headers
pf_mock.get.side_effect = mock_get_data_label
json_mock.side_effect = mock_get_obs_metadata
file_type_mock.return_value = 'application/fits'
with TemporaryDirectory() as tmp_dir_name:
test_config = mc.Config()
test_config.task_types = [mc.TaskType.SCRAPE]
test_config.use_local_files = True
test_config.data_sources = data_sources
test_config.working_directory = tmp_dir_name
test_config.proxy_fqn = f'{tmp_dir_name}/test_proxy.pem'
with open(test_config.proxy_fqn, 'w') as f:
f.write('test content')
observation = None
in_fqn = expected_fqn.replace('.expected.', '.in.')
if os.path.exists(in_fqn):
observation = mc.read_obs_from_file(in_fqn)
actual_fqn = expected_fqn.replace('expected', 'actual')
if os.path.exists(actual_fqn):
os.unlink(actual_fqn)
for entry in test_set:
filter_cache = svofps.FilterMetadataCache(svofps_mock)
metadata_reader = gemini_metadata.GeminiFileMetadataReader(
Mock(), pf_mock, filter_cache)
test_metadata = gemini_metadata.GeminiMetadataLookup(
metadata_reader)
test_builder = builder.GemObsIDBuilder(test_config,
metadata_reader,
test_metadata)
storage_name = test_builder.build(entry)
client_mock = Mock()
client_mock.metadata_client = headers_mock
kwargs = {
'storage_name': storage_name,
'metadata_reader': metadata_reader,
'clients': client_mock,
}
logging.getLogger('caom2utils.caom2blueprint', ).setLevel(
logging.INFO)
logging.getLogger('GeminiFits2caom2Visitor').setLevel(logging.INFO)
logging.getLogger('ValueRepairCache').setLevel(logging.INFO)
logging.getLogger('root').setLevel(logging.INFO)
# logging.getLogger('Gmos').setLevel(logging.INFO)
observation = fits2caom2_augmentation.visit(observation, **kwargs)
compare(expected_fqn, actual_fqn, observation)
def test_going_public(
pf_mock,
get_pi_mock,
svofps_mock,
headers_mock,
json_mock,
file_type_mock,
remote_headers_mock,
):
# test case is the data is going public, so the file needs to be
# retrieved from archive.gemini.edu. The mocks are for the case
# where the headers are not yet available from CADC.
test_fid = 'N20150929S0013'
test_f_name = f'{test_fid}.fits'
mock_return_fqn = f'{gem_mocks.TEST_DATA_DIR}/GMOS/{test_f_name}.header'
remote_headers_mock.side_effect = [
ac.make_headers_from_file(mock_return_fqn)
]
expected_fqn = f'{gem_mocks.TEST_DATA_DIR}/GMOS/{test_fid}.expected.xml'
get_pi_mock.side_effect = gem_mocks.mock_get_pi_metadata
svofps_mock.side_effect = gem_mocks.mock_get_votable
headers_mock.side_effect = exceptions.UnexpectedException
pf_mock.get.side_effect = gem_mocks.mock_get_data_label
json_mock.side_effect = gem_mocks.mock_get_obs_metadata
file_type_mock.return_value = 'application/fits'
with TemporaryDirectory() as tmp_dir_name:
test_config = mc.Config()
test_config.task_types = [mc.TaskType.SCRAPE]
test_config.use_local_files = True
test_config.data_sources = [f'{gem_mocks.TEST_DATA_DIR}/broken_files']
test_config.working_directory = tmp_dir_name
test_config.proxy_fqn = f'{tmp_dir_name}/test_proxy.pem'
with open(test_config.proxy_fqn, 'w') as f:
f.write('test content')
in_fqn = expected_fqn.replace('.expected.', '.in.')
observation = mc.read_obs_from_file(in_fqn)
actual_fqn = expected_fqn.replace('expected', 'actual')
if exists(actual_fqn):
unlink(actual_fqn)
filter_cache = svofps.FilterMetadataCache(svofps_mock)
metadata_reader = gemini_metadata.GeminiStorageClientReader(
Mock(), Mock(), pf_mock, filter_cache
)
test_metadata = gemini_metadata.GeminiMetadataLookup(metadata_reader)
test_builder = builder.GemObsIDBuilder(
test_config, metadata_reader, test_metadata
)
storage_name = test_builder.build(mock_return_fqn)
client_mock = Mock()
client_mock.metadata_client = headers_mock
kwargs = {
'storage_name': storage_name,
'metadata_reader': metadata_reader,
'clients': client_mock,
}
ignore = fits2caom2_augmentation.visit(observation, **kwargs)
assert remote_headers_mock.called, 'expect remote header retrieval'
remote_headers_mock.assert_called_with(
'N20150929S0013.fits.header', ANY, ANY
), 'wrong remote header args'