def test_pull_v_augmentation(put_mock, http_mock):
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
test_uri = f'{SCHEME}:{COLLECTION}/{TEST_PRODUCT_ID}.fits'
for plane in obs.planes.values():
for artifact in plane.artifacts.values():
artifact.uri = test_uri
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
cadc_client_mock = Mock()
kwargs = {'working_directory': TEST_DATA_DIR,
'cadc_client': cadc_client_mock,
'observable': test_observable}
result = pull_v_augmentation.visit(obs, **kwargs)
test_url = f'{pull_augmentation.FILE_URL}/{TEST_PRODUCT_ID}.fits'
test_prev = f'{TEST_DATA_DIR}/{TEST_PRODUCT_ID}.fits'
http_mock.assert_called_with(test_url, test_prev), 'mock not called'
assert put_mock.called, 'put mock not called'
args, kwargs = put_mock.call_args
assert args[1] == TEST_DATA_DIR, 'wrong working dir'
assert args[2] == f'{TEST_PRODUCT_ID}.fits', 'wrong file name'
assert args[3] == test_uri, 'wrong storage name'
assert result is not None, 'expect a result'
assert result['observation'] == 0, 'no updated metadata'
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, \
'no new artifacts'
def test_pull_augmentation():
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
cadc_client_mock = Mock()
kwargs = {'working_directory': TEST_DATA_DIR,
'cadc_client': cadc_client_mock,
'stream': 'stream',
'observable': test_observable}
with patch('caom2pipe.manage_composable.http_get') as http_mock, \
patch('caom2pipe.manage_composable.data_put') as ad_put_mock:
cadc_client_mock.return_value.data_get.return_value = mc.CadcException(
'test')
# no scheme from cadc client
cadc_client_mock.get_file_info.return_value = {'md5sum': '1234'}
result = pull_augmentation.visit(obs, **kwargs)
test_url = f'{pull_augmentation.FILE_URL}/{TEST_PRODUCT_ID}.fits'
test_prev = f'{TEST_DATA_DIR}/{TEST_PRODUCT_ID}.fits'
http_mock.assert_called_with(test_url, test_prev), 'mock not called'
assert ad_put_mock.called, 'ad put mock not called'
assert result is not None, 'expect a result'
assert result['observation'] == 0, 'no updated metadata'
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, \
'no new artifacts'
def test_store(put_mock):
test_config = mc.Config()
test_config.logging_level = 'ERROR'
test_config.working_directory = '/tmp'
test_url = 'https://archive-new.nrao.edu/vlass/quicklook/VLASS2.1/' \
'T10t12/VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1/' \
'VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1.I.iter1.image.' \
'pbcor.tt0.rms.subim.fits'
test_storage_name = VlassName(url=test_url, entry=test_url)
transferrer = Mock()
cred_param = Mock()
cadc_data_client = Mock()
caom_repo_client = Mock()
observable = mc.Observable(mc.Rejected('/tmp/rejected.yml'),
mc.Metrics(test_config))
test_subject = ec.Store(test_config, test_storage_name, APPLICATION,
cred_param, cadc_data_client, caom_repo_client,
observable, transferrer)
test_subject.execute(None)
assert put_mock.called, 'expect a call'
args, kwargs = put_mock.call_args
assert args[2] == test_storage_name.file_name, 'wrong file name'
assert transferrer.get.called, 'expect a transfer call'
args, kwargs = transferrer.get.call_args
assert args[0] == test_url, 'wrong source parameter'
assert args[1] == f'/tmp/{test_storage_name.obs_id}/' \
f'{test_storage_name.file_name}',\
'wrong destination parameter'
def test_provenance_augmentation(dmf_mock, headers_mock, access_mock,
builder_mock, repo_get_mock, test_fqn):
builder_mock.return_value._get_obs_id.return_value = None
access_mock.return_value = 'https://localhost'
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_config.task_types = [mc.TaskType.VISIT]
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
headers_mock.return_value.get_head.side_effect = _get_headers_mock
dmf_mock.get.side_effect = _get_obs_id_mock
repo_get_mock.side_effect = _repo_get_mock
getcwd_orig = os.getcwd
os.getcwd = Mock(return_value=test_main_app.TEST_DATA_DIR)
temp = os.path.basename(test_fqn).replace('.expected.xml', '.fits')
test_storage_name = GemProcName(entry=temp)
try:
test_obs = mc.read_obs_from_file(test_fqn)
assert not test_obs.target.moving, 'initial conditions moving target'
kwargs = {
'storage_name': test_storage_name,
'working_directory': test_main_app.TEST_DATA_DIR,
'observable': test_observable,
'caom_repo_client': Mock(),
}
test_result = provenance_augmentation.visit(test_obs, **kwargs)
assert test_result is not None, 'expect a result'
assert test_result.get('provenance') == 2, 'wrong result'
assert len(test_obs.members) == 1, 'wrong membership'
assert test_obs.target.moving, 'should be changed'
finally:
os.getcwd = getcwd_orig
def test_preview_augment_delete_preview():
# plane starts with a preview artifact, but it represents a non-existent
# file, so remove the artifact from the CAOM observation
test_product_id = 'S20080610S0045'
fqn = os.path.join(TEST_DATA_DIR, 'GS-2008A-C-5-35-002.fits.xml')
obs = mc.read_obs_from_file(fqn)
assert len(obs.planes[test_product_id].artifacts) == 2, 'initial condition'
test_rejected = mc.Rejected('/tmp/nonexistent')
test_rejected.content = {
'bad_metadata': [],
'no_preview':
['S20080610S0043.jpg',
'S20080610S0041.jpg',
'S20080610S0044.jpg',
'S20080610S0045.jpg']}
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
kwargs = {'working_directory': TEST_DATA_DIR,
'cadc_client': None,
'stream': 'stream',
'observable': test_observable}
result = preview_augmentation.visit(obs, **kwargs)
assert result is not None, 'expect a result'
assert result['artifacts'] == 1, 'wrong result'
assert len(obs.planes[test_product_id].artifacts) == 1, 'post condition'
def test_store(put_mock):
test_config = mc.Config()
test_config.logging_level = 'ERROR'
test_config.working_directory = '/tmp'
test_fqn = '/users/OpenData_DonneesOuvertes/pub/NEOSSAT/ASTRO/2019/' \
'268/NEOS_SCI_2019268004930_clean.fits'
test_storage_name = NEOSSatName(file_name=test_fqn, entry=test_fqn)
transferrer = Mock()
cred_param = Mock()
cadc_data_client = Mock()
caom_repo_client = Mock()
observable = mc.Observable(mc.Rejected('/tmp/rejected.yml'),
mc.Metrics(test_config))
test_subject = ec.Store(test_config, test_storage_name, APPLICATION,
cred_param, cadc_data_client, caom_repo_client,
observable, transferrer)
test_subject.execute(None)
assert put_mock.called, 'expect a call'
args, kwargs = put_mock.call_args
assert args[2] == test_storage_name.file_name, 'wrong file name'
assert transferrer.get.called, 'expect a transfer call'
args, kwargs = transferrer.get.call_args
import logging
logging.error(args)
assert args[0] == test_fqn, 'wrong source parameter'
assert args[1] == f'/tmp/{test_storage_name.obs_id}/' \
f'{test_storage_name.file_name}', \
'wrong destination parameter'
def test_meta_update_observation_direct(test_config):
test_cred_param = '--cert /usr/src/app/cadcproxy.pem'
data_client_mock = Mock()
repo_client_mock = Mock()
test_sn = tc.TestStorageName()
test_observation = mc.read_obs_from_file(
f'{tc.TEST_DATA_DIR}/fpf_start_obs.xml'
)
try:
test_observable = mc.Observable(
mc.Rejected(test_config.rejected_fqn), mc.Metrics(test_config)
)
test_executor = ec.MetaUpdateObservation(
test_config,
test_sn,
TEST_APP,
test_cred_param,
data_client_mock,
repo_client_mock,
test_observation,
[],
observable=test_observable,
)
test_executor.execute(None)
assert repo_client_mock.update.called, 'repo update called'
finally:
pass
def test_provenance_augmentation(obs_id_mock, repo_get_mock, headers_mock,
test_fqn):
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
headers_mock.side_effect = _get_headers_mock
repo_get_mock.side_effect = _repo_get_mock
obs_id_mock.side_effect = _get_obs_id_mock
getcwd_orig = os.getcwd
os.getcwd = Mock(return_value=test_main_app.TEST_DATA_DIR)
try:
test_obs = mc.read_obs_from_file(test_fqn)
assert not test_obs.target.moving, 'initial conditions moving target'
kwargs = {
'science_file':
os.path.basename(test_fqn).replace('.expected.xml', '.fits'),
'working_directory':
test_main_app.TEST_DATA_DIR,
'observable':
test_observable,
'caom_repo_client':
Mock()
}
test_result = provenance_augmentation.visit(test_obs, **kwargs)
assert test_result is not None, 'expect a result'
assert test_result.get('provenance') == 2, 'wrong result'
assert len(test_obs.members) == 1, 'wrong membership'
assert test_obs.target.moving, 'should be changed'
finally:
os.getcwd = getcwd_orig
def __init__(self):
super(GeminiValidator, self).__init__(
source_name=main_app.COLLECTION, scheme=gem_name.SCHEME,
preview_suffix='_th.jpg')
self._gofr = external_metadata.get_gofr()
self._max_date = datetime.fromtimestamp(
self._gofr.get_max_timestamp()).date()
logging.error(f'max date is {self._max_date}')
self._rejected = mc.Rejected(self._config.rejected_fqn)
self._logger = logging.getLogger(__name__)
def test_preview_augment(http_mock):
# this should result in two new artifacts being added to the plane
# one for a thumbnail and one for a preview
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_metrics = mc.Metrics(test_config)
test_observable = mc.Observable(test_rejected, test_metrics)
cadc_client_mock = Mock()
clients_mock = Mock()
clients_mock.data_client = cadc_client_mock
test_storage_name = gem_name.GemName(file_name=f'{TEST_PRODUCT_ID}.fits')
kwargs = {
'working_directory': '/test_files',
'clients': clients_mock,
'observable': test_observable,
'storage_name': test_storage_name,
}
test_prev = f'/test_files/{TEST_PRODUCT_ID}.jpg'
if os.path.exists(test_prev):
os.unlink(test_prev)
try:
cadc_client_mock.get.side_effect = exceptions.UnexpectedException(
'test')
http_mock.side_effect = _get_mock
obs = preview_augmentation.visit(obs, **kwargs)
test_url = (f'{preview_augmentation.PREVIEW_URL}'
f'{TEST_PRODUCT_ID}.fits')
assert http_mock.called, 'http mock should be called'
http_mock.assert_called_with(test_url, test_prev), 'mock not called'
assert cadc_client_mock.put.called, 'put mock not called'
cadc_client_mock.put.assert_called_with(
'/test_files',
'cadc:GEMINI/GN2001BQ013-04_th.jpg',
), 'wrong put arguments'
assert obs is not None, 'expect a result'
assert (len(
obs.planes[TEST_PRODUCT_ID].artifacts) == 3), 'two new artifacts'
prev_uri = mc.build_uri(COLLECTION, f'{TEST_PRODUCT_ID}.jpg', SCHEME)
thumb_uri = mc.build_uri(COLLECTION, f'{TEST_PRODUCT_ID}_th.jpg',
'cadc')
assert (prev_uri
in obs.planes[TEST_PRODUCT_ID].artifacts.keys()), 'no preview'
assert (thumb_uri
in obs.planes[TEST_PRODUCT_ID].artifacts), 'no thumbnail'
finally:
if os.path.exists(test_prev):
os.unlink(test_prev)
def test_preview_augment_known_no_preview():
# rejected file exists that says there's a preview known to not
# exist, so trying to generate a thumbnail will result in no
# change to the plane/artifact structure
try:
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert (len(
obs.planes[TEST_PRODUCT_ID].artifacts) == 1), 'initial condition'
if os.path.exists(REJECTED_FILE):
os.unlink(REJECTED_FILE)
test_rejected = mc.Rejected(REJECTED_FILE)
test_rejected.record(mc.Rejected.NO_PREVIEW, f'{TEST_PRODUCT_ID}.jpg')
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
test_storage_name = gem_name.GemName(file_name=TEST_FP_FILE)
cadc_client_mock = Mock()
clients_mock = Mock()
clients_mock.data_client = cadc_client_mock
kwargs = {
'working_directory': TEST_DATA_DIR,
'clients': clients_mock,
'stream': 'stream',
'observable': test_observable,
'storage_name': test_storage_name,
}
with patch('caom2pipe.manage_composable.http_get') as http_mock, patch(
'caom2pipe.manage_composable.data_put') as ad_put_mock, patch(
'caom2pipe.manage_composable.get_artifact_metadata'
) as art_mock, patch(
'caom2pipe.manage_composable.exec_cmd') as exec_mock:
cadc_client_mock.return_value.data_get.return_value = (
mc.CadcException('test'))
obs = preview_augmentation.visit(obs, **kwargs)
assert not http_mock.called, 'http mock should not be called'
assert not ad_put_mock.called, 'ad put mock should not be called'
assert not art_mock.called, 'art mock should not be called'
assert not exec_mock.called, 'exec mock should not be called'
assert obs is not None, 'expect a result'
assert (len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1
), 'no new artifacts'
test_rejected.persist_state()
assert os.path.exists(REJECTED_FILE)
finally:
if os.path.exists(REJECTED_FILE):
os.unlink(REJECTED_FILE)
def test_preview_augment_v(put_mock, http_mock):
# this should result in two new artifacts being added to the plane
# one for a thumbnail and one for a preview
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_metrics = mc.Metrics(test_config)
test_observable = mc.Observable(test_rejected, test_metrics)
cadc_client_mock = Mock()
kwargs = {'working_directory': '/test_files',
'cadc_client': cadc_client_mock,
'observable': test_observable}
test_prev = f'/test_files/{TEST_PRODUCT_ID}.jpg'
if os.path.exists(test_prev):
os.unlink(test_prev)
try:
cadc_client_mock.return_value.data_get.return_value = \
mc.CadcException('test')
http_mock.side_effect = _get_mock
result = preview_v_augmentation.visit(obs, **kwargs)
test_url = f'{preview_augmentation.PREVIEW_URL}' \
f'{TEST_PRODUCT_ID}.fits'
assert http_mock.called, 'http mock should be called'
http_mock.assert_called_with(test_url, test_prev), \
'mock not called'
assert put_mock.called, 'put mock not called'
put_mock.assert_called_with(
cadc_client_mock, '/test_files', 'GN2001BQ013-04_th.jpg',
'ad:GEM/GN2001BQ013-04_th.jpg', metrics=test_metrics)
assert result is not None, 'expect a result'
assert result['artifacts'] == 2, 'artifacts should be added'
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 3, \
'two new artifacts'
prev_uri = mc.build_uri(
ARCHIVE, f'{TEST_PRODUCT_ID}.jpg', SCHEME)
thumb_uri = mc.build_uri(ARCHIVE, f'{TEST_PRODUCT_ID}_th.jpg')
assert prev_uri in obs.planes[TEST_PRODUCT_ID].artifacts.keys(), \
'no preview'
assert thumb_uri in obs.planes[TEST_PRODUCT_ID].artifacts, \
'no thumbnail'
finally:
if os.path.exists(test_prev):
os.unlink(test_prev)
def test_pull_augmentation(http_mock, json_mock, header_mock, file_type_mock):
obs = mc.read_obs_from_file(TEST_OBS_AD_URI_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
original_uri = 'gemini:GEMINI/GN2001BQ013-04.fits'
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
assert (
original_uri
in obs.planes[TEST_PRODUCT_ID].artifacts.keys()), 'initial condition'
test_uri = f'{SCHEME}:{COLLECTION}/{TEST_PRODUCT_ID}.fits'
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
cadc_client_mock = Mock()
cadc_client_mock.return_value.info.return_value = None
clients_mock = Mock()
clients_mock.data_client = cadc_client_mock
json_mock.side_effect = gem_mocks.mock_retrieve_json
filter_cache = svofps.FilterMetadataCache(Mock())
test_reader = gemini_metadata.GeminiFileMetadataReader(
Mock(), Mock(), filter_cache)
test_fqn = f'{gem_mocks.TEST_DATA_DIR}/GMOS/GN2001BQ013-04.fits.header'
test_storage_name = gem_name.GemName(file_name='GN2001BQ013-04.fits')
header_mock.side_effect = gem_mocks._mock_headers
file_type_mock.return_values = 'application/fits'
test_reader.set(test_storage_name)
kwargs = {
'working_directory': TEST_DATA_DIR,
'clients': clients_mock,
'observable': test_observable,
'metadata_reader': test_reader,
'storage_name': test_storage_name,
}
obs = pull_augmentation.visit(obs, **kwargs)
test_url = f'{pull_augmentation.FILE_URL}/{TEST_PRODUCT_ID}.fits'
test_prev = f'{TEST_DATA_DIR}/{TEST_PRODUCT_ID}.fits'
http_mock.assert_called_with(test_url, test_prev), 'mock not called'
assert cadc_client_mock.put.called, 'put mock not called'
cadc_client_mock.put.assert_called_with(
TEST_DATA_DIR, 'gemini:GEMINI/GN2001BQ013-04.fits'), 'wrong put args'
assert obs is not None, 'expect a result'
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'no new artifacts'
try:
ignore = obs.planes[TEST_PRODUCT_ID].artifacts[test_uri]
except KeyError as ke:
# because CAOM does magic
result = obs.planes[TEST_PRODUCT_ID].artifacts[original_uri]
assert result.uri == test_uri, f'wrong uri {result.uri}'
def test_cadc_transfer(client_mock, caps_mock):
caps_mock.return_value = 'https://sc2.canfar.net/sc2repo'
test_subject = tc.CadcTransfer()
assert test_subject is not None, 'expect a result'
test_config = mc.Config()
test_config.rejected_fqn = '/tmp/rejected.yml'
test_observable = mc.Observable(mc.Rejected(test_config.rejected_fqn),
mc.Metrics(test_config))
test_subject.observable = test_observable
test_subject.cadc_client = client_mock
test_source = 'ad:TEST/test_file.fits'
test_destination = '/tmp/test_file.fits'
test_subject.get(test_source, test_destination)
assert client_mock.get.called, 'should have been called'
client_mock.get.assert_called_with(
'/tmp', 'ad:TEST/test_file.fits'), 'wrong parameters'
def __init__(self):
super(GeminiValidator, self).__init__(
source_name=COLLECTION,
scheme=SCHEME,
preview_suffix='_th.jpg',
)
config = mc.Config()
config.get_executors()
# TODO
# external_metadata.init_global(config)
# self._gofr = external_metadata.get_gofr(config)
# self._max_date = datetime.fromtimestamp(
# self._gofr.get_max_timestamp()
# ).date()
logging.info(f'max date is {self._max_date}')
self._rejected = mc.Rejected(self._config.rejected_fqn)
self._logger = logging.getLogger(__name__)
def test_preview_augment_unknown_no_preview():
# what happens when it's not known that there's no preview
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
# make sure the rejected file is empty
if os.path.exists(REJECTED_FILE):
os.unlink(REJECTED_FILE)
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
test_storage_name = gem_name.GemName(file_name=f'{TEST_PRODUCT_ID}.fits')
cadc_client_mock = Mock()
clients_mock = Mock()
clients_mock.data_client = cadc_client_mock
kwargs = {
'working_directory': TEST_DATA_DIR,
'clients': clients_mock,
'stream': 'stream',
'observable': test_observable,
'storage_name': test_storage_name,
}
with patch(
'caom2pipe.manage_composable.http_get',
side_effect=mc.CadcException(
'Not Found for url: https://archive.gemini.edu/preview'),
) as http_mock, patch(
'caom2pipe.manage_composable.data_put') as ad_put_mock, patch(
'caom2pipe.manage_composable.get_artifact_metadata'
) as art_mock, patch(
'caom2pipe.manage_composable.exec_cmd') as exec_mock:
cadc_client_mock.get.side_effect = exceptions.UnexpectedException(
'test')
obs = preview_augmentation.visit(obs, **kwargs)
assert obs is not None, 'expect result'
test_url = f'{preview_augmentation.PREVIEW_URL}{TEST_PRODUCT_ID}.fits'
test_prev = f'{TEST_DATA_DIR}/{TEST_PRODUCT_ID}.jpg'
http_mock.assert_called_with(test_url, test_prev), 'mock not called'
assert not ad_put_mock.called, 'ad put mock should not be called'
assert not art_mock.called, 'art mock should not be called'
assert not exec_mock.called, 'exec mock should not be called'
def test_cadc_transfer(data_get_mock, caps_mock):
caps_mock.return_value = 'https://sc2.canfar.net/sc2repo'
data_get_mock.side_effect = Mock(autospec=True)
test_subject = tc.CadcTransfer()
assert test_subject is not None, 'expect a result'
test_config = mc.Config()
test_config.rejected_fqn = '/tmp/rejected.yml'
test_observable = mc.Observable(mc.Rejected(test_config.rejected_fqn),
mc.Metrics(test_config))
test_subject.observable = test_observable
test_subject.cadc_client = Mock()
test_source = 'ad:TEST/test_file.fits'
test_destination = '/tmp/test_file.fits'
test_subject.get(test_source, test_destination)
assert data_get_mock.called, 'should have been called'
args, kwargs = data_get_mock.call_args
assert args[1] == '/tmp', 'wrong dir name'
assert args[2] == 'test_file.fits', 'wrong file name'
assert args[3] == 'TEST', 'wrong archive name'
def test_preview_augmentation(access_mock):
access_mock.return_value = 'https://localhost'
test_fqn = f'{TEST_DATA_DIR}/preview_augmentation_start.xml'
test_science_f_name = (
'VLASS1.1.ql.T01t01.J000228-363000.10.2048.v1.I.iter1.image.pbcor.'
'tt0.subim.fits')
test_storage_name = sn.VlassName(test_science_f_name)
test_obs = mc.read_obs_from_file(test_fqn)
test_config = mc.Config()
test_rejected = mc.Rejected(f'{TEST_DATA_DIR}/rejected.yml')
test_metrics = mc.Metrics(test_config)
test_observable = mc.Observable(test_rejected, test_metrics)
kwargs = {
'stream': None,
'observable': test_observable,
'storage_name': test_storage_name,
'working_directory': '/test_files',
}
test_subject = preview_augmentation.VlassPreview(**kwargs)
assert test_subject is not None, 'need a test subject'
assert len(test_obs.planes) == 1, 'wrong number of planes'
assert (len(test_obs.planes[test_storage_name.product_id].artifacts) == 4
), 'wrong starting # of artifacts'
test_obs = test_subject.visit(test_obs)
assert test_obs is not None, 'expect a result'
assert (len(test_obs.planes[test_storage_name.product_id].artifacts) == 6
), 'wrong ending # of artifacts'
test_report = test_subject.report
assert test_report is not None
assert 'artifacts' in test_report
assert test_report.get('artifacts') == 2, 'wrong report count'
# does artifact re-naming work?
test_url = (f'https://archive-new.nrao.edu/vlass/quicklook/VLASS1.1/'
f'T01t01/VLASS1.1.ql.T01t01.J000228-363000.10.2048.v1/'
f'{test_science_f_name}')
kwargs = {'url': test_url}
test_obs = cleanup_augmentation.visit(test_obs, **kwargs)
test_artifacts = test_obs.planes[test_storage_name.product_id].artifacts
assert len(test_artifacts) == 4, 'wrong ending conditions'
assert test_storage_name.prev_uri in test_artifacts, 'missing preview'
assert test_storage_name.thumb_uri in test_artifacts, 'missing thumbnail'
def test_preview_augmentation(data_client_mock, tap_mock):
getcwd_orig = os.getcwd
os.getcwd = Mock(return_value=test_main_app.TEST_DATA_DIR)
tap_mock.side_effect = _tap_mock
data_client_mock.return_value.info.side_effect = (
test_main_app._get_file_info)
test_f_id = 'rnN20140428S0181_ronchi'
test_f_name = f'{test_f_id}.fits'
test_obs = mc.read_obs_from_file(
f'{test_main_app.TEST_DATA_DIR}/{test_f_id}.expected.xml')
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_config.get_executors()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
external_metadata.init_global(test_config)
test_builder = builder.GemProcBuilder(test_config)
test_fqn = os.path.join(test_main_app.TEST_DATA_DIR, test_f_name)
test_storage_name = test_builder.build(test_fqn)
kwargs = {
'working_directory': TEST_FILES_DIR,
'cadc_client': None,
'stream': 'stream',
'observable': test_observable,
'storage_name': test_storage_name,
}
try:
start_ts = datetime.utcnow().timestamp()
test_result = preview_augmentation.visit(test_obs, **kwargs)
end_ts = datetime.utcnow().timestamp()
logging.error(f'{test_f_name} execution time {end_ts - start_ts}')
except Exception as e:
logging.error(e)
logging.error(traceback.format_exc())
assert False
finally:
os.getcwd = getcwd_orig
assert test_result is not None, 'expect a result'
assert test_result.get('artifacts') == 2, 'wrong result'
def test_http_transfer(get_mock):
test_source = 'http://localhost/test_file.fits'
test_destination = '/tmp/test_file.fits'
if not os.path.exists(test_destination):
with open(test_destination, 'w') as f:
f.write('test content')
get_mock.side_effect = Mock(autospec=True)
test_config = mc.Config()
test_config.working_directory = test_conf.TEST_DATA_DIR
test_config.netrc_file = 'test_netrc'
test_config.rejected_fqn = '/tmp/rejected.yml'
test_observable = mc.Observable(mc.Rejected(test_config.rejected_fqn),
mc.Metrics(test_config))
test_subject = tc.HttpTransfer()
assert test_subject is not None, 'expect a result'
test_subject.observable = test_observable
with pytest.raises(mc.CadcException):
test_subject.get(test_source, test_destination)
assert get_mock.called, 'should have been called'
args, kwargs = get_mock.call_args
assert args[1] == test_source, 'wrong source name'
assert args[2] == test_destination, 'wrong dest name'
def test_preview_augment_unknown_no_preview():
# what happens when it's not known that there's no preview
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
# make sure the rejected file is empty
if os.path.exists(REJECTED_FILE):
os.unlink(REJECTED_FILE)
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
cadc_client_mock = Mock()
kwargs = {'working_directory': TEST_DATA_DIR,
'cadc_client': cadc_client_mock,
'stream': 'stream',
'observable': test_observable}
with patch('caom2pipe.manage_composable.http_get',
side_effect=mc.CadcException(
'Internal Server Error for url: '
'https://archive.gemini.edu/preview')) as http_mock, \
patch('caom2pipe.manage_composable.data_put') as ad_put_mock, \
patch('caom2pipe.manage_composable.get_artifact_metadata') as \
art_mock, \
patch('caom2pipe.manage_composable.exec_cmd') as exec_mock:
cadc_client_mock.return_value.data_get.return_value = mc.CadcException(
'test')
result = preview_augmentation.visit(obs, **kwargs)
assert result is not None, 'expect result'
# 0 because the preview artifact doesn't already exist
assert result['artifacts'] == 0, 'wrong result'
test_url = f'{preview_augmentation.PREVIEW_URL}{TEST_PRODUCT_ID}.fits'
test_prev = f'{TEST_DATA_DIR}/{TEST_PRODUCT_ID}.jpg'
http_mock.assert_called_with(test_url, test_prev), 'mock not called'
assert not ad_put_mock.called, 'ad put mock should not be called'
assert not art_mock.called, 'art mock should not be called'
assert not exec_mock.called, 'exec mock should not be called'
def test_store():
test_config = mc.Config()
test_config.logging_level = 'ERROR'
test_config.working_directory = '/tmp'
test_url = ('https://archive-new.nrao.edu/vlass/quicklook/VLASS2.1/'
'T10t12/VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1/'
'VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1.I.iter1.image.'
'pbcor.tt0.rms.subim.fits')
test_storage_name = VlassName(test_url)
transferrer = Mock()
cadc_data_client = Mock()
observable = mc.Observable(mc.Rejected('/tmp/rejected.yml'),
mc.Metrics(test_config))
test_subject = ec.Store(
test_config,
test_storage_name,
cadc_data_client,
observable,
transferrer,
)
test_subject.execute(None)
assert cadc_data_client.put.called, 'expect a call'
cadc_data_client.put.assert_called_with(
'/tmp/VLASS2.1.T10t12.J073401-033000',
f'{SCHEME}:VLASS/VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1.I.'
f'iter1.image.pbcor.tt0.rms.subim.fits',
None,
), 'wrong put args'
assert transferrer.get.called, 'expect a transfer call'
test_f_name = ('VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1.I.iter1.'
'image.pbcor.tt0.rms.subim.fits')
transferrer.get.assert_called_with(
f'https://archive-new.nrao.edu/vlass/quicklook/VLASS2.1/T10t12/'
f'VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1/{test_f_name}',
f'/tmp/VLASS2.1.T10t12.J073401-033000/{test_f_name}',
), 'wrong transferrer args'
def test_preview_augment():
# this should result in three new artifacts being added to every plane:
# one for a thumbnail and two for previews (one zoom)
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
test_files = {
'visit_obs_start_wircam.xml':
['1151210o.fits.fz', '1151210s.fits.fz', '1151210m.fits.fz',
'1151210p.fits.fz', '1151210w.fits.fz', '1151210y.fits.fz',
'1151210g.fits.fz'],
'visit_obs_start_megacam_sci.xml':
['1927963f.fits.fz', '1927963o.fits.fz', '1927963p.fits.fz'],
'visit_obs_start_megacam_cal.xml':
['979412b.fits.fz', '979412o.fits.fz', '979412p.fits.fz'],
'visit_obs_start_sitelle_calibrated_cube.xml':
['2359320p.fits'],
'visit_obs_start_sitelle.xml':
['2359320o.fits.fz'],
'visit_obs_start_espadons.xml':
['2460606i.fits.gz', '2460606o.fits.gz'],
'visit_obs_start_espadons_cal.xml':
['1001063b.fits.gz'],
'visit_obs_start_spirou.xml': ['2401734o.fits', '2401734e.fits',
'2401734r.fits', '2401734s.fits',
'2401734t.fits', '2401734v.fits'],
'visit_obs_start_2hdus_mega.xml': ['1821271p.fits.fz']
}
test_checksums = {
'ad:CFHT/2460606i_preview_1024.jpg':
'md5:eeb2558c76c83a1b56e0afa3d96b0fa0',
'ad:CFHT/2460606i_preview_256.jpg':
'md5:1084a1e58d246c35fae43ca021916c6a',
'ad:CFHT/2460606o_preview_1024.jpg':
'md5:eeeb436f0e6b1cb48aca5a9ba5d1ab3a',
'ad:CFHT/2460606o_preview_256.jpg':
'md5:3879c153776ee0569ee33bc8a35b5e75',
'ad:CFHT/2460606o_preview_zoom_1024.jpg':
'md5:1d5c9e5ee1586ddd6ce5c53b17f8f5ec',
'ad:CFHT/1001063b_preview_1024.jpg':
'md5:d457fa8ecdd397790de65f8bd3a80d06',
'ad:CFHT/1001063b_preview_256.jpg':
'md5:3ae90e1e120fb97216ddb2d965e9423e',
'ad:CFHT/1001063b_preview_zoom_1024.jpg':
'md5:a2d62e654838a620dcf97425753481ca',
'ad:CFHT/979412b_preview_1024.jpg':
'md5:8b3e4ff2c210b3861748423339a71cc5',
'ad:CFHT/979412b_preview_256.jpg':
'md5:893a9cd18f66fab907573bf1e664148d',
'ad:CFHT/979412b_preview_zoom_1024.jpg':
'md5:4f21d6676b0749c48b4872699f7aff7c',
'ad:CFHT/979412o_preview_1024.jpg':
'md5:379a7b21022a0e758fbdea6130c07233',
'ad:CFHT/979412o_preview_256.jpg':
'md5:59c2ecedda2a2560cafdca543533e190',
'ad:CFHT/979412o_preview_zoom_1024.jpg':
'md5:6f3a3a39155a426a365f13f6d10982a9',
'ad:CFHT/979412p_preview_1024.jpg':
'md5:d449d26987a4d9e220d249f8573025ee',
'ad:CFHT/979412p_preview_256.jpg':
'md5:102be79a6ee26a7e5c2ee7e40623d452',
'ad:CFHT/979412p_preview_zoom_1024.jpg':
'md5:7e09407afb9e8148ad36ba56933e1822',
'ad:CFHT/1927963f_preview_1024.jpg':
'md5:9afd99f2d60de3bc168832ce37200508',
'ad:CFHT/1927963f_preview_256.jpg':
'md5:5de596f4b5ab1ae8bfab1de5550ec56d',
'ad:CFHT/1927963f_preview_zoom_1024.jpg':
'md5:ca705c70eb0d320856e346662a83f1ba',
'ad:CFHT/1927963o_preview_1024.jpg':
'md5:d8a9c9c6dde50c59f75a1b384300d41a',
'ad:CFHT/1927963o_preview_256.jpg':
'md5:84cfc76e094678e61eaa9f9421e30dd9',
'ad:CFHT/1927963o_preview_zoom_1024.jpg':
'md5:843109c217104034fa3b029949fa277e',
'ad:CFHT/1927963p_preview_1024.jpg':
'md5:61b903863106438cfa2771731ed8690d',
'ad:CFHT/1927963p_preview_256.jpg':
'md5:01e4de796ff3f20949d02fe776b5f699',
'ad:CFHT/1927963p_preview_zoom_1024.jpg':
'md5:91c4a735d507d3c11482179d725a4166',
'ad:CFHT/2359320o_preview_1024.jpg':
'md5:45b164799c67751c5a90255c11ccd9e5',
'ad:CFHT/2359320o_preview_256.jpg':
'md5:dce23df89c3424bc67d733a7784a13d3',
'ad:CFHT/2359320o_preview_zoom_1024.jpg':
'md5:5cc26a79326b42c6a17bca8e0d3fb1fb',
# 'ad:CFHT/2359320p_preview_1024.jpg': ####
# 'md5:b09f54461fb10259f4958ff9ab1e8179',
# 'ad:CFHT/2359320p_preview_256.jpg':
# 'md5:b037076c0d9488d013052d6750e55aeb',
# 'ad:CFHT/2359320p_preview_zoom_1024.jpg':
# 'md5:514aa25fec1b74fe942ccc8ce8e552f7',
'ad:CFHT/2401734e_preview_1024.jpg':
'md5:a3b272473e959cbc5496c1d562f16c0d',
'ad:CFHT/2401734e_preview_256.jpg':
'md5:781a1c86e727f82ec17a98b5eaf2b7d0',
'ad:CFHT/2401734o_preview_1024.jpg':
'md5:1b84806bcd9ed45e850afa802499e239',
'ad:CFHT/2401734o_preview_256.jpg':
'md5:b1e8aa9b1f0275505660435d3ee6443e',
'ad:CFHT/2401734o_preview_zoom_1024.jpg':
'md5:18839f9482dd3420f0d6ca3c167e6575',
'ad:CFHT/2401734r_preview_1024.jpg':
'md5:3c0dc874898cefe2ca46195e2e19b763',
'ad:CFHT/2401734r_preview_256.jpg':
'md5:3b06668db0e8d9819cc1c50100bd85f4',
'ad:CFHT/2401734r_preview_zoom_1024.jpg':
'md5:32b54773700dd652e244632f51aca5ba',
'ad:CFHT/2401734s_preview_1024.jpg':
'md5:30a76081e4e6f5ee332e8e076feff2a7',
'ad:CFHT/2401734s_preview_256.jpg':
'md5:f32f4288aebc1f7cea577d6d217929c9',
'ad:CFHT/2401734t_preview_1024.jpg':
'md5:3a5409aa4cd728e0a92f50a3eff88427',
'ad:CFHT/2401734t_preview_256.jpg':
'md5:f90654bad03ba8c2e4796ab4acfb5264',
'ad:CFHT/1151210g_preview_1024.jpg':
'md5:ac415da2e2d9847b81c59477c73f26ce',
'ad:CFHT/1151210g_preview_256.jpg':
'md5:56ad048a81c7c16f0801ad8b22fc943f',
'ad:CFHT/1151210g_preview_zoom_1024.jpg':
'md5:0486b7e6572bb0bd8750ff16159ee5ad',
'ad:CFHT/1151210m_preview_1024.jpg':
'md5:02abd143e36acd593cab8eeea458eaf2',
'ad:CFHT/1151210m_preview_256.jpg':
'md5:99d5b739c4ab1ef69aa0b34a0696f9cf',
'ad:CFHT/1151210m_preview_zoom_1024.jpg':
'md5:d55b984b8dcb39baa0d5ba0a34051ffa',
'ad:CFHT/1151210w_preview_1024.jpg':
'md5:542b11ae4e21edc19380e0b37dec129b',
'ad:CFHT/1151210w_preview_256.jpg':
'md5:a10b7c6b8f101f52ab16d0da0649451a',
'ad:CFHT/1151210w_preview_zoom_1024.jpg':
'md5:e79bb2ac4230d5587f97a9b838d6f88d',
'ad:CFHT/1151210o_preview_1024.jpg':
'md5:302ce34edd7c69da23119d001e85d39a',
'ad:CFHT/1151210o_preview_256.jpg':
'md5:137fa8a405cdf03f827000908831ece9',
'ad:CFHT/1151210o_preview_zoom_1024.jpg':
'md5:e656f21a21a4a41e088cccf87ec76580',
'ad:CFHT/1151210p_preview_1024.jpg':
'md5:f05cb0ad85ac610ef9bc3cf1ea00c853',
'ad:CFHT/1151210p_preview_256.jpg':
'md5:73aa50e0d59a69452ae8be06de9c6317',
'ad:CFHT/1151210p_preview_zoom_1024.jpg':
'md5:e56690286391a74840bc07158cedaeec',
'ad:CFHT/1151210s_preview_1024.jpg':
'md5:7d7019efd828ff748fece2b778c5fecb',
'ad:CFHT/1151210s_preview_256.jpg':
'md5:43051edaa8dbb6436d4dcb32fc5a2bd0',
'ad:CFHT/1151210s_preview_zoom_1024.jpg':
'md5:4f20f5586eaa43abc5de977e60cea807',
'ad:CFHT/1151210y_preview_1024.jpg':
'md5:931dd9cabffe9e470a556327c08c6b96',
'ad:CFHT/1151210y_preview_256.jpg':
'md5:45df1e0a5dd4b4b058b7e20cdc55b9f7',
'ad:CFHT/1151210y_preview_zoom_1024.jpg':
'md5:53cfa0d681c46fedfc2bf719e88231b5'
}
kwargs = {'working_directory': TEST_FILES_DIR,
'cadc_client': None,
'stream': 'stream',
'observable': test_observable}
for entry in glob.glob(f'{TEST_FILES_DIR}/*.jpg'):
os.unlink(entry)
checksum_failures = []
for key, value in test_files.items():
obs = mc.read_obs_from_file(f'{test_main_app.TEST_DATA_DIR}/{key}')
if 'wircam' in key:
instrument = md.Inst.WIRCAM
elif 'mega' in key:
instrument = md.Inst.MEGACAM
elif 'sitelle' in key:
instrument = md.Inst.SITELLE
elif 'espadons' in key:
instrument = md.Inst.ESPADONS
elif 'spirou' in key:
instrument = md.Inst.SPIROU
else:
assert False, 'do not understand instrument'
for f_name in value:
kwargs['science_file'] = f_name
test_name = cfht_name.CFHTName(file_name=f_name,
instrument=instrument)
check_number = 1
if test_name.suffix == 'p' and instrument is md.Inst.SITELLE:
check_number = 3
assert len(obs.planes[test_name.product_id].artifacts) == \
check_number, f'initial condition {f_name}'
try:
from datetime import datetime
import logging
start_ts = datetime.utcnow().timestamp()
test_result = preview_augmentation.visit(obs, **kwargs)
end_ts = datetime.utcnow().timestamp()
logging.error(f'{f_name} execution time {end_ts - start_ts}')
except Exception as e:
import logging
logging.error(e)
import traceback
logging.error(traceback.format_exc())
assert False
assert test_result is not None, f'expect a result {f_name}'
check_number = 3
end_artifact_count = 4
f_name_list = [test_name.prev_uri, test_name.thumb_uri,
test_name.zoom_uri]
if ((instrument is md.Inst.ESPADONS and test_name.suffix == 'i') or
(instrument is md.Inst.SPIROU and
test_name.suffix in ['e', 'p', 's', 't', 'v'])):
check_number = 2
end_artifact_count = 3
f_name_list = [test_name.prev_uri, test_name.thumb_uri]
assert test_result['artifacts'] == check_number, \
f'artifacts should be added {f_name}'
if test_name.suffix == 'p' and instrument is md.Inst.SITELLE:
end_artifact_count = 6
assert len(obs.planes[test_name.product_id].artifacts) == \
end_artifact_count, f'new artifacts {f_name}'
for p in f_name_list:
# assert p in \
# obs.planes[test_name.product_id].artifacts.keys(), \
# f'no preview {p}'
if p in obs.planes[test_name.product_id].artifacts.keys():
artifact = obs.planes[test_name.product_id].artifacts[p]
# because 2359320p_preview_1024 keeps changing ....
if artifact.uri in test_checksums:
if artifact.content_checksum.uri != test_checksums[p]:
checksum_failures.append(
f'{p} expected {test_checksums[p]} actual '
f'{artifact.content_checksum.uri}')
assert len(checksum_failures) == 0, \
'\n'.join(ii for ii in checksum_failures)
def test_preview_augment():
# this should result in two new artifacts being added to the plane
# one for a thumbnail and one for a preview
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
cadc_client_mock = Mock()
kwargs = {
'working_directory': '/test_files',
'cadc_client': cadc_client_mock,
'stream': 'stream',
'observable': test_observable,
}
test_prev = f'/test_files/{TEST_PRODUCT_ID}.jpg'
if os.path.exists(test_prev):
os.unlink(test_prev)
try:
with patch('caom2pipe.manage_composable.http_get') as http_mock, \
patch('caom2pipe.manage_composable.data_put') as ad_put_mock, \
patch(
'caom2pipe.manage_composable.get_artifact_metadata'
) as art_mock:
def _art_mock(
fqn,
product_type,
release_type,
uri,
art_ignore,
):
if '_th' in uri:
return Artifact(
uri,
ProductType.PREVIEW,
ReleaseType.DATA,
'image/jpeg',
13,
ChecksumURI('md5:13'),
)
else:
return Artifact(
uri,
ProductType.PREVIEW,
ReleaseType.DATA,
'image/jpeg',
12,
ChecksumURI('md5:12'),
)
cadc_client_mock.return_value.data_get.return_value = \
mc.CadcException('test')
art_mock.side_effect = _art_mock
http_mock.side_effect = _get_mock
result = preview_augmentation.visit(obs, **kwargs)
test_url = f'{preview_augmentation.PREVIEW_URL}' \
f'{TEST_PRODUCT_ID}.fits'
assert http_mock.called, 'http mock should be called'
http_mock.assert_called_with(
test_url, test_prev
), 'mock not called'
assert ad_put_mock.called, 'ad put mock not called'
assert art_mock.called, 'art mock not called'
assert result is not None, 'expect a result'
assert result['artifacts'] == 2, 'artifacts should be added'
assert (
len(obs.planes[TEST_PRODUCT_ID].artifacts) == 3
), 'two new artifacts'
prev_uri = mc.build_uri(
ARCHIVE, f'{TEST_PRODUCT_ID}.jpg', SCHEME
)
thumb_uri = mc.build_uri(ARCHIVE, f'{TEST_PRODUCT_ID}_th.jpg')
assert (
prev_uri in obs.planes[TEST_PRODUCT_ID].artifacts.keys()
), 'no preview'
assert (
thumb_uri in obs.planes[TEST_PRODUCT_ID].artifacts
), 'no thumbnail'
finally:
if os.path.exists(test_prev):
os.unlink(test_prev)
def test_visit():
class TestVisitor(mc.PreviewVisitor):
def __init__(self, **kwargs):
super(TestVisitor, self).__init__(archive='VLASS', **kwargs)
def generate_plots(self, obs_id):
fqn = f'{self._working_dir}/{self._storage_name.prev}'
with open(fqn, 'w') as f:
f.write('test content')
self.add_preview(
self._storage_name.prev_uri,
self._storage_name.prev,
ProductType.THUMBNAIL,
ReleaseType.META,
)
return 1
class VisitStorageName(tc.TestStorageName):
def __init__(self):
super(VisitStorageName, self).__init__()
self._source_names = [self.file_uri]
@property
def product_id(self):
return test_product_id
@property
def file_uri(self):
return f'ad:VLASS/{test_file_name}'
test_rejected = mc.Rejected(f'{tc.TEST_DATA_DIR}/rejected.yml')
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
cadc_client_mock = Mock()
test_product_id = 'VLASS1.2.T07t14.J084202-123000.quicklook.v1'
test_file_name = ('VLASS1.2.ql.T07t14.J084202-123000.10.2048.v1.I.iter1.'
'image.pbcor.tt0.subim.fits')
storage_name = VisitStorageName()
kwargs = {
'working_directory': tc.TEST_FILES_DIR,
'cadc_client': cadc_client_mock,
'stream': 'stream',
'observable': test_observable,
'storage_name': storage_name,
}
obs = mc.read_obs_from_file(f'{tc.TEST_DATA_DIR}/fpf_start_obs.xml')
assert len(obs.planes[test_product_id].artifacts) == 2, 'initial condition'
try:
test_subject = TestVisitor(**kwargs)
test_result = test_subject.visit(obs)
except Exception as e:
assert False, f'{str(e)}'
assert test_result is not None, f'expect a result'
check_number = 1
end_artifact_count = 3
expected_call_count = 1
assert test_result['artifacts'] == check_number, 'artifact not added'
assert (len(obs.planes[test_product_id].artifacts) == end_artifact_count
), f'new artifacts'
test_preview_uri = 'cadc:TEST/test_obs_id_prev.jpg'
assert (test_preview_uri
in obs.planes[test_product_id].artifacts.keys()), 'no preview'
assert cadc_client_mock.put.called, 'put mock not called'
assert (cadc_client_mock.put.call_count == expected_call_count
), 'put called wrong number of times'
# it's an ad call, so there's a stream parameter
cadc_client_mock.put.assert_called_with('/test_files',
'cadc:TEST/test_obs_id_prev.jpg',
'stream')
def test_visit():
# this should result in three new artifacts being added to every plane:
# one for a thumbnail and two for previews (one zoom)
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
test_files = {
# processed spectrum
'visit_obs_start_v.xml': [
'dao_c122_2007_000882_v.fits',
'dao_c122_2007_000882.fits.gz',
],
# processed image
'visit_obs_start_a.xml': [
'dao_c182_2016_004034_a.fits',
'dao_c182_2016_004034.fits.gz',
],
'dao_r182_1989_000369.xml': ['dao_r182_1989_000369.fits.gz'],
'dao_r122_1989_003111.xml': ['dao_r122_1989_003111.fits.gz'],
'dao_c122_2017_011124.xml': ['dao_c122_2017_011124.fits.gz'],
'dao_c182_2017_010870.xml': ['dao_c182_2017_010870.fits.gz'],
'dao_c182_2017_019322.xml': ['dao_c182_2017_019322.fits.gz'],
'dao_c182_2017_016292.xml': ['dao_c182_2017_016292.fits.gz'],
'visit_obs_start_e.xml': [
'dao_c122_2007_000881.fits.gz',
'dao_c122_2007_000881_e.fits',
],
'sky_cam_start.xml': ['a2020_06_17_07_00_01.fits']
}
test_checksums = {
'cadc:DAO/dao_c122_2007_000881_e_256.png':
'md5:e68a3c389bbdf53cde0cca073c7a37c1',
'cadc:DAO/dao_c122_2007_000881_e_1024.png':
'md5:48065b2feb0fda3d7561c02b2890efaa',
'cadc:DAO/dao_c122_2007_000882_256.png':
'md5:1268e5b79e463ca75580d7e538d88a2a',
'cadc:DAO/dao_c122_2007_000882_1024.png':
'md5:5694ee03486899d3b14b404caab22505',
'cadc:DAO/dao_c122_2017_011124_256.png':
'md5:9793efcdeec043be8f0c77c7bf875cca',
'cadc:DAO/dao_c122_2017_011124_1024.png':
'md5:4d77005756fbed7fc3ae525591cc07db',
'cadc:DAO/dao_c182_2016_004034_256.png':
'md5:5c6b6816f8c0a54a0ff041b29247bdd5',
'cadc:DAO/dao_c182_2016_004034_1024.png':
'md5:455bddac53488acca46758fbc1d2b096',
'cadc:DAO/dao_c182_2017_010870_256.png':
'md5:106a6e0837fd00e971a37462266c2f66',
'cadc:DAO/dao_c182_2017_010870_1024.png':
'md5:bc92d495aa0e56eb4ea35f409932391e',
'cadc:DAO/dao_c182_2017_016292_256.png':
'md5:ef6bcc6501db0f64dfc7741f5ab37870',
'cadc:DAO/dao_c182_2017_016292_1024.png':
'md5:6c425b3606dd598be33b61dfef51d132',
'cadc:DAO/dao_c182_2017_019322_256.png':
'md5:dfca85e675f29a33ecdbabc91ed274bf',
'cadc:DAO/dao_c182_2017_019322_1024.png':
'md5:2818ddf6acb0f36655fbdfc9e19098e7',
'cadc:DAO/dao_r122_1989_003111_256.png':
'md5:7cb2344a84d195228d7fb866a83d031a',
'cadc:DAO/dao_r122_1989_003111_1024.png':
'md5:a864bc14955bc1f8524e8829c3ee2ee9',
'cadc:DAO/dao_r182_1989_000369_256.png':
'md5:5ef8f53784376b31a12dccb8671aca5e',
'cadc:DAO/dao_r182_1989_000369_1024.png':
'md5:2e14aefda2a74b5575f6a4d8d1b1b3cb',
'cadc:DAO/dao_c122_2007_000882_v_1024.png':
'md5:37cfb96bce27da93ede75eadabad7b8f',
'cadc:DAO/a2020_06_17_07_00_01_1024.png':
'md5:37cfb96bce27da93ede75eadabad7b8f',
}
kwargs = {
'working_directory': TEST_FILES_DIR,
'cadc_client': None,
'stream': 'stream',
'observable': test_observable,
}
for entry in glob.glob(f'{TEST_FILES_DIR}/*.png'):
os.unlink(entry)
for key, value in test_files.items():
obs = mc.read_obs_from_file(
f'{test_fits2caom2_augmentation.TEST_DATA_DIR}/previews/{key}')
for f_name in value:
test_name = dao_name.DAOName(f_name)
kwargs['storage_name'] = test_name
try:
ignore = preview_augmentation.visit(obs, **kwargs)
f_name_list = [test_name.prev_uri, test_name.thumb_uri]
for p in f_name_list:
artifact = obs.planes[test_name.product_id].artifacts[p]
# assert artifact.content_checksum.uri ==
# test_checksums[p], \
# f'wrong checksum {p} {artifact.content_checksum} ' \
# f'{test_checksums[p]}'
except Exception as e:
assert False, f'key {key} value {value} f_name {f_name} {str(e)}'
def test_preview_augment_failure(http_mock):
# mimic 'Not Found' behaviour
# this should result in no new artifacts being added to the plane
# but a record for 'no preview exists at Gemini' added to the
# record
def _failure_mock(ignore_url, ignore_local_fqn):
raise mc.CadcException(
'Could not retrieve /usr/src/app/N20211007A0003/'
'N20211007A0003b.jpg from '
'https://archive.gemini.edu/preview/N20211007A0003b.fits. Failed '
'with 404 Client Error: Not Found for url: '
'https://archive.gemini.edu/preview/N20211007A0003b.fits')
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_metrics = mc.Metrics(test_config)
test_observable = mc.Observable(test_rejected, test_metrics)
cadc_client_mock = Mock()
clients_mock = Mock()
clients_mock.data_client = cadc_client_mock
test_storage_name = gem_name.GemName(file_name=f'{TEST_PRODUCT_ID}.fits')
kwargs = {
'working_directory': '/test_files',
'clients': clients_mock,
'observable': test_observable,
'storage_name': test_storage_name,
}
test_prev = f'/test_files/{TEST_PRODUCT_ID}.jpg'
if os.path.exists(test_prev):
os.unlink(test_prev)
try:
cadc_client_mock.get.side_effect = exceptions.UnexpectedException(
'test')
http_mock.side_effect = _failure_mock
obs = preview_augmentation.visit(obs, **kwargs)
test_url = (f'{preview_augmentation.PREVIEW_URL}'
f'{TEST_PRODUCT_ID}.fits')
assert http_mock.called, 'http mock should be called'
http_mock.assert_called_with(test_url, test_prev), 'mock not called'
assert not cadc_client_mock.put.called, 'put mock should not be called'
assert obs is not None, 'expect a result'
assert (len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1
), 'same as the pre-condition'
prev_uri = mc.build_uri(COLLECTION, f'{TEST_PRODUCT_ID}.jpg', SCHEME)
thumb_uri = mc.build_uri(COLLECTION, f'{TEST_PRODUCT_ID}_th.jpg',
'cadc')
assert (prev_uri not in obs.planes[TEST_PRODUCT_ID].artifacts.keys()
), 'should be no preview'
assert (thumb_uri not in obs.planes[TEST_PRODUCT_ID].artifacts
), 'should be no thumbnail'
assert not (
test_rejected.is_no_preview(prev_uri)), 'preview should be tracked'
assert http_mock.call_count == 1, 'wrong number of calls'
# now try again to generate the preview, and ensure that the
# rejected tracking is working
obs = preview_augmentation.visit(obs, **kwargs)
assert obs is not None, 'expect a result the second time'
assert http_mock.call_count == 1, 'never even tried to retrieve it'
finally:
if os.path.exists(test_prev):
os.unlink(test_prev)
