def test_provenance_augmentation(dmf_mock, headers_mock, access_mock,
builder_mock, repo_get_mock, test_fqn):
builder_mock.return_value._get_obs_id.return_value = None
access_mock.return_value = 'https://localhost'
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_config.task_types = [mc.TaskType.VISIT]
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
headers_mock.return_value.get_head.side_effect = _get_headers_mock
dmf_mock.get.side_effect = _get_obs_id_mock
repo_get_mock.side_effect = _repo_get_mock
getcwd_orig = os.getcwd
os.getcwd = Mock(return_value=test_main_app.TEST_DATA_DIR)
temp = os.path.basename(test_fqn).replace('.expected.xml', '.fits')
test_storage_name = GemProcName(entry=temp)
try:
test_obs = mc.read_obs_from_file(test_fqn)
assert not test_obs.target.moving, 'initial conditions moving target'
kwargs = {
'storage_name': test_storage_name,
'working_directory': test_main_app.TEST_DATA_DIR,
'observable': test_observable,
'caom_repo_client': Mock(),
}
test_result = provenance_augmentation.visit(test_obs, **kwargs)
assert test_result is not None, 'expect a result'
assert test_result.get('provenance') == 2, 'wrong result'
assert len(test_obs.members) == 1, 'wrong membership'
assert test_obs.target.moving, 'should be changed'
finally:
os.getcwd = getcwd_orig
def test_store(put_mock):
test_config = mc.Config()
test_config.logging_level = 'ERROR'
test_config.working_directory = '/tmp'
test_url = 'https://archive-new.nrao.edu/vlass/quicklook/VLASS2.1/' \
'T10t12/VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1/' \
'VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1.I.iter1.image.' \
'pbcor.tt0.rms.subim.fits'
test_storage_name = VlassName(url=test_url, entry=test_url)
transferrer = Mock()
cred_param = Mock()
cadc_data_client = Mock()
caom_repo_client = Mock()
observable = mc.Observable(mc.Rejected('/tmp/rejected.yml'),
mc.Metrics(test_config))
test_subject = ec.Store(test_config, test_storage_name, APPLICATION,
cred_param, cadc_data_client, caom_repo_client,
observable, transferrer)
test_subject.execute(None)
assert put_mock.called, 'expect a call'
args, kwargs = put_mock.call_args
assert args[2] == test_storage_name.file_name, 'wrong file name'
assert transferrer.get.called, 'expect a transfer call'
args, kwargs = transferrer.get.call_args
assert args[0] == test_url, 'wrong source parameter'
assert args[1] == f'/tmp/{test_storage_name.obs_id}/' \
f'{test_storage_name.file_name}',\
'wrong destination parameter'
def test_meta_update_observation_direct(test_config):
test_cred_param = '--cert /usr/src/app/cadcproxy.pem'
data_client_mock = Mock()
repo_client_mock = Mock()
test_sn = tc.TestStorageName()
test_observation = mc.read_obs_from_file(
f'{tc.TEST_DATA_DIR}/fpf_start_obs.xml'
)
try:
test_observable = mc.Observable(
mc.Rejected(test_config.rejected_fqn), mc.Metrics(test_config)
)
test_executor = ec.MetaUpdateObservation(
test_config,
test_sn,
TEST_APP,
test_cred_param,
data_client_mock,
repo_client_mock,
test_observation,
[],
observable=test_observable,
)
test_executor.execute(None)
assert repo_client_mock.update.called, 'repo update called'
finally:
pass
def test_preview_augment(http_mock):
# this should result in two new artifacts being added to the plane
# one for a thumbnail and one for a preview
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_metrics = mc.Metrics(test_config)
test_observable = mc.Observable(test_rejected, test_metrics)
cadc_client_mock = Mock()
clients_mock = Mock()
clients_mock.data_client = cadc_client_mock
test_storage_name = gem_name.GemName(file_name=f'{TEST_PRODUCT_ID}.fits')
kwargs = {
'working_directory': '/test_files',
'clients': clients_mock,
'observable': test_observable,
'storage_name': test_storage_name,
}
test_prev = f'/test_files/{TEST_PRODUCT_ID}.jpg'
if os.path.exists(test_prev):
os.unlink(test_prev)
try:
cadc_client_mock.get.side_effect = exceptions.UnexpectedException(
'test')
http_mock.side_effect = _get_mock
obs = preview_augmentation.visit(obs, **kwargs)
test_url = (f'{preview_augmentation.PREVIEW_URL}'
f'{TEST_PRODUCT_ID}.fits')
assert http_mock.called, 'http mock should be called'
http_mock.assert_called_with(test_url, test_prev), 'mock not called'
assert cadc_client_mock.put.called, 'put mock not called'
cadc_client_mock.put.assert_called_with(
'/test_files',
'cadc:GEMINI/GN2001BQ013-04_th.jpg',
), 'wrong put arguments'
assert obs is not None, 'expect a result'
assert (len(
obs.planes[TEST_PRODUCT_ID].artifacts) == 3), 'two new artifacts'
prev_uri = mc.build_uri(COLLECTION, f'{TEST_PRODUCT_ID}.jpg', SCHEME)
thumb_uri = mc.build_uri(COLLECTION, f'{TEST_PRODUCT_ID}_th.jpg',
'cadc')
assert (prev_uri
in obs.planes[TEST_PRODUCT_ID].artifacts.keys()), 'no preview'
assert (thumb_uri
in obs.planes[TEST_PRODUCT_ID].artifacts), 'no thumbnail'
finally:
if os.path.exists(test_prev):
os.unlink(test_prev)
def test_preview_augment_plane():
omm_name = OmmName(file_name=TEST_FILE, entry=TEST_FILE)
preview = os.path.join(TEST_FILES_DIR, omm_name.prev)
thumb = os.path.join(TEST_FILES_DIR, omm_name.thumb)
if os.path.exists(preview):
os.remove(preview)
if os.path.exists(thumb):
os.remove(thumb)
test_fqn = os.path.join(TEST_DATA_DIR,
f'{omm_name.product_id}.expected.xml')
test_obs = mc.read_obs_from_file(test_fqn)
assert len(test_obs.planes[omm_name.product_id].artifacts) == 1
preva = 'cadc:OMM/C170324_0054_SCI_prev.jpg'
thumba = 'cadc:OMM/C170324_0054_SCI_prev_256.jpg'
test_config = mc.Config()
test_config.observe_execution = True
test_metrics = mc.Metrics(test_config)
test_observable = mc.Observable(rejected=None, metrics=test_metrics)
test_kwargs = {
'working_directory': TEST_FILES_DIR,
'cadc_client': None,
'observable': test_observable,
'storage_name': omm_name,
}
test_result = preview_augmentation.visit(test_obs, **test_kwargs)
assert test_result is not None, 'expected a visit return value'
assert os.path.exists(preview)
assert os.path.exists(thumb)
test_plane = test_result.planes[omm_name.product_id]
assert test_plane.artifacts[preva].content_checksum == ChecksumURI(
'md5:f37d21c53055498d1b5cb7753e1c6d6f'), 'prev checksum failure'
assert test_plane.artifacts[thumba].content_checksum == ChecksumURI(
'md5:19661c3c2508ecc22425ee2a05881ed4'), 'thumb checksum failure'
# now do updates
test_obs.planes[
omm_name.product_id].artifacts[preva].content_checksum = ChecksumURI(
'de9f39804f172682ea9b001f8ca11f15')
test_obs.planes[
omm_name.product_id].artifacts[thumba].content_checksum = ChecksumURI(
'cd118dae04391f6bea93ba4bf2711adf')
test_result = preview_augmentation.visit(test_obs, **test_kwargs)
assert test_result is not None, 'expected update visit return value'
assert len(test_result.planes[omm_name.product_id].artifacts) == 3
assert os.path.exists(preview)
assert os.path.exists(thumb)
assert test_plane.artifacts[preva].content_checksum == ChecksumURI(
'md5:f37d21c53055498d1b5cb7753e1c6d6f'), 'prev update failed'
assert test_plane.artifacts[thumba].content_checksum == ChecksumURI(
'md5:19661c3c2508ecc22425ee2a05881ed4'), 'prev_256 update failed'
assert len(test_metrics.history) == 0, 'wrong history, client is not None'
def test_pull_augmentation(http_mock, json_mock, header_mock, file_type_mock):
obs = mc.read_obs_from_file(TEST_OBS_AD_URI_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
original_uri = 'gemini:GEMINI/GN2001BQ013-04.fits'
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
assert (
original_uri
in obs.planes[TEST_PRODUCT_ID].artifacts.keys()), 'initial condition'
test_uri = f'{SCHEME}:{COLLECTION}/{TEST_PRODUCT_ID}.fits'
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
cadc_client_mock = Mock()
cadc_client_mock.return_value.info.return_value = None
clients_mock = Mock()
clients_mock.data_client = cadc_client_mock
json_mock.side_effect = gem_mocks.mock_retrieve_json
filter_cache = svofps.FilterMetadataCache(Mock())
test_reader = gemini_metadata.GeminiFileMetadataReader(
Mock(), Mock(), filter_cache)
test_fqn = f'{gem_mocks.TEST_DATA_DIR}/GMOS/GN2001BQ013-04.fits.header'
test_storage_name = gem_name.GemName(file_name='GN2001BQ013-04.fits')
header_mock.side_effect = gem_mocks._mock_headers
file_type_mock.return_values = 'application/fits'
test_reader.set(test_storage_name)
kwargs = {
'working_directory': TEST_DATA_DIR,
'clients': clients_mock,
'observable': test_observable,
'metadata_reader': test_reader,
'storage_name': test_storage_name,
}
obs = pull_augmentation.visit(obs, **kwargs)
test_url = f'{pull_augmentation.FILE_URL}/{TEST_PRODUCT_ID}.fits'
test_prev = f'{TEST_DATA_DIR}/{TEST_PRODUCT_ID}.fits'
http_mock.assert_called_with(test_url, test_prev), 'mock not called'
assert cadc_client_mock.put.called, 'put mock not called'
cadc_client_mock.put.assert_called_with(
TEST_DATA_DIR, 'gemini:GEMINI/GN2001BQ013-04.fits'), 'wrong put args'
assert obs is not None, 'expect a result'
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'no new artifacts'
try:
ignore = obs.planes[TEST_PRODUCT_ID].artifacts[test_uri]
except KeyError as ke:
# because CAOM does magic
result = obs.planes[TEST_PRODUCT_ID].artifacts[original_uri]
assert result.uri == test_uri, f'wrong uri {result.uri}'
def test_cadc_transfer(client_mock, caps_mock):
caps_mock.return_value = 'https://sc2.canfar.net/sc2repo'
test_subject = tc.CadcTransfer()
assert test_subject is not None, 'expect a result'
test_config = mc.Config()
test_config.rejected_fqn = '/tmp/rejected.yml'
test_observable = mc.Observable(mc.Rejected(test_config.rejected_fqn),
mc.Metrics(test_config))
test_subject.observable = test_observable
test_subject.cadc_client = client_mock
test_source = 'ad:TEST/test_file.fits'
test_destination = '/tmp/test_file.fits'
test_subject.get(test_source, test_destination)
assert client_mock.get.called, 'should have been called'
client_mock.get.assert_called_with(
'/tmp', 'ad:TEST/test_file.fits'), 'wrong parameters'
def test_preview_augment_unknown_no_preview():
# what happens when it's not known that there's no preview
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
# make sure the rejected file is empty
if os.path.exists(REJECTED_FILE):
os.unlink(REJECTED_FILE)
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
test_storage_name = gem_name.GemName(file_name=f'{TEST_PRODUCT_ID}.fits')
cadc_client_mock = Mock()
clients_mock = Mock()
clients_mock.data_client = cadc_client_mock
kwargs = {
'working_directory': TEST_DATA_DIR,
'clients': clients_mock,
'stream': 'stream',
'observable': test_observable,
'storage_name': test_storage_name,
}
with patch(
'caom2pipe.manage_composable.http_get',
side_effect=mc.CadcException(
'Not Found for url: https://archive.gemini.edu/preview'),
) as http_mock, patch(
'caom2pipe.manage_composable.data_put') as ad_put_mock, patch(
'caom2pipe.manage_composable.get_artifact_metadata'
) as art_mock, patch(
'caom2pipe.manage_composable.exec_cmd') as exec_mock:
cadc_client_mock.get.side_effect = exceptions.UnexpectedException(
'test')
obs = preview_augmentation.visit(obs, **kwargs)
assert obs is not None, 'expect result'
test_url = f'{preview_augmentation.PREVIEW_URL}{TEST_PRODUCT_ID}.fits'
test_prev = f'{TEST_DATA_DIR}/{TEST_PRODUCT_ID}.jpg'
http_mock.assert_called_with(test_url, test_prev), 'mock not called'
assert not ad_put_mock.called, 'ad put mock should not be called'
assert not art_mock.called, 'art mock should not be called'
assert not exec_mock.called, 'exec mock should not be called'
def test_preview_augmentation(access_mock):
access_mock.return_value = 'https://localhost'
test_fqn = f'{TEST_DATA_DIR}/preview_augmentation_start.xml'
test_science_f_name = (
'VLASS1.1.ql.T01t01.J000228-363000.10.2048.v1.I.iter1.image.pbcor.'
'tt0.subim.fits')
test_storage_name = sn.VlassName(test_science_f_name)
test_obs = mc.read_obs_from_file(test_fqn)
test_config = mc.Config()
test_rejected = mc.Rejected(f'{TEST_DATA_DIR}/rejected.yml')
test_metrics = mc.Metrics(test_config)
test_observable = mc.Observable(test_rejected, test_metrics)
kwargs = {
'stream': None,
'observable': test_observable,
'storage_name': test_storage_name,
'working_directory': '/test_files',
}
test_subject = preview_augmentation.VlassPreview(**kwargs)
assert test_subject is not None, 'need a test subject'
assert len(test_obs.planes) == 1, 'wrong number of planes'
assert (len(test_obs.planes[test_storage_name.product_id].artifacts) == 4
), 'wrong starting # of artifacts'
test_obs = test_subject.visit(test_obs)
assert test_obs is not None, 'expect a result'
assert (len(test_obs.planes[test_storage_name.product_id].artifacts) == 6
), 'wrong ending # of artifacts'
test_report = test_subject.report
assert test_report is not None
assert 'artifacts' in test_report
assert test_report.get('artifacts') == 2, 'wrong report count'
# does artifact re-naming work?
test_url = (f'https://archive-new.nrao.edu/vlass/quicklook/VLASS1.1/'
f'T01t01/VLASS1.1.ql.T01t01.J000228-363000.10.2048.v1/'
f'{test_science_f_name}')
kwargs = {'url': test_url}
test_obs = cleanup_augmentation.visit(test_obs, **kwargs)
test_artifacts = test_obs.planes[test_storage_name.product_id].artifacts
assert len(test_artifacts) == 4, 'wrong ending conditions'
assert test_storage_name.prev_uri in test_artifacts, 'missing preview'
assert test_storage_name.thumb_uri in test_artifacts, 'missing thumbnail'
def test_store():
test_config = mc.Config()
test_config.logging_level = 'ERROR'
test_config.working_directory = '/tmp'
test_url = ('https://archive-new.nrao.edu/vlass/quicklook/VLASS2.1/'
'T10t12/VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1/'
'VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1.I.iter1.image.'
'pbcor.tt0.rms.subim.fits')
test_storage_name = VlassName(test_url)
transferrer = Mock()
cadc_data_client = Mock()
observable = mc.Observable(mc.Rejected('/tmp/rejected.yml'),
mc.Metrics(test_config))
test_subject = ec.Store(
test_config,
test_storage_name,
cadc_data_client,
observable,
transferrer,
)
test_subject.execute(None)
assert cadc_data_client.put.called, 'expect a call'
cadc_data_client.put.assert_called_with(
'/tmp/VLASS2.1.T10t12.J073401-033000',
f'{SCHEME}:VLASS/VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1.I.'
f'iter1.image.pbcor.tt0.rms.subim.fits',
None,
), 'wrong put args'
assert transferrer.get.called, 'expect a transfer call'
test_f_name = ('VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1.I.iter1.'
'image.pbcor.tt0.rms.subim.fits')
transferrer.get.assert_called_with(
f'https://archive-new.nrao.edu/vlass/quicklook/VLASS2.1/T10t12/'
f'VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1/{test_f_name}',
f'/tmp/VLASS2.1.T10t12.J073401-033000/{test_f_name}',
), 'wrong transferrer args'
def test_visit():
class TestVisitor(mc.PreviewVisitor):
def __init__(self, **kwargs):
super(TestVisitor, self).__init__(archive='VLASS', **kwargs)
def generate_plots(self, obs_id):
fqn = f'{self._working_dir}/{self._storage_name.prev}'
with open(fqn, 'w') as f:
f.write('test content')
self.add_preview(
self._storage_name.prev_uri,
self._storage_name.prev,
ProductType.THUMBNAIL,
ReleaseType.META,
)
return 1
class VisitStorageName(tc.TestStorageName):
def __init__(self):
super(VisitStorageName, self).__init__()
self._source_names = [self.file_uri]
@property
def product_id(self):
return test_product_id
@property
def file_uri(self):
return f'ad:VLASS/{test_file_name}'
test_rejected = mc.Rejected(f'{tc.TEST_DATA_DIR}/rejected.yml')
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
cadc_client_mock = Mock()
test_product_id = 'VLASS1.2.T07t14.J084202-123000.quicklook.v1'
test_file_name = ('VLASS1.2.ql.T07t14.J084202-123000.10.2048.v1.I.iter1.'
'image.pbcor.tt0.subim.fits')
storage_name = VisitStorageName()
kwargs = {
'working_directory': tc.TEST_FILES_DIR,
'cadc_client': cadc_client_mock,
'stream': 'stream',
'observable': test_observable,
'storage_name': storage_name,
}
obs = mc.read_obs_from_file(f'{tc.TEST_DATA_DIR}/fpf_start_obs.xml')
assert len(obs.planes[test_product_id].artifacts) == 2, 'initial condition'
try:
test_subject = TestVisitor(**kwargs)
test_result = test_subject.visit(obs)
except Exception as e:
assert False, f'{str(e)}'
assert test_result is not None, f'expect a result'
check_number = 1
end_artifact_count = 3
expected_call_count = 1
assert test_result['artifacts'] == check_number, 'artifact not added'
assert (len(obs.planes[test_product_id].artifacts) == end_artifact_count
), f'new artifacts'
test_preview_uri = 'cadc:TEST/test_obs_id_prev.jpg'
assert (test_preview_uri
in obs.planes[test_product_id].artifacts.keys()), 'no preview'
assert cadc_client_mock.put.called, 'put mock not called'
assert (cadc_client_mock.put.call_count == expected_call_count
), 'put called wrong number of times'
# it's an ad call, so there's a stream parameter
cadc_client_mock.put.assert_called_with('/test_files',
'cadc:TEST/test_obs_id_prev.jpg',
'stream')
def test_preview_augment():
# this should result in three new artifacts being added to every plane:
# one for a thumbnail and two for previews (one zoom)
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
test_files = {
'visit_obs_start_wircam.xml':
['1151210o.fits.fz', '1151210s.fits.fz', '1151210m.fits.fz',
'1151210p.fits.fz', '1151210w.fits.fz', '1151210y.fits.fz',
'1151210g.fits.fz'],
'visit_obs_start_megacam_sci.xml':
['1927963f.fits.fz', '1927963o.fits.fz', '1927963p.fits.fz'],
'visit_obs_start_megacam_cal.xml':
['979412b.fits.fz', '979412o.fits.fz', '979412p.fits.fz'],
'visit_obs_start_sitelle_calibrated_cube.xml':
['2359320p.fits'],
'visit_obs_start_sitelle.xml':
['2359320o.fits.fz'],
'visit_obs_start_espadons.xml':
['2460606i.fits.gz', '2460606o.fits.gz'],
'visit_obs_start_espadons_cal.xml':
['1001063b.fits.gz'],
'visit_obs_start_spirou.xml': ['2401734o.fits', '2401734e.fits',
'2401734r.fits', '2401734s.fits',
'2401734t.fits', '2401734v.fits'],
'visit_obs_start_2hdus_mega.xml': ['1821271p.fits.fz']
}
test_checksums = {
'ad:CFHT/2460606i_preview_1024.jpg':
'md5:eeb2558c76c83a1b56e0afa3d96b0fa0',
'ad:CFHT/2460606i_preview_256.jpg':
'md5:1084a1e58d246c35fae43ca021916c6a',
'ad:CFHT/2460606o_preview_1024.jpg':
'md5:eeeb436f0e6b1cb48aca5a9ba5d1ab3a',
'ad:CFHT/2460606o_preview_256.jpg':
'md5:3879c153776ee0569ee33bc8a35b5e75',
'ad:CFHT/2460606o_preview_zoom_1024.jpg':
'md5:1d5c9e5ee1586ddd6ce5c53b17f8f5ec',
'ad:CFHT/1001063b_preview_1024.jpg':
'md5:d457fa8ecdd397790de65f8bd3a80d06',
'ad:CFHT/1001063b_preview_256.jpg':
'md5:3ae90e1e120fb97216ddb2d965e9423e',
'ad:CFHT/1001063b_preview_zoom_1024.jpg':
'md5:a2d62e654838a620dcf97425753481ca',
'ad:CFHT/979412b_preview_1024.jpg':
'md5:8b3e4ff2c210b3861748423339a71cc5',
'ad:CFHT/979412b_preview_256.jpg':
'md5:893a9cd18f66fab907573bf1e664148d',
'ad:CFHT/979412b_preview_zoom_1024.jpg':
'md5:4f21d6676b0749c48b4872699f7aff7c',
'ad:CFHT/979412o_preview_1024.jpg':
'md5:379a7b21022a0e758fbdea6130c07233',
'ad:CFHT/979412o_preview_256.jpg':
'md5:59c2ecedda2a2560cafdca543533e190',
'ad:CFHT/979412o_preview_zoom_1024.jpg':
'md5:6f3a3a39155a426a365f13f6d10982a9',
'ad:CFHT/979412p_preview_1024.jpg':
'md5:d449d26987a4d9e220d249f8573025ee',
'ad:CFHT/979412p_preview_256.jpg':
'md5:102be79a6ee26a7e5c2ee7e40623d452',
'ad:CFHT/979412p_preview_zoom_1024.jpg':
'md5:7e09407afb9e8148ad36ba56933e1822',
'ad:CFHT/1927963f_preview_1024.jpg':
'md5:9afd99f2d60de3bc168832ce37200508',
'ad:CFHT/1927963f_preview_256.jpg':
'md5:5de596f4b5ab1ae8bfab1de5550ec56d',
'ad:CFHT/1927963f_preview_zoom_1024.jpg':
'md5:ca705c70eb0d320856e346662a83f1ba',
'ad:CFHT/1927963o_preview_1024.jpg':
'md5:d8a9c9c6dde50c59f75a1b384300d41a',
'ad:CFHT/1927963o_preview_256.jpg':
'md5:84cfc76e094678e61eaa9f9421e30dd9',
'ad:CFHT/1927963o_preview_zoom_1024.jpg':
'md5:843109c217104034fa3b029949fa277e',
'ad:CFHT/1927963p_preview_1024.jpg':
'md5:61b903863106438cfa2771731ed8690d',
'ad:CFHT/1927963p_preview_256.jpg':
'md5:01e4de796ff3f20949d02fe776b5f699',
'ad:CFHT/1927963p_preview_zoom_1024.jpg':
'md5:91c4a735d507d3c11482179d725a4166',
'ad:CFHT/2359320o_preview_1024.jpg':
'md5:45b164799c67751c5a90255c11ccd9e5',
'ad:CFHT/2359320o_preview_256.jpg':
'md5:dce23df89c3424bc67d733a7784a13d3',
'ad:CFHT/2359320o_preview_zoom_1024.jpg':
'md5:5cc26a79326b42c6a17bca8e0d3fb1fb',
# 'ad:CFHT/2359320p_preview_1024.jpg': ####
# 'md5:b09f54461fb10259f4958ff9ab1e8179',
# 'ad:CFHT/2359320p_preview_256.jpg':
# 'md5:b037076c0d9488d013052d6750e55aeb',
# 'ad:CFHT/2359320p_preview_zoom_1024.jpg':
# 'md5:514aa25fec1b74fe942ccc8ce8e552f7',
'ad:CFHT/2401734e_preview_1024.jpg':
'md5:a3b272473e959cbc5496c1d562f16c0d',
'ad:CFHT/2401734e_preview_256.jpg':
'md5:781a1c86e727f82ec17a98b5eaf2b7d0',
'ad:CFHT/2401734o_preview_1024.jpg':
'md5:1b84806bcd9ed45e850afa802499e239',
'ad:CFHT/2401734o_preview_256.jpg':
'md5:b1e8aa9b1f0275505660435d3ee6443e',
'ad:CFHT/2401734o_preview_zoom_1024.jpg':
'md5:18839f9482dd3420f0d6ca3c167e6575',
'ad:CFHT/2401734r_preview_1024.jpg':
'md5:3c0dc874898cefe2ca46195e2e19b763',
'ad:CFHT/2401734r_preview_256.jpg':
'md5:3b06668db0e8d9819cc1c50100bd85f4',
'ad:CFHT/2401734r_preview_zoom_1024.jpg':
'md5:32b54773700dd652e244632f51aca5ba',
'ad:CFHT/2401734s_preview_1024.jpg':
'md5:30a76081e4e6f5ee332e8e076feff2a7',
'ad:CFHT/2401734s_preview_256.jpg':
'md5:f32f4288aebc1f7cea577d6d217929c9',
'ad:CFHT/2401734t_preview_1024.jpg':
'md5:3a5409aa4cd728e0a92f50a3eff88427',
'ad:CFHT/2401734t_preview_256.jpg':
'md5:f90654bad03ba8c2e4796ab4acfb5264',
'ad:CFHT/1151210g_preview_1024.jpg':
'md5:ac415da2e2d9847b81c59477c73f26ce',
'ad:CFHT/1151210g_preview_256.jpg':
'md5:56ad048a81c7c16f0801ad8b22fc943f',
'ad:CFHT/1151210g_preview_zoom_1024.jpg':
'md5:0486b7e6572bb0bd8750ff16159ee5ad',
'ad:CFHT/1151210m_preview_1024.jpg':
'md5:02abd143e36acd593cab8eeea458eaf2',
'ad:CFHT/1151210m_preview_256.jpg':
'md5:99d5b739c4ab1ef69aa0b34a0696f9cf',
'ad:CFHT/1151210m_preview_zoom_1024.jpg':
'md5:d55b984b8dcb39baa0d5ba0a34051ffa',
'ad:CFHT/1151210w_preview_1024.jpg':
'md5:542b11ae4e21edc19380e0b37dec129b',
'ad:CFHT/1151210w_preview_256.jpg':
'md5:a10b7c6b8f101f52ab16d0da0649451a',
'ad:CFHT/1151210w_preview_zoom_1024.jpg':
'md5:e79bb2ac4230d5587f97a9b838d6f88d',
'ad:CFHT/1151210o_preview_1024.jpg':
'md5:302ce34edd7c69da23119d001e85d39a',
'ad:CFHT/1151210o_preview_256.jpg':
'md5:137fa8a405cdf03f827000908831ece9',
'ad:CFHT/1151210o_preview_zoom_1024.jpg':
'md5:e656f21a21a4a41e088cccf87ec76580',
'ad:CFHT/1151210p_preview_1024.jpg':
'md5:f05cb0ad85ac610ef9bc3cf1ea00c853',
'ad:CFHT/1151210p_preview_256.jpg':
'md5:73aa50e0d59a69452ae8be06de9c6317',
'ad:CFHT/1151210p_preview_zoom_1024.jpg':
'md5:e56690286391a74840bc07158cedaeec',
'ad:CFHT/1151210s_preview_1024.jpg':
'md5:7d7019efd828ff748fece2b778c5fecb',
'ad:CFHT/1151210s_preview_256.jpg':
'md5:43051edaa8dbb6436d4dcb32fc5a2bd0',
'ad:CFHT/1151210s_preview_zoom_1024.jpg':
'md5:4f20f5586eaa43abc5de977e60cea807',
'ad:CFHT/1151210y_preview_1024.jpg':
'md5:931dd9cabffe9e470a556327c08c6b96',
'ad:CFHT/1151210y_preview_256.jpg':
'md5:45df1e0a5dd4b4b058b7e20cdc55b9f7',
'ad:CFHT/1151210y_preview_zoom_1024.jpg':
'md5:53cfa0d681c46fedfc2bf719e88231b5'
}
kwargs = {'working_directory': TEST_FILES_DIR,
'cadc_client': None,
'stream': 'stream',
'observable': test_observable}
for entry in glob.glob(f'{TEST_FILES_DIR}/*.jpg'):
os.unlink(entry)
checksum_failures = []
for key, value in test_files.items():
obs = mc.read_obs_from_file(f'{test_main_app.TEST_DATA_DIR}/{key}')
if 'wircam' in key:
instrument = md.Inst.WIRCAM
elif 'mega' in key:
instrument = md.Inst.MEGACAM
elif 'sitelle' in key:
instrument = md.Inst.SITELLE
elif 'espadons' in key:
instrument = md.Inst.ESPADONS
elif 'spirou' in key:
instrument = md.Inst.SPIROU
else:
assert False, 'do not understand instrument'
for f_name in value:
kwargs['science_file'] = f_name
test_name = cfht_name.CFHTName(file_name=f_name,
instrument=instrument)
check_number = 1
if test_name.suffix == 'p' and instrument is md.Inst.SITELLE:
check_number = 3
assert len(obs.planes[test_name.product_id].artifacts) == \
check_number, f'initial condition {f_name}'
try:
from datetime import datetime
import logging
start_ts = datetime.utcnow().timestamp()
test_result = preview_augmentation.visit(obs, **kwargs)
end_ts = datetime.utcnow().timestamp()
logging.error(f'{f_name} execution time {end_ts - start_ts}')
except Exception as e:
import logging
logging.error(e)
import traceback
logging.error(traceback.format_exc())
assert False
assert test_result is not None, f'expect a result {f_name}'
check_number = 3
end_artifact_count = 4
f_name_list = [test_name.prev_uri, test_name.thumb_uri,
test_name.zoom_uri]
if ((instrument is md.Inst.ESPADONS and test_name.suffix == 'i') or
(instrument is md.Inst.SPIROU and
test_name.suffix in ['e', 'p', 's', 't', 'v'])):
check_number = 2
end_artifact_count = 3
f_name_list = [test_name.prev_uri, test_name.thumb_uri]
assert test_result['artifacts'] == check_number, \
f'artifacts should be added {f_name}'
if test_name.suffix == 'p' and instrument is md.Inst.SITELLE:
end_artifact_count = 6
assert len(obs.planes[test_name.product_id].artifacts) == \
end_artifact_count, f'new artifacts {f_name}'
for p in f_name_list:
# assert p in \
# obs.planes[test_name.product_id].artifacts.keys(), \
# f'no preview {p}'
if p in obs.planes[test_name.product_id].artifacts.keys():
artifact = obs.planes[test_name.product_id].artifacts[p]
# because 2359320p_preview_1024 keeps changing ....
if artifact.uri in test_checksums:
if artifact.content_checksum.uri != test_checksums[p]:
checksum_failures.append(
f'{p} expected {test_checksums[p]} actual '
f'{artifact.content_checksum.uri}')
assert len(checksum_failures) == 0, \
'\n'.join(ii for ii in checksum_failures)
