def test_preview_augment_delete_preview():
# plane starts with a preview artifact, but it represents a non-existent
# file, so remove the artifact from the CAOM observation
test_product_id = 'S20080610S0045'
fqn = os.path.join(TEST_DATA_DIR, 'GS-2008A-C-5-35-002.fits.xml')
obs = mc.read_obs_from_file(fqn)
assert len(obs.planes[test_product_id].artifacts) == 2, 'initial condition'
test_rejected = mc.Rejected('/tmp/nonexistent')
test_rejected.content = {
'bad_metadata': [],
'no_preview':
['S20080610S0043.jpg',
'S20080610S0041.jpg',
'S20080610S0044.jpg',
'S20080610S0045.jpg']}
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
kwargs = {'working_directory': TEST_DATA_DIR,
'cadc_client': None,
'stream': 'stream',
'observable': test_observable}
result = preview_augmentation.visit(obs, **kwargs)
assert result is not None, 'expect a result'
assert result['artifacts'] == 1, 'wrong result'
assert len(obs.planes[test_product_id].artifacts) == 1, 'post condition'
def test_preview_augment(http_mock):
# this should result in two new artifacts being added to the plane
# one for a thumbnail and one for a preview
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_metrics = mc.Metrics(test_config)
test_observable = mc.Observable(test_rejected, test_metrics)
cadc_client_mock = Mock()
clients_mock = Mock()
clients_mock.data_client = cadc_client_mock
test_storage_name = gem_name.GemName(file_name=f'{TEST_PRODUCT_ID}.fits')
kwargs = {
'working_directory': '/test_files',
'clients': clients_mock,
'observable': test_observable,
'storage_name': test_storage_name,
}
test_prev = f'/test_files/{TEST_PRODUCT_ID}.jpg'
if os.path.exists(test_prev):
os.unlink(test_prev)
try:
cadc_client_mock.get.side_effect = exceptions.UnexpectedException(
'test')
http_mock.side_effect = _get_mock
obs = preview_augmentation.visit(obs, **kwargs)
test_url = (f'{preview_augmentation.PREVIEW_URL}'
f'{TEST_PRODUCT_ID}.fits')
assert http_mock.called, 'http mock should be called'
http_mock.assert_called_with(test_url, test_prev), 'mock not called'
assert cadc_client_mock.put.called, 'put mock not called'
cadc_client_mock.put.assert_called_with(
'/test_files',
'cadc:GEMINI/GN2001BQ013-04_th.jpg',
), 'wrong put arguments'
assert obs is not None, 'expect a result'
assert (len(
obs.planes[TEST_PRODUCT_ID].artifacts) == 3), 'two new artifacts'
prev_uri = mc.build_uri(COLLECTION, f'{TEST_PRODUCT_ID}.jpg', SCHEME)
thumb_uri = mc.build_uri(COLLECTION, f'{TEST_PRODUCT_ID}_th.jpg',
'cadc')
assert (prev_uri
in obs.planes[TEST_PRODUCT_ID].artifacts.keys()), 'no preview'
assert (thumb_uri
in obs.planes[TEST_PRODUCT_ID].artifacts), 'no thumbnail'
finally:
if os.path.exists(test_prev):
os.unlink(test_prev)
def test_preview_augment_known_no_preview():
# rejected file exists that says there's a preview known to not
# exist, so trying to generate a thumbnail will result in no
# change to the plane/artifact structure
try:
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert (len(
obs.planes[TEST_PRODUCT_ID].artifacts) == 1), 'initial condition'
if os.path.exists(REJECTED_FILE):
os.unlink(REJECTED_FILE)
test_rejected = mc.Rejected(REJECTED_FILE)
test_rejected.record(mc.Rejected.NO_PREVIEW, f'{TEST_PRODUCT_ID}.jpg')
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
test_storage_name = gem_name.GemName(file_name=TEST_FP_FILE)
cadc_client_mock = Mock()
clients_mock = Mock()
clients_mock.data_client = cadc_client_mock
kwargs = {
'working_directory': TEST_DATA_DIR,
'clients': clients_mock,
'stream': 'stream',
'observable': test_observable,
'storage_name': test_storage_name,
}
with patch('caom2pipe.manage_composable.http_get') as http_mock, patch(
'caom2pipe.manage_composable.data_put') as ad_put_mock, patch(
'caom2pipe.manage_composable.get_artifact_metadata'
) as art_mock, patch(
'caom2pipe.manage_composable.exec_cmd') as exec_mock:
cadc_client_mock.return_value.data_get.return_value = (
mc.CadcException('test'))
obs = preview_augmentation.visit(obs, **kwargs)
assert not http_mock.called, 'http mock should not be called'
assert not ad_put_mock.called, 'ad put mock should not be called'
assert not art_mock.called, 'art mock should not be called'
assert not exec_mock.called, 'exec mock should not be called'
assert obs is not None, 'expect a result'
assert (len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1
), 'no new artifacts'
test_rejected.persist_state()
assert os.path.exists(REJECTED_FILE)
finally:
if os.path.exists(REJECTED_FILE):
os.unlink(REJECTED_FILE)
def test_preview_augment_plane(mock_obs_id):
mock_obs_id.return_value = TEST_OBS
thumb = os.path.join(TESTDATA_DIR,
'GMOS/{}'.format(GemName(TEST_FILE).thumb))
if os.path.exists(thumb):
os.remove(thumb)
test_fqn = os.path.join(TESTDATA_DIR, '{}/{}.in.xml'.format(
'GMOS', TEST_OBS))
test_obs = mc.read_obs_from_file(test_fqn)
assert len(test_obs.planes[TEST_OBS].artifacts) == 2
thumba = '{}:{}/N20131203S0006_th.jpg'.format(SCHEME, ARCHIVE)
test_kwargs = {'working_directory': '{}/GMOS'.format(TESTDATA_DIR),
'cadc_client': None,
'stream': 'default'}
test_result = preview_augmentation.visit(test_obs, **test_kwargs)
assert test_result is not None, 'expected a visit return value'
assert test_result['artifacts'] == 1
assert len(test_obs.planes[TEST_OBS].artifacts) == 3
assert os.path.exists(thumb)
assert test_obs.planes[TEST_OBS].artifacts[thumba].content_checksum == \
ChecksumURI('md5:a8c106c04db4c148695787bfc364cbd8'), \
'thumb checksum failure'
# now do updates
test_obs.planes[TEST_OBS].artifacts[thumba].content_checksum = \
ChecksumURI('19661c3c2508ecc22425ee2a05881ed4')
test_result = preview_augmentation.visit(test_obs, **test_kwargs)
assert test_result is not None, 'expected update visit return value'
assert test_result['artifacts'] == 1
assert len(test_obs.planes) == 1
assert len(test_obs.planes[TEST_OBS].artifacts) == 3
assert os.path.exists(thumb)
assert test_obs.planes[TEST_OBS].artifacts[thumba].content_checksum == \
ChecksumURI('md5:a8c106c04db4c148695787bfc364cbd8'), \
'thumb update failed'
def test_preview_augment_unknown_no_preview():
# what happens when it's not known that there's no preview
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
# make sure the rejected file is empty
if os.path.exists(REJECTED_FILE):
os.unlink(REJECTED_FILE)
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
test_storage_name = gem_name.GemName(file_name=f'{TEST_PRODUCT_ID}.fits')
cadc_client_mock = Mock()
clients_mock = Mock()
clients_mock.data_client = cadc_client_mock
kwargs = {
'working_directory': TEST_DATA_DIR,
'clients': clients_mock,
'stream': 'stream',
'observable': test_observable,
'storage_name': test_storage_name,
}
with patch(
'caom2pipe.manage_composable.http_get',
side_effect=mc.CadcException(
'Not Found for url: https://archive.gemini.edu/preview'),
) as http_mock, patch(
'caom2pipe.manage_composable.data_put') as ad_put_mock, patch(
'caom2pipe.manage_composable.get_artifact_metadata'
) as art_mock, patch(
'caom2pipe.manage_composable.exec_cmd') as exec_mock:
cadc_client_mock.get.side_effect = exceptions.UnexpectedException(
'test')
obs = preview_augmentation.visit(obs, **kwargs)
assert obs is not None, 'expect result'
test_url = f'{preview_augmentation.PREVIEW_URL}{TEST_PRODUCT_ID}.fits'
test_prev = f'{TEST_DATA_DIR}/{TEST_PRODUCT_ID}.jpg'
http_mock.assert_called_with(test_url, test_prev), 'mock not called'
assert not ad_put_mock.called, 'ad put mock should not be called'
assert not art_mock.called, 'art mock should not be called'
assert not exec_mock.called, 'exec mock should not be called'
def test_preview_augment_unknown_no_preview():
# what happens when it's not known that there's no preview
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
# make sure the rejected file is empty
if os.path.exists(REJECTED_FILE):
os.unlink(REJECTED_FILE)
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
cadc_client_mock = Mock()
kwargs = {'working_directory': TEST_DATA_DIR,
'cadc_client': cadc_client_mock,
'stream': 'stream',
'observable': test_observable}
with patch('caom2pipe.manage_composable.http_get',
side_effect=mc.CadcException(
'Internal Server Error for url: '
'https://archive.gemini.edu/preview')) as http_mock, \
patch('caom2pipe.manage_composable.data_put') as ad_put_mock, \
patch('caom2pipe.manage_composable.get_artifact_metadata') as \
art_mock, \
patch('caom2pipe.manage_composable.exec_cmd') as exec_mock:
cadc_client_mock.return_value.data_get.return_value = mc.CadcException(
'test')
result = preview_augmentation.visit(obs, **kwargs)
assert result is not None, 'expect result'
# 0 because the preview artifact doesn't already exist
assert result['artifacts'] == 0, 'wrong result'
test_url = f'{preview_augmentation.PREVIEW_URL}{TEST_PRODUCT_ID}.fits'
test_prev = f'{TEST_DATA_DIR}/{TEST_PRODUCT_ID}.jpg'
http_mock.assert_called_with(test_url, test_prev), 'mock not called'
assert not ad_put_mock.called, 'ad put mock should not be called'
assert not art_mock.called, 'art mock should not be called'
assert not exec_mock.called, 'exec mock should not be called'
def test_preview_augment():
# this should result in two new artifacts being added to the plane
# one for a thumbnail and one for a preview
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
cadc_client_mock = Mock()
kwargs = {
'working_directory': '/test_files',
'cadc_client': cadc_client_mock,
'stream': 'stream',
'observable': test_observable,
}
test_prev = f'/test_files/{TEST_PRODUCT_ID}.jpg'
if os.path.exists(test_prev):
os.unlink(test_prev)
try:
with patch('caom2pipe.manage_composable.http_get') as http_mock, \
patch('caom2pipe.manage_composable.data_put') as ad_put_mock, \
patch(
'caom2pipe.manage_composable.get_artifact_metadata'
) as art_mock:
def _art_mock(
fqn,
product_type,
release_type,
uri,
art_ignore,
):
if '_th' in uri:
return Artifact(
uri,
ProductType.PREVIEW,
ReleaseType.DATA,
'image/jpeg',
13,
ChecksumURI('md5:13'),
)
else:
return Artifact(
uri,
ProductType.PREVIEW,
ReleaseType.DATA,
'image/jpeg',
12,
ChecksumURI('md5:12'),
)
cadc_client_mock.return_value.data_get.return_value = \
mc.CadcException('test')
art_mock.side_effect = _art_mock
http_mock.side_effect = _get_mock
result = preview_augmentation.visit(obs, **kwargs)
test_url = f'{preview_augmentation.PREVIEW_URL}' \
f'{TEST_PRODUCT_ID}.fits'
assert http_mock.called, 'http mock should be called'
http_mock.assert_called_with(
test_url, test_prev
), 'mock not called'
assert ad_put_mock.called, 'ad put mock not called'
assert art_mock.called, 'art mock not called'
assert result is not None, 'expect a result'
assert result['artifacts'] == 2, 'artifacts should be added'
assert (
len(obs.planes[TEST_PRODUCT_ID].artifacts) == 3
), 'two new artifacts'
prev_uri = mc.build_uri(
ARCHIVE, f'{TEST_PRODUCT_ID}.jpg', SCHEME
)
thumb_uri = mc.build_uri(ARCHIVE, f'{TEST_PRODUCT_ID}_th.jpg')
assert (
prev_uri in obs.planes[TEST_PRODUCT_ID].artifacts.keys()
), 'no preview'
assert (
thumb_uri in obs.planes[TEST_PRODUCT_ID].artifacts
), 'no thumbnail'
finally:
if os.path.exists(test_prev):
os.unlink(test_prev)
def test_preview_augment_failure(http_mock):
# mimic 'Not Found' behaviour
# this should result in no new artifacts being added to the plane
# but a record for 'no preview exists at Gemini' added to the
# record
def _failure_mock(ignore_url, ignore_local_fqn):
raise mc.CadcException(
'Could not retrieve /usr/src/app/N20211007A0003/'
'N20211007A0003b.jpg from '
'https://archive.gemini.edu/preview/N20211007A0003b.fits. Failed '
'with 404 Client Error: Not Found for url: '
'https://archive.gemini.edu/preview/N20211007A0003b.fits')
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_metrics = mc.Metrics(test_config)
test_observable = mc.Observable(test_rejected, test_metrics)
cadc_client_mock = Mock()
clients_mock = Mock()
clients_mock.data_client = cadc_client_mock
test_storage_name = gem_name.GemName(file_name=f'{TEST_PRODUCT_ID}.fits')
kwargs = {
'working_directory': '/test_files',
'clients': clients_mock,
'observable': test_observable,
'storage_name': test_storage_name,
}
test_prev = f'/test_files/{TEST_PRODUCT_ID}.jpg'
if os.path.exists(test_prev):
os.unlink(test_prev)
try:
cadc_client_mock.get.side_effect = exceptions.UnexpectedException(
'test')
http_mock.side_effect = _failure_mock
obs = preview_augmentation.visit(obs, **kwargs)
test_url = (f'{preview_augmentation.PREVIEW_URL}'
f'{TEST_PRODUCT_ID}.fits')
assert http_mock.called, 'http mock should be called'
http_mock.assert_called_with(test_url, test_prev), 'mock not called'
assert not cadc_client_mock.put.called, 'put mock should not be called'
assert obs is not None, 'expect a result'
assert (len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1
), 'same as the pre-condition'
prev_uri = mc.build_uri(COLLECTION, f'{TEST_PRODUCT_ID}.jpg', SCHEME)
thumb_uri = mc.build_uri(COLLECTION, f'{TEST_PRODUCT_ID}_th.jpg',
'cadc')
assert (prev_uri not in obs.planes[TEST_PRODUCT_ID].artifacts.keys()
), 'should be no preview'
assert (thumb_uri not in obs.planes[TEST_PRODUCT_ID].artifacts
), 'should be no thumbnail'
assert not (
test_rejected.is_no_preview(prev_uri)), 'preview should be tracked'
assert http_mock.call_count == 1, 'wrong number of calls'
# now try again to generate the preview, and ensure that the
# rejected tracking is working
obs = preview_augmentation.visit(obs, **kwargs)
assert obs is not None, 'expect a result the second time'
assert http_mock.call_count == 1, 'never even tried to retrieve it'
finally:
if os.path.exists(test_prev):
os.unlink(test_prev)
