def test_provenance_augmentation(dmf_mock, headers_mock, access_mock,
builder_mock, repo_get_mock, test_fqn):
builder_mock.return_value._get_obs_id.return_value = None
access_mock.return_value = 'https://localhost'
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_config.task_types = [mc.TaskType.VISIT]
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
headers_mock.return_value.get_head.side_effect = _get_headers_mock
dmf_mock.get.side_effect = _get_obs_id_mock
repo_get_mock.side_effect = _repo_get_mock
getcwd_orig = os.getcwd
os.getcwd = Mock(return_value=test_main_app.TEST_DATA_DIR)
temp = os.path.basename(test_fqn).replace('.expected.xml', '.fits')
test_storage_name = GemProcName(entry=temp)
try:
test_obs = mc.read_obs_from_file(test_fqn)
assert not test_obs.target.moving, 'initial conditions moving target'
kwargs = {
'storage_name': test_storage_name,
'working_directory': test_main_app.TEST_DATA_DIR,
'observable': test_observable,
'caom_repo_client': Mock(),
}
test_result = provenance_augmentation.visit(test_obs, **kwargs)
assert test_result is not None, 'expect a result'
assert test_result.get('provenance') == 2, 'wrong result'
assert len(test_obs.members) == 1, 'wrong membership'
assert test_obs.target.moving, 'should be changed'
finally:
os.getcwd = getcwd_orig
def test_pull_augmentation():
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
cadc_client_mock = Mock()
kwargs = {'working_directory': TEST_DATA_DIR,
'cadc_client': cadc_client_mock,
'stream': 'stream',
'observable': test_observable}
with patch('caom2pipe.manage_composable.http_get') as http_mock, \
patch('caom2pipe.manage_composable.data_put') as ad_put_mock:
cadc_client_mock.return_value.data_get.return_value = mc.CadcException(
'test')
# no scheme from cadc client
cadc_client_mock.get_file_info.return_value = {'md5sum': '1234'}
result = pull_augmentation.visit(obs, **kwargs)
test_url = f'{pull_augmentation.FILE_URL}/{TEST_PRODUCT_ID}.fits'
test_prev = f'{TEST_DATA_DIR}/{TEST_PRODUCT_ID}.fits'
http_mock.assert_called_with(test_url, test_prev), 'mock not called'
assert ad_put_mock.called, 'ad put mock not called'
assert result is not None, 'expect a result'
assert result['observation'] == 0, 'no updated metadata'
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, \
'no new artifacts'
def test_pull_v_augmentation(put_mock, http_mock):
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
test_uri = f'{SCHEME}:{COLLECTION}/{TEST_PRODUCT_ID}.fits'
for plane in obs.planes.values():
for artifact in plane.artifacts.values():
artifact.uri = test_uri
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
cadc_client_mock = Mock()
kwargs = {'working_directory': TEST_DATA_DIR,
'cadc_client': cadc_client_mock,
'observable': test_observable}
result = pull_v_augmentation.visit(obs, **kwargs)
test_url = f'{pull_augmentation.FILE_URL}/{TEST_PRODUCT_ID}.fits'
test_prev = f'{TEST_DATA_DIR}/{TEST_PRODUCT_ID}.fits'
http_mock.assert_called_with(test_url, test_prev), 'mock not called'
assert put_mock.called, 'put mock not called'
args, kwargs = put_mock.call_args
assert args[1] == TEST_DATA_DIR, 'wrong working dir'
assert args[2] == f'{TEST_PRODUCT_ID}.fits', 'wrong file name'
assert args[3] == test_uri, 'wrong storage name'
assert result is not None, 'expect a result'
assert result['observation'] == 0, 'no updated metadata'
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, \
'no new artifacts'
def test_store(put_mock):
test_config = mc.Config()
test_config.logging_level = 'ERROR'
test_config.working_directory = '/tmp'
test_url = 'https://archive-new.nrao.edu/vlass/quicklook/VLASS2.1/' \
'T10t12/VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1/' \
'VLASS2.1.ql.T10t12.J073401-033000.10.2048.v1.I.iter1.image.' \
'pbcor.tt0.rms.subim.fits'
test_storage_name = VlassName(url=test_url, entry=test_url)
transferrer = Mock()
cred_param = Mock()
cadc_data_client = Mock()
caom_repo_client = Mock()
observable = mc.Observable(mc.Rejected('/tmp/rejected.yml'),
mc.Metrics(test_config))
test_subject = ec.Store(test_config, test_storage_name, APPLICATION,
cred_param, cadc_data_client, caom_repo_client,
observable, transferrer)
test_subject.execute(None)
assert put_mock.called, 'expect a call'
args, kwargs = put_mock.call_args
assert args[2] == test_storage_name.file_name, 'wrong file name'
assert transferrer.get.called, 'expect a transfer call'
args, kwargs = transferrer.get.call_args
assert args[0] == test_url, 'wrong source parameter'
assert args[1] == f'/tmp/{test_storage_name.obs_id}/' \
f'{test_storage_name.file_name}',\
'wrong destination parameter'
def test_preview_augment_delete_preview():
# plane starts with a preview artifact, but it represents a non-existent
# file, so remove the artifact from the CAOM observation
test_product_id = 'S20080610S0045'
fqn = os.path.join(TEST_DATA_DIR, 'GS-2008A-C-5-35-002.fits.xml')
obs = mc.read_obs_from_file(fqn)
assert len(obs.planes[test_product_id].artifacts) == 2, 'initial condition'
test_rejected = mc.Rejected('/tmp/nonexistent')
test_rejected.content = {
'bad_metadata': [],
'no_preview':
['S20080610S0043.jpg',
'S20080610S0041.jpg',
'S20080610S0044.jpg',
'S20080610S0045.jpg']}
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
kwargs = {'working_directory': TEST_DATA_DIR,
'cadc_client': None,
'stream': 'stream',
'observable': test_observable}
result = preview_augmentation.visit(obs, **kwargs)
assert result is not None, 'expect a result'
assert result['artifacts'] == 1, 'wrong result'
assert len(obs.planes[test_product_id].artifacts) == 1, 'post condition'
def test_meta_update_observation_direct(test_config):
test_cred_param = '--cert /usr/src/app/cadcproxy.pem'
data_client_mock = Mock()
repo_client_mock = Mock()
test_sn = tc.TestStorageName()
test_observation = mc.read_obs_from_file(
f'{tc.TEST_DATA_DIR}/fpf_start_obs.xml'
)
try:
test_observable = mc.Observable(
mc.Rejected(test_config.rejected_fqn), mc.Metrics(test_config)
)
test_executor = ec.MetaUpdateObservation(
test_config,
test_sn,
TEST_APP,
test_cred_param,
data_client_mock,
repo_client_mock,
test_observation,
[],
observable=test_observable,
)
test_executor.execute(None)
assert repo_client_mock.update.called, 'repo update called'
finally:
pass
def test_store(put_mock):
test_config = mc.Config()
test_config.logging_level = 'ERROR'
test_config.working_directory = '/tmp'
test_fqn = '/users/OpenData_DonneesOuvertes/pub/NEOSSAT/ASTRO/2019/' \
'268/NEOS_SCI_2019268004930_clean.fits'
test_storage_name = NEOSSatName(file_name=test_fqn, entry=test_fqn)
transferrer = Mock()
cred_param = Mock()
cadc_data_client = Mock()
caom_repo_client = Mock()
observable = mc.Observable(mc.Rejected('/tmp/rejected.yml'),
mc.Metrics(test_config))
test_subject = ec.Store(test_config, test_storage_name, APPLICATION,
cred_param, cadc_data_client, caom_repo_client,
observable, transferrer)
test_subject.execute(None)
assert put_mock.called, 'expect a call'
args, kwargs = put_mock.call_args
assert args[2] == test_storage_name.file_name, 'wrong file name'
assert transferrer.get.called, 'expect a transfer call'
args, kwargs = transferrer.get.call_args
import logging
logging.error(args)
assert args[0] == test_fqn, 'wrong source parameter'
assert args[1] == f'/tmp/{test_storage_name.obs_id}/' \
f'{test_storage_name.file_name}', \
'wrong destination parameter'
def test_look_pull_and_put_v(http_mock, mock_client):
stat_orig = os.stat
os.stat = Mock()
os.stat.return_value = Mock(st_size=1234)
try:
test_storage_name = 'cadc:GEMINI/TEST.fits'
f_name = 'test_f_name.fits'
url = f'https://localhost/{f_name}'
test_config = mc.Config()
test_config.observe_execution = True
test_metrics = mc.Metrics(test_config)
mock_client.get_node.side_effect = tc.mock_get_node
mock_client.copy.return_value = 1234
assert len(test_metrics.history) == 0, 'initial history conditions'
assert len(test_metrics.failures) == 0, 'initial failure conditions'
mc.look_pull_and_put_v(
test_storage_name,
f_name,
tc.TEST_DATA_DIR,
url,
mock_client,
'md5:01234',
test_metrics,
)
test_fqn = os.path.join(tc.TEST_DATA_DIR, f_name)
mock_client.copy.assert_called_with(
test_fqn, destination=test_storage_name), 'mock not called'
http_mock.assert_called_with(url, os.path.join(
tc.TEST_DATA_DIR, f_name)), 'http mock not called'
assert len(test_metrics.history) == 1, 'history conditions'
assert len(test_metrics.failures) == 0, 'failure conditions'
finally:
os.stat = stat_orig
def test_provenance_augmentation(obs_id_mock, repo_get_mock, headers_mock,
test_fqn):
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
headers_mock.side_effect = _get_headers_mock
repo_get_mock.side_effect = _repo_get_mock
obs_id_mock.side_effect = _get_obs_id_mock
getcwd_orig = os.getcwd
os.getcwd = Mock(return_value=test_main_app.TEST_DATA_DIR)
try:
test_obs = mc.read_obs_from_file(test_fqn)
assert not test_obs.target.moving, 'initial conditions moving target'
kwargs = {
'science_file':
os.path.basename(test_fqn).replace('.expected.xml', '.fits'),
'working_directory':
test_main_app.TEST_DATA_DIR,
'observable':
test_observable,
'caom_repo_client':
Mock()
}
test_result = provenance_augmentation.visit(test_obs, **kwargs)
assert test_result is not None, 'expect a result'
assert test_result.get('provenance') == 2, 'wrong result'
assert len(test_obs.members) == 1, 'wrong membership'
assert test_obs.target.moving, 'should be changed'
finally:
os.getcwd = getcwd_orig
def _filter_result(self):
config = mc.Config()
config.get_executors()
subject = clc.define_subject(config)
caom_client = CAOM2RepoClient(subject)
metrics = mc.Metrics(config)
for entry in self._source:
if VlassValidator._later_version_at_cadc(entry, caom_client,
metrics):
self._source.remove(entry)
def test_preview_augment(http_mock):
# this should result in two new artifacts being added to the plane
# one for a thumbnail and one for a preview
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_metrics = mc.Metrics(test_config)
test_observable = mc.Observable(test_rejected, test_metrics)
cadc_client_mock = Mock()
clients_mock = Mock()
clients_mock.data_client = cadc_client_mock
test_storage_name = gem_name.GemName(file_name=f'{TEST_PRODUCT_ID}.fits')
kwargs = {
'working_directory': '/test_files',
'clients': clients_mock,
'observable': test_observable,
'storage_name': test_storage_name,
}
test_prev = f'/test_files/{TEST_PRODUCT_ID}.jpg'
if os.path.exists(test_prev):
os.unlink(test_prev)
try:
cadc_client_mock.get.side_effect = exceptions.UnexpectedException(
'test')
http_mock.side_effect = _get_mock
obs = preview_augmentation.visit(obs, **kwargs)
test_url = (f'{preview_augmentation.PREVIEW_URL}'
f'{TEST_PRODUCT_ID}.fits')
assert http_mock.called, 'http mock should be called'
http_mock.assert_called_with(test_url, test_prev), 'mock not called'
assert cadc_client_mock.put.called, 'put mock not called'
cadc_client_mock.put.assert_called_with(
'/test_files',
'cadc:GEMINI/GN2001BQ013-04_th.jpg',
), 'wrong put arguments'
assert obs is not None, 'expect a result'
assert (len(
obs.planes[TEST_PRODUCT_ID].artifacts) == 3), 'two new artifacts'
prev_uri = mc.build_uri(COLLECTION, f'{TEST_PRODUCT_ID}.jpg', SCHEME)
thumb_uri = mc.build_uri(COLLECTION, f'{TEST_PRODUCT_ID}_th.jpg',
'cadc')
assert (prev_uri
in obs.planes[TEST_PRODUCT_ID].artifacts.keys()), 'no preview'
assert (thumb_uri
in obs.planes[TEST_PRODUCT_ID].artifacts), 'no thumbnail'
finally:
if os.path.exists(test_prev):
os.unlink(test_prev)
def test_preview_augment_plane():
omm_name = OmmName(file_name=TEST_FILE, entry=TEST_FILE)
preview = os.path.join(TEST_FILES_DIR, omm_name.prev)
thumb = os.path.join(TEST_FILES_DIR, omm_name.thumb)
if os.path.exists(preview):
os.remove(preview)
if os.path.exists(thumb):
os.remove(thumb)
test_fqn = os.path.join(TEST_DATA_DIR,
f'{omm_name.product_id}.expected.xml')
test_obs = mc.read_obs_from_file(test_fqn)
assert len(test_obs.planes[omm_name.product_id].artifacts) == 1
preva = 'cadc:OMM/C170324_0054_SCI_prev.jpg'
thumba = 'cadc:OMM/C170324_0054_SCI_prev_256.jpg'
test_config = mc.Config()
test_config.observe_execution = True
test_metrics = mc.Metrics(test_config)
test_observable = mc.Observable(rejected=None, metrics=test_metrics)
test_kwargs = {
'working_directory': TEST_FILES_DIR,
'cadc_client': None,
'observable': test_observable,
'storage_name': omm_name,
}
test_result = preview_augmentation.visit(test_obs, **test_kwargs)
assert test_result is not None, 'expected a visit return value'
assert os.path.exists(preview)
assert os.path.exists(thumb)
test_plane = test_result.planes[omm_name.product_id]
assert test_plane.artifacts[preva].content_checksum == ChecksumURI(
'md5:f37d21c53055498d1b5cb7753e1c6d6f'), 'prev checksum failure'
assert test_plane.artifacts[thumba].content_checksum == ChecksumURI(
'md5:19661c3c2508ecc22425ee2a05881ed4'), 'thumb checksum failure'
# now do updates
test_obs.planes[
omm_name.product_id].artifacts[preva].content_checksum = ChecksumURI(
'de9f39804f172682ea9b001f8ca11f15')
test_obs.planes[
omm_name.product_id].artifacts[thumba].content_checksum = ChecksumURI(
'cd118dae04391f6bea93ba4bf2711adf')
test_result = preview_augmentation.visit(test_obs, **test_kwargs)
assert test_result is not None, 'expected update visit return value'
assert len(test_result.planes[omm_name.product_id].artifacts) == 3
assert os.path.exists(preview)
assert os.path.exists(thumb)
assert test_plane.artifacts[preva].content_checksum == ChecksumURI(
'md5:f37d21c53055498d1b5cb7753e1c6d6f'), 'prev update failed'
assert test_plane.artifacts[thumba].content_checksum == ChecksumURI(
'md5:19661c3c2508ecc22425ee2a05881ed4'), 'prev_256 update failed'
assert len(test_metrics.history) == 0, 'wrong history, client is not None'
def declare_client(config):
"""Common code to set the client used for interacting with CADC
storage."""
subject = define_subject(config)
cadc_client = StorageClientWrapper(
using_storage_inventory=config.features.supports_latest_client,
resource_id=config.storage_inventory_resource_id,
subject=subject,
metrics=mc.Metrics(config),
)
return cadc_client
def __init__(self, config):
super(CFHTBuilder, self).__init__()
self._config = config
self._data_client = None
self._repo_client = None
self._metrics = mc.Metrics(self._config)
if not self._config.use_local_files:
subject = mc.define_subject(self._config)
self._data_client = CadcDataClient(subject)
self._repo_client = CAOM2RepoClient(
subject, resource_id=self._config.resource_id)
self._logger = logging.getLogger(__name__)
def test_client_get_failure(mock_client):
test_config = mc.Config()
test_config.observe_execution = True
test_metrics = mc.Metrics(test_config)
with pytest.raises(mc.CadcException):
mc.client_get(
mock_client,
tc.TEST_DATA_DIR,
'TEST_get.fits',
'TEST',
test_metrics,
)
assert len(test_metrics.failures) == 1, 'wrong failures'
assert test_metrics.failures['vos']['copy']['TEST_get.fits'] == 1, 'count'
def test_client_get_failure(mock_client):
test_config = mc.Config()
test_config.observe_execution = True
test_metrics = mc.Metrics(test_config)
with pytest.raises(mc.CadcException):
clc.si_client_get(
mock_client,
os.path.join(tc.TEST_DATA_DIR, 'TEST_get.fits'),
'cadc:TEST/TEST_get.fits',
test_metrics,
)
assert len(test_metrics.failures) == 1, 'wrong failures'
assert test_metrics.failures['si']['cadcget'][
'TEST_get.fits'] == 1, 'count'
def test_preview_augment_known_no_preview():
# rejected file exists that says there's a preview known to not
# exist, so trying to generate a thumbnail will result in no
# change to the plane/artifact structure
try:
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert (len(
obs.planes[TEST_PRODUCT_ID].artifacts) == 1), 'initial condition'
if os.path.exists(REJECTED_FILE):
os.unlink(REJECTED_FILE)
test_rejected = mc.Rejected(REJECTED_FILE)
test_rejected.record(mc.Rejected.NO_PREVIEW, f'{TEST_PRODUCT_ID}.jpg')
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
test_storage_name = gem_name.GemName(file_name=TEST_FP_FILE)
cadc_client_mock = Mock()
clients_mock = Mock()
clients_mock.data_client = cadc_client_mock
kwargs = {
'working_directory': TEST_DATA_DIR,
'clients': clients_mock,
'stream': 'stream',
'observable': test_observable,
'storage_name': test_storage_name,
}
with patch('caom2pipe.manage_composable.http_get') as http_mock, patch(
'caom2pipe.manage_composable.data_put') as ad_put_mock, patch(
'caom2pipe.manage_composable.get_artifact_metadata'
) as art_mock, patch(
'caom2pipe.manage_composable.exec_cmd') as exec_mock:
cadc_client_mock.return_value.data_get.return_value = (
mc.CadcException('test'))
obs = preview_augmentation.visit(obs, **kwargs)
assert not http_mock.called, 'http mock should not be called'
assert not ad_put_mock.called, 'ad put mock should not be called'
assert not art_mock.called, 'art mock should not be called'
assert not exec_mock.called, 'exec mock should not be called'
assert obs is not None, 'expect a result'
assert (len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1
), 'no new artifacts'
test_rejected.persist_state()
assert os.path.exists(REJECTED_FILE)
finally:
if os.path.exists(REJECTED_FILE):
os.unlink(REJECTED_FILE)
def test_preview_augment_v(put_mock, http_mock):
# this should result in two new artifacts being added to the plane
# one for a thumbnail and one for a preview
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_metrics = mc.Metrics(test_config)
test_observable = mc.Observable(test_rejected, test_metrics)
cadc_client_mock = Mock()
kwargs = {'working_directory': '/test_files',
'cadc_client': cadc_client_mock,
'observable': test_observable}
test_prev = f'/test_files/{TEST_PRODUCT_ID}.jpg'
if os.path.exists(test_prev):
os.unlink(test_prev)
try:
cadc_client_mock.return_value.data_get.return_value = \
mc.CadcException('test')
http_mock.side_effect = _get_mock
result = preview_v_augmentation.visit(obs, **kwargs)
test_url = f'{preview_augmentation.PREVIEW_URL}' \
f'{TEST_PRODUCT_ID}.fits'
assert http_mock.called, 'http mock should be called'
http_mock.assert_called_with(test_url, test_prev), \
'mock not called'
assert put_mock.called, 'put mock not called'
put_mock.assert_called_with(
cadc_client_mock, '/test_files', 'GN2001BQ013-04_th.jpg',
'ad:GEM/GN2001BQ013-04_th.jpg', metrics=test_metrics)
assert result is not None, 'expect a result'
assert result['artifacts'] == 2, 'artifacts should be added'
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 3, \
'two new artifacts'
prev_uri = mc.build_uri(
ARCHIVE, f'{TEST_PRODUCT_ID}.jpg', SCHEME)
thumb_uri = mc.build_uri(ARCHIVE, f'{TEST_PRODUCT_ID}_th.jpg')
assert prev_uri in obs.planes[TEST_PRODUCT_ID].artifacts.keys(), \
'no preview'
assert thumb_uri in obs.planes[TEST_PRODUCT_ID].artifacts, \
'no thumbnail'
finally:
if os.path.exists(test_prev):
os.unlink(test_prev)
def test_pull_augmentation(http_mock, json_mock, header_mock, file_type_mock):
obs = mc.read_obs_from_file(TEST_OBS_AD_URI_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
original_uri = 'gemini:GEMINI/GN2001BQ013-04.fits'
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
assert (
original_uri
in obs.planes[TEST_PRODUCT_ID].artifacts.keys()), 'initial condition'
test_uri = f'{SCHEME}:{COLLECTION}/{TEST_PRODUCT_ID}.fits'
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
cadc_client_mock = Mock()
cadc_client_mock.return_value.info.return_value = None
clients_mock = Mock()
clients_mock.data_client = cadc_client_mock
json_mock.side_effect = gem_mocks.mock_retrieve_json
filter_cache = svofps.FilterMetadataCache(Mock())
test_reader = gemini_metadata.GeminiFileMetadataReader(
Mock(), Mock(), filter_cache)
test_fqn = f'{gem_mocks.TEST_DATA_DIR}/GMOS/GN2001BQ013-04.fits.header'
test_storage_name = gem_name.GemName(file_name='GN2001BQ013-04.fits')
header_mock.side_effect = gem_mocks._mock_headers
file_type_mock.return_values = 'application/fits'
test_reader.set(test_storage_name)
kwargs = {
'working_directory': TEST_DATA_DIR,
'clients': clients_mock,
'observable': test_observable,
'metadata_reader': test_reader,
'storage_name': test_storage_name,
}
obs = pull_augmentation.visit(obs, **kwargs)
test_url = f'{pull_augmentation.FILE_URL}/{TEST_PRODUCT_ID}.fits'
test_prev = f'{TEST_DATA_DIR}/{TEST_PRODUCT_ID}.fits'
http_mock.assert_called_with(test_url, test_prev), 'mock not called'
assert cadc_client_mock.put.called, 'put mock not called'
cadc_client_mock.put.assert_called_with(
TEST_DATA_DIR, 'gemini:GEMINI/GN2001BQ013-04.fits'), 'wrong put args'
assert obs is not None, 'expect a result'
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'no new artifacts'
try:
ignore = obs.planes[TEST_PRODUCT_ID].artifacts[test_uri]
except KeyError as ke:
# because CAOM does magic
result = obs.planes[TEST_PRODUCT_ID].artifacts[original_uri]
assert result.uri == test_uri, f'wrong uri {result.uri}'
def test_cadc_transfer(client_mock, caps_mock):
caps_mock.return_value = 'https://sc2.canfar.net/sc2repo'
test_subject = tc.CadcTransfer()
assert test_subject is not None, 'expect a result'
test_config = mc.Config()
test_config.rejected_fqn = '/tmp/rejected.yml'
test_observable = mc.Observable(mc.Rejected(test_config.rejected_fqn),
mc.Metrics(test_config))
test_subject.observable = test_observable
test_subject.cadc_client = client_mock
test_source = 'ad:TEST/test_file.fits'
test_destination = '/tmp/test_file.fits'
test_subject.get(test_source, test_destination)
assert client_mock.get.called, 'should have been called'
client_mock.get.assert_called_with(
'/tmp', 'ad:TEST/test_file.fits'), 'wrong parameters'
def _init(self, config):
if mc.TaskType.SCRAPE in config.task_types:
self._logger.info(
f'SCRAPE\'ing data - no clients will be initialized.'
)
else:
subject = define_subject(config)
self._metadata_client = CAOM2RepoClient(
subject, config.logging_level, config.resource_id
)
self._data_client = declare_client(config)
if config.tap_id is not None:
self._query_client = CadcTapClient(
subject=subject, resource_id=config.tap_id
)
self._metrics = mc.Metrics(config)
def test_preview_augment_unknown_no_preview():
# what happens when it's not known that there's no preview
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
# make sure the rejected file is empty
if os.path.exists(REJECTED_FILE):
os.unlink(REJECTED_FILE)
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
test_storage_name = gem_name.GemName(file_name=f'{TEST_PRODUCT_ID}.fits')
cadc_client_mock = Mock()
clients_mock = Mock()
clients_mock.data_client = cadc_client_mock
kwargs = {
'working_directory': TEST_DATA_DIR,
'clients': clients_mock,
'stream': 'stream',
'observable': test_observable,
'storage_name': test_storage_name,
}
with patch(
'caom2pipe.manage_composable.http_get',
side_effect=mc.CadcException(
'Not Found for url: https://archive.gemini.edu/preview'),
) as http_mock, patch(
'caom2pipe.manage_composable.data_put') as ad_put_mock, patch(
'caom2pipe.manage_composable.get_artifact_metadata'
) as art_mock, patch(
'caom2pipe.manage_composable.exec_cmd') as exec_mock:
cadc_client_mock.get.side_effect = exceptions.UnexpectedException(
'test')
obs = preview_augmentation.visit(obs, **kwargs)
assert obs is not None, 'expect result'
test_url = f'{preview_augmentation.PREVIEW_URL}{TEST_PRODUCT_ID}.fits'
test_prev = f'{TEST_DATA_DIR}/{TEST_PRODUCT_ID}.jpg'
http_mock.assert_called_with(test_url, test_prev), 'mock not called'
assert not ad_put_mock.called, 'ad put mock should not be called'
assert not art_mock.called, 'art mock should not be called'
assert not exec_mock.called, 'exec mock should not be called'
def test_cadc_transfer(data_get_mock, caps_mock):
caps_mock.return_value = 'https://sc2.canfar.net/sc2repo'
data_get_mock.side_effect = Mock(autospec=True)
test_subject = tc.CadcTransfer()
assert test_subject is not None, 'expect a result'
test_config = mc.Config()
test_config.rejected_fqn = '/tmp/rejected.yml'
test_observable = mc.Observable(mc.Rejected(test_config.rejected_fqn),
mc.Metrics(test_config))
test_subject.observable = test_observable
test_subject.cadc_client = Mock()
test_source = 'ad:TEST/test_file.fits'
test_destination = '/tmp/test_file.fits'
test_subject.get(test_source, test_destination)
assert data_get_mock.called, 'should have been called'
args, kwargs = data_get_mock.call_args
assert args[1] == '/tmp', 'wrong dir name'
assert args[2] == 'test_file.fits', 'wrong file name'
assert args[3] == 'TEST', 'wrong archive name'
def test_preview_augmentation(access_mock):
access_mock.return_value = 'https://localhost'
test_fqn = f'{TEST_DATA_DIR}/preview_augmentation_start.xml'
test_science_f_name = (
'VLASS1.1.ql.T01t01.J000228-363000.10.2048.v1.I.iter1.image.pbcor.'
'tt0.subim.fits')
test_storage_name = sn.VlassName(test_science_f_name)
test_obs = mc.read_obs_from_file(test_fqn)
test_config = mc.Config()
test_rejected = mc.Rejected(f'{TEST_DATA_DIR}/rejected.yml')
test_metrics = mc.Metrics(test_config)
test_observable = mc.Observable(test_rejected, test_metrics)
kwargs = {
'stream': None,
'observable': test_observable,
'storage_name': test_storage_name,
'working_directory': '/test_files',
}
test_subject = preview_augmentation.VlassPreview(**kwargs)
assert test_subject is not None, 'need a test subject'
assert len(test_obs.planes) == 1, 'wrong number of planes'
assert (len(test_obs.planes[test_storage_name.product_id].artifacts) == 4
), 'wrong starting # of artifacts'
test_obs = test_subject.visit(test_obs)
assert test_obs is not None, 'expect a result'
assert (len(test_obs.planes[test_storage_name.product_id].artifacts) == 6
), 'wrong ending # of artifacts'
test_report = test_subject.report
assert test_report is not None
assert 'artifacts' in test_report
assert test_report.get('artifacts') == 2, 'wrong report count'
# does artifact re-naming work?
test_url = (f'https://archive-new.nrao.edu/vlass/quicklook/VLASS1.1/'
f'T01t01/VLASS1.1.ql.T01t01.J000228-363000.10.2048.v1/'
f'{test_science_f_name}')
kwargs = {'url': test_url}
test_obs = cleanup_augmentation.visit(test_obs, **kwargs)
test_artifacts = test_obs.planes[test_storage_name.product_id].artifacts
assert len(test_artifacts) == 4, 'wrong ending conditions'
assert test_storage_name.prev_uri in test_artifacts, 'missing preview'
assert test_storage_name.thumb_uri in test_artifacts, 'missing thumbnail'
def test_preview_augmentation(data_client_mock, tap_mock):
getcwd_orig = os.getcwd
os.getcwd = Mock(return_value=test_main_app.TEST_DATA_DIR)
tap_mock.side_effect = _tap_mock
data_client_mock.return_value.info.side_effect = (
test_main_app._get_file_info)
test_f_id = 'rnN20140428S0181_ronchi'
test_f_name = f'{test_f_id}.fits'
test_obs = mc.read_obs_from_file(
f'{test_main_app.TEST_DATA_DIR}/{test_f_id}.expected.xml')
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_config.get_executors()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
external_metadata.init_global(test_config)
test_builder = builder.GemProcBuilder(test_config)
test_fqn = os.path.join(test_main_app.TEST_DATA_DIR, test_f_name)
test_storage_name = test_builder.build(test_fqn)
kwargs = {
'working_directory': TEST_FILES_DIR,
'cadc_client': None,
'stream': 'stream',
'observable': test_observable,
'storage_name': test_storage_name,
}
try:
start_ts = datetime.utcnow().timestamp()
test_result = preview_augmentation.visit(test_obs, **kwargs)
end_ts = datetime.utcnow().timestamp()
logging.error(f'{test_f_name} execution time {end_ts - start_ts}')
except Exception as e:
logging.error(e)
logging.error(traceback.format_exc())
assert False
finally:
os.getcwd = getcwd_orig
assert test_result is not None, 'expect a result'
assert test_result.get('artifacts') == 2, 'wrong result'
def _update_ngvs_time(chunk, provenance, obs_id):
logging.debug(f'Begin _update_ngvs_time for {obs_id}')
if (chunk is not None and provenance is not None and
len(provenance.inputs) > 0):
# bounds = ctor
config = mc.Config()
config.get_executors()
subject = mc.define_subject(config)
client = CAOM2RepoClient(
subject, config.logging_level, 'ivo://cadc.nrc.ca/ams')
metrics = mc.Metrics(config)
bounds = CoordBounds1D()
min_date = 0
max_date = sys.float_info.max
exposure = 0
for entry in provenance.inputs:
ip_obs_id, ip_product_id = mc.CaomName.decompose_provenance_input(
entry.uri)
logging.info(f'Retrieving provenance metadata for {ip_obs_id}.')
ip_obs = mc.repo_get(client, 'CFHT', ip_obs_id, metrics)
if ip_obs is not None:
ip_plane = ip_obs.planes.get(ip_product_id)
if (ip_plane is not None and ip_plane.time is not None and
ip_plane.time.bounds is not None):
bounds.samples.append(CoordRange1D(
RefCoord(pix=0.5, val=ip_plane.time.bounds.lower),
RefCoord(pix=1.5, val=ip_plane.time.bounds.upper)))
min_date = min(ip_plane.time.bounds.lower, min_date)
max_date = max(ip_plane.time.bounds.upper, max_date)
exposure += ip_plane.time.exposure
axis = Axis(ctype='TIME', cunit='d')
time_axis = CoordAxis1D(axis=axis,
error=None,
range=None,
bounds=bounds,
function=None)
temporal_wcs = TemporalWCS(axis=time_axis, timesys=None, trefpos=None,
mjdref=None, exposure=mc.to_float(exposure),
resolution=None)
chunk.time = temporal_wcs
logging.debug(f'End _update_ngvs_time.')
def test_repo_delete(mock_client):
test_config = mc.Config()
test_config.observe_execution = True
test_metrics = mc.Metrics(test_config)
assert len(test_metrics.history) == 0, 'initial history conditions'
assert len(test_metrics.failures) == 0, 'initial failure conditions'
mc.repo_delete(mock_client, 'coll', 'test_id', test_metrics)
mock_client.delete.assert_called_with('coll', 'test_id'), 'mock not called'
assert len(test_metrics.history) == 1, 'history conditions'
assert len(test_metrics.failures) == 0, 'failure conditions'
assert 'caom2' in test_metrics.history, 'history'
assert 'delete' in test_metrics.history['caom2'], 'delete'
assert 'test_id' in test_metrics.history['caom2']['delete'], 'obs id'
mock_client.reset_mock()
mock_client.delete.side_effect = Exception('boo')
with pytest.raises(mc.CadcException):
mc.repo_delete(mock_client, 'coll', 'test_id', test_metrics)
assert len(test_metrics.failures) == 1, 'should have failure counts'
def test_http_transfer(get_mock):
test_source = 'http://localhost/test_file.fits'
test_destination = '/tmp/test_file.fits'
if not os.path.exists(test_destination):
with open(test_destination, 'w') as f:
f.write('test content')
get_mock.side_effect = Mock(autospec=True)
test_config = mc.Config()
test_config.working_directory = test_conf.TEST_DATA_DIR
test_config.netrc_file = 'test_netrc'
test_config.rejected_fqn = '/tmp/rejected.yml'
test_observable = mc.Observable(mc.Rejected(test_config.rejected_fqn),
mc.Metrics(test_config))
test_subject = tc.HttpTransfer()
assert test_subject is not None, 'expect a result'
test_subject.observable = test_observable
with pytest.raises(mc.CadcException):
test_subject.get(test_source, test_destination)
assert get_mock.called, 'should have been called'
args, kwargs = get_mock.call_args
assert args[1] == test_source, 'wrong source name'
assert args[2] == test_destination, 'wrong dest name'
def test_repo_update(mock_client):
test_obs = mc.read_obs_from_file(TEST_OBS_FILE)
test_config = mc.Config()
test_config.observe_execution = True
test_metrics = mc.Metrics(test_config)
assert len(test_metrics.history) == 0, 'initial history conditions'
assert len(test_metrics.failures) == 0, 'initial failure conditions'
mc.repo_update(mock_client, test_obs, test_metrics)
mock_client.update.assert_called_with(test_obs), 'mock not called'
assert len(test_metrics.history) == 1, 'history conditions'
assert len(test_metrics.failures) == 0, 'failure conditions'
assert 'caom2' in test_metrics.history, 'history'
assert 'update' in test_metrics.history['caom2'], 'update'
assert 'test_obs_id' in test_metrics.history['caom2']['update'], 'obs id'
mock_client.reset_mock()
mock_client.update.side_effect = Exception('boo')
with pytest.raises(mc.CadcException):
mc.repo_update(mock_client, test_obs, test_metrics)
assert len(test_metrics.failures) == 1, 'should have failure counts'
def test_preview_augment_unknown_no_preview():
# what happens when it's not known that there's no preview
obs = mc.read_obs_from_file(TEST_OBS_FILE)
obs.planes[TEST_PRODUCT_ID].data_release = datetime.utcnow()
assert len(obs.planes[TEST_PRODUCT_ID].artifacts) == 1, 'initial condition'
# make sure the rejected file is empty
if os.path.exists(REJECTED_FILE):
os.unlink(REJECTED_FILE)
test_rejected = mc.Rejected(REJECTED_FILE)
test_config = mc.Config()
test_observable = mc.Observable(test_rejected, mc.Metrics(test_config))
cadc_client_mock = Mock()
kwargs = {'working_directory': TEST_DATA_DIR,
'cadc_client': cadc_client_mock,
'stream': 'stream',
'observable': test_observable}
with patch('caom2pipe.manage_composable.http_get',
side_effect=mc.CadcException(
'Internal Server Error for url: '
'https://archive.gemini.edu/preview')) as http_mock, \
patch('caom2pipe.manage_composable.data_put') as ad_put_mock, \
patch('caom2pipe.manage_composable.get_artifact_metadata') as \
art_mock, \
patch('caom2pipe.manage_composable.exec_cmd') as exec_mock:
cadc_client_mock.return_value.data_get.return_value = mc.CadcException(
'test')
result = preview_augmentation.visit(obs, **kwargs)
assert result is not None, 'expect result'
# 0 because the preview artifact doesn't already exist
assert result['artifacts'] == 0, 'wrong result'
test_url = f'{preview_augmentation.PREVIEW_URL}{TEST_PRODUCT_ID}.fits'
test_prev = f'{TEST_DATA_DIR}/{TEST_PRODUCT_ID}.jpg'
http_mock.assert_called_with(test_url, test_prev), 'mock not called'
assert not ad_put_mock.called, 'ad put mock should not be called'
assert not art_mock.called, 'art mock should not be called'
assert not exec_mock.called, 'exec mock should not be called'
