def look_pull_and_put(storage_name, fqn, url, clients, checksum):
"""Checks to see if a file exists at CADC. If yes, stop. If no,
pull via https to local storage, then put to CADC storage.
:param storage_name Artifact URI as the file will appear at CADC
:param fqn name on disk for caching between the
pull and the put
:param url for retrieving the file externally, if it does not exist
:param clients GemClientCollection instance
:param checksum what the CAOM observation says the checksum should be -
just the checksum part of ChecksumURI please, or the comparison will
always fail.
"""
cadc_meta = clients.data_client.info(storage_name)
if (checksum is not None and cadc_meta is not None
and cadc_meta.md5sum.replace('md5:', '') != checksum
) or cadc_meta is None:
logging.debug(
f'Different checksums: Source {checksum}, CADC {cadc_meta}')
mc.http_get(url, fqn)
clients.data_client.put(os.path.dirname(fqn), storage_name)
logging.info(f'Retrieved {os.path.basename(fqn)} for storage as '
f'{storage_name}')
else:
logging.info(f'{os.path.basename(fqn)} already exists at CADC.')
def get(self, source, dest_fqn):
"""
:param source: HTTP URL
:param dest_fqn: fully-qualified string that represents file name
:return:
"""
self._logger.debug(f'Retrieve {source}')
mc.http_get(source, dest_fqn)
if '.fits' in dest_fqn:
self.check(dest_fqn)
self._logger.debug(f'Successfully retrieved {source}')
def _retrieve_from_gemini(
gem_name,
observable,
plane,
preview_fqn,
):
temp = basename(gem_name.file_name)
preview_url = f'{PREVIEW_URL}{temp}'
try:
mc.http_get(preview_url, preview_fqn)
except Exception as e:
if observable.rejected.check_and_record(str(e), gem_name.prev):
_check_for_delete(gem_name.prev, gem_name.prev_uri, observable,
plane)
else:
raise e
def _retrieve_from_gemini(
gem_name, observable, plane, preview_fqn,
):
preview_url = f'{PREVIEW_URL}{plane.product_id}.fits'
new_retrieval = False
try:
mc.http_get(preview_url, preview_fqn)
new_retrieval = True
except Exception as e:
if observable.rejected.check_and_record(str(e), gem_name.prev):
_check_for_delete(
gem_name.prev, gem_name.prev_uri, observable, plane
)
else:
raise e
return new_retrieval
def test_http_get(mock_req):
# Response mock
class Object(object):
def __init__(self):
self.headers = {
'Date': 'Thu, 25 Jul 2019 16:10:02 GMT',
'Server': 'Apache',
'Content-Length': '4',
'Cache-Control': 'no-cache',
'Expired': '-1',
'Content-Disposition':
'attachment; filename="S20080610S0045.fits"',
'Connection': 'close',
'Content-Type': 'application/fits',
}
def raise_for_status(self):
pass
def iter_content(self, chunk_size):
return ['aaa'.encode(), 'bbb'.encode()]
def __enter__(self):
return self
def __exit__(self, a, b, c):
return None
test_object = Object()
mock_req.return_value = test_object
# wrong size
with pytest.raises(mc.CadcException) as ex:
mc.http_get('https://localhost/index.html', '/tmp/abc')
assert mock_req.called, 'mock not called'
assert 'File size error' in repr(ex.value), 'wrong failure'
mock_req.reset_mock()
# wrong checksum
test_object.headers['Content-Length'] = 6
test_object.headers['Content-Checksum'] = 'md5:abc'
with pytest.raises(mc.CadcException) as ex:
mc.http_get('https://localhost/index.html', '/tmp/abc')
assert mock_req.called, 'mock not called'
assert 'File checksum error' in repr(ex.value), 'wrong failure'
mock_req.reset_mock()
# checks succeed
test_object.headers[
'Content-Checksum'] = '6547436690a26a399603a7096e876a2d'
mc.http_get('https://localhost/index.html', '/tmp/abc')
assert mock_req.called, 'mock not called'