def put_cadc_file(filename,
stream,
subject,
mime_type=None,
mime_encoding=None):
"""
Transfers a file to the CADC archive
:param filename -- name of the file
:param stream -- the name of archive stream at the CADC
:param subject -- subject (type cadcutils.net.Subject) executing the
command.
:param mime_type -- file MIME type
:param mime_encoding - file MIME encoding
"""
size = os.stat(filename).st_size
transfer_result = {
'path': filename,
'size': round((size / 1024.0 / 1024.0), 2)
}
try:
archive = etrans_config.get('etransfer', 'archive')
if not archive:
raise RuntimeError('Name of archive not found')
start = time.time()
data_client = CadcDataClient(subject)
data_client.put_file(archive,
filename,
archive_stream=stream,
mime_type=mime_type,
mime_encoding=mime_encoding)
duration = time.time() - start
transfer_result['success'] = True
transfer_result['time'] = duration
transfer_result['speed'] = round(size / 1024 / 1024 / duration, 2)
except Exception as e:
transfer_result['success'] = False
transfer_result['message'] = str(e)
raise ProcError('Error transferring file: ' + str(e))
finally:
_get_transfer_log().info('{} - {}'.format(LOG_PUT_LABEL,
json.dumps(transfer_result)))
def test_put_file(basews_mock):
client = CadcDataClient(auth.Subject())
# test a put
file_name = '/tmp/putfile.txt'
file_content = 'ABCDEFGH12345'
hash_md5 = hashlib.md5()
hash_md5.update(file_content.encode())
hash_md5 = hash_md5.hexdigest()
# write the file
with open(file_name, 'w') as f:
f.write(file_content)
put_mock = Mock()
basews_mock.return_value.put = put_mock
with pytest.raises(exceptions.UnauthorizedException):
client.put_file('TEST', 'putfile', file_name)
client._data_client.subject.anon = False # authenticate the user
transf_end_point = 'http://test.ca/endpoint'
def mock_get_trans_protocols(archive, file_name, is_get, headers):
protocol = Mock()
protocol.endpoint = '{}/{}'.format(transf_end_point, file_name)
return [protocol]
client._get_transfer_protocols = mock_get_trans_protocols
client.put_file('TEST', file_name)
# Note Content* headers automatically created by cadc-data except when
# MAGIC_WANT -- libmagic not present
put_mock.assert_called_with('{}/{}'.format(transf_end_point,
os.path.basename(file_name)),
data=ANY,
headers={
'Content-Type': 'text/plain',
'Content-Encoding': 'us-ascii',
'Content-MD5': '{}'.format(hash_md5)
})
# mimic libmagic missing
cadcdata.core.MAGIC_WARN = 'Some warning'
put_mock.reset_mock()
client.put_file('TEST', file_name)
put_mock.assert_called_with(
'{}/{}'.format(transf_end_point, os.path.basename(file_name)),
data=ANY,
headers={'Content-MD5': '835e7e6cd54e18ae21d50af963b0c32b'})
cadcdata.core.MAGIC_WARN = None
# specify an archive stream and override the name of the file
input_name = 'abc'
client.put_file('TEST',
file_name,
archive_stream='default',
input_name=input_name)
put_mock.assert_called_with('{}/{}'.format(transf_end_point, input_name),
data=ANY,
headers={
'Content-Encoding': 'us-ascii',
'X-CADC-Stream': 'default',
'Content-Type': 'text/plain',
'Content-MD5': '{}'.format(hash_md5)
})
# specify the mime types
client.put_file('TEST',
file_name,
archive_stream='default',
mime_type='ASCII',
mime_encoding='GZIP')
put_mock.assert_called_with('{}/{}'.format(transf_end_point,
os.path.basename(file_name)),
data=ANY,
headers={
'Content-Encoding': 'GZIP',
'X-CADC-Stream': 'default',
'Content-Type': 'ASCII',
'Content-MD5': '{}'.format(hash_md5)
})
os.remove(file_name)
class StorageClientWrapper:
"""
Wrap the choice between CadcDataClient and StorageInventoryClient.
"""
def __init__(
self,
subject,
using_storage_inventory=True,
resource_id='ivo://cadc.nrc.ca/uvic/minoc',
metrics=None,
):
"""
:param subject: net.Subject instance for authentication and
authorization
:param using_storage_inventory: if True will use
StorageInventoryClient for file operations at CADC. If False will
use CadcDataClient.
:param resource_id: str identifies the StorageInventoryClient
endpoint. If using_storage_inventory is set to False, it's
un-necessary.
:param metrics: caom2pipe.manaage_composable.Metrics instance. If set,
will track execution times, by action, from the beginning of
the method invocation to the end of the method invocation,
success or failure. Defaults to None, because fits2caom2 is
a stand-alone application.
"""
if using_storage_inventory:
self._cadc_client = StorageInventoryClient(
subject=subject, resource_id=resource_id
)
else:
self._cadc_client = CadcDataClient(subject=subject)
self._use_si = using_storage_inventory
self._metrics = metrics
self._logger = logging.getLogger(self.__class__.__name__)
def _add_fail_metric(self, action, name):
"""Single location for the check for a self._metrics member in the
failure case."""
if self._metrics is not None:
client_name = 'si' if self._use_si else 'data'
self._metrics.observe_failure(action, client_name, name)
def _add_metric(self, action, name, start, value):
"""Single location for the check for a self._metrics member in the
success case."""
if self._metrics is not None:
client_name = 'si' if self._use_si else 'data'
self._metrics.observe(
start,
StorageClientWrapper._current(),
value,
action,
client_name,
name,
)
def get(self, working_directory, uri):
"""
Retrieve data.
:param working_directory: str where the file will be retrieved to.
Assumes the same machine as this function is being called from.
:param uri: str this is an Artifact URI, representing the file to
be retrieved.
"""
self._logger.debug(f'Being get for {uri} in {working_directory}')
start = StorageClientWrapper._current()
try:
archive, f_name = self._decompose(uri)
fqn = path.join(working_directory, f_name)
if self._use_si:
self._cadc_client.cadcget(uri, dest=fqn)
else:
self._cadc_client.get_file(archive, f_name, destination=fqn)
except Exception as e:
self._add_fail_metric('get', uri)
self._logger.debug(traceback.format_exc())
raise exceptions.UnexpectedException(
f'Did not retrieve {uri} because {e}'
)
self._add_metric('get', uri, start, stat(fqn).st_size)
self._logger.debug('End get')
def get_head(self, uri):
"""
Retrieve FITS file header data.
:param uri: str that is an Artifact URI, representing the file for
which to retrieve headers
:return: list of fits.Header instances
"""
self._logger.debug(f'Begin get_head for {uri}')
start = StorageClientWrapper._current()
try:
b = BytesIO()
b.name = uri
if self._use_si:
self._cadc_client.cadcget(uri, b, fhead=True)
else:
archive, f_name = StorageClientWrapper._decompose(uri)
self._cadc_client.get_file(archive, f_name, b, fhead=True)
fits_header = b.getvalue().decode('ascii')
b.close()
self._add_metric('get_head', uri, start, len(fits_header))
temp = make_headers_from_string(fits_header)
self._logger.debug('End get_head')
return temp
except Exception as e:
self._add_fail_metric('get_header', uri)
self._logger.debug(traceback.format_exc())
self._logger.error(e)
raise exceptions.UnexpectedException(
f'Did not retrieve {uri} header because {e}'
)
def info(self, uri):
"""
Retrieve the descriptive metdata associated with a file.
:param uri: str that is an Artifact URI, representing the file for
which to retrieve metadata
:return: cadcdata.FileInfo instance, no scheme for md5sum
"""
self._logger.debug(f'Begin info for {uri}')
try:
if self._use_si:
result = self._cadc_client.cadcinfo(uri)
# make the result look like the other possible ways to
# obtain metadata
result.md5sum = result.md5sum.replace('md5:', '')
else:
archive, f_name = StorageClientWrapper._decompose(uri)
temp = self._cadc_client.get_file_info(archive, f_name)
result = FileInfo(
id=uri,
size=temp.get('size'),
file_type=temp.get('type'),
md5sum=temp.get('md5sum').replace('md5:', '')
)
except exceptions.NotFoundException:
self._logger.info(f'cadcinfo:: {uri} not found')
result = None
self._logger.debug('End info')
return result
def put(self, working_directory, uri, stream='default'):
"""
Store a file at CADC.
:param working_directory: str fully-qualified name of where to find
the file on the local machine
:param uri: str that is an Artifact URI, representing the file to
be stored at CADC.
:param stream: str representing the namespace used by the
CadcDataClient. Not required if using the StorageInventoryClient.
'default' is default name for a lately-created ad archive.
"""
self._logger.debug(f'Begin put for {uri} in {working_directory}')
start = self._current()
cwd = getcwd()
archive, f_name = StorageClientWrapper._decompose(uri)
fqn = path.join(working_directory, f_name)
chdir(working_directory)
try:
local_meta = get_local_file_info(fqn)
encoding = get_file_encoding(fqn)
if self._use_si:
replace = True
cadc_meta = self.info(uri)
if cadc_meta is None:
replace = False
self._logger.debug(
f'uri {uri} src {fqn} replace {replace} file_type '
f'{local_meta.file_type} encoding {encoding} md5_checksum '
f'{local_meta.md5sum}'
)
self._cadc_client.cadcput(
uri,
src=fqn,
replace=replace,
file_type=local_meta.file_type,
file_encoding=encoding,
md5_checksum=local_meta.md5sum,
)
else:
archive, f_name = self._decompose(uri)
# libmagic does a worse job with guessing file types
# than ad for .fits.gz => it will say 'binary'
self._logger.debug(
f'archive {archive} f_name {f_name} archive_stream '
f'{stream} mime_type {local_meta.file_type} '
f'mime_encoding {encoding} md5_check True '
)
self._cadc_client.put_file(
archive,
f_name,
archive_stream=stream,
mime_type=local_meta.file_type,
mime_encoding=encoding,
md5_check=True,
)
self._logger.info(f'Stored {fqn} at CADC.')
except Exception as e:
self._add_fail_metric('put', uri)
self._logger.debug(traceback.format_exc())
self._logger.error(e)
raise exceptions.UnexpectedException(
f'Failed to store data with {e}'
)
finally:
chdir(cwd)
self._add_metric('put', uri, start, local_meta.size)
self._logger.debug('End put')
def remove(self, uri):
"""
Delete a file from CADC storage.
:param uri: str that is an Artifact URI, representing the file to
be removed from CADC.
"""
self._logger.debug(f'Begin remove for {uri}')
start = StorageClientWrapper._current()
if self._use_si:
try:
self._cadc_client.cadcremove(uri)
except Exception as e:
self._add_fail_metric('remove', uri)
self._logger.debug(traceback.format_exc())
self._logger.error(e)
raise exceptions.UnexpectedException(
f'Did not remove {uri} because {e}'
)
else:
raise NotImplementedError(
'No remove functionality for CadcDataClient'
)
self._add_metric('remove', uri, start, value=None)
self._logger.debug('End remove')
@staticmethod
def _current():
"""Encapsulate returning UTC now in microsecond resolution."""
return datetime.now(tz=timezone.utc).timestamp()
@staticmethod
def _decompose(uri):
temp = urlparse(uri)
return path.dirname(temp.path), path.basename(temp.path)
