def put_cadc_file(filename, stream, subject, mime_type=None, mime_encoding=None): """ Transfers a file to the CADC archive :param filename -- name of the file :param stream -- the name of archive stream at the CADC :param subject -- subject (type cadcutils.net.Subject) executing the command. :param mime_type -- file MIME type :param mime_encoding - file MIME encoding """ size = os.stat(filename).st_size transfer_result = { 'path': filename, 'size': round((size / 1024.0 / 1024.0), 2) } try: archive = etrans_config.get('etransfer', 'archive') if not archive: raise RuntimeError('Name of archive not found') start = time.time() data_client = CadcDataClient(subject) data_client.put_file(archive, filename, archive_stream=stream, mime_type=mime_type, mime_encoding=mime_encoding) duration = time.time() - start transfer_result['success'] = True transfer_result['time'] = duration transfer_result['speed'] = round(size / 1024 / 1024 / duration, 2) except Exception as e: transfer_result['success'] = False transfer_result['message'] = str(e) raise ProcError('Error transferring file: ' + str(e)) finally: _get_transfer_log().info('{} - {}'.format(LOG_PUT_LABEL, json.dumps(transfer_result)))
def test_put_file(basews_mock): client = CadcDataClient(auth.Subject()) # test a put file_name = '/tmp/putfile.txt' file_content = 'ABCDEFGH12345' hash_md5 = hashlib.md5() hash_md5.update(file_content.encode()) hash_md5 = hash_md5.hexdigest() # write the file with open(file_name, 'w') as f: f.write(file_content) put_mock = Mock() basews_mock.return_value.put = put_mock with pytest.raises(exceptions.UnauthorizedException): client.put_file('TEST', 'putfile', file_name) client._data_client.subject.anon = False # authenticate the user transf_end_point = 'http://test.ca/endpoint' def mock_get_trans_protocols(archive, file_name, is_get, headers): protocol = Mock() protocol.endpoint = '{}/{}'.format(transf_end_point, file_name) return [protocol] client._get_transfer_protocols = mock_get_trans_protocols client.put_file('TEST', file_name) # Note Content* headers automatically created by cadc-data except when # MAGIC_WANT -- libmagic not present put_mock.assert_called_with('{}/{}'.format(transf_end_point, os.path.basename(file_name)), data=ANY, headers={ 'Content-Type': 'text/plain', 'Content-Encoding': 'us-ascii', 'Content-MD5': '{}'.format(hash_md5) }) # mimic libmagic missing cadcdata.core.MAGIC_WARN = 'Some warning' put_mock.reset_mock() client.put_file('TEST', file_name) put_mock.assert_called_with( '{}/{}'.format(transf_end_point, os.path.basename(file_name)), data=ANY, headers={'Content-MD5': '835e7e6cd54e18ae21d50af963b0c32b'}) cadcdata.core.MAGIC_WARN = None # specify an archive stream and override the name of the file input_name = 'abc' client.put_file('TEST', file_name, archive_stream='default', input_name=input_name) put_mock.assert_called_with('{}/{}'.format(transf_end_point, input_name), data=ANY, headers={ 'Content-Encoding': 'us-ascii', 'X-CADC-Stream': 'default', 'Content-Type': 'text/plain', 'Content-MD5': '{}'.format(hash_md5) }) # specify the mime types client.put_file('TEST', file_name, archive_stream='default', mime_type='ASCII', mime_encoding='GZIP') put_mock.assert_called_with('{}/{}'.format(transf_end_point, os.path.basename(file_name)), data=ANY, headers={ 'Content-Encoding': 'GZIP', 'X-CADC-Stream': 'default', 'Content-Type': 'ASCII', 'Content-MD5': '{}'.format(hash_md5) }) os.remove(file_name)
class StorageClientWrapper: """ Wrap the choice between CadcDataClient and StorageInventoryClient. """ def __init__( self, subject, using_storage_inventory=True, resource_id='ivo://cadc.nrc.ca/uvic/minoc', metrics=None, ): """ :param subject: net.Subject instance for authentication and authorization :param using_storage_inventory: if True will use StorageInventoryClient for file operations at CADC. If False will use CadcDataClient. :param resource_id: str identifies the StorageInventoryClient endpoint. If using_storage_inventory is set to False, it's un-necessary. :param metrics: caom2pipe.manaage_composable.Metrics instance. If set, will track execution times, by action, from the beginning of the method invocation to the end of the method invocation, success or failure. Defaults to None, because fits2caom2 is a stand-alone application. """ if using_storage_inventory: self._cadc_client = StorageInventoryClient( subject=subject, resource_id=resource_id ) else: self._cadc_client = CadcDataClient(subject=subject) self._use_si = using_storage_inventory self._metrics = metrics self._logger = logging.getLogger(self.__class__.__name__) def _add_fail_metric(self, action, name): """Single location for the check for a self._metrics member in the failure case.""" if self._metrics is not None: client_name = 'si' if self._use_si else 'data' self._metrics.observe_failure(action, client_name, name) def _add_metric(self, action, name, start, value): """Single location for the check for a self._metrics member in the success case.""" if self._metrics is not None: client_name = 'si' if self._use_si else 'data' self._metrics.observe( start, StorageClientWrapper._current(), value, action, client_name, name, ) def get(self, working_directory, uri): """ Retrieve data. :param working_directory: str where the file will be retrieved to. Assumes the same machine as this function is being called from. :param uri: str this is an Artifact URI, representing the file to be retrieved. """ self._logger.debug(f'Being get for {uri} in {working_directory}') start = StorageClientWrapper._current() try: archive, f_name = self._decompose(uri) fqn = path.join(working_directory, f_name) if self._use_si: self._cadc_client.cadcget(uri, dest=fqn) else: self._cadc_client.get_file(archive, f_name, destination=fqn) except Exception as e: self._add_fail_metric('get', uri) self._logger.debug(traceback.format_exc()) raise exceptions.UnexpectedException( f'Did not retrieve {uri} because {e}' ) self._add_metric('get', uri, start, stat(fqn).st_size) self._logger.debug('End get') def get_head(self, uri): """ Retrieve FITS file header data. :param uri: str that is an Artifact URI, representing the file for which to retrieve headers :return: list of fits.Header instances """ self._logger.debug(f'Begin get_head for {uri}') start = StorageClientWrapper._current() try: b = BytesIO() b.name = uri if self._use_si: self._cadc_client.cadcget(uri, b, fhead=True) else: archive, f_name = StorageClientWrapper._decompose(uri) self._cadc_client.get_file(archive, f_name, b, fhead=True) fits_header = b.getvalue().decode('ascii') b.close() self._add_metric('get_head', uri, start, len(fits_header)) temp = make_headers_from_string(fits_header) self._logger.debug('End get_head') return temp except Exception as e: self._add_fail_metric('get_header', uri) self._logger.debug(traceback.format_exc()) self._logger.error(e) raise exceptions.UnexpectedException( f'Did not retrieve {uri} header because {e}' ) def info(self, uri): """ Retrieve the descriptive metdata associated with a file. :param uri: str that is an Artifact URI, representing the file for which to retrieve metadata :return: cadcdata.FileInfo instance, no scheme for md5sum """ self._logger.debug(f'Begin info for {uri}') try: if self._use_si: result = self._cadc_client.cadcinfo(uri) # make the result look like the other possible ways to # obtain metadata result.md5sum = result.md5sum.replace('md5:', '') else: archive, f_name = StorageClientWrapper._decompose(uri) temp = self._cadc_client.get_file_info(archive, f_name) result = FileInfo( id=uri, size=temp.get('size'), file_type=temp.get('type'), md5sum=temp.get('md5sum').replace('md5:', '') ) except exceptions.NotFoundException: self._logger.info(f'cadcinfo:: {uri} not found') result = None self._logger.debug('End info') return result def put(self, working_directory, uri, stream='default'): """ Store a file at CADC. :param working_directory: str fully-qualified name of where to find the file on the local machine :param uri: str that is an Artifact URI, representing the file to be stored at CADC. :param stream: str representing the namespace used by the CadcDataClient. Not required if using the StorageInventoryClient. 'default' is default name for a lately-created ad archive. """ self._logger.debug(f'Begin put for {uri} in {working_directory}') start = self._current() cwd = getcwd() archive, f_name = StorageClientWrapper._decompose(uri) fqn = path.join(working_directory, f_name) chdir(working_directory) try: local_meta = get_local_file_info(fqn) encoding = get_file_encoding(fqn) if self._use_si: replace = True cadc_meta = self.info(uri) if cadc_meta is None: replace = False self._logger.debug( f'uri {uri} src {fqn} replace {replace} file_type ' f'{local_meta.file_type} encoding {encoding} md5_checksum ' f'{local_meta.md5sum}' ) self._cadc_client.cadcput( uri, src=fqn, replace=replace, file_type=local_meta.file_type, file_encoding=encoding, md5_checksum=local_meta.md5sum, ) else: archive, f_name = self._decompose(uri) # libmagic does a worse job with guessing file types # than ad for .fits.gz => it will say 'binary' self._logger.debug( f'archive {archive} f_name {f_name} archive_stream ' f'{stream} mime_type {local_meta.file_type} ' f'mime_encoding {encoding} md5_check True ' ) self._cadc_client.put_file( archive, f_name, archive_stream=stream, mime_type=local_meta.file_type, mime_encoding=encoding, md5_check=True, ) self._logger.info(f'Stored {fqn} at CADC.') except Exception as e: self._add_fail_metric('put', uri) self._logger.debug(traceback.format_exc()) self._logger.error(e) raise exceptions.UnexpectedException( f'Failed to store data with {e}' ) finally: chdir(cwd) self._add_metric('put', uri, start, local_meta.size) self._logger.debug('End put') def remove(self, uri): """ Delete a file from CADC storage. :param uri: str that is an Artifact URI, representing the file to be removed from CADC. """ self._logger.debug(f'Begin remove for {uri}') start = StorageClientWrapper._current() if self._use_si: try: self._cadc_client.cadcremove(uri) except Exception as e: self._add_fail_metric('remove', uri) self._logger.debug(traceback.format_exc()) self._logger.error(e) raise exceptions.UnexpectedException( f'Did not remove {uri} because {e}' ) else: raise NotImplementedError( 'No remove functionality for CadcDataClient' ) self._add_metric('remove', uri, start, value=None) self._logger.debug('End remove') @staticmethod def _current(): """Encapsulate returning UTC now in microsecond resolution.""" return datetime.now(tz=timezone.utc).timestamp() @staticmethod def _decompose(uri): temp = urlparse(uri) return path.dirname(temp.path), path.basename(temp.path)