def check_status(self, response):
"""
Check the response status. Maps the application related requests
error status into Exceptions and raises the others
:param response: response
:return:
"""
try:
response.raise_for_status()
except requests.HTTPError as e:
if e.response.status_code == requests.codes.not_found:
raise exceptions.NotFoundException(orig_exception=e)
elif e.response.status_code == requests.codes.unauthorized:
raise exceptions.UnauthorizedException(orig_exception=e)
elif e.response.status_code == requests.codes.forbidden:
raise exceptions.ForbiddenException(orig_exception=e)
elif e.response.status_code == requests.codes.bad_request:
raise exceptions.BadRequestException(orig_exception=e)
elif e.response.status_code == requests.codes.conflict:
raise exceptions.AlreadyExistsException(orig_exception=e)
elif e.response.status_code == \
requests.codes.internal_server_error:
raise exceptions.InternalServerException(orig_exception=e)
elif e.response.status_code == \
requests.codes.request_entity_too_large:
raise exceptions.ByteLimitException(orig_exception=e)
elif self.retry and e.response.status_code in self.retry_errors:
raise e
else:
raise exceptions.UnexpectedException(orig_exception=e)
def __init__(self, resource_id, subject, agent, retry=True, host=None,
session_headers=None):
"""
Client constructor
:param resource_id -- ID of the resource being accessed (URI format)
as it appears in the registry.
:param subject -- The subject that is using the service
:type subject: cadcutil.auth.Subject
:param agent -- Name of the agent (application) that accesses the
service and its version, e.g. foo/1.0.2
:type agent: Subject
:param retry -- True if the client retries on transient errors False
otherwise
:param host -- override the name of the host the service is running on
(for testing purposes)
:param session_headers -- Headers used throughout the session -
dictionary format expected.
"""
self.logger = logging.getLogger('BaseWsClient')
logging.getLogger('BaseWsClient').addHandler(logging.NullHandler())
if resource_id is None:
raise ValueError('No resource ID provided')
if agent is None or not agent:
raise ValueError('agent is None or empty string')
self._session = None
self.subject = subject
self.resource_id = resource_id
self.retry = retry
self.session_headers = session_headers
# agent is / delimited key value pairs, separated by a space,
# containing the application name and version,
# plus the name and version of application libraries.
# eg: foo/1.0.2 foo-lib/1.2.3
self.agent = agent
# Get the package name and version, plus any imported libraries.
self.package_info = "cadcutils/{} requests/{}".format(
cadctools_version.version,
requests.__version__)
self.python_info = "{}/{}".format(platform.python_implementation(),
platform.python_version())
self.system_info = "{}/{}".format(platform.system(),
platform.version())
o_s = sys.platform
if o_s.lower().startswith('linux'):
distname, version, osid = distro.linux_distribution()
self.os_info = "{} {}".format(distname, version)
elif o_s == "darwin":
release, version, machine = platform.mac_ver()
self.os_info = "Mac OS X {}".format(release)
elif o_s.lower().startswith("win32"):
release, version, csd, ptype = platform.win32_ver()
self.os_info = "{} {}".format(release, version)
# build the corresponding capabilities instance
self.caps = WsCapabilities(self, host)
self._host = host
if resource_id.startswith('http'):
self._host = resource_id
if self._host is None:
base_url = self.caps.get_access_url(SERVICE_AVAILABILITY_ID)
self._host = urlparse(base_url).hostname
# Clients should add entries to this dict for specialized
# conversion of HTTP error codes into particular exceptions.
#
# Use this form to include a search string in the response to
# handle multiple possibilities for a single HTTP code.
# XXX : {'SEARCHSTRING1' : exceptionInstance1,
# 'SEARCHSTRING2' : exceptionInstance2}
#
# Otherwise provide a simple HTTP code -> exception mapping
# XXX : exceptionInstance
#
# The actual conversion is performed by get_exception()
self._HTTP_STATUS_CODE_EXCEPTIONS = {
401: exceptions.UnauthorizedException()}
def put_file(self,
archive,
src_file,
archive_stream=None,
mime_type=None,
mime_encoding=None,
md5_check=True,
input_name=None):
"""
Puts a file into the archive storage
:param archive: name of the archive
:param src_file: location of the source file
:param archive_stream: specific archive stream
:param mime_type: file mime type
:param mime_encoding: file mime encoding
:param md5_check: if True, calculate the md5sum before sending the file
Server will fail if it receives a corrupted file.
:param input_name: name to use in the archive overriding the actual
file name.
"""
if not archive:
raise AttributeError('No archive specified')
# We actually raise an exception here since the web
# service will normally respond with a 200 for an
# anonymous put, though not provide any endpoints.
if self._data_client.subject.anon:
raise exceptions.UnauthorizedException(
'Must be authenticated to put data')
self.logger.debug('PUT {}/{}'.format(archive, src_file))
headers = {}
if md5_check:
# calculate the md5sum
md5sum = self._get_md5sum(src_file)
headers['Content-MD5'] = md5sum
logger.debug('Set Content-MD5: {}'.format(md5sum))
if archive_stream is not None:
headers[ARCHIVE_STREAM_HTTP_HEADER] = str(archive_stream)
if mime_type is not None:
mtype = mime_type
elif MAGIC_WARN:
mtype = None
logger.warning(MAGIC_WARN)
else:
m = magic.Magic(mime=True)
mtype = m.from_file(os.path.realpath(src_file))
if mtype is not None:
headers['Content-Type'] = mtype
logger.debug('Set MIME type: {}'.format(mtype))
if mime_encoding:
mencoding = mime_encoding
elif MAGIC_WARN:
mencoding = None
if mtype:
logger.warning(MAGIC_WARN)
else:
m = magic.Magic(mime_encoding=True)
mencoding = m.from_file(os.path.realpath(src_file))
if mencoding:
headers['Content-Encoding'] = mencoding
logger.debug('Set MIME encoding: {}'.format(mencoding))
fname = input_name
if not fname:
fname = os.path.basename(src_file)
protocols = self._get_transfer_protocols(archive,
fname,
is_get=False,
headers=headers)
if len(protocols) == 0:
raise exceptions.HttpException('No URLs available to put data to')
# get the list of transfer points
for protocol in protocols:
url = protocol.endpoint
if url is None:
self.logger.debug('No endpoint for URI, skipping.')
continue
self.logger.debug('PUT to URL {}'.format(url))
try:
start = time.time()
with open(src_file, 'rb') as f:
self._data_client.put(url, headers=headers, data=f)
duration = time.time() - start
stat_info = os.stat(src_file)
self.logger.info(
('Successfully uploaded archive/file {}/{} in {}s '
'(avg. speed: {}MB/s)').format(
archive, src_file, round(duration, 2),
round(stat_info.st_size / 1024 / 1024 / duration, 2)))
return
except (exceptions.HttpException, socket.timeout) as e:
# try a different URL
self.logger.info(
'WARN: Cannot put data to {}. Exception: {}'.format(
url, e))
self.logger.warn('Try the next URL')
continue
raise exceptions.HttpException(
'Unable to put data from any of the available URLs')