def _read_metadata_footer(self, mime_documents_iter): try: with ChunkReadTimeout(self.client_timeout): footer_hdrs, footer_iter = next(mime_documents_iter) except ChunkReadTimeout: raise HTTPClientDisconnect() except StopIteration: raise HTTPBadRequest(body="couldn't find footer MIME doc") timeout_reader = self._make_timeout_reader(footer_iter) try: footer_body = ''.join(iter(timeout_reader, '')) except ChunkReadTimeout: raise HTTPClientDisconnect() footer_md5 = footer_hdrs.get('Content-MD5') if not footer_md5: raise HTTPBadRequest(body="no Content-MD5 in footer") if footer_md5 != md5(footer_body).hexdigest(): raise HTTPUnprocessableEntity(body="footer MD5 mismatch") try: return HeaderKeyDict(json.loads(footer_body)) except ValueError: raise HTTPBadRequest("invalid JSON for footer doc")
def _store_object(self, req, data_source, headers): kwargs = {} content_type = req.headers.get('content-type', 'octet/stream') policy = None container_info = self.container_info(self.account_name, self.container_name, req) if 'X-Oio-Storage-Policy' in req.headers: policy = req.headers.get('X-Oio-Storage-Policy') if not self.app.POLICIES.get_by_name(policy): raise HTTPBadRequest( "invalid policy '%s', must be in %s" % (policy, self.app.POLICIES.by_name.keys())) else: try: policy_index = int( req.headers.get('X-Backend-Storage-Policy-Index', container_info['storage_policy'])) except TypeError: policy_index = 0 if policy_index != 0: policy = self.app.POLICIES.get_by_index(policy_index).name else: content_length = int(req.headers.get('content-length', 0)) policy = self._get_auto_policy_from_size(content_length) ct_props = {'properties': {}, 'system': {}} metadata = self.load_object_metadata(headers) oio_headers = {REQID_HEADER: self.trans_id} oio_cache = req.environ.get('oio.cache') perfdata = req.environ.get('swift.perfdata') # only send headers if needed if SUPPORT_VERSIONING and headers.get(FORCEVERSIONING_HEADER): oio_headers[FORCEVERSIONING_HEADER] = \ headers.get(FORCEVERSIONING_HEADER) if req.environ.get('oio.force-version'): # In a case of MPU, it contains version of the UploadId # to be able to include version-id of MPU in S3 reponse kwargs['version'] = req.environ.get('oio.force-version') # In case a shard is being created, save the name of the S3 bucket # in a container property. This will be used when aggregating # container statistics to make bucket statistics. if BUCKET_NAME_HEADER in headers: bname = headers[BUCKET_NAME_HEADER] # FIXME(FVE): the segments container is not part of another bucket! # We should not have to strip this here. if bname and bname.endswith(MULTIUPLOAD_SUFFIX): bname = bname[:-len(MULTIUPLOAD_SUFFIX)] ct_props['system'][BUCKET_NAME_PROP] = bname try: _chunks, _size, checksum, _meta = self._object_create( self.account_name, self.container_name, obj_name=self.object_name, file_or_path=data_source, mime_type=content_type, policy=policy, headers=oio_headers, etag=req.headers.get('etag', '').strip('"'), properties=metadata, container_properties=ct_props, cache=oio_cache, perfdata=perfdata, **kwargs) # TODO(FVE): when oio-sds supports it, do that in a callback # passed to object_create (or whatever upload method supports it) footer_md = self.load_object_metadata(self._get_footers(req)) if footer_md: self.app.storage.object_set_properties(self.account_name, self.container_name, self.object_name, version=_meta.get( 'version', None), properties=footer_md, headers=oio_headers, cache=oio_cache, perfdata=perfdata) except exceptions.Conflict: raise HTTPConflict(request=req) except exceptions.PreconditionFailed: raise HTTPPreconditionFailed(request=req) except SourceReadTimeout as err: self.app.logger.warning(_('ERROR Client read timeout (%s)'), err) self.app.logger.increment('client_timeouts') raise HTTPRequestTimeout(request=req) except exceptions.SourceReadError: req.client_disconnect = True self.app.logger.warning( _('Client disconnected without sending last chunk')) self.app.logger.increment('client_disconnects') raise HTTPClientDisconnect(request=req) except exceptions.EtagMismatch: return HTTPUnprocessableEntity(request=req) except (exceptions.ServiceBusy, exceptions.OioTimeout, exceptions.DeadlineReached): raise except exceptions.NoSuchContainer: raise HTTPNotFound(request=req) except exceptions.ClientException as err: # 481 = CODE_POLICY_NOT_SATISFIABLE if err.status == 481: raise exceptions.ServiceBusy() self.app.logger.exception( _('ERROR Exception transferring data %s'), {'path': req.path}) raise HTTPInternalServerError(request=req) except Exception: self.app.logger.exception( _('ERROR Exception transferring data %s'), {'path': req.path}) raise HTTPInternalServerError(request=req) last_modified = int(_meta.get('mtime', math.ceil(time.time()))) # FIXME(FVE): if \x10 character in object name, decode version # number and set it in the response headers, instead of the oio # version number. version_id = _meta.get('version', 'null') resp = HTTPCreated(request=req, etag=checksum, last_modified=last_modified, headers={'x-object-sysmeta-version-id': version_id}) return resp
def _link_object(self, req): _, container, obj = req.headers['Oio-Copy-From'].split('/', 2) from_account = req.headers.get('X-Copy-From-Account', self.account_name) self.app.logger.info( "Creating link from %s/%s/%s to %s/%s/%s", # Existing from_account, container, obj, # New self.account_name, self.container_name, self.object_name) storage = self.app.storage if req.headers.get('Range'): raise Exception("Fast Copy with Range is unsupported") ranges = ranges_from_http_header(req.headers.get('Range')) if len(ranges) != 1: raise HTTPInternalServerError( request=req, body="mutiple ranges unsupported") ranges = ranges[0] else: ranges = None headers = self._prepare_headers(req) metadata = self.load_object_metadata(headers) oio_headers = {REQID_HEADER: self.trans_id} oio_cache = req.environ.get('oio.cache') perfdata = req.environ.get('swift.perfdata') # FIXME(FVE): use object_show, cache in req.environ version = obj_version_from_env(req.environ) props = storage.object_get_properties(from_account, container, obj, headers=oio_headers, version=version, cache=oio_cache, perfdata=perfdata) if props['properties'].get(SLO, None): raise Exception("Fast Copy with SLO is unsupported") else: if ranges: raise HTTPInternalServerError( request=req, body="no range supported with single object") try: # TODO check return code (values ?) link_meta = storage.object_link(from_account, container, obj, self.account_name, self.container_name, self.object_name, headers=oio_headers, properties=metadata, properties_directive='REPLACE', target_version=version, cache=oio_cache, perfdata=perfdata) # TODO(FVE): this exception catching block has to be refactored # TODO check which ones are ok or make non sense except exceptions.Conflict: raise HTTPConflict(request=req) except exceptions.PreconditionFailed: raise HTTPPreconditionFailed(request=req) except exceptions.SourceReadError: req.client_disconnect = True self.app.logger.warning( _('Client disconnected without sending last chunk')) self.app.logger.increment('client_disconnects') raise HTTPClientDisconnect(request=req) except exceptions.EtagMismatch: return HTTPUnprocessableEntity(request=req) except (exceptions.ServiceBusy, exceptions.OioTimeout, exceptions.DeadlineReached): raise except (exceptions.NoSuchContainer, exceptions.NotFound): raise HTTPNotFound(request=req) except exceptions.ClientException as err: # 481 = CODE_POLICY_NOT_SATISFIABLE if err.status == 481: raise exceptions.ServiceBusy() self.app.logger.exception( _('ERROR Exception transferring data %s'), {'path': req.path}) raise HTTPInternalServerError(request=req) except Exception: self.app.logger.exception( _('ERROR Exception transferring data %s'), {'path': req.path}) raise HTTPInternalServerError(request=req) resp = HTTPCreated(request=req, etag=link_meta['hash']) return resp
def PUT(self, request): """Handle HTTP PUT requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = \ get_name_and_placement(request, 5, 5, True) req_timestamp = valid_timestamp(request) error_response = check_object_creation(request, obj) if error_response: return error_response new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') try: fsize = request.message_length() except ValueError as e: return HTTPBadRequest(body=str(e), request=request, content_type='text/plain') try: disk_file = self.get_diskfile(device, partition, account, container, obj, policy_idx=policy_idx) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) except (DiskFileNotExist, DiskFileQuarantined): orig_metadata = {} # Checks for If-None-Match if request.if_none_match is not None and orig_metadata: if '*' in request.if_none_match: # File exists already so return 412 return HTTPPreconditionFailed(request=request) if orig_metadata.get('ETag') in request.if_none_match: # The current ETag matches, so return 412 return HTTPPreconditionFailed(request=request) orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0)) if orig_timestamp >= req_timestamp: return HTTPConflict( request=request, headers={'X-Backend-Timestamp': orig_timestamp.internal}) orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) upload_expiration = time.time() + self.max_upload_time etag = md5() elapsed_time = 0 try: with disk_file.create(size=fsize) as writer: upload_size = 0 def timeout_reader(): with ChunkReadTimeout(self.client_timeout): return request.environ['wsgi.input'].read( self.network_chunk_size) try: for chunk in iter(lambda: timeout_reader(), ''): start_time = time.time() if start_time > upload_expiration: self.logger.increment('PUT.timeouts') return HTTPRequestTimeout(request=request) etag.update(chunk) upload_size = writer.write(chunk) elapsed_time += time.time() - start_time except ChunkReadTimeout: return HTTPRequestTimeout(request=request) if upload_size: self.logger.transfer_rate('PUT.' + device + '.timing', elapsed_time, upload_size) if fsize is not None and fsize != upload_size: return HTTPClientDisconnect(request=request) etag = etag.hexdigest() if 'etag' in request.headers and \ request.headers['etag'].lower() != etag: return HTTPUnprocessableEntity(request=request) metadata = { 'X-Timestamp': request.timestamp.internal, 'Content-Type': request.headers['content-type'], 'ETag': etag, 'Content-Length': str(upload_size), } metadata.update(val for val in request.headers.iteritems() if is_sys_or_user_meta('object', val[0])) headers_to_copy = (request.headers.get( 'X-Backend-Replication-Headers', '').split() + list(self.allowed_headers)) for header_key in headers_to_copy: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] writer.put(metadata) except (DiskFileXattrNotSupported, DiskFileNoSpace): return HTTPInsufficientStorage(drive=device, request=request) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update('PUT', new_delete_at, account, container, obj, request, device, policy_idx) if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device, policy_idx) self.container_update( 'PUT', account, container, obj, request, HeaderKeyDict({ 'x-size': metadata['Content-Length'], 'x-content-type': metadata['Content-Type'], 'x-timestamp': metadata['X-Timestamp'], 'x-etag': metadata['ETag'] }), device, policy_idx) return HTTPCreated(request=request, etag=etag)
def PUT(self, request): """Handle HTTP PUT requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) if 'x-timestamp' not in request.headers or \ not check_float(request.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=request, content_type='text/plain') error_response = check_object_creation(request, obj) if error_response: return error_response new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') try: fsize = request.message_length() except ValueError as e: return HTTPBadRequest(body=str(e), request=request, content_type='text/plain') try: disk_file = self._diskfile(device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) old_delete_at = int(disk_file.metadata.get('X-Delete-At') or 0) orig_timestamp = disk_file.metadata.get('X-Timestamp') if orig_timestamp and orig_timestamp >= request.headers['x-timestamp']: return HTTPConflict(request=request) upload_expiration = time.time() + self.max_upload_time etag = md5() elapsed_time = 0 try: with disk_file.create(size=fsize) as writer: reader = request.environ['wsgi.input'].read for chunk in iter(lambda: reader(self.network_chunk_size), ''): start_time = time.time() if start_time > upload_expiration: self.logger.increment('PUT.timeouts') return HTTPRequestTimeout(request=request) etag.update(chunk) writer.write(chunk) sleep() elapsed_time += time.time() - start_time upload_size = writer.upload_size if upload_size: self.logger.transfer_rate('PUT.' + device + '.timing', elapsed_time, upload_size) if fsize is not None and fsize != upload_size: return HTTPClientDisconnect(request=request) etag = etag.hexdigest() if 'etag' in request.headers and \ request.headers['etag'].lower() != etag: return HTTPUnprocessableEntity(request=request) metadata = { 'X-Timestamp': request.headers['x-timestamp'], 'Content-Type': request.headers['content-type'], 'ETag': etag, 'Content-Length': str(upload_size), } metadata.update(val for val in request.headers.iteritems() if val[0].lower().startswith('x-object-meta-') and len(val[0]) > 14) for header_key in self.allowed_headers: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] writer.put(metadata) except DiskFileNoSpace: return HTTPInsufficientStorage(drive=device, request=request) if old_delete_at != new_delete_at: if new_delete_at: self.delete_at_update('PUT', new_delete_at, account, container, obj, request, device) if old_delete_at: self.delete_at_update('DELETE', old_delete_at, account, container, obj, request, device) if not orig_timestamp or \ orig_timestamp < request.headers['x-timestamp']: self.container_update( 'PUT', account, container, obj, request, HeaderKeyDict({ 'x-size': disk_file.metadata['Content-Length'], 'x-content-type': disk_file.metadata['Content-Type'], 'x-timestamp': disk_file.metadata['X-Timestamp'], 'x-etag': disk_file.metadata['ETag'] }), device) resp = HTTPCreated(request=request, etag=etag) return resp
def footers_callback(footers): if inner_callback: # pass on footers dict to any other callback that was # registered before this one. It may override any footers that # were set. inner_callback(footers) plaintext_etag = None if self.body_crypto_ctxt: plaintext_etag = self.plaintext_md5.hexdigest() # If client (or other middleware) supplied etag, then validate # against plaintext etag etag_to_check = footers.get('Etag') or client_etag if (etag_to_check is not None and plaintext_etag != etag_to_check): raise HTTPUnprocessableEntity(request=Request(self.env)) # override any previous notion of etag with the ciphertext etag footers['Etag'] = self.ciphertext_md5.hexdigest() # Encrypt the plaintext etag using the object key and persist # as sysmeta along with the crypto parameters that were used. encrypted_etag, etag_crypto_meta = encrypt_header_val( self.crypto, plaintext_etag, self.keys['object']) footers['X-Object-Sysmeta-Crypto-Etag'] = \ append_crypto_meta(encrypted_etag, etag_crypto_meta) footers['X-Object-Sysmeta-Crypto-Body-Meta'] = \ dump_crypto_meta(self.body_crypto_meta) # Also add an HMAC of the etag for use when evaluating # conditional requests footers['X-Object-Sysmeta-Crypto-Etag-Mac'] = _hmac_etag( self.keys['object'], plaintext_etag) else: # No data was read from body, nothing was encrypted, so don't # set any crypto sysmeta for the body, but do re-instate any # etag provided in inbound request if other middleware has not # already set a value. if client_etag is not None: footers.setdefault('Etag', client_etag) # When deciding on the etag that should appear in container # listings, look for: # * override in the footer, otherwise # * override in the header, and finally # * MD5 of the plaintext received # This may be None if no override was set and no data was read container_listing_etag = footers.get( 'X-Object-Sysmeta-Container-Update-Override-Etag', container_listing_etag_header) or plaintext_etag if (container_listing_etag is not None and (container_listing_etag != MD5_OF_EMPTY_STRING or plaintext_etag)): # Encrypt the container-listing etag using the container key # and a random IV, and use it to override the container update # value, with the crypto parameters appended. We use the # container key here so that only that key is required to # decrypt all etag values in a container listing when handling # a container GET request. Don't encrypt an EMPTY_ETAG # unless there actually was some body content, in which case # the container-listing etag is possibly conveying some # non-obvious information. val, crypto_meta = encrypt_header_val(self.crypto, container_listing_etag, self.keys['container']) crypto_meta['key_id'] = self.keys['id'] footers['X-Object-Sysmeta-Container-Update-Override-Etag'] = \ append_crypto_meta(val, crypto_meta)
def PUT(self, request): """Handle HTTP PUT requests for the Swift Object Server.""" device, partition, account, container, obj, policy = \ get_name_and_placement(request, 5, 5, True) req_timestamp = valid_timestamp(request) error_response = check_object_creation(request, obj) if error_response: return error_response new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') try: fsize = request.message_length() except ValueError as e: return HTTPBadRequest(body=str(e), request=request, content_type='text/plain') # In case of multipart-MIME put, the proxy sends a chunked request, # but may let us know the real content length so we can verify that # we have enough disk space to hold the object. if fsize is None: fsize = request.headers.get('X-Backend-Obj-Content-Length') if fsize is not None: try: fsize = int(fsize) except ValueError as e: return HTTPBadRequest(body=str(e), request=request, content_type='text/plain') # SSYNC will include Frag-Index header for subrequests to primary # nodes; handoff nodes should 409 subrequests to over-write an # existing data fragment until they offloaded the existing fragment frag_index = request.headers.get('X-Backend-Ssync-Frag-Index') try: disk_file = self.get_diskfile( device, partition, account, container, obj, policy=policy, frag_index=frag_index) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) except (DiskFileNotExist, DiskFileQuarantined): orig_metadata = {} # Checks for If-None-Match if request.if_none_match is not None and orig_metadata: if '*' in request.if_none_match: # File exists already so return 412 return HTTPPreconditionFailed(request=request) if orig_metadata.get('ETag') in request.if_none_match: # The current ETag matches, so return 412 return HTTPPreconditionFailed(request=request) orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0)) if orig_timestamp >= req_timestamp: return HTTPConflict( request=request, headers={'X-Backend-Timestamp': orig_timestamp.internal}) orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) upload_expiration = time.time() + self.max_upload_time etag = md5() elapsed_time = 0 try: with disk_file.create(size=fsize) as writer: upload_size = 0 # If the proxy wants to send us object metadata after the # object body, it sets some headers. We have to tell the # proxy, in the 100 Continue response, that we're able to # parse a multipart MIME document and extract the object and # metadata from it. If we don't, then the proxy won't # actually send the footer metadata. have_metadata_footer = False use_multiphase_commit = False mime_documents_iter = iter([]) obj_input = request.environ['wsgi.input'] hundred_continue_headers = [] if config_true_value( request.headers.get( 'X-Backend-Obj-Multiphase-Commit')): use_multiphase_commit = True hundred_continue_headers.append( ('X-Obj-Multiphase-Commit', 'yes')) if config_true_value( request.headers.get('X-Backend-Obj-Metadata-Footer')): have_metadata_footer = True hundred_continue_headers.append( ('X-Obj-Metadata-Footer', 'yes')) if have_metadata_footer or use_multiphase_commit: obj_input.set_hundred_continue_response_headers( hundred_continue_headers) mime_boundary = request.headers.get( 'X-Backend-Obj-Multipart-Mime-Boundary') if not mime_boundary: return HTTPBadRequest("no MIME boundary") try: with ChunkReadTimeout(self.client_timeout): mime_documents_iter = iter_mime_headers_and_bodies( request.environ['wsgi.input'], mime_boundary, self.network_chunk_size) _junk_hdrs, obj_input = next(mime_documents_iter) except ChunkReadTimeout: return HTTPRequestTimeout(request=request) timeout_reader = self._make_timeout_reader(obj_input) try: for chunk in iter(timeout_reader, ''): start_time = time.time() if start_time > upload_expiration: self.logger.increment('PUT.timeouts') return HTTPRequestTimeout(request=request) etag.update(chunk) upload_size = writer.write(chunk) elapsed_time += time.time() - start_time except ChunkReadTimeout: return HTTPRequestTimeout(request=request) if upload_size: self.logger.transfer_rate( 'PUT.' + device + '.timing', elapsed_time, upload_size) if fsize is not None and fsize != upload_size: return HTTPClientDisconnect(request=request) footer_meta = {} if have_metadata_footer: footer_meta = self._read_metadata_footer( mime_documents_iter) request_etag = (footer_meta.get('etag') or request.headers.get('etag', '')).lower() etag = etag.hexdigest() if request_etag and request_etag != etag: return HTTPUnprocessableEntity(request=request) metadata = { 'X-Timestamp': request.timestamp.internal, 'Content-Type': request.headers['content-type'], 'ETag': etag, 'Content-Length': str(upload_size), } metadata.update(val for val in request.headers.items() if is_sys_or_user_meta('object', val[0])) metadata.update(val for val in footer_meta.items() if is_sys_or_user_meta('object', val[0])) headers_to_copy = ( request.headers.get( 'X-Backend-Replication-Headers', '').split() + list(self.allowed_headers)) for header_key in headers_to_copy: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] writer.put(metadata) # if the PUT requires a two-phase commit (a data and a commit # phase) send the proxy server another 100-continue response # to indicate that we are finished writing object data if use_multiphase_commit: request.environ['wsgi.input'].\ send_hundred_continue_response() if not self._read_put_commit_message(mime_documents_iter): return HTTPServerError(request=request) # got 2nd phase confirmation, write a timestamp.durable # state file to indicate a successful PUT writer.commit(request.timestamp) # Drain any remaining MIME docs from the socket. There # shouldn't be any, but we must read the whole request body. try: while True: with ChunkReadTimeout(self.client_timeout): _junk_hdrs, _junk_body = next(mime_documents_iter) drain(_junk_body, self.network_chunk_size, self.client_timeout) except ChunkReadTimeout: raise HTTPClientDisconnect() except StopIteration: pass except (DiskFileXattrNotSupported, DiskFileNoSpace): return HTTPInsufficientStorage(drive=device, request=request) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update( 'PUT', new_delete_at, account, container, obj, request, device, policy) if orig_delete_at: self.delete_at_update( 'DELETE', orig_delete_at, account, container, obj, request, device, policy) update_headers = HeaderKeyDict({ 'x-size': metadata['Content-Length'], 'x-content-type': metadata['Content-Type'], 'x-timestamp': metadata['X-Timestamp'], 'x-etag': metadata['ETag']}) # apply any container update header overrides sent with request self._check_container_override(update_headers, request.headers) self._check_container_override(update_headers, footer_meta) self.container_update( 'PUT', account, container, obj, request, update_headers, device, policy) return HTTPCreated(request=request, etag=etag)
def _store_object(self, req, data_source, headers): content_type = req.headers.get('content-type', 'octet/stream') storage = self.app.storage policy = None container_info = self.container_info(self.account_name, self.container_name, req) if 'X-Oio-Storage-Policy' in req.headers: policy = req.headers.get('X-Oio-Storage-Policy') if not self.app.POLICIES.get_by_name(policy): raise HTTPBadRequest( "invalid policy '%s', must be in %s" % (policy, self.app.POLICIES.by_name.keys())) else: try: policy_index = int( req.headers.get('X-Backend-Storage-Policy-Index', container_info['storage_policy'])) except TypeError: policy_index = 0 if policy_index != 0: policy = self.app.POLICIES.get_by_index(policy_index).name else: content_length = int(req.headers.get('content-length', 0)) policy = self._get_auto_policy_from_size(content_length) metadata = self.load_object_metadata(headers) oio_headers = {'X-oio-req-id': self.trans_id} try: # FIXME(FVE): use 'properties' instead of 'metadata' # as soon as we require oio>=4.2.0 _chunks, _size, checksum = storage.object_create( self.account_name, self.container_name, obj_name=self.object_name, file_or_path=data_source, mime_type=content_type, policy=policy, headers=oio_headers, etag=req.headers.get('etag', '').strip('"'), metadata=metadata) # TODO(FVE): when oio-sds supports it, do that in a callback # passed to object_create (or whatever upload method supports it) footer_md = self.load_object_metadata(self._get_footers(req)) if footer_md: storage.object_set_properties(self.account_name, self.container_name, self.object_name, properties=footer_md) except exceptions.Conflict: raise HTTPConflict(request=req) except exceptions.PreconditionFailed: raise HTTPPreconditionFailed(request=req) except (SourceReadTimeout, GreenSourceReadTimeout) as err: self.app.logger.warning(_('ERROR Client read timeout (%s)'), err) self.app.logger.increment('client_timeouts') raise HTTPRequestTimeout(request=req) except exceptions.SourceReadError: req.client_disconnect = True self.app.logger.warning( _('Client disconnected without sending last chunk')) self.app.logger.increment('client_disconnects') raise HTTPClientDisconnect(request=req) except exceptions.EtagMismatch: return HTTPUnprocessableEntity(request=req) except (exceptions.ServiceBusy, exceptions.OioTimeout): raise # TODO(FVE): exceptions.NotFound is never raised from oio # Remove when dependency is oio>=4.2.0 except (exceptions.NoSuchContainer, exceptions.NotFound): raise HTTPNotFound(request=req) except exceptions.ClientException as err: # 481 = CODE_POLICY_NOT_SATISFIABLE if err.status == 481: raise exceptions.ServiceBusy() self.app.logger.exception( _('ERROR Exception transferring data %s'), {'path': req.path}) raise HTTPInternalServerError(request=req) except Exception: self.app.logger.exception( _('ERROR Exception transferring data %s'), {'path': req.path}) raise HTTPInternalServerError(request=req) # TODO(FVE): use the timestamp of the object. # Unfortunately the oio-sds API does not return the actual ctime. resp = HTTPCreated(request=req, etag=checksum, last_modified=int(time.time())) return resp
def _link_object(self, req): _, container, obj = req.headers['Oio-Copy-From'].split('/', 2) from_account = req.headers.get('X-Copy-From-Account', self.account_name) self.app.logger.info("LINK (%s,%s,%s) TO (%s,%s,%s)", from_account, self.container_name, self.object_name, self.account_name, container, obj) storage = self.app.storage if req.headers.get('Range'): raise Exception("Fast Copy with Range is unsupported") ranges = ranges_from_http_header(req.headers.get('Range')) if len(ranges) != 1: raise HTTPInternalServerError( request=req, body="mutiple ranges unsupported") ranges = ranges[0] else: ranges = None headers = self._prepare_headers(req) metadata = self.load_object_metadata(headers) oio_headers = {'X-oio-req-id': self.trans_id} # FIXME(FVE): use object_show, cache in req.environ props = storage.object_get_properties(from_account, container, obj) if props['properties'].get(SLO, None): raise Exception("Fast Copy with SLO is unsupported") if ranges is None: raise HTTPInternalServerError(request=req, body="LINK a MPU requires range") self.app.logger.debug("LINK, original object is a SLO") # retrieve manifest _, data = storage.object_fetch(from_account, container, obj) manifest = json.loads("".join(data)) offset = 0 # identify segment to copy for entry in manifest: if (ranges[0] == offset and ranges[1] + 1 == offset + entry['bytes']): _, container, obj = entry['name'].split('/', 2) checksum = entry['hash'] self.app.logger.info("LINK SLO (%s,%s,%s) TO (%s,%s,%s)", from_account, self.container_name, self.object_name, self.account_name, container, obj) break offset += entry['bytes'] else: raise HTTPInternalServerError(request=req, body="no segment matching range") else: checksum = props['hash'] if ranges: raise HTTPInternalServerError( request=req, body="no range supported with single object") try: # TODO check return code (values ?) storage.object_fastcopy(from_account, container, obj, self.account_name, self.container_name, self.object_name, headers=oio_headers, properties=metadata, properties_directive='REPLACE') # TODO(FVE): this exception catching block has to be refactored # TODO check which ones are ok or make non sense except exceptions.Conflict: raise HTTPConflict(request=req) except exceptions.PreconditionFailed: raise HTTPPreconditionFailed(request=req) except exceptions.SourceReadError: req.client_disconnect = True self.app.logger.warning( _('Client disconnected without sending last chunk')) self.app.logger.increment('client_disconnects') raise HTTPClientDisconnect(request=req) except exceptions.EtagMismatch: return HTTPUnprocessableEntity(request=req) except (exceptions.ServiceBusy, exceptions.OioTimeout): raise except (exceptions.NoSuchContainer, exceptions.NotFound): raise HTTPNotFound(request=req) except exceptions.ClientException as err: # 481 = CODE_POLICY_NOT_SATISFIABLE if err.status == 481: raise exceptions.ServiceBusy() self.app.logger.exception( _('ERROR Exception transferring data %s'), {'path': req.path}) raise HTTPInternalServerError(request=req) except Exception: self.app.logger.exception( _('ERROR Exception transferring data %s'), {'path': req.path}) raise HTTPInternalServerError(request=req) resp = HTTPCreated(request=req, etag=checksum) return resp
drop_buffer_cache(fd, last_sync, upload_size - last_sync) last_sync = upload_size sleep() elapsed_time += time.time() - start_time if upload_size: self.logger.transfer_rate('PUT.' + device + '.timing', elapsed_time, upload_size) if 'content-length' in request.headers and \ int(request.headers['content-length']) != upload_size: return HTTPClientDisconnect(request=request) etag = etag.hexdigest() if 'etag' in request.headers and \ request.headers['etag'].lower() != etag: return HTTPUnprocessableEntity(request=request) metadata = { 'X-Timestamp': request.headers['x-timestamp'], 'Content-Type': request.headers['content-type'], 'ETag': etag, 'Content-Length': str(upload_size), } metadata.update(val for val in request.headers.iteritems() if val[0].lower().startswith('x-object-meta-') and len(val[0]) > 14) for header_key in self.allowed_headers: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] old_delete_at = int(file.metadata.get('X-Delete-At') or 0) if old_delete_at != new_delete_at:
def _store_object(self, req, data_source, headers): content_type = req.headers.get('content-type', 'octet/stream') storage = self.app.storage policy = None container_info = self.container_info(self.account_name, self.container_name, req) if 'X-Oio-Storage-Policy' in req.headers: policy = req.headers.get('X-Oio-Storage-Policy') if not self.app.POLICIES.get_by_name(policy): raise HTTPBadRequest( "invalid policy '%s', must be in %s" % (policy, self.app.POLICIES.by_name.keys())) else: try: policy_index = int( req.headers.get('X-Backend-Storage-Policy-Index', container_info['storage_policy'])) except TypeError: policy_index = 0 if policy_index != 0: policy = self.app.POLICIES.get_by_index(policy_index).name else: content_length = int(req.headers.get('content-length', 0)) policy = self._get_auto_policy_from_size(content_length) metadata = self.load_object_metadata(headers) oio_headers = {REQID_HEADER: self.trans_id} # only send headers if needed if SUPPORT_VERSIONING and headers.get(FORCEVERSIONING_HEADER): oio_headers[FORCEVERSIONING_HEADER] = \ headers.get(FORCEVERSIONING_HEADER) try: _chunks, _size, checksum, _meta = self._object_create( self.account_name, self.container_name, obj_name=self.object_name, file_or_path=data_source, mime_type=content_type, policy=policy, headers=oio_headers, etag=req.headers.get('etag', '').strip('"'), properties=metadata) # TODO(FVE): when oio-sds supports it, do that in a callback # passed to object_create (or whatever upload method supports it) footer_md = self.load_object_metadata(self._get_footers(req)) if footer_md: storage.object_set_properties(self.account_name, self.container_name, self.object_name, version=_meta.get( 'version', None), properties=footer_md) except exceptions.Conflict: raise HTTPConflict(request=req) except exceptions.PreconditionFailed: raise HTTPPreconditionFailed(request=req) except SourceReadTimeout as err: self.app.logger.warning(_('ERROR Client read timeout (%s)'), err) self.app.logger.increment('client_timeouts') raise HTTPRequestTimeout(request=req) except exceptions.SourceReadError: req.client_disconnect = True self.app.logger.warning( _('Client disconnected without sending last chunk')) self.app.logger.increment('client_disconnects') raise HTTPClientDisconnect(request=req) except exceptions.EtagMismatch: return HTTPUnprocessableEntity(request=req) except (exceptions.ServiceBusy, exceptions.OioTimeout, exceptions.DeadlineReached): raise except exceptions.NoSuchContainer: raise HTTPNotFound(request=req) except exceptions.ClientException as err: # 481 = CODE_POLICY_NOT_SATISFIABLE if err.status == 481: raise exceptions.ServiceBusy() self.app.logger.exception( _('ERROR Exception transferring data %s'), {'path': req.path}) raise HTTPInternalServerError(request=req) except Exception: self.app.logger.exception( _('ERROR Exception transferring data %s'), {'path': req.path}) raise HTTPInternalServerError(request=req) last_modified = int(_meta.get('mtime', math.ceil(time.time()))) resp = HTTPCreated(request=req, etag=checksum, last_modified=last_modified, headers={ 'x-object-sysmeta-version-id': _meta.get('version', None) }) return resp
def _store_object(self, req, data_source, headers): content_type = req.headers.get('content-type', 'octet/stream') storage = self.app.storage policy = None container_info = self.container_info(self.account_name, self.container_name, req) if 'X-Oio-Storage-Policy' in req.headers: policy = req.headers.get('X-Oio-Storage-Policy') if not self.app.POLICIES.get_by_name(policy): raise HTTPBadRequest( "invalid policy '%s', must be in %s" % (policy, self.app.POLICIES.by_name.keys())) else: try: policy_index = int( req.headers.get('X-Backend-Storage-Policy-Index', container_info['storage_policy'])) except TypeError: policy_index = 0 if policy_index != 0: policy = self.app.POLICIES.get_by_index(policy_index).name else: content_length = int(req.headers.get('content-length', 0)) policy = self._get_auto_policy_from_size(content_length) metadata = self.load_object_metadata(headers) # TODO actually support if-none-match try: chunks, size, checksum = storage.object_create( self.account_name, self.container_name, obj_name=self.object_name, file_or_path=data_source, mime_type=content_type, policy=policy, etag=req.headers.get('etag', '').strip('"'), metadata=metadata) except exceptions.PreconditionFailed: raise HTTPPreconditionFailed(request=req) except SourceReadTimeout as err: self.app.logger.warning(_('ERROR Client read timeout (%ss)'), err.seconds) self.app.logger.increment('client_timeouts') raise HTTPRequestTimeout(request=req) except exceptions.SourceReadError: req.client_disconnect = True self.app.logger.warning( _('Client disconnected without sending last chunk')) self.app.logger.increment('client_disconnects') raise HTTPClientDisconnect(request=req) except exceptions.EtagMismatch: return HTTPUnprocessableEntity(request=req) except exceptions.OioTimeout: self.app.logger.exception( _('ERROR Exception causing client disconnect')) raise HTTPClientDisconnect(request=req) except ServiceBusy: raise except Exception: self.app.logger.exception( _('ERROR Exception transferring data %s'), {'path': req.path}) raise HTTPInternalServerError(request=req) resp = HTTPCreated(request=req, etag=checksum) return resp
def resp_iter(total_size=total_size): # wsgi won't propagate start_response calls until some data has # been yielded so make sure first heartbeat is sent immediately if heartbeat: yield ' ' last_yield_time = time.time() # BEGIN: New OpenIO code sub_req = make_subrequest( req.environ, path='%s?format=json&prefix=%s&limit=%d' % (segments_container_path, seg_prefix, self.max_manifest_segments), method='GET', headers={'x-auth-token': req.headers.get('x-auth-token')}, agent='%(orig)s SLO MultipartPUT', swift_source='SLO') sub_req.environ.setdefault('oio.query', {}) # All meta2 databases may not be synchronized sub_req.environ['oio.query']['force_master'] = True sub_req.environ['oio.query']['slo'] = True list_seg_resp = sub_req.get_response(self) with closing_if_possible(list_seg_resp.app_iter): segments_resp = json.loads(list_seg_resp.body) seg_resp_dict = dict() for seg_resp in segments_resp: obj_name = '/'.join(('', segments_container, seg_resp['name'])) seg_resp_dict[obj_name] = seg_resp for obj_name in path2indices: now = time.time() if heartbeat and (now - last_yield_time > self.yield_frequency): # Make sure we've called start_response before # sending data yield ' ' last_yield_time = now for i in path2indices[obj_name]: if not list_seg_resp.is_success: problem_segments.append( [quote(obj_name), list_seg_resp.status]) segment_length = 0 seg_data = None else: seg_resp = seg_resp_dict.get(obj_name) if seg_resp: segment_length, seg_data = validate_seg_dict( parsed_data[i], seg_resp, (i == len(parsed_data) - 1)) else: problem_segments.append([quote(obj_name), 404]) segment_length = 0 seg_data = None data_for_storage[i] = seg_data total_size += segment_length # END: New OpenIO code if problem_segments: err = HTTPBadRequest(content_type=out_content_type) resp_dict = {} if heartbeat: resp_dict['Response Status'] = err.status resp_dict['Response Body'] = err.body or '\n'.join( RESPONSE_REASONS.get(err.status_int, [''])) else: start_response(err.status, [(h, v) for h, v in err.headers.items() if h.lower() != 'content-length']) yield separator + get_response_body( out_content_type, resp_dict, problem_segments, 'upload') return slo_etag = md5() for seg_data in data_for_storage: if 'data' in seg_data: raw_data = base64.b64decode(seg_data['data']) slo_etag.update(md5(raw_data).hexdigest()) elif seg_data.get('range'): slo_etag.update('%s:%s;' % (seg_data['hash'], seg_data['range'])) else: slo_etag.update(seg_data['hash']) slo_etag = slo_etag.hexdigest() client_etag = req.headers.get('Etag') if client_etag and client_etag.strip('"') != slo_etag: err = HTTPUnprocessableEntity(request=req) if heartbeat: yield separator + get_response_body( out_content_type, { 'Response Status': err.status, 'Response Body': err.body or '\n'.join( RESPONSE_REASONS.get(err.status_int, [''])), }, problem_segments, 'upload') else: for chunk in err(req.environ, start_response): yield chunk return json_data = json.dumps(data_for_storage) if six.PY3: json_data = json_data.encode('utf-8') req.body = json_data req.headers.update({ SYSMETA_SLO_ETAG: slo_etag, SYSMETA_SLO_SIZE: total_size, 'X-Static-Large-Object': 'True', 'Etag': md5(json_data).hexdigest(), }) # Ensure container listings have both etags. However, if any # middleware to the left of us touched the base value, trust them. override_header = 'X-Object-Sysmeta-Container-Update-Override-Etag' val, sep, params = req.headers.get(override_header, '').partition(';') req.headers[override_header] = '%s; slo_etag=%s' % ( (val or req.headers['Etag']) + sep + params, slo_etag) env = req.environ if not env.get('CONTENT_TYPE'): guessed_type, _junk = mimetypes.guess_type(req.path_info) env['CONTENT_TYPE'] = (guessed_type or 'application/octet-stream') env['swift.content_type_overridden'] = True env['CONTENT_TYPE'] += ";swift_bytes=%d" % total_size resp = req.get_response(self.app) resp_dict = {'Response Status': resp.status} if resp.is_success: resp.etag = slo_etag resp_dict['Etag'] = resp.headers['Etag'] resp_dict['Last Modified'] = resp.headers['Last-Modified'] if heartbeat: resp_dict['Response Body'] = resp.body yield separator + get_response_body(out_content_type, resp_dict, [], 'upload') else: for chunk in resp(req.environ, start_response): yield chunk