def _validate_and_prep_request_headers(req): """ Validate that the value from x-symlink-target header is well formatted and that the x-symlink-target-etag header (if present) does not contain problematic characters. We assume the caller ensures that x-symlink-target header is present in req.headers. :param req: HTTP request object :returns: a tuple, the full versioned path to the object (as a WSGI string) and the X-Symlink-Target-Etag header value which may be None :raise: HTTPPreconditionFailed if x-symlink-target value is not well formatted. :raise: HTTPBadRequest if the x-symlink-target value points to the request path. :raise: HTTPBadRequest if the x-symlink-target-etag value contains a semicolon, double-quote, or backslash. """ # N.B. check_path_header doesn't assert the leading slash and # copy middleware may accept the format. In the symlink, API # says apparently to use "container/object" format so add the # validation first, here. error_body = 'X-Symlink-Target header must be of the form ' \ '<container name>/<object name>' if wsgi_unquote(req.headers[TGT_OBJ_SYMLINK_HDR]).startswith('/'): raise HTTPPreconditionFailed(body=error_body, request=req, content_type='text/plain') # check container and object format container, obj = check_path_header(req, TGT_OBJ_SYMLINK_HDR, 2, error_body) req.headers[TGT_OBJ_SYMLINK_HDR] = wsgi_quote('%s/%s' % (container, obj)) # Check account format if it exists account = check_account_format( req, wsgi_unquote(req.headers[TGT_ACCT_SYMLINK_HDR])) \ if TGT_ACCT_SYMLINK_HDR in req.headers else None # Extract request path _junk, req_acc, req_cont, req_obj = req.split_path(4, 4, True) if account: req.headers[TGT_ACCT_SYMLINK_HDR] = wsgi_quote(account) else: account = req_acc # Check if symlink targets the symlink itself or not if (account, container, obj) == (req_acc, req_cont, req_obj): raise HTTPBadRequest(body='Symlink cannot target itself', request=req, content_type='text/plain') etag = req.headers.get(TGT_ETAG_SYMLINK_HDR, None) if etag and any(c in etag for c in ';"\\'): # See cgi.parse_header for why the above chars are problematic raise HTTPBadRequest(body='Bad %s format' % TGT_ETAG_SYMLINK_HDR.title(), request=req, content_type='text/plain') if not (etag or req.headers.get('Content-Type')): req.headers['Content-Type'] = 'application/symlink' return '/v1/%s/%s/%s' % (account, container, obj), etag
def handle_COPY(self, req, start_response): if not req.headers.get('Destination'): return HTTPPreconditionFailed(request=req, body='Destination header required' )(req.environ, start_response) dest_account = self.account_name if 'Destination-Account' in req.headers: dest_account = req.headers.get('Destination-Account') dest_account = check_account_format(req, dest_account) req.headers['X-Copy-From-Account'] = self.account_name self.account_name = dest_account del req.headers['Destination-Account'] dest_container, dest_object = _check_destination_header(req) source = '/%s/%s' % (self.container_name, self.object_name) self.container_name = dest_container self.object_name = dest_object # re-write the existing request as a PUT instead of creating a new one req.method = 'PUT' # As this the path info is updated with destination container, # the proxy server app will use the right object controller # implementation corresponding to the container's policy type. ver, _junk = req.split_path(1, 2, rest_with_last=True) req.path_info = '/%s/%s/%s/%s' % \ (ver, dest_account, dest_container, dest_object) req.headers['Content-Length'] = 0 req.headers['X-Copy-From'] = quote(source) del req.headers['Destination'] return self.handle_PUT(req, start_response)
def COPY(self, req): """HTTP COPY request handler.""" if not req.headers.get('Destination'): return HTTPPreconditionFailed(request=req, body='Destination header required') dest_account = self.account_name if 'Destination-Account' in req.headers: dest_account = req.headers.get('Destination-Account') dest_account = check_account_format(req, dest_account) req.headers['X-Copy-From-Account'] = self.account_name self.account_name = dest_account del req.headers['Destination-Account'] dest_container, dest_object = check_destination_header(req) source = '/%s/%s' % (self.container_name, self.object_name) self.container_name = dest_container self.object_name = dest_object # re-write the existing request as a PUT instead of creating a new one # since this one is already attached to the posthooklogger req.method = 'PUT' req.path_info = '/v1/%s/%s/%s' % \ (dest_account, dest_container, dest_object) req.headers['Content-Length'] = 0 req.headers['X-Copy-From'] = quote(source) del req.headers['Destination'] return self.PUT(req)
def COPY(self, req): """HTTP COPY request handler.""" if not req.headers.get('Destination'): return HTTPPreconditionFailed(request=req, body='Destination header required') dest_account = self.account_name if 'Destination-Account' in req.headers: dest_account = req.headers.get('Destination-Account') dest_account = check_account_format(req, dest_account) req.headers['X-Copy-From-Account'] = self.account_name self.account_name = dest_account del req.headers['Destination-Account'] dest_container, dest_object = check_destination_header(req) source = '/%s/%s' % (self.container_name, self.object_name) self.container_name = dest_container self.object_name = dest_object # re-write the existing request as a PUT instead of creating a new one # since this one is already attached to the posthooklogger req.method = 'PUT' req.path_info = '/v1/%s/%s/%s' % \ (dest_account, dest_container, dest_object) req.headers['Content-Length'] = 0 req.headers['X-Copy-From'] = quote(source) del req.headers['Destination'] return self.PUT(req)
def object_request(self, req, version, account, container, obj, allow_versioned_writes): account_name = unquote(account) container_name = unquote(container) object_name = unquote(obj) container_info = None resp = None is_enabled = config_true_value(allow_versioned_writes) if req.method in ('PUT', 'DELETE'): container_info = get_container_info( req.environ, self.app) elif req.method == 'COPY' and 'Destination' in req.headers: if 'Destination-Account' in req.headers: account_name = req.headers.get('Destination-Account') account_name = check_account_format(req, account_name) container_name, object_name = check_destination_header(req) req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % ( version, account_name, container_name, object_name) container_info = get_container_info( req.environ, self.app) if not container_info: return self.app # To maintain backwards compatibility, container version # location could be stored as sysmeta or not, need to check both. # If stored as sysmeta, check if middleware is enabled. If sysmeta # is not set, but versions property is set in container_info, then # for backwards compatibility feature is enabled. object_versions = container_info.get( 'sysmeta', {}).get('versions-location') if object_versions and isinstance(object_versions, six.text_type): object_versions = object_versions.encode('utf-8') elif not object_versions: object_versions = container_info.get('versions') # if allow_versioned_writes is not set in the configuration files # but 'versions' is configured, enable feature to maintain # backwards compatibility if not allow_versioned_writes and object_versions: is_enabled = True if is_enabled and object_versions: object_versions = unquote(object_versions) vw_ctx = VersionedWritesContext(self.app, self.logger) if req.method in ('PUT', 'COPY'): policy_idx = req.headers.get( 'X-Backend-Storage-Policy-Index', container_info['storage_policy']) resp = vw_ctx.handle_obj_versions_put( req, object_versions, object_name, policy_idx) else: # handle DELETE resp = vw_ctx.handle_obj_versions_delete( req, object_versions, account_name, container_name, object_name) if resp: return resp else: return self.app
def object_request(self, req, version, account, container, obj, allow_versioned_writes): account_name = unquote(account) container_name = unquote(container) object_name = unquote(obj) container_info = None resp = None is_enabled = config_true_value(allow_versioned_writes) if req.method in ('PUT', 'DELETE'): container_info = get_container_info( req.environ, self.app) elif req.method == 'COPY' and 'Destination' in req.headers: if 'Destination-Account' in req.headers: account_name = req.headers.get('Destination-Account') account_name = check_account_format(req, account_name) container_name, object_name = check_destination_header(req) req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % ( version, account_name, container_name, object_name) container_info = get_container_info( req.environ, self.app) if not container_info: return self.app # To maintain backwards compatibility, container version # location could be stored as sysmeta or not, need to check both. # If stored as sysmeta, check if middleware is enabled. If sysmeta # is not set, but versions property is set in container_info, then # for backwards compatibility feature is enabled. object_versions = container_info.get( 'sysmeta', {}).get('versions-location') if object_versions and isinstance(object_versions, unicode): object_versions = object_versions.encode('utf-8') elif not object_versions: object_versions = container_info.get('versions') # if allow_versioned_writes is not set in the configuration files # but 'versions' is configured, enable feature to maintain # backwards compatibility if not allow_versioned_writes and object_versions: is_enabled = True if is_enabled and object_versions: object_versions = unquote(object_versions) vw_ctx = VersionedWritesContext(self.app, self.logger) if req.method in ('PUT', 'COPY'): policy_idx = req.headers.get( 'X-Backend-Storage-Policy-Index', container_info['storage_policy']) resp = vw_ctx.handle_obj_versions_put( req, object_versions, object_name, policy_idx) else: # handle DELETE resp = vw_ctx.handle_obj_versions_delete( req, object_versions, account_name, container_name, object_name) if resp: return resp else: return self.app
def _check_symlink_header(req): """ Validate that the value from x-symlink-target header is well formatted. We assume the caller ensures that x-symlink-target header is present in req.headers. :param req: HTTP request object :raise: HTTPPreconditionFailed if x-symlink-target value is not well formatted. :raise: HTTPBadRequest if the x-symlink-target value points to the request path. """ # N.B. check_path_header doesn't assert the leading slash and # copy middleware may accept the format. In the symlink, API # says apparently to use "container/object" format so add the # validation first, here. error_body = 'X-Symlink-Target header must be of the form ' \ '<container name>/<object name>' try: if wsgi_unquote(req.headers[TGT_OBJ_SYMLINK_HDR]).startswith('/'): raise HTTPPreconditionFailed(body=error_body, request=req, content_type='text/plain') except TypeError: raise HTTPPreconditionFailed(body=error_body, request=req, content_type='text/plain') # check container and object format container, obj = check_path_header(req, TGT_OBJ_SYMLINK_HDR, 2, error_body) req.headers[TGT_OBJ_SYMLINK_HDR] = wsgi_quote('%s/%s' % (container, obj)) # Check account format if it exists try: account = check_account_format( req, wsgi_unquote(req.headers[TGT_ACCT_SYMLINK_HDR])) \ if TGT_ACCT_SYMLINK_HDR in req.headers else None except TypeError: raise HTTPPreconditionFailed( body='Account name cannot contain slashes', request=req, content_type='text/plain') # Extract request path _junk, req_acc, req_cont, req_obj = req.split_path(4, 4, True) if account: req.headers[TGT_ACCT_SYMLINK_HDR] = wsgi_quote(account) else: account = req_acc # Check if symlink targets the symlink itself or not if (account, container, obj) == (req_acc, req_cont, req_obj): raise HTTPBadRequest(body='Symlink cannot target itself', request=req, content_type='text/plain')
def object_request(self, req, api_version, account, container, obj, allow_versioned_writes): account_name = unquote(account) container_name = unquote(container) object_name = unquote(obj) container_info = None resp = None is_enabled = config_true_value(allow_versioned_writes) if req.method in ('PUT', 'DELETE'): container_info = get_container_info(req.environ, self.app) elif req.method == 'COPY' and 'Destination' in req.headers: if 'Destination-Account' in req.headers: account_name = req.headers.get('Destination-Account') account_name = check_account_format(req, account_name) container_name, object_name = check_destination_header(req) req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % ( api_version, account_name, container_name, object_name) container_info = get_container_info(req.environ, self.app) if not container_info: return self.app # To maintain backwards compatibility, container version # location could be stored as sysmeta or not, need to check both. # If stored as sysmeta, check if middleware is enabled. If sysmeta # is not set, but versions property is set in container_info, then # for backwards compatibility feature is enabled. versions_cont = container_info.get('sysmeta', {}).get('versions-location') if not versions_cont: versions_cont = container_info.get('versions') # if allow_versioned_writes is not set in the configuration files # but 'versions' is configured, enable feature to maintain # backwards compatibility if not allow_versioned_writes and versions_cont: is_enabled = True if is_enabled and versions_cont: versions_cont = unquote(versions_cont).split('/')[0] vw_ctx = VersionedWritesContext(self.app, self.logger) if req.method in ('PUT', 'COPY'): resp = vw_ctx.handle_obj_versions_put(req, versions_cont, api_version, account_name, object_name) else: # handle DELETE resp = vw_ctx.handle_obj_versions_delete( req, versions_cont, api_version, account_name, container_name, object_name) if resp: return resp else: return self.app
def _check_symlink_header(req): """ Validate that the value from x-symlink-target header is well formatted. We assume the caller ensures that x-symlink-target header is present in req.headers. :param req: HTTP request object :raise: HTTPPreconditionFailed if x-symlink-target value is not well formatted. :raise: HTTPBadRequest if the x-symlink-target value points to the request path. """ # N.B. check_path_header doesn't assert the leading slash and # copy middleware may accept the format. In the symlink, API # says apparently to use "container/object" format so add the # validation first, here. if unquote(req.headers[TGT_OBJ_SYMLINK_HDR]).startswith('/'): raise HTTPPreconditionFailed( body='X-Symlink-Target header must be of the ' 'form <container name>/<object name>', request=req, content_type='text/plain') # check container and object format container, obj = check_path_header( req, TGT_OBJ_SYMLINK_HDR, 2, 'X-Symlink-Target header must be of the ' 'form <container name>/<object name>') # Check account format if it exists account = check_account_format( req, unquote(req.headers[TGT_ACCT_SYMLINK_HDR])) \ if TGT_ACCT_SYMLINK_HDR in req.headers else None # Extract request path _junk, req_acc, req_cont, req_obj = req.split_path(4, 4, True) if not account: account = req_acc # Check if symlink targets the symlink itself or not if (account, container, obj) == (req_acc, req_cont, req_obj): raise HTTPBadRequest( body='Symlink cannot target itself', request=req, content_type='text/plain')
def __call__(self, env, start_response): req = Request(env) if req.method not in self.write_methods: return self.app(env, start_response) try: version, account, container, obj = req.split_path(1, 4, True) except ValueError: return self.app(env, start_response) if req.method == 'COPY' and 'Destination-Account' in req.headers: dest_account = req.headers.get('Destination-Account') account = check_account_format(req, dest_account) if self.account_read_only(req, account): msg = 'Writes are disabled for this account.' return HTTPMethodNotAllowed(body=msg)(env, start_response) return self.app(env, start_response)
def __call__(self, env, start_response): req = Request(env) if req.method not in self.write_methods: return self.app(env, start_response) try: version, account, container, obj = req.split_path(1, 4, True) except ValueError: return self.app(env, start_response) if req.method == 'COPY' and 'Destination-Account' in req.headers: dest_account = req.headers.get('Destination-Account') account = check_account_format(req, dest_account) if self.account_read_only(req, account): msg = 'Writes are disabled for this account.' return HTTPMethodNotAllowed(body=msg)(env, start_response) return self.app(env, start_response)
def PUT(self, req): """HTTP PUT request handler.""" if req.if_none_match is not None and '*' not in req.if_none_match: # Sending an etag with if-none-match isn't currently supported return HTTPBadRequest(request=req, content_type='text/plain', body='If-None-Match only supports *') if 'swift.authorize' in req.environ: aresp = req.environ['swift.authorize'](req) if aresp: return aresp # Sometimes the 'content-type' header exists, but is set to None. content_type_manually_set = True detect_content_type = \ config_true_value(req.headers.get('x-detect-content-type')) if detect_content_type or not req.headers.get('content-type'): guessed_type, _junk = mimetypes.guess_type(req.path_info) req.headers['Content-Type'] = guessed_type or \ 'application/octet-stream' if detect_content_type: req.headers.pop('x-detect-content-type') else: content_type_manually_set = False error_response = check_object_creation(req, self.object_name) or \ check_content_type(req) if error_response: return error_response req.headers['X-Timestamp'] = Timestamp(time.time()).internal stream = req.environ['wsgi.input'] source_header = req.headers.get('X-Copy-From') source_resp = None if source_header: if req.environ.get('swift.orig_req_method', req.method) != 'POST': req.environ.setdefault('swift.log_info', []).append( 'x-copy-from:%s' % source_header) ver, acct, _rest = req.split_path(2, 3, True) src_account_name = req.headers.get('X-Copy-From-Account', None) if src_account_name: src_account_name = check_account_format(req, src_account_name) else: src_account_name = acct src_container_name, src_obj_name = check_copy_from_header(req) source_header = '/%s/%s/%s/%s' % (ver, src_account_name, src_container_name, src_obj_name) source_req = req.copy_get() # make sure the source request uses it's container_info source_req.headers.pop('X-Backend-Storage-Policy-Index', None) source_req.path_info = source_header source_req.headers['X-Newest'] = 'true' orig_obj_name = self.object_name orig_container_name = self.container_name orig_account_name = self.account_name self.object_name = src_obj_name self.container_name = src_container_name self.account_name = src_account_name sink_req = Request.blank(req.path_info, environ=req.environ, headers=req.headers) source_resp = self.GET(source_req) # This gives middlewares a way to change the source; for example, # this lets you COPY a SLO manifest and have the new object be the # concatenation of the segments (like what a GET request gives # the client), not a copy of the manifest file. hook = req.environ.get( 'swift.copy_hook', (lambda source_req, source_resp, sink_req: source_resp)) source_resp = hook(source_req, source_resp, sink_req) if source_resp.status_int >= HTTP_MULTIPLE_CHOICES: return source_resp self.object_name = orig_obj_name self.container_name = orig_container_name self.account_name = orig_account_name stream = IterO(source_resp.app_iter) sink_req.content_length = source_resp.content_length if sink_req.content_length is None: # This indicates a transfer-encoding: chunked source object, # which currently only happens because there are more than # CONTAINER_LISTING_LIMIT segments in a segmented object. In # this case, we're going to refuse to do the server-side copy. return HTTPRequestEntityTooLarge(request=req) if sink_req.content_length > constraints.MAX_FILE_SIZE: return HTTPRequestEntityTooLarge(request=req) sink_req.etag = source_resp.etag # we no longer need the X-Copy-From header del sink_req.headers['X-Copy-From'] if 'X-Copy-From-Account' in sink_req.headers: del sink_req.headers['X-Copy-From-Account'] if not content_type_manually_set: sink_req.headers['Content-Type'] = \ source_resp.headers['Content-Type'] if config_true_value( sink_req.headers.get('x-fresh-metadata', 'false')): # post-as-copy: ignore new sysmeta, copy existing sysmeta condition = lambda k: is_sys_meta('object', k) remove_items(sink_req.headers, condition) copy_header_subset(source_resp, sink_req, condition) else: # copy/update existing sysmeta and user meta copy_headers_into(source_resp, sink_req) copy_headers_into(req, sink_req) # copy over x-static-large-object for POSTs and manifest copies if 'X-Static-Large-Object' in source_resp.headers and \ req.params.get('multipart-manifest') == 'get': sink_req.headers['X-Static-Large-Object'] = \ source_resp.headers['X-Static-Large-Object'] req = sink_req content_length = req.content_length content_type = req.headers.get('content-type', 'octet/stream') storage = self.app.storage if content_length is None: content_length = 0 try: chunks, size, checksum = storage.object_create(self.account_name, self.container_name, obj_name=self.object_name, file_or_path=stream, content_length=content_length, content_type=content_type) except exceptions.NoSuchContainer: return HTTPNotFound(request=req) except exceptions.ClientReadTimeout: return HTTPRequestTimeout(request=req) resp = HTTPCreated(request=req, etag=checksum) if source_header: acct, path = source_header.split('/', 3)[2:4] resp.headers['X-Copied-From-Account'] = quote(acct) resp.headers['X-Copied-From'] = quote(path) if 'last-modified' in source_resp.headers: resp.headers['X-Copied-From-Last-Modified'] = \ source_resp.headers['last-modified'] copy_headers_into(req, resp) resp.last_modified = math.ceil( float(Timestamp(req.headers['X-Timestamp']))) return resp
def PUT(self, req): """HTTP PUT request handler.""" if req.if_none_match is not None and '*' not in req.if_none_match: # Sending an etag with if-none-match isn't currently supported return HTTPBadRequest(request=req, content_type='text/plain', body='If-None-Match only supports *') container_info = self.container_info( self.account_name, self.container_name, req) policy_index = req.headers.get('X-Backend-Storage-Policy-Index', container_info['storage_policy']) obj_ring = self.app.get_object_ring(policy_index) # pass the policy index to storage nodes via req header req.headers['X-Backend-Storage-Policy-Index'] = policy_index container_partition = container_info['partition'] containers = container_info['nodes'] req.acl = container_info['write_acl'] req.environ['swift_sync_key'] = container_info['sync_key'] object_versions = container_info['versions'] if 'swift.authorize' in req.environ: aresp = req.environ['swift.authorize'](req) if aresp: return aresp if not containers: return HTTPNotFound(request=req) # Sometimes the 'content-type' header exists, but is set to None. content_type_manually_set = True detect_content_type = \ config_true_value(req.headers.get('x-detect-content-type')) if detect_content_type or not req.headers.get('content-type'): guessed_type, _junk = mimetypes.guess_type(req.path_info) req.headers['Content-Type'] = guessed_type or \ 'application/octet-stream' if detect_content_type: req.headers.pop('x-detect-content-type') else: content_type_manually_set = False error_response = check_object_creation(req, self.object_name) or \ check_content_type(req) if error_response: return error_response partition, nodes = obj_ring.get_nodes( self.account_name, self.container_name, self.object_name) # do a HEAD request for container sync and checking object versions if 'x-timestamp' in req.headers or \ (object_versions and not req.environ.get('swift_versioned_copy')): # make sure proxy-server uses the right policy index _headers = {'X-Backend-Storage-Policy-Index': policy_index, 'X-Newest': 'True'} hreq = Request.blank(req.path_info, headers=_headers, environ={'REQUEST_METHOD': 'HEAD'}) hresp = self.GETorHEAD_base( hreq, _('Object'), obj_ring, partition, hreq.swift_entity_path) # Used by container sync feature if 'x-timestamp' in req.headers: try: req_timestamp = Timestamp(req.headers['X-Timestamp']) if hresp.environ and 'swift_x_timestamp' in hresp.environ and \ hresp.environ['swift_x_timestamp'] >= req_timestamp: return HTTPAccepted(request=req) except ValueError: return HTTPBadRequest( request=req, content_type='text/plain', body='X-Timestamp should be a UNIX timestamp float value; ' 'was %r' % req.headers['x-timestamp']) req.headers['X-Timestamp'] = req_timestamp.internal else: req.headers['X-Timestamp'] = Timestamp(time.time()).internal if object_versions and not req.environ.get('swift_versioned_copy'): if hresp.status_int != HTTP_NOT_FOUND: # This is a version manifest and needs to be handled # differently. First copy the existing data to a new object, # then write the data from this request to the version manifest # object. lcontainer = object_versions.split('/')[0] prefix_len = '%03x' % len(self.object_name) lprefix = prefix_len + self.object_name + '/' ts_source = hresp.environ.get('swift_x_timestamp') if ts_source is None: ts_source = time.mktime(time.strptime( hresp.headers['last-modified'], '%a, %d %b %Y %H:%M:%S GMT')) new_ts = Timestamp(ts_source).internal vers_obj_name = lprefix + new_ts copy_headers = { 'Destination': '%s/%s' % (lcontainer, vers_obj_name)} copy_environ = {'REQUEST_METHOD': 'COPY', 'swift_versioned_copy': True } copy_req = Request.blank(req.path_info, headers=copy_headers, environ=copy_environ) copy_resp = self.COPY(copy_req) if is_client_error(copy_resp.status_int): # missing container or bad permissions return HTTPPreconditionFailed(request=req) elif not is_success(copy_resp.status_int): # could not copy the data, bail return HTTPServiceUnavailable(request=req) reader = req.environ['wsgi.input'].read data_source = iter(lambda: reader(self.app.client_chunk_size), '') source_header = req.headers.get('X-Copy-From') source_resp = None if source_header: if req.environ.get('swift.orig_req_method', req.method) != 'POST': req.environ.setdefault('swift.log_info', []).append( 'x-copy-from:%s' % source_header) ver, acct, _rest = req.split_path(2, 3, True) src_account_name = req.headers.get('X-Copy-From-Account', None) if src_account_name: src_account_name = check_account_format(req, src_account_name) else: src_account_name = acct src_container_name, src_obj_name = check_copy_from_header(req) source_header = '/%s/%s/%s/%s' % (ver, src_account_name, src_container_name, src_obj_name) source_req = req.copy_get() # make sure the source request uses it's container_info source_req.headers.pop('X-Backend-Storage-Policy-Index', None) source_req.path_info = source_header source_req.headers['X-Newest'] = 'true' orig_obj_name = self.object_name orig_container_name = self.container_name orig_account_name = self.account_name self.object_name = src_obj_name self.container_name = src_container_name self.account_name = src_account_name sink_req = Request.blank(req.path_info, environ=req.environ, headers=req.headers) source_resp = self.GET(source_req) # This gives middlewares a way to change the source; for example, # this lets you COPY a SLO manifest and have the new object be the # concatenation of the segments (like what a GET request gives # the client), not a copy of the manifest file. hook = req.environ.get( 'swift.copy_hook', (lambda source_req, source_resp, sink_req: source_resp)) source_resp = hook(source_req, source_resp, sink_req) if source_resp.status_int >= HTTP_MULTIPLE_CHOICES: return source_resp self.object_name = orig_obj_name self.container_name = orig_container_name self.account_name = orig_account_name data_source = iter(source_resp.app_iter) sink_req.content_length = source_resp.content_length if sink_req.content_length is None: # This indicates a transfer-encoding: chunked source object, # which currently only happens because there are more than # CONTAINER_LISTING_LIMIT segments in a segmented object. In # this case, we're going to refuse to do the server-side copy. return HTTPRequestEntityTooLarge(request=req) if sink_req.content_length > constraints.MAX_FILE_SIZE: return HTTPRequestEntityTooLarge(request=req) sink_req.etag = source_resp.etag # we no longer need the X-Copy-From header del sink_req.headers['X-Copy-From'] if 'X-Copy-From-Account' in sink_req.headers: del sink_req.headers['X-Copy-From-Account'] if not content_type_manually_set: sink_req.headers['Content-Type'] = \ source_resp.headers['Content-Type'] if config_true_value( sink_req.headers.get('x-fresh-metadata', 'false')): # post-as-copy: ignore new sysmeta, copy existing sysmeta condition = lambda k: is_sys_meta('object', k) remove_items(sink_req.headers, condition) copy_header_subset(source_resp, sink_req, condition) else: # copy/update existing sysmeta and user meta copy_headers_into(source_resp, sink_req) copy_headers_into(req, sink_req) # copy over x-static-large-object for POSTs and manifest copies if 'X-Static-Large-Object' in source_resp.headers and \ req.params.get('multipart-manifest') == 'get': sink_req.headers['X-Static-Large-Object'] = \ source_resp.headers['X-Static-Large-Object'] req = sink_req req, delete_at_container, delete_at_part, \ delete_at_nodes = self._config_obj_expiration(req) node_iter = GreenthreadSafeIterator( self.iter_nodes_local_first(obj_ring, partition)) pile = GreenPile(len(nodes)) te = req.headers.get('transfer-encoding', '') chunked = ('chunked' in te) outgoing_headers = self._backend_requests( req, len(nodes), container_partition, containers, delete_at_container, delete_at_part, delete_at_nodes) for nheaders in outgoing_headers: # RFC2616:8.2.3 disallows 100-continue without a body if (req.content_length > 0) or chunked: nheaders['Expect'] = '100-continue' pile.spawn(self._connect_put_node, node_iter, partition, req.swift_entity_path, nheaders, self.app.logger.thread_locals) conns = [conn for conn in pile if conn] min_conns = quorum_size(len(nodes)) if req.if_none_match is not None and '*' in req.if_none_match: statuses = [conn.resp.status for conn in conns if conn.resp] if HTTP_PRECONDITION_FAILED in statuses: # If we find any copy of the file, it shouldn't be uploaded self.app.logger.debug( _('Object PUT returning 412, %(statuses)r'), {'statuses': statuses}) return HTTPPreconditionFailed(request=req) if len(conns) < min_conns: self.app.logger.error( _('Object PUT returning 503, %(conns)s/%(nodes)s ' 'required connections'), {'conns': len(conns), 'nodes': min_conns}) return HTTPServiceUnavailable(request=req) bytes_transferred = 0 try: with ContextPool(len(nodes)) as pool: for conn in conns: conn.failed = False conn.queue = Queue(self.app.put_queue_depth) pool.spawn(self._send_file, conn, req.path) while True: with ChunkReadTimeout(self.app.client_timeout): try: chunk = next(data_source) except StopIteration: if chunked: for conn in conns: conn.queue.put('0\r\n\r\n') break bytes_transferred += len(chunk) if bytes_transferred > constraints.MAX_FILE_SIZE: return HTTPRequestEntityTooLarge(request=req) for conn in list(conns): if not conn.failed: conn.queue.put( '%x\r\n%s\r\n' % (len(chunk), chunk) if chunked else chunk) else: conns.remove(conn) if len(conns) < min_conns: self.app.logger.error(_( 'Object PUT exceptions during' ' send, %(conns)s/%(nodes)s required connections'), {'conns': len(conns), 'nodes': min_conns}) return HTTPServiceUnavailable(request=req) for conn in conns: if conn.queue.unfinished_tasks: conn.queue.join() conns = [conn for conn in conns if not conn.failed] except ChunkReadTimeout as err: self.app.logger.warn( _('ERROR Client read timeout (%ss)'), err.seconds) self.app.logger.increment('client_timeouts') return HTTPRequestTimeout(request=req) except (Exception, Timeout): self.app.logger.exception( _('ERROR Exception causing client disconnect')) return HTTPClientDisconnect(request=req) if req.content_length and bytes_transferred < req.content_length: req.client_disconnect = True self.app.logger.warn( _('Client disconnected without sending enough data')) self.app.logger.increment('client_disconnects') return HTTPClientDisconnect(request=req) statuses, reasons, bodies, etags = self._get_put_responses(req, conns, nodes) if len(etags) > 1: self.app.logger.error( _('Object servers returned %s mismatched etags'), len(etags)) return HTTPServerError(request=req) etag = etags.pop() if len(etags) else None resp = self.best_response(req, statuses, reasons, bodies, _('Object PUT'), etag=etag) if source_header: acct, path = source_header.split('/', 3)[2:4] resp.headers['X-Copied-From-Account'] = quote(acct) resp.headers['X-Copied-From'] = quote(path) if 'last-modified' in source_resp.headers: resp.headers['X-Copied-From-Last-Modified'] = \ source_resp.headers['last-modified'] copy_headers_into(req, resp) resp.last_modified = math.ceil( float(Timestamp(req.headers['X-Timestamp']))) return resp
def handle_PUT(self, req, start_response): if req.content_length: return HTTPBadRequest(body='Copy requests require a zero byte ' 'body', request=req, content_type='text/plain')(req.environ, start_response) # Form the path of source object to be fetched ver, acct, _rest = req.split_path(2, 3, True) src_account_name = req.headers.get('X-Copy-From-Account') if src_account_name: src_account_name = check_account_format(req, src_account_name) else: src_account_name = acct src_container_name, src_obj_name = _check_copy_from_header(req) source_path = '/%s/%s/%s/%s' % (ver, src_account_name, src_container_name, src_obj_name) if req.environ.get('swift.orig_req_method', req.method) != 'POST': self.logger.info("Copying object from %s to %s" % (source_path, req.path)) # GET the source object, bail out on error ssc_ctx = ServerSideCopyWebContext(self.app, self.logger) source_resp = self._get_source_object(ssc_ctx, source_path, req) if source_resp.status_int >= HTTP_MULTIPLE_CHOICES: return source_resp(source_resp.environ, start_response) # Create a new Request object based on the original request instance. # This will preserve original request environ including headers. sink_req = Request.blank(req.path_info, environ=req.environ) def is_object_sysmeta(k): return is_sys_meta('object', k) if config_true_value(req.headers.get('x-fresh-metadata', 'false')): # x-fresh-metadata only applies to copy, not post-as-copy: ignore # existing user metadata, update existing sysmeta with new copy_header_subset(source_resp, sink_req, is_object_sysmeta) copy_header_subset(req, sink_req, is_object_sysmeta) else: # First copy existing sysmeta, user meta and other headers from the # source to the sink, apart from headers that are conditionally # copied below and timestamps. exclude_headers = ('x-static-large-object', 'x-object-manifest', 'etag', 'content-type', 'x-timestamp', 'x-backend-timestamp') copy_header_subset(source_resp, sink_req, lambda k: k.lower() not in exclude_headers) # now update with original req headers sink_req.headers.update(req.headers) params = sink_req.params if params.get('multipart-manifest') == 'get': if 'X-Static-Large-Object' in source_resp.headers: params['multipart-manifest'] = 'put' if 'X-Object-Manifest' in source_resp.headers: del params['multipart-manifest'] sink_req.headers['X-Object-Manifest'] = \ source_resp.headers['X-Object-Manifest'] sink_req.params = params # Set swift.source, data source, content length and etag # for the PUT request sink_req.environ['swift.source'] = 'SSC' sink_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter) sink_req.content_length = source_resp.content_length if (source_resp.status_int == HTTP_OK and 'X-Static-Large-Object' not in source_resp.headers and ('X-Object-Manifest' not in source_resp.headers or req.params.get('multipart-manifest') == 'get')): # copy source etag so that copied content is verified, unless: # - not a 200 OK response: source etag may not match the actual # content, for example with a 206 Partial Content response to a # ranged request # - SLO manifest: etag cannot be specified in manifest PUT; SLO # generates its own etag value which may differ from source # - SLO: etag in SLO response is not hash of actual content # - DLO: etag in DLO response is not hash of actual content sink_req.headers['Etag'] = source_resp.etag else: # since we're not copying the source etag, make sure that any # container update override values are not copied. remove_items( sink_req.headers, lambda k: k.startswith( 'X-Object-Sysmeta-Container-Update-Override-')) # We no longer need these headers sink_req.headers.pop('X-Copy-From', None) sink_req.headers.pop('X-Copy-From-Account', None) # If the copy request does not explicitly override content-type, # use the one present in the source object. if not req.headers.get('content-type'): sink_req.headers['Content-Type'] = \ source_resp.headers['Content-Type'] # Create response headers for PUT response resp_headers = self._create_response_headers(source_path, source_resp, sink_req) put_resp = ssc_ctx.send_put_req(sink_req, resp_headers, start_response) close_if_possible(source_resp.app_iter) return put_resp
def handle_PUT(self, req, start_response): if req.content_length: return HTTPBadRequest(body='Copy requests require a zero byte ' 'body', request=req, content_type='text/plain')(req.environ, start_response) # Form the path of source object to be fetched ver, acct, _rest = req.split_path(2, 3, True) src_account_name = req.headers.get('X-Copy-From-Account') if src_account_name: src_account_name = check_account_format(req, src_account_name) else: src_account_name = acct src_container_name, src_obj_name = _check_copy_from_header(req) source_path = '/%s/%s/%s/%s' % (ver, src_account_name, src_container_name, src_obj_name) if req.environ.get('swift.orig_req_method', req.method) != 'POST': self.logger.info("Copying object from %s to %s" % (source_path, req.path)) # GET the source object, bail out on error ssc_ctx = ServerSideCopyWebContext(self.app, self.logger) source_resp = self._get_source_object(ssc_ctx, source_path, req) if source_resp.status_int >= HTTP_MULTIPLE_CHOICES: return source_resp(source_resp.environ, start_response) # Create a new Request object based on the original request instance. # This will preserve original request environ including headers. sink_req = Request.blank(req.path_info, environ=req.environ) def is_object_sysmeta(k): return is_sys_meta('object', k) if 'swift.post_as_copy' in sink_req.environ: # Post-as-copy: ignore new sysmeta, copy existing sysmeta remove_items(sink_req.headers, is_object_sysmeta) copy_header_subset(source_resp, sink_req, is_object_sysmeta) elif config_true_value(req.headers.get('x-fresh-metadata', 'false')): # x-fresh-metadata only applies to copy, not post-as-copy: ignore # existing user metadata, update existing sysmeta with new copy_header_subset(source_resp, sink_req, is_object_sysmeta) copy_header_subset(req, sink_req, is_object_sysmeta) else: # First copy existing sysmeta, user meta and other headers from the # source to the sink, apart from headers that are conditionally # copied below and timestamps. exclude_headers = ('x-static-large-object', 'x-object-manifest', 'etag', 'content-type', 'x-timestamp', 'x-backend-timestamp') copy_header_subset(source_resp, sink_req, lambda k: k.lower() not in exclude_headers) # now update with original req headers sink_req.headers.update(req.headers) params = sink_req.params if params.get('multipart-manifest') == 'get': if 'X-Static-Large-Object' in source_resp.headers: params['multipart-manifest'] = 'put' if 'X-Object-Manifest' in source_resp.headers: del params['multipart-manifest'] if 'swift.post_as_copy' not in sink_req.environ: sink_req.headers['X-Object-Manifest'] = \ source_resp.headers['X-Object-Manifest'] sink_req.params = params # Set swift.source, data source, content length and etag # for the PUT request sink_req.environ['swift.source'] = 'SSC' sink_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter) sink_req.content_length = source_resp.content_length if (source_resp.status_int == HTTP_OK and 'X-Static-Large-Object' not in source_resp.headers and ('X-Object-Manifest' not in source_resp.headers or req.params.get('multipart-manifest') == 'get')): # copy source etag so that copied content is verified, unless: # - not a 200 OK response: source etag may not match the actual # content, for example with a 206 Partial Content response to a # ranged request # - SLO manifest: etag cannot be specified in manifest PUT; SLO # generates its own etag value which may differ from source # - SLO: etag in SLO response is not hash of actual content # - DLO: etag in DLO response is not hash of actual content sink_req.headers['Etag'] = source_resp.etag else: # since we're not copying the source etag, make sure that any # container update override values are not copied. remove_items(sink_req.headers, lambda k: k.startswith( 'X-Object-Sysmeta-Container-Update-Override-')) # We no longer need these headers sink_req.headers.pop('X-Copy-From', None) sink_req.headers.pop('X-Copy-From-Account', None) # If the copy request does not explicitly override content-type, # use the one present in the source object. if not req.headers.get('content-type'): sink_req.headers['Content-Type'] = \ source_resp.headers['Content-Type'] # Create response headers for PUT response resp_headers = self._create_response_headers(source_path, source_resp, sink_req) put_resp = ssc_ctx.send_put_req(sink_req, resp_headers, start_response) close_if_possible(source_resp.app_iter) return put_resp
def PUT(self, req): """HTTP PUT request handler.""" if req.if_none_match is not None and '*' not in req.if_none_match: # Sending an etag with if-none-match isn't currently supported return HTTPBadRequest(request=req, content_type='text/plain', body='If-None-Match only supports *') container_info = self.container_info(self.account_name, self.container_name, req) policy_index = req.headers.get('X-Backend-Storage-Policy-Index', container_info['storage_policy']) obj_ring = self.app.get_object_ring(policy_index) # pass the policy index to storage nodes via req header req.headers['X-Backend-Storage-Policy-Index'] = policy_index container_partition = container_info['partition'] containers = container_info['nodes'] req.acl = container_info['write_acl'] req.environ['swift_sync_key'] = container_info['sync_key'] object_versions = container_info['versions'] if 'swift.authorize' in req.environ: aresp = req.environ['swift.authorize'](req) if aresp: return aresp if not containers: return HTTPNotFound(request=req) # Sometimes the 'content-type' header exists, but is set to None. content_type_manually_set = True detect_content_type = \ config_true_value(req.headers.get('x-detect-content-type')) if detect_content_type or not req.headers.get('content-type'): guessed_type, _junk = mimetypes.guess_type(req.path_info) req.headers['Content-Type'] = guessed_type or \ 'application/octet-stream' if detect_content_type: req.headers.pop('x-detect-content-type') else: content_type_manually_set = False error_response = check_object_creation(req, self.object_name) or \ check_content_type(req) if error_response: return error_response partition, nodes = obj_ring.get_nodes(self.account_name, self.container_name, self.object_name) #################################### CHANGED_CODE ############################################################ # Change the nodes list to contain only one dictionary item instead of the original 3 returned by the ring. d = dict() # d[partition] = nodes[1:] # f.write(str(d)+"\n") # f.close() print("===Original Nodes===") print(nodes) temp_nodes = [] flag = 0 f = open("/home/hduser/swift/swift/proxy/controllers/spindowndevices", "r") sdlist = f.read().split("\n") print("===Spun down devices===:", sdlist) f.close() upnodes = [item for item in nodes if item['device'] not in sdlist] downnodes = [item for item in nodes if item['device'] in sdlist] temp_nodes = upnodes if (len(downnodes) > 0): d = ast.literal_eval( open("/home/hduser/swift/swift/proxy/controllers/nodes.txt", "r").read()) # d_temp=pickle.load("/home/hduser/swift/proxy/controllers/nodes.p","rb") # print("===Current dict===:",d) for item in downnodes: if (partition in d): d[partition].append(item) # print("===Modified dict===:",d) else: d[partition] = [item] # print("===Modified dict===:",d) # pickle.dump(d,open("/home/hduser/nodes.p","wb")) # print("Before writing:",d) fo = open("/home/hduser/swift/swift/proxy/controllers/nodes.txt", "w") fo.write(str(d) + "\n") fo.close() # pickle.dump(d,open("/home/hduser/swift/swift/proxy/controllers/nodes.p","wb")) ## Old method, IGNORE # for item in nodes: # device = item['device'] # if(device not in sdlist): # # if(os.path.ismount("path")) # temp_nodes.append(item) # flag = 1 # break # else: # pickle.dump(d,open("/home/hduser/nodes.p","wb")) # # d = pickle.load(open("/home/hduser/nodes.p","rb")) # import ast # d = ast.literal_eval(open("/home/hduser/nodes.txt","r").read()) # print("===Current dict===:",d) # if(partition in d): # print("In IF") # d[partition].append(item) # print("===Modified dict===:",d) # else: # print("In ELSE") # d[partition] = [item] # print("===Modified dict===:",d) # pickle.dump(d,open("/home/hduser/nodes.p","wb")) # fo = open("/home/hduser/nodes.txt","w") # fo.write(str(d)+"\n") # Code to spin up a device if none are running already. if (len(upnodes) == 0): dev = nodes[0]['device'] print("===ALL NODES DOWN===") print("===Mounting device===", dev) os.system("mount /dev/" + str(dev)) print('===In controller PUT===:') print("===Partition===", partition) nodes = temp_nodes print('===In controller PUT===:') print("===Partition===", partition) nodes = temp_nodes print("===Nodes===:", nodes) check_ssd() ############################################ CHANGED_CODE ######################################################## # do a HEAD request for checking object versions if object_versions and not req.environ.get('swift_versioned_copy'): # make sure proxy-server uses the right policy index _headers = { 'X-Backend-Storage-Policy-Index': policy_index, 'X-Newest': 'True' } hreq = Request.blank(req.path_info, headers=_headers, environ={'REQUEST_METHOD': 'HEAD'}) hresp = self.GETorHEAD_base(hreq, _('Object'), obj_ring, partition, hreq.swift_entity_path) # Used by container sync feature if 'x-timestamp' in req.headers: try: req_timestamp = Timestamp(req.headers['X-Timestamp']) except ValueError: return HTTPBadRequest( request=req, content_type='text/plain', body='X-Timestamp should be a UNIX timestamp float value; ' 'was %r' % req.headers['x-timestamp']) req.headers['X-Timestamp'] = req_timestamp.internal else: req.headers['X-Timestamp'] = Timestamp(time.time()).internal if object_versions and not req.environ.get('swift_versioned_copy'): is_manifest = 'X-Object-Manifest' in req.headers or \ 'X-Object-Manifest' in hresp.headers if hresp.status_int != HTTP_NOT_FOUND and not is_manifest: # This is a version manifest and needs to be handled # differently. First copy the existing data to a new object, # then write the data from this request to the version manifest # object. lcontainer = object_versions.split('/')[0] prefix_len = '%03x' % len(self.object_name) lprefix = prefix_len + self.object_name + '/' ts_source = hresp.environ.get('swift_x_timestamp') if ts_source is None: ts_source = time.mktime( time.strptime(hresp.headers['last-modified'], '%a, %d %b %Y %H:%M:%S GMT')) new_ts = Timestamp(ts_source).internal vers_obj_name = lprefix + new_ts copy_headers = { 'Destination': '%s/%s' % (lcontainer, vers_obj_name) } copy_environ = { 'REQUEST_METHOD': 'COPY', 'swift_versioned_copy': True } copy_req = Request.blank(req.path_info, headers=copy_headers, environ=copy_environ) copy_resp = self.COPY(copy_req) if is_client_error(copy_resp.status_int): # missing container or bad permissions return HTTPPreconditionFailed(request=req) elif not is_success(copy_resp.status_int): # could not copy the data, bail return HTTPServiceUnavailable(request=req) reader = req.environ['wsgi.input'].read data_source = iter(lambda: reader(self.app.client_chunk_size), '') source_header = req.headers.get('X-Copy-From') source_resp = None if source_header: if req.environ.get('swift.orig_req_method', req.method) != 'POST': req.environ.setdefault('swift.log_info', []).append( 'x-copy-from:%s' % source_header) ver, acct, _rest = req.split_path(2, 3, True) src_account_name = req.headers.get('X-Copy-From-Account', None) if src_account_name: src_account_name = check_account_format(req, src_account_name) else: src_account_name = acct src_container_name, src_obj_name = check_copy_from_header(req) source_header = '/%s/%s/%s/%s' % (ver, src_account_name, src_container_name, src_obj_name) source_req = req.copy_get() # make sure the source request uses it's container_info source_req.headers.pop('X-Backend-Storage-Policy-Index', None) source_req.path_info = source_header source_req.headers['X-Newest'] = 'true' orig_obj_name = self.object_name orig_container_name = self.container_name orig_account_name = self.account_name self.object_name = src_obj_name self.container_name = src_container_name self.account_name = src_account_name sink_req = Request.blank(req.path_info, environ=req.environ, headers=req.headers) source_resp = self.GET(source_req) # This gives middlewares a way to change the source; for example, # this lets you COPY a SLO manifest and have the new object be the # concatenation of the segments (like what a GET request gives # the client), not a copy of the manifest file. hook = req.environ.get( 'swift.copy_hook', (lambda source_req, source_resp, sink_req: source_resp)) source_resp = hook(source_req, source_resp, sink_req) if source_resp.status_int >= HTTP_MULTIPLE_CHOICES: return source_resp self.object_name = orig_obj_name self.container_name = orig_container_name self.account_name = orig_account_name data_source = iter(source_resp.app_iter) sink_req.content_length = source_resp.content_length if sink_req.content_length is None: # This indicates a transfer-encoding: chunked source object, # which currently only happens because there are more than # CONTAINER_LISTING_LIMIT segments in a segmented object. In # this case, we're going to refuse to do the server-side copy. return HTTPRequestEntityTooLarge(request=req) if sink_req.content_length > constraints.MAX_FILE_SIZE: return HTTPRequestEntityTooLarge(request=req) sink_req.etag = source_resp.etag # we no longer need the X-Copy-From header del sink_req.headers['X-Copy-From'] if 'X-Copy-From-Account' in sink_req.headers: del sink_req.headers['X-Copy-From-Account'] if not content_type_manually_set: sink_req.headers['Content-Type'] = \ source_resp.headers['Content-Type'] if config_true_value( sink_req.headers.get('x-fresh-metadata', 'false')): # post-as-copy: ignore new sysmeta, copy existing sysmeta condition = lambda k: is_sys_meta('object', k) remove_items(sink_req.headers, condition) copy_header_subset(source_resp, sink_req, condition) else: # copy/update existing sysmeta and user meta copy_headers_into(source_resp, sink_req) copy_headers_into(req, sink_req) # copy over x-static-large-object for POSTs and manifest copies if 'X-Static-Large-Object' in source_resp.headers and \ req.params.get('multipart-manifest') == 'get': sink_req.headers['X-Static-Large-Object'] = \ source_resp.headers['X-Static-Large-Object'] req = sink_req req, delete_at_container, delete_at_part, \ delete_at_nodes = self._config_obj_expiration(req) node_iter = GreenthreadSafeIterator( self.iter_nodes_local_first(obj_ring, partition)) pile = GreenPile(len(nodes)) te = req.headers.get('transfer-encoding', '') chunked = ('chunked' in te) outgoing_headers = self._backend_requests( req, len(nodes), container_partition, containers, delete_at_container, delete_at_part, delete_at_nodes) for nheaders in outgoing_headers: # RFC2616:8.2.3 disallows 100-continue without a body if (req.content_length > 0) or chunked: nheaders['Expect'] = '100-continue' ################################# CHANGED_CODE ################################################################### # Replaced node_iter by nodes in the following line to make sure that a new list with different order isnt used. # Change from node_iter to nodes to make sure it writes to the same device. # Without this, it gets a new list of nodes from the ring in a different order and connects to the first one. pile.spawn(self._connect_put_node, nodes, partition, req.swift_entity_path, nheaders, self.app.logger.thread_locals) ################################# CHANGED_CODE ################################################################### conns = [conn for conn in pile if conn] min_conns = quorum_size(len(nodes)) if req.if_none_match is not None and '*' in req.if_none_match: statuses = [conn.resp.status for conn in conns if conn.resp] if HTTP_PRECONDITION_FAILED in statuses: # If we find any copy of the file, it shouldn't be uploaded self.app.logger.debug( _('Object PUT returning 412, %(statuses)r'), {'statuses': statuses}) return HTTPPreconditionFailed(request=req) if any(conn for conn in conns if conn.resp and conn.resp.status == HTTP_CONFLICT): timestamps = [ HeaderKeyDict( conn.resp.getheaders()).get('X-Backend-Timestamp') for conn in conns if conn.resp ] self.app.logger.debug( _('Object PUT returning 202 for 409: ' '%(req_timestamp)s <= %(timestamps)r'), { 'req_timestamp': req.timestamp.internal, 'timestamps': ', '.join(timestamps) }) return HTTPAccepted(request=req) if len(conns) < min_conns: self.app.logger.error( _('Object PUT returning 503, %(conns)s/%(nodes)s ' 'required connections'), { 'conns': len(conns), 'nodes': min_conns }) return HTTPServiceUnavailable(request=req) bytes_transferred = 0 try: with ContextPool(len(nodes)) as pool: for conn in conns: conn.failed = False conn.queue = Queue(self.app.put_queue_depth) pool.spawn(self._send_file, conn, req.path) while True: with ChunkReadTimeout(self.app.client_timeout): try: chunk = next(data_source) except StopIteration: if chunked: for conn in conns: conn.queue.put('0\r\n\r\n') break bytes_transferred += len(chunk) if bytes_transferred > constraints.MAX_FILE_SIZE: return HTTPRequestEntityTooLarge(request=req) for conn in list(conns): if not conn.failed: conn.queue.put('%x\r\n%s\r\n' % (len(chunk), chunk) if chunked else chunk) else: conns.remove(conn) if len(conns) < min_conns: self.app.logger.error( _('Object PUT exceptions during' ' send, %(conns)s/%(nodes)s required connections' ), { 'conns': len(conns), 'nodes': min_conns }) return HTTPServiceUnavailable(request=req) for conn in conns: if conn.queue.unfinished_tasks: conn.queue.join() conns = [conn for conn in conns if not conn.failed] except ChunkReadTimeout as err: self.app.logger.warn(_('ERROR Client read timeout (%ss)'), err.seconds) self.app.logger.increment('client_timeouts') return HTTPRequestTimeout(request=req) except (Exception, Timeout): self.app.logger.exception( _('ERROR Exception causing client disconnect')) return HTTPClientDisconnect(request=req) if req.content_length and bytes_transferred < req.content_length: req.client_disconnect = True self.app.logger.warn( _('Client disconnected without sending enough data')) self.app.logger.increment('client_disconnects') return HTTPClientDisconnect(request=req) statuses, reasons, bodies, etags = self._get_put_responses( req, conns, nodes) if len(etags) > 1: self.app.logger.error( _('Object servers returned %s mismatched etags'), len(etags)) return HTTPServerError(request=req) etag = etags.pop() if len(etags) else None resp = self.best_response(req, statuses, reasons, bodies, _('Object PUT'), etag=etag) if source_header: acct, path = source_header.split('/', 3)[2:4] resp.headers['X-Copied-From-Account'] = quote(acct) resp.headers['X-Copied-From'] = quote(path) if 'last-modified' in source_resp.headers: resp.headers['X-Copied-From-Last-Modified'] = \ source_resp.headers['last-modified'] copy_headers_into(req, resp) resp.last_modified = math.ceil( float(Timestamp(req.headers['X-Timestamp']))) return resp
def handle_PUT(self, req, start_response): if req.content_length: return HTTPBadRequest( body="Copy requests require a zero byte " "body", request=req, content_type="text/plain" )(req.environ, start_response) # Form the path of source object to be fetched ver, acct, _rest = req.split_path(2, 3, True) src_account_name = req.headers.get("X-Copy-From-Account") if src_account_name: src_account_name = check_account_format(req, src_account_name) else: src_account_name = acct src_container_name, src_obj_name = _check_copy_from_header(req) source_path = "/%s/%s/%s/%s" % (ver, src_account_name, src_container_name, src_obj_name) if req.environ.get("swift.orig_req_method", req.method) != "POST": self.logger.info("Copying object from %s to %s" % (source_path, req.path)) # GET the source object, bail out on error ssc_ctx = ServerSideCopyWebContext(self.app, self.logger) source_resp = self._get_source_object(ssc_ctx, source_path, req) if source_resp.status_int >= HTTP_MULTIPLE_CHOICES: close_if_possible(source_resp.app_iter) return source_resp(source_resp.environ, start_response) # Create a new Request object based on the original req instance. # This will preserve env and headers. sink_req = Request.blank(req.path_info, environ=req.environ, headers=req.headers) params = sink_req.params if params.get("multipart-manifest") == "get": if "X-Static-Large-Object" in source_resp.headers: params["multipart-manifest"] = "put" if "X-Object-Manifest" in source_resp.headers: del params["multipart-manifest"] sink_req.headers["X-Object-Manifest"] = source_resp.headers["X-Object-Manifest"] sink_req.params = params # Set data source, content length and etag for the PUT request sink_req.environ["wsgi.input"] = FileLikeIter(source_resp.app_iter) sink_req.content_length = source_resp.content_length if ( source_resp.status_int == HTTP_OK and "X-Static-Large-Object" not in source_resp.headers and ("X-Object-Manifest" not in source_resp.headers or req.params.get("multipart-manifest") == "get") ): # copy source etag so that copied content is verified, unless: # - not a 200 OK response: source etag may not match the actual # content, for example with a 206 Partial Content response to a # ranged request # - SLO manifest: etag cannot be specified in manifest PUT; SLO # generates its own etag value which may differ from source # - SLO: etag in SLO response is not hash of actual content # - DLO: etag in DLO response is not hash of actual content sink_req.headers["Etag"] = source_resp.etag else: # since we're not copying the source etag, make sure that any # container update override values are not copied. remove_items(source_resp.headers, lambda k: k.startswith("X-Object-Sysmeta-Container-Update-Override-")) # We no longer need these headers sink_req.headers.pop("X-Copy-From", None) sink_req.headers.pop("X-Copy-From-Account", None) # If the copy request does not explicitly override content-type, # use the one present in the source object. if not req.headers.get("content-type"): sink_req.headers["Content-Type"] = source_resp.headers["Content-Type"] fresh_meta_flag = config_true_value(sink_req.headers.get("x-fresh-metadata", "false")) if fresh_meta_flag or "swift.post_as_copy" in sink_req.environ: # Post-as-copy: ignore new sysmeta, copy existing sysmeta condition = lambda k: is_sys_meta("object", k) remove_items(sink_req.headers, condition) copy_header_subset(source_resp, sink_req, condition) else: # Copy/update existing sysmeta, transient-sysmeta and user meta _copy_headers(source_resp.headers, sink_req.headers) # Copy/update new metadata provided in request if any _copy_headers(req.headers, sink_req.headers) # Create response headers for PUT response resp_headers = self._create_response_headers(source_path, source_resp, sink_req) put_resp = ssc_ctx.send_put_req(sink_req, resp_headers, start_response) close_if_possible(source_resp.app_iter) return put_resp
def __call__(self, req): try: (version, account, container, obj) = req.split_path(3, 4, True) except ValueError: return self.app # verify new quota headers are properly formatted if not obj and req.method in ('PUT', 'POST'): val = req.headers.get('X-Container-Meta-Quota-Bytes') if val and not val.isdigit(): return HTTPBadRequest(body='Invalid bytes quota.') val = req.headers.get('X-Container-Meta-Quota-Count') if val and not val.isdigit(): return HTTPBadRequest(body='Invalid count quota.') # check user uploads against quotas elif obj and req.method in ('PUT', 'COPY'): container_info = None if req.method == 'PUT': container_info = get_container_info( req.environ, self.app, swift_source='CQ') if req.method == 'COPY' and 'Destination' in req.headers: dest_account = account if 'Destination-Account' in req.headers: dest_account = req.headers.get('Destination-Account') dest_account = check_account_format(req, dest_account) dest_container, dest_object = check_destination_header(req) path_info = req.environ['PATH_INFO'] req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % ( version, dest_account, dest_container, dest_object) try: container_info = get_container_info( req.environ, self.app, swift_source='CQ') finally: req.environ['PATH_INFO'] = path_info if not container_info or not is_success(container_info['status']): # this will hopefully 404 later return self.app if 'quota-bytes' in container_info.get('meta', {}) and \ 'bytes' in container_info and \ container_info['meta']['quota-bytes'].isdigit(): content_length = (req.content_length or 0) if 'x-copy-from' in req.headers or req.method == 'COPY': if 'x-copy-from' in req.headers: container, obj = check_copy_from_header(req) path = '/%s/%s/%s/%s' % (version, account, container, obj) object_info = get_object_info(req.environ, self.app, path) if not object_info or not object_info['length']: content_length = 0 else: content_length = int(object_info['length']) new_size = int(container_info['bytes']) + content_length if int(container_info['meta']['quota-bytes']) < new_size: return self.bad_response(req, container_info) if 'quota-count' in container_info.get('meta', {}) and \ 'object_count' in container_info and \ container_info['meta']['quota-count'].isdigit(): new_count = int(container_info['object_count']) + 1 if int(container_info['meta']['quota-count']) < new_count: return self.bad_response(req, container_info) return self.app
def _handle_copy_request(self, req): """ This method handles copying objects based on values set in the headers 'X-Copy-From' and 'X-Copy-From-Account' This method was added as part of the refactoring of the PUT method and the functionality is expected to be moved to middleware """ if req.environ.get('swift.orig_req_method', req.method) != 'POST': req.environ.setdefault('swift.log_info', []).append( 'x-copy-from:%s' % req.headers['X-Copy-From']) ver, acct, _rest = req.split_path(2, 3, True) src_account_name = req.headers.get('X-Copy-From-Account', None) if src_account_name: src_account_name = check_account_format(req, src_account_name) else: src_account_name = acct src_container_name, src_obj_name = check_copy_from_header(req) source_header = '/%s/%s/%s/%s' % ( ver, src_account_name, src_container_name, src_obj_name) source_req = req.copy_get() # make sure the source request uses it's container_info source_req.headers.pop('X-Backend-Storage-Policy-Index', None) source_req.path_info = source_header source_req.headers['X-Newest'] = 'true' orig_obj_name = self.object_name orig_container_name = self.container_name orig_account_name = self.account_name sink_req = Request.blank(req.path_info, environ=req.environ, headers=req.headers) self.object_name = src_obj_name self.container_name = src_container_name self.account_name = src_account_name source_resp = self.GET(source_req) # This gives middlewares a way to change the source; for example, # this lets you COPY a SLO manifest and have the new object be the # concatenation of the segments (like what a GET request gives # the client), not a copy of the manifest file. hook = req.environ.get( 'swift.copy_hook', (lambda source_req, source_resp, sink_req: source_resp)) source_resp = hook(source_req, source_resp, sink_req) # reset names self.object_name = orig_obj_name self.container_name = orig_container_name self.account_name = orig_account_name if source_resp.status_int >= HTTP_MULTIPLE_CHOICES: # this is a bit of ugly code, but I'm willing to live with it # until copy request handling moves to middleware return source_resp, None, None, None if source_resp.content_length is None: # This indicates a transfer-encoding: chunked source object, # which currently only happens because there are more than # CONTAINER_LISTING_LIMIT segments in a segmented object. In # this case, we're going to refuse to do the server-side copy. raise HTTPRequestEntityTooLarge(request=req) if source_resp.content_length > constraints.MAX_FILE_SIZE: raise HTTPRequestEntityTooLarge(request=req) data_source = iter(source_resp.app_iter) sink_req.content_length = source_resp.content_length sink_req.etag = source_resp.etag # we no longer need the X-Copy-From header del sink_req.headers['X-Copy-From'] if 'X-Copy-From-Account' in sink_req.headers: del sink_req.headers['X-Copy-From-Account'] if not req.content_type_manually_set: sink_req.headers['Content-Type'] = \ source_resp.headers['Content-Type'] if config_true_value( sink_req.headers.get('x-fresh-metadata', 'false')): # post-as-copy: ignore new sysmeta, copy existing sysmeta condition = lambda k: is_sys_meta('object', k) remove_items(sink_req.headers, condition) copy_header_subset(source_resp, sink_req, condition) else: # copy/update existing sysmeta and user meta copy_headers_into(source_resp, sink_req) copy_headers_into(req, sink_req) # copy over x-static-large-object for POSTs and manifest copies if 'X-Static-Large-Object' in source_resp.headers and \ req.params.get('multipart-manifest') == 'get': sink_req.headers['X-Static-Large-Object'] = \ source_resp.headers['X-Static-Large-Object'] req = sink_req def update_response(req, resp): acct, path = source_resp.environ['PATH_INFO'].split('/', 3)[2:4] resp.headers['X-Copied-From-Account'] = quote(acct) resp.headers['X-Copied-From'] = quote(path) if 'last-modified' in source_resp.headers: resp.headers['X-Copied-From-Last-Modified'] = \ source_resp.headers['last-modified'] copy_headers_into(req, resp) return resp # this is a bit of ugly code, but I'm willing to live with it # until copy request handling moves to middleware return None, req, data_source, update_response
def handle_PUT(self, req, start_response): if req.content_length: return HTTPBadRequest(body='Copy requests require a zero byte ' 'body', request=req, content_type='text/plain')(req.environ, start_response) # Form the path of source object to be fetched ver, acct, _rest = req.split_path(2, 3, True) src_account_name = req.headers.get('X-Copy-From-Account') if src_account_name: src_account_name = check_account_format(req, src_account_name) else: src_account_name = acct src_container_name, src_obj_name = _check_copy_from_header(req) source_path = '/%s/%s/%s/%s' % (ver, src_account_name, src_container_name, src_obj_name) if req.environ.get('swift.orig_req_method', req.method) != 'POST': self.logger.info("Copying object from %s to %s" % (source_path, req.path)) # GET the source object, bail out on error ssc_ctx = ServerSideCopyWebContext(self.app, self.logger) source_resp = self._get_source_object(ssc_ctx, source_path, req) if source_resp.status_int >= HTTP_MULTIPLE_CHOICES: close_if_possible(source_resp.app_iter) return source_resp(source_resp.environ, start_response) # Create a new Request object based on the original req instance. # This will preserve env and headers. sink_req = Request.blank(req.path_info, environ=req.environ, headers=req.headers) params = sink_req.params if params.get('multipart-manifest') == 'get': if 'X-Static-Large-Object' in source_resp.headers: params['multipart-manifest'] = 'put' if 'X-Object-Manifest' in source_resp.headers: del params['multipart-manifest'] sink_req.headers['X-Object-Manifest'] = \ source_resp.headers['X-Object-Manifest'] sink_req.params = params # Set data source, content length and etag for the PUT request sink_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter) sink_req.content_length = source_resp.content_length sink_req.etag = source_resp.etag # We no longer need these headers sink_req.headers.pop('X-Copy-From', None) sink_req.headers.pop('X-Copy-From-Account', None) # If the copy request does not explicitly override content-type, # use the one present in the source object. if not req.headers.get('content-type'): sink_req.headers['Content-Type'] = \ source_resp.headers['Content-Type'] fresh_meta_flag = config_true_value( sink_req.headers.get('x-fresh-metadata', 'false')) if fresh_meta_flag or 'swift.post_as_copy' in sink_req.environ: # Post-as-copy: ignore new sysmeta, copy existing sysmeta condition = lambda k: is_sys_meta('object', k) remove_items(sink_req.headers, condition) copy_header_subset(source_resp, sink_req, condition) else: # Copy/update existing sysmeta and user meta _copy_headers_into(source_resp, sink_req) # Copy/update new metadata provided in request if any _copy_headers_into(req, sink_req) # Create response headers for PUT response resp_headers = self._create_response_headers(source_path, source_resp, sink_req) put_resp = ssc_ctx.send_put_req(sink_req, resp_headers, start_response) close_if_possible(source_resp.app_iter) return put_resp
def _handle_copy_request(self, req): """ This method handles copying objects based on values set in the headers 'X-Copy-From' and 'X-Copy-From-Account' This method was added as part of the refactoring of the PUT method and the functionality is expected to be moved to middleware """ if req.environ.get('swift.orig_req_method', req.method) != 'POST': req.environ.setdefault('swift.log_info', []).append( 'x-copy-from:%s' % req.headers['X-Copy-From']) ver, acct, _rest = req.split_path(2, 3, True) src_account_name = req.headers.get('X-Copy-From-Account', None) if src_account_name: src_account_name = check_account_format(req, src_account_name) else: src_account_name = acct src_container_name, src_obj_name = check_copy_from_header(req) source_header = '/%s/%s/%s/%s' % (ver, src_account_name, src_container_name, src_obj_name) source_req = req.copy_get() # make sure the source request uses it's container_info source_req.headers.pop('X-Backend-Storage-Policy-Index', None) source_req.path_info = source_header source_req.headers['X-Newest'] = 'true' orig_obj_name = self.object_name orig_container_name = self.container_name orig_account_name = self.account_name sink_req = Request.blank(req.path_info, environ=req.environ, headers=req.headers) self.object_name = src_obj_name self.container_name = src_container_name self.account_name = src_account_name source_resp = self.GET(source_req) # This gives middlewares a way to change the source; for example, # this lets you COPY a SLO manifest and have the new object be the # concatenation of the segments (like what a GET request gives # the client), not a copy of the manifest file. hook = req.environ.get( 'swift.copy_hook', (lambda source_req, source_resp, sink_req: source_resp)) source_resp = hook(source_req, source_resp, sink_req) # reset names self.object_name = orig_obj_name self.container_name = orig_container_name self.account_name = orig_account_name if source_resp.status_int >= HTTP_MULTIPLE_CHOICES: # this is a bit of ugly code, but I'm willing to live with it # until copy request handling moves to middleware return source_resp, None, None, None if source_resp.content_length is None: # This indicates a transfer-encoding: chunked source object, # which currently only happens because there are more than # CONTAINER_LISTING_LIMIT segments in a segmented object. In # this case, we're going to refuse to do the server-side copy. raise HTTPRequestEntityTooLarge(request=req) if source_resp.content_length > constraints.MAX_FILE_SIZE: raise HTTPRequestEntityTooLarge(request=req) data_source = iter(source_resp.app_iter) sink_req.content_length = source_resp.content_length sink_req.etag = source_resp.etag # we no longer need the X-Copy-From header del sink_req.headers['X-Copy-From'] if 'X-Copy-From-Account' in sink_req.headers: del sink_req.headers['X-Copy-From-Account'] if not req.content_type_manually_set: sink_req.headers['Content-Type'] = \ source_resp.headers['Content-Type'] if config_true_value(sink_req.headers.get('x-fresh-metadata', 'false')): # post-as-copy: ignore new sysmeta, copy existing sysmeta condition = lambda k: is_sys_meta('object', k) remove_items(sink_req.headers, condition) copy_header_subset(source_resp, sink_req, condition) else: # copy/update existing sysmeta and user meta copy_headers_into(source_resp, sink_req) copy_headers_into(req, sink_req) # copy over x-static-large-object for POSTs and manifest copies if 'X-Static-Large-Object' in source_resp.headers and \ req.params.get('multipart-manifest') == 'get': sink_req.headers['X-Static-Large-Object'] = \ source_resp.headers['X-Static-Large-Object'] req = sink_req def update_response(req, resp): acct, path = source_resp.environ['PATH_INFO'].split('/', 3)[2:4] resp.headers['X-Copied-From-Account'] = quote(acct) resp.headers['X-Copied-From'] = quote(path) if 'last-modified' in source_resp.headers: resp.headers['X-Copied-From-Last-Modified'] = \ source_resp.headers['last-modified'] copy_headers_into(req, resp) return resp # this is a bit of ugly code, but I'm willing to live with it # until copy request handling moves to middleware return None, req, data_source, update_response