def _validate_and_prep_request_headers(req):
"""
Validate that the value from x-symlink-target header is well formatted
and that the x-symlink-target-etag header (if present) does not contain
problematic characters. We assume the caller ensures that
x-symlink-target header is present in req.headers.
:param req: HTTP request object
:returns: a tuple, the full versioned path to the object (as a WSGI string)
and the X-Symlink-Target-Etag header value which may be None
:raise: HTTPPreconditionFailed if x-symlink-target value
is not well formatted.
:raise: HTTPBadRequest if the x-symlink-target value points to the request
path.
:raise: HTTPBadRequest if the x-symlink-target-etag value contains
a semicolon, double-quote, or backslash.
"""
# N.B. check_path_header doesn't assert the leading slash and
# copy middleware may accept the format. In the symlink, API
# says apparently to use "container/object" format so add the
# validation first, here.
error_body = 'X-Symlink-Target header must be of the form ' \
'<container name>/<object name>'
if wsgi_unquote(req.headers[TGT_OBJ_SYMLINK_HDR]).startswith('/'):
raise HTTPPreconditionFailed(body=error_body,
request=req,
content_type='text/plain')
# check container and object format
container, obj = check_path_header(req, TGT_OBJ_SYMLINK_HDR, 2, error_body)
req.headers[TGT_OBJ_SYMLINK_HDR] = wsgi_quote('%s/%s' % (container, obj))
# Check account format if it exists
account = check_account_format(
req, wsgi_unquote(req.headers[TGT_ACCT_SYMLINK_HDR])) \
if TGT_ACCT_SYMLINK_HDR in req.headers else None
# Extract request path
_junk, req_acc, req_cont, req_obj = req.split_path(4, 4, True)
if account:
req.headers[TGT_ACCT_SYMLINK_HDR] = wsgi_quote(account)
else:
account = req_acc
# Check if symlink targets the symlink itself or not
if (account, container, obj) == (req_acc, req_cont, req_obj):
raise HTTPBadRequest(body='Symlink cannot target itself',
request=req,
content_type='text/plain')
etag = req.headers.get(TGT_ETAG_SYMLINK_HDR, None)
if etag and any(c in etag for c in ';"\\'):
# See cgi.parse_header for why the above chars are problematic
raise HTTPBadRequest(body='Bad %s format' %
TGT_ETAG_SYMLINK_HDR.title(),
request=req,
content_type='text/plain')
if not (etag or req.headers.get('Content-Type')):
req.headers['Content-Type'] = 'application/symlink'
return '/v1/%s/%s/%s' % (account, container, obj), etag
def handle_COPY(self, req, start_response):
if not req.headers.get('Destination'):
return HTTPPreconditionFailed(request=req,
body='Destination header required'
)(req.environ, start_response)
dest_account = self.account_name
if 'Destination-Account' in req.headers:
dest_account = req.headers.get('Destination-Account')
dest_account = check_account_format(req, dest_account)
req.headers['X-Copy-From-Account'] = self.account_name
self.account_name = dest_account
del req.headers['Destination-Account']
dest_container, dest_object = _check_destination_header(req)
source = '/%s/%s' % (self.container_name, self.object_name)
self.container_name = dest_container
self.object_name = dest_object
# re-write the existing request as a PUT instead of creating a new one
req.method = 'PUT'
# As this the path info is updated with destination container,
# the proxy server app will use the right object controller
# implementation corresponding to the container's policy type.
ver, _junk = req.split_path(1, 2, rest_with_last=True)
req.path_info = '/%s/%s/%s/%s' % \
(ver, dest_account, dest_container, dest_object)
req.headers['Content-Length'] = 0
req.headers['X-Copy-From'] = quote(source)
del req.headers['Destination']
return self.handle_PUT(req, start_response)
def COPY(self, req):
"""HTTP COPY request handler."""
if not req.headers.get('Destination'):
return HTTPPreconditionFailed(request=req,
body='Destination header required')
dest_account = self.account_name
if 'Destination-Account' in req.headers:
dest_account = req.headers.get('Destination-Account')
dest_account = check_account_format(req, dest_account)
req.headers['X-Copy-From-Account'] = self.account_name
self.account_name = dest_account
del req.headers['Destination-Account']
dest_container, dest_object = check_destination_header(req)
source = '/%s/%s' % (self.container_name, self.object_name)
self.container_name = dest_container
self.object_name = dest_object
# re-write the existing request as a PUT instead of creating a new one
# since this one is already attached to the posthooklogger
req.method = 'PUT'
req.path_info = '/v1/%s/%s/%s' % \
(dest_account, dest_container, dest_object)
req.headers['Content-Length'] = 0
req.headers['X-Copy-From'] = quote(source)
del req.headers['Destination']
return self.PUT(req)
def COPY(self, req):
"""HTTP COPY request handler."""
if not req.headers.get('Destination'):
return HTTPPreconditionFailed(request=req,
body='Destination header required')
dest_account = self.account_name
if 'Destination-Account' in req.headers:
dest_account = req.headers.get('Destination-Account')
dest_account = check_account_format(req, dest_account)
req.headers['X-Copy-From-Account'] = self.account_name
self.account_name = dest_account
del req.headers['Destination-Account']
dest_container, dest_object = check_destination_header(req)
source = '/%s/%s' % (self.container_name, self.object_name)
self.container_name = dest_container
self.object_name = dest_object
# re-write the existing request as a PUT instead of creating a new one
# since this one is already attached to the posthooklogger
req.method = 'PUT'
req.path_info = '/v1/%s/%s/%s' % \
(dest_account, dest_container, dest_object)
req.headers['Content-Length'] = 0
req.headers['X-Copy-From'] = quote(source)
del req.headers['Destination']
return self.PUT(req)
def object_request(self, req, version, account, container, obj,
allow_versioned_writes):
account_name = unquote(account)
container_name = unquote(container)
object_name = unquote(obj)
container_info = None
resp = None
is_enabled = config_true_value(allow_versioned_writes)
if req.method in ('PUT', 'DELETE'):
container_info = get_container_info(
req.environ, self.app)
elif req.method == 'COPY' and 'Destination' in req.headers:
if 'Destination-Account' in req.headers:
account_name = req.headers.get('Destination-Account')
account_name = check_account_format(req, account_name)
container_name, object_name = check_destination_header(req)
req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % (
version, account_name, container_name, object_name)
container_info = get_container_info(
req.environ, self.app)
if not container_info:
return self.app
# To maintain backwards compatibility, container version
# location could be stored as sysmeta or not, need to check both.
# If stored as sysmeta, check if middleware is enabled. If sysmeta
# is not set, but versions property is set in container_info, then
# for backwards compatibility feature is enabled.
object_versions = container_info.get(
'sysmeta', {}).get('versions-location')
if object_versions and isinstance(object_versions, six.text_type):
object_versions = object_versions.encode('utf-8')
elif not object_versions:
object_versions = container_info.get('versions')
# if allow_versioned_writes is not set in the configuration files
# but 'versions' is configured, enable feature to maintain
# backwards compatibility
if not allow_versioned_writes and object_versions:
is_enabled = True
if is_enabled and object_versions:
object_versions = unquote(object_versions)
vw_ctx = VersionedWritesContext(self.app, self.logger)
if req.method in ('PUT', 'COPY'):
policy_idx = req.headers.get(
'X-Backend-Storage-Policy-Index',
container_info['storage_policy'])
resp = vw_ctx.handle_obj_versions_put(
req, object_versions, object_name, policy_idx)
else: # handle DELETE
resp = vw_ctx.handle_obj_versions_delete(
req, object_versions, account_name,
container_name, object_name)
if resp:
return resp
else:
return self.app
def object_request(self, req, version, account, container, obj,
allow_versioned_writes):
account_name = unquote(account)
container_name = unquote(container)
object_name = unquote(obj)
container_info = None
resp = None
is_enabled = config_true_value(allow_versioned_writes)
if req.method in ('PUT', 'DELETE'):
container_info = get_container_info(
req.environ, self.app)
elif req.method == 'COPY' and 'Destination' in req.headers:
if 'Destination-Account' in req.headers:
account_name = req.headers.get('Destination-Account')
account_name = check_account_format(req, account_name)
container_name, object_name = check_destination_header(req)
req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % (
version, account_name, container_name, object_name)
container_info = get_container_info(
req.environ, self.app)
if not container_info:
return self.app
# To maintain backwards compatibility, container version
# location could be stored as sysmeta or not, need to check both.
# If stored as sysmeta, check if middleware is enabled. If sysmeta
# is not set, but versions property is set in container_info, then
# for backwards compatibility feature is enabled.
object_versions = container_info.get(
'sysmeta', {}).get('versions-location')
if object_versions and isinstance(object_versions, unicode):
object_versions = object_versions.encode('utf-8')
elif not object_versions:
object_versions = container_info.get('versions')
# if allow_versioned_writes is not set in the configuration files
# but 'versions' is configured, enable feature to maintain
# backwards compatibility
if not allow_versioned_writes and object_versions:
is_enabled = True
if is_enabled and object_versions:
object_versions = unquote(object_versions)
vw_ctx = VersionedWritesContext(self.app, self.logger)
if req.method in ('PUT', 'COPY'):
policy_idx = req.headers.get(
'X-Backend-Storage-Policy-Index',
container_info['storage_policy'])
resp = vw_ctx.handle_obj_versions_put(
req, object_versions, object_name, policy_idx)
else: # handle DELETE
resp = vw_ctx.handle_obj_versions_delete(
req, object_versions, account_name,
container_name, object_name)
if resp:
return resp
else:
return self.app
def _check_symlink_header(req):
"""
Validate that the value from x-symlink-target header is
well formatted. We assume the caller ensures that
x-symlink-target header is present in req.headers.
:param req: HTTP request object
:raise: HTTPPreconditionFailed if x-symlink-target value
is not well formatted.
:raise: HTTPBadRequest if the x-symlink-target value points to the request
path.
"""
# N.B. check_path_header doesn't assert the leading slash and
# copy middleware may accept the format. In the symlink, API
# says apparently to use "container/object" format so add the
# validation first, here.
error_body = 'X-Symlink-Target header must be of the form ' \
'<container name>/<object name>'
try:
if wsgi_unquote(req.headers[TGT_OBJ_SYMLINK_HDR]).startswith('/'):
raise HTTPPreconditionFailed(body=error_body,
request=req,
content_type='text/plain')
except TypeError:
raise HTTPPreconditionFailed(body=error_body,
request=req,
content_type='text/plain')
# check container and object format
container, obj = check_path_header(req, TGT_OBJ_SYMLINK_HDR, 2, error_body)
req.headers[TGT_OBJ_SYMLINK_HDR] = wsgi_quote('%s/%s' % (container, obj))
# Check account format if it exists
try:
account = check_account_format(
req, wsgi_unquote(req.headers[TGT_ACCT_SYMLINK_HDR])) \
if TGT_ACCT_SYMLINK_HDR in req.headers else None
except TypeError:
raise HTTPPreconditionFailed(
body='Account name cannot contain slashes',
request=req,
content_type='text/plain')
# Extract request path
_junk, req_acc, req_cont, req_obj = req.split_path(4, 4, True)
if account:
req.headers[TGT_ACCT_SYMLINK_HDR] = wsgi_quote(account)
else:
account = req_acc
# Check if symlink targets the symlink itself or not
if (account, container, obj) == (req_acc, req_cont, req_obj):
raise HTTPBadRequest(body='Symlink cannot target itself',
request=req,
content_type='text/plain')
def object_request(self, req, api_version, account, container, obj,
allow_versioned_writes):
account_name = unquote(account)
container_name = unquote(container)
object_name = unquote(obj)
container_info = None
resp = None
is_enabled = config_true_value(allow_versioned_writes)
if req.method in ('PUT', 'DELETE'):
container_info = get_container_info(req.environ, self.app)
elif req.method == 'COPY' and 'Destination' in req.headers:
if 'Destination-Account' in req.headers:
account_name = req.headers.get('Destination-Account')
account_name = check_account_format(req, account_name)
container_name, object_name = check_destination_header(req)
req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % (
api_version, account_name, container_name, object_name)
container_info = get_container_info(req.environ, self.app)
if not container_info:
return self.app
# To maintain backwards compatibility, container version
# location could be stored as sysmeta or not, need to check both.
# If stored as sysmeta, check if middleware is enabled. If sysmeta
# is not set, but versions property is set in container_info, then
# for backwards compatibility feature is enabled.
versions_cont = container_info.get('sysmeta',
{}).get('versions-location')
if not versions_cont:
versions_cont = container_info.get('versions')
# if allow_versioned_writes is not set in the configuration files
# but 'versions' is configured, enable feature to maintain
# backwards compatibility
if not allow_versioned_writes and versions_cont:
is_enabled = True
if is_enabled and versions_cont:
versions_cont = unquote(versions_cont).split('/')[0]
vw_ctx = VersionedWritesContext(self.app, self.logger)
if req.method in ('PUT', 'COPY'):
resp = vw_ctx.handle_obj_versions_put(req, versions_cont,
api_version,
account_name,
object_name)
else: # handle DELETE
resp = vw_ctx.handle_obj_versions_delete(
req, versions_cont, api_version, account_name,
container_name, object_name)
if resp:
return resp
else:
return self.app
def _check_symlink_header(req):
"""
Validate that the value from x-symlink-target header is
well formatted. We assume the caller ensures that
x-symlink-target header is present in req.headers.
:param req: HTTP request object
:raise: HTTPPreconditionFailed if x-symlink-target value
is not well formatted.
:raise: HTTPBadRequest if the x-symlink-target value points to the request
path.
"""
# N.B. check_path_header doesn't assert the leading slash and
# copy middleware may accept the format. In the symlink, API
# says apparently to use "container/object" format so add the
# validation first, here.
if unquote(req.headers[TGT_OBJ_SYMLINK_HDR]).startswith('/'):
raise HTTPPreconditionFailed(
body='X-Symlink-Target header must be of the '
'form <container name>/<object name>',
request=req, content_type='text/plain')
# check container and object format
container, obj = check_path_header(
req, TGT_OBJ_SYMLINK_HDR, 2,
'X-Symlink-Target header must be of the '
'form <container name>/<object name>')
# Check account format if it exists
account = check_account_format(
req, unquote(req.headers[TGT_ACCT_SYMLINK_HDR])) \
if TGT_ACCT_SYMLINK_HDR in req.headers else None
# Extract request path
_junk, req_acc, req_cont, req_obj = req.split_path(4, 4, True)
if not account:
account = req_acc
# Check if symlink targets the symlink itself or not
if (account, container, obj) == (req_acc, req_cont, req_obj):
raise HTTPBadRequest(
body='Symlink cannot target itself',
request=req, content_type='text/plain')
def __call__(self, env, start_response):
req = Request(env)
if req.method not in self.write_methods:
return self.app(env, start_response)
try:
version, account, container, obj = req.split_path(1, 4, True)
except ValueError:
return self.app(env, start_response)
if req.method == 'COPY' and 'Destination-Account' in req.headers:
dest_account = req.headers.get('Destination-Account')
account = check_account_format(req, dest_account)
if self.account_read_only(req, account):
msg = 'Writes are disabled for this account.'
return HTTPMethodNotAllowed(body=msg)(env, start_response)
return self.app(env, start_response)
def __call__(self, env, start_response):
req = Request(env)
if req.method not in self.write_methods:
return self.app(env, start_response)
try:
version, account, container, obj = req.split_path(1, 4, True)
except ValueError:
return self.app(env, start_response)
if req.method == 'COPY' and 'Destination-Account' in req.headers:
dest_account = req.headers.get('Destination-Account')
account = check_account_format(req, dest_account)
if self.account_read_only(req, account):
msg = 'Writes are disabled for this account.'
return HTTPMethodNotAllowed(body=msg)(env, start_response)
return self.app(env, start_response)
def PUT(self, req):
"""HTTP PUT request handler."""
if req.if_none_match is not None and '*' not in req.if_none_match:
# Sending an etag with if-none-match isn't currently supported
return HTTPBadRequest(request=req, content_type='text/plain',
body='If-None-Match only supports *')
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
return aresp
# Sometimes the 'content-type' header exists, but is set to None.
content_type_manually_set = True
detect_content_type = \
config_true_value(req.headers.get('x-detect-content-type'))
if detect_content_type or not req.headers.get('content-type'):
guessed_type, _junk = mimetypes.guess_type(req.path_info)
req.headers['Content-Type'] = guessed_type or \
'application/octet-stream'
if detect_content_type:
req.headers.pop('x-detect-content-type')
else:
content_type_manually_set = False
error_response = check_object_creation(req, self.object_name) or \
check_content_type(req)
if error_response:
return error_response
req.headers['X-Timestamp'] = Timestamp(time.time()).internal
stream = req.environ['wsgi.input']
source_header = req.headers.get('X-Copy-From')
source_resp = None
if source_header:
if req.environ.get('swift.orig_req_method', req.method) != 'POST':
req.environ.setdefault('swift.log_info', []).append(
'x-copy-from:%s' % source_header)
ver, acct, _rest = req.split_path(2, 3, True)
src_account_name = req.headers.get('X-Copy-From-Account', None)
if src_account_name:
src_account_name = check_account_format(req, src_account_name)
else:
src_account_name = acct
src_container_name, src_obj_name = check_copy_from_header(req)
source_header = '/%s/%s/%s/%s' % (ver, src_account_name,
src_container_name, src_obj_name)
source_req = req.copy_get()
# make sure the source request uses it's container_info
source_req.headers.pop('X-Backend-Storage-Policy-Index', None)
source_req.path_info = source_header
source_req.headers['X-Newest'] = 'true'
orig_obj_name = self.object_name
orig_container_name = self.container_name
orig_account_name = self.account_name
self.object_name = src_obj_name
self.container_name = src_container_name
self.account_name = src_account_name
sink_req = Request.blank(req.path_info,
environ=req.environ, headers=req.headers)
source_resp = self.GET(source_req)
# This gives middlewares a way to change the source; for example,
# this lets you COPY a SLO manifest and have the new object be the
# concatenation of the segments (like what a GET request gives
# the client), not a copy of the manifest file.
hook = req.environ.get(
'swift.copy_hook',
(lambda source_req, source_resp, sink_req: source_resp))
source_resp = hook(source_req, source_resp, sink_req)
if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
return source_resp
self.object_name = orig_obj_name
self.container_name = orig_container_name
self.account_name = orig_account_name
stream = IterO(source_resp.app_iter)
sink_req.content_length = source_resp.content_length
if sink_req.content_length is None:
# This indicates a transfer-encoding: chunked source object,
# which currently only happens because there are more than
# CONTAINER_LISTING_LIMIT segments in a segmented object. In
# this case, we're going to refuse to do the server-side copy.
return HTTPRequestEntityTooLarge(request=req)
if sink_req.content_length > constraints.MAX_FILE_SIZE:
return HTTPRequestEntityTooLarge(request=req)
sink_req.etag = source_resp.etag
# we no longer need the X-Copy-From header
del sink_req.headers['X-Copy-From']
if 'X-Copy-From-Account' in sink_req.headers:
del sink_req.headers['X-Copy-From-Account']
if not content_type_manually_set:
sink_req.headers['Content-Type'] = \
source_resp.headers['Content-Type']
if config_true_value(
sink_req.headers.get('x-fresh-metadata', 'false')):
# post-as-copy: ignore new sysmeta, copy existing sysmeta
condition = lambda k: is_sys_meta('object', k)
remove_items(sink_req.headers, condition)
copy_header_subset(source_resp, sink_req, condition)
else:
# copy/update existing sysmeta and user meta
copy_headers_into(source_resp, sink_req)
copy_headers_into(req, sink_req)
# copy over x-static-large-object for POSTs and manifest copies
if 'X-Static-Large-Object' in source_resp.headers and \
req.params.get('multipart-manifest') == 'get':
sink_req.headers['X-Static-Large-Object'] = \
source_resp.headers['X-Static-Large-Object']
req = sink_req
content_length = req.content_length
content_type = req.headers.get('content-type', 'octet/stream')
storage = self.app.storage
if content_length is None:
content_length = 0
try:
chunks, size, checksum = storage.object_create(self.account_name, self.container_name,
obj_name=self.object_name,
file_or_path=stream,
content_length=content_length,
content_type=content_type)
except exceptions.NoSuchContainer:
return HTTPNotFound(request=req)
except exceptions.ClientReadTimeout:
return HTTPRequestTimeout(request=req)
resp = HTTPCreated(request=req, etag=checksum)
if source_header:
acct, path = source_header.split('/', 3)[2:4]
resp.headers['X-Copied-From-Account'] = quote(acct)
resp.headers['X-Copied-From'] = quote(path)
if 'last-modified' in source_resp.headers:
resp.headers['X-Copied-From-Last-Modified'] = \
source_resp.headers['last-modified']
copy_headers_into(req, resp)
resp.last_modified = math.ceil(
float(Timestamp(req.headers['X-Timestamp'])))
return resp
def PUT(self, req):
"""HTTP PUT request handler."""
if req.if_none_match is not None and '*' not in req.if_none_match:
# Sending an etag with if-none-match isn't currently supported
return HTTPBadRequest(request=req, content_type='text/plain',
body='If-None-Match only supports *')
container_info = self.container_info(
self.account_name, self.container_name, req)
policy_index = req.headers.get('X-Backend-Storage-Policy-Index',
container_info['storage_policy'])
obj_ring = self.app.get_object_ring(policy_index)
# pass the policy index to storage nodes via req header
req.headers['X-Backend-Storage-Policy-Index'] = policy_index
container_partition = container_info['partition']
containers = container_info['nodes']
req.acl = container_info['write_acl']
req.environ['swift_sync_key'] = container_info['sync_key']
object_versions = container_info['versions']
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
return aresp
if not containers:
return HTTPNotFound(request=req)
# Sometimes the 'content-type' header exists, but is set to None.
content_type_manually_set = True
detect_content_type = \
config_true_value(req.headers.get('x-detect-content-type'))
if detect_content_type or not req.headers.get('content-type'):
guessed_type, _junk = mimetypes.guess_type(req.path_info)
req.headers['Content-Type'] = guessed_type or \
'application/octet-stream'
if detect_content_type:
req.headers.pop('x-detect-content-type')
else:
content_type_manually_set = False
error_response = check_object_creation(req, self.object_name) or \
check_content_type(req)
if error_response:
return error_response
partition, nodes = obj_ring.get_nodes(
self.account_name, self.container_name, self.object_name)
# do a HEAD request for container sync and checking object versions
if 'x-timestamp' in req.headers or \
(object_versions and not
req.environ.get('swift_versioned_copy')):
# make sure proxy-server uses the right policy index
_headers = {'X-Backend-Storage-Policy-Index': policy_index,
'X-Newest': 'True'}
hreq = Request.blank(req.path_info, headers=_headers,
environ={'REQUEST_METHOD': 'HEAD'})
hresp = self.GETorHEAD_base(
hreq, _('Object'), obj_ring, partition,
hreq.swift_entity_path)
# Used by container sync feature
if 'x-timestamp' in req.headers:
try:
req_timestamp = Timestamp(req.headers['X-Timestamp'])
if hresp.environ and 'swift_x_timestamp' in hresp.environ and \
hresp.environ['swift_x_timestamp'] >= req_timestamp:
return HTTPAccepted(request=req)
except ValueError:
return HTTPBadRequest(
request=req, content_type='text/plain',
body='X-Timestamp should be a UNIX timestamp float value; '
'was %r' % req.headers['x-timestamp'])
req.headers['X-Timestamp'] = req_timestamp.internal
else:
req.headers['X-Timestamp'] = Timestamp(time.time()).internal
if object_versions and not req.environ.get('swift_versioned_copy'):
if hresp.status_int != HTTP_NOT_FOUND:
# This is a version manifest and needs to be handled
# differently. First copy the existing data to a new object,
# then write the data from this request to the version manifest
# object.
lcontainer = object_versions.split('/')[0]
prefix_len = '%03x' % len(self.object_name)
lprefix = prefix_len + self.object_name + '/'
ts_source = hresp.environ.get('swift_x_timestamp')
if ts_source is None:
ts_source = time.mktime(time.strptime(
hresp.headers['last-modified'],
'%a, %d %b %Y %H:%M:%S GMT'))
new_ts = Timestamp(ts_source).internal
vers_obj_name = lprefix + new_ts
copy_headers = {
'Destination': '%s/%s' % (lcontainer, vers_obj_name)}
copy_environ = {'REQUEST_METHOD': 'COPY',
'swift_versioned_copy': True
}
copy_req = Request.blank(req.path_info, headers=copy_headers,
environ=copy_environ)
copy_resp = self.COPY(copy_req)
if is_client_error(copy_resp.status_int):
# missing container or bad permissions
return HTTPPreconditionFailed(request=req)
elif not is_success(copy_resp.status_int):
# could not copy the data, bail
return HTTPServiceUnavailable(request=req)
reader = req.environ['wsgi.input'].read
data_source = iter(lambda: reader(self.app.client_chunk_size), '')
source_header = req.headers.get('X-Copy-From')
source_resp = None
if source_header:
if req.environ.get('swift.orig_req_method', req.method) != 'POST':
req.environ.setdefault('swift.log_info', []).append(
'x-copy-from:%s' % source_header)
ver, acct, _rest = req.split_path(2, 3, True)
src_account_name = req.headers.get('X-Copy-From-Account', None)
if src_account_name:
src_account_name = check_account_format(req, src_account_name)
else:
src_account_name = acct
src_container_name, src_obj_name = check_copy_from_header(req)
source_header = '/%s/%s/%s/%s' % (ver, src_account_name,
src_container_name, src_obj_name)
source_req = req.copy_get()
# make sure the source request uses it's container_info
source_req.headers.pop('X-Backend-Storage-Policy-Index', None)
source_req.path_info = source_header
source_req.headers['X-Newest'] = 'true'
orig_obj_name = self.object_name
orig_container_name = self.container_name
orig_account_name = self.account_name
self.object_name = src_obj_name
self.container_name = src_container_name
self.account_name = src_account_name
sink_req = Request.blank(req.path_info,
environ=req.environ, headers=req.headers)
source_resp = self.GET(source_req)
# This gives middlewares a way to change the source; for example,
# this lets you COPY a SLO manifest and have the new object be the
# concatenation of the segments (like what a GET request gives
# the client), not a copy of the manifest file.
hook = req.environ.get(
'swift.copy_hook',
(lambda source_req, source_resp, sink_req: source_resp))
source_resp = hook(source_req, source_resp, sink_req)
if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
return source_resp
self.object_name = orig_obj_name
self.container_name = orig_container_name
self.account_name = orig_account_name
data_source = iter(source_resp.app_iter)
sink_req.content_length = source_resp.content_length
if sink_req.content_length is None:
# This indicates a transfer-encoding: chunked source object,
# which currently only happens because there are more than
# CONTAINER_LISTING_LIMIT segments in a segmented object. In
# this case, we're going to refuse to do the server-side copy.
return HTTPRequestEntityTooLarge(request=req)
if sink_req.content_length > constraints.MAX_FILE_SIZE:
return HTTPRequestEntityTooLarge(request=req)
sink_req.etag = source_resp.etag
# we no longer need the X-Copy-From header
del sink_req.headers['X-Copy-From']
if 'X-Copy-From-Account' in sink_req.headers:
del sink_req.headers['X-Copy-From-Account']
if not content_type_manually_set:
sink_req.headers['Content-Type'] = \
source_resp.headers['Content-Type']
if config_true_value(
sink_req.headers.get('x-fresh-metadata', 'false')):
# post-as-copy: ignore new sysmeta, copy existing sysmeta
condition = lambda k: is_sys_meta('object', k)
remove_items(sink_req.headers, condition)
copy_header_subset(source_resp, sink_req, condition)
else:
# copy/update existing sysmeta and user meta
copy_headers_into(source_resp, sink_req)
copy_headers_into(req, sink_req)
# copy over x-static-large-object for POSTs and manifest copies
if 'X-Static-Large-Object' in source_resp.headers and \
req.params.get('multipart-manifest') == 'get':
sink_req.headers['X-Static-Large-Object'] = \
source_resp.headers['X-Static-Large-Object']
req = sink_req
req, delete_at_container, delete_at_part, \
delete_at_nodes = self._config_obj_expiration(req)
node_iter = GreenthreadSafeIterator(
self.iter_nodes_local_first(obj_ring, partition))
pile = GreenPile(len(nodes))
te = req.headers.get('transfer-encoding', '')
chunked = ('chunked' in te)
outgoing_headers = self._backend_requests(
req, len(nodes), container_partition, containers,
delete_at_container, delete_at_part, delete_at_nodes)
for nheaders in outgoing_headers:
# RFC2616:8.2.3 disallows 100-continue without a body
if (req.content_length > 0) or chunked:
nheaders['Expect'] = '100-continue'
pile.spawn(self._connect_put_node, node_iter, partition,
req.swift_entity_path, nheaders,
self.app.logger.thread_locals)
conns = [conn for conn in pile if conn]
min_conns = quorum_size(len(nodes))
if req.if_none_match is not None and '*' in req.if_none_match:
statuses = [conn.resp.status for conn in conns if conn.resp]
if HTTP_PRECONDITION_FAILED in statuses:
# If we find any copy of the file, it shouldn't be uploaded
self.app.logger.debug(
_('Object PUT returning 412, %(statuses)r'),
{'statuses': statuses})
return HTTPPreconditionFailed(request=req)
if len(conns) < min_conns:
self.app.logger.error(
_('Object PUT returning 503, %(conns)s/%(nodes)s '
'required connections'),
{'conns': len(conns), 'nodes': min_conns})
return HTTPServiceUnavailable(request=req)
bytes_transferred = 0
try:
with ContextPool(len(nodes)) as pool:
for conn in conns:
conn.failed = False
conn.queue = Queue(self.app.put_queue_depth)
pool.spawn(self._send_file, conn, req.path)
while True:
with ChunkReadTimeout(self.app.client_timeout):
try:
chunk = next(data_source)
except StopIteration:
if chunked:
for conn in conns:
conn.queue.put('0\r\n\r\n')
break
bytes_transferred += len(chunk)
if bytes_transferred > constraints.MAX_FILE_SIZE:
return HTTPRequestEntityTooLarge(request=req)
for conn in list(conns):
if not conn.failed:
conn.queue.put(
'%x\r\n%s\r\n' % (len(chunk), chunk)
if chunked else chunk)
else:
conns.remove(conn)
if len(conns) < min_conns:
self.app.logger.error(_(
'Object PUT exceptions during'
' send, %(conns)s/%(nodes)s required connections'),
{'conns': len(conns), 'nodes': min_conns})
return HTTPServiceUnavailable(request=req)
for conn in conns:
if conn.queue.unfinished_tasks:
conn.queue.join()
conns = [conn for conn in conns if not conn.failed]
except ChunkReadTimeout as err:
self.app.logger.warn(
_('ERROR Client read timeout (%ss)'), err.seconds)
self.app.logger.increment('client_timeouts')
return HTTPRequestTimeout(request=req)
except (Exception, Timeout):
self.app.logger.exception(
_('ERROR Exception causing client disconnect'))
return HTTPClientDisconnect(request=req)
if req.content_length and bytes_transferred < req.content_length:
req.client_disconnect = True
self.app.logger.warn(
_('Client disconnected without sending enough data'))
self.app.logger.increment('client_disconnects')
return HTTPClientDisconnect(request=req)
statuses, reasons, bodies, etags = self._get_put_responses(req, conns,
nodes)
if len(etags) > 1:
self.app.logger.error(
_('Object servers returned %s mismatched etags'), len(etags))
return HTTPServerError(request=req)
etag = etags.pop() if len(etags) else None
resp = self.best_response(req, statuses, reasons, bodies,
_('Object PUT'), etag=etag)
if source_header:
acct, path = source_header.split('/', 3)[2:4]
resp.headers['X-Copied-From-Account'] = quote(acct)
resp.headers['X-Copied-From'] = quote(path)
if 'last-modified' in source_resp.headers:
resp.headers['X-Copied-From-Last-Modified'] = \
source_resp.headers['last-modified']
copy_headers_into(req, resp)
resp.last_modified = math.ceil(
float(Timestamp(req.headers['X-Timestamp'])))
return resp
def handle_PUT(self, req, start_response):
if req.content_length:
return HTTPBadRequest(body='Copy requests require a zero byte '
'body',
request=req,
content_type='text/plain')(req.environ,
start_response)
# Form the path of source object to be fetched
ver, acct, _rest = req.split_path(2, 3, True)
src_account_name = req.headers.get('X-Copy-From-Account')
if src_account_name:
src_account_name = check_account_format(req, src_account_name)
else:
src_account_name = acct
src_container_name, src_obj_name = _check_copy_from_header(req)
source_path = '/%s/%s/%s/%s' % (ver, src_account_name,
src_container_name, src_obj_name)
if req.environ.get('swift.orig_req_method', req.method) != 'POST':
self.logger.info("Copying object from %s to %s" %
(source_path, req.path))
# GET the source object, bail out on error
ssc_ctx = ServerSideCopyWebContext(self.app, self.logger)
source_resp = self._get_source_object(ssc_ctx, source_path, req)
if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
return source_resp(source_resp.environ, start_response)
# Create a new Request object based on the original request instance.
# This will preserve original request environ including headers.
sink_req = Request.blank(req.path_info, environ=req.environ)
def is_object_sysmeta(k):
return is_sys_meta('object', k)
if config_true_value(req.headers.get('x-fresh-metadata', 'false')):
# x-fresh-metadata only applies to copy, not post-as-copy: ignore
# existing user metadata, update existing sysmeta with new
copy_header_subset(source_resp, sink_req, is_object_sysmeta)
copy_header_subset(req, sink_req, is_object_sysmeta)
else:
# First copy existing sysmeta, user meta and other headers from the
# source to the sink, apart from headers that are conditionally
# copied below and timestamps.
exclude_headers = ('x-static-large-object', 'x-object-manifest',
'etag', 'content-type', 'x-timestamp',
'x-backend-timestamp')
copy_header_subset(source_resp, sink_req,
lambda k: k.lower() not in exclude_headers)
# now update with original req headers
sink_req.headers.update(req.headers)
params = sink_req.params
if params.get('multipart-manifest') == 'get':
if 'X-Static-Large-Object' in source_resp.headers:
params['multipart-manifest'] = 'put'
if 'X-Object-Manifest' in source_resp.headers:
del params['multipart-manifest']
sink_req.headers['X-Object-Manifest'] = \
source_resp.headers['X-Object-Manifest']
sink_req.params = params
# Set swift.source, data source, content length and etag
# for the PUT request
sink_req.environ['swift.source'] = 'SSC'
sink_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter)
sink_req.content_length = source_resp.content_length
if (source_resp.status_int == HTTP_OK
and 'X-Static-Large-Object' not in source_resp.headers
and ('X-Object-Manifest' not in source_resp.headers
or req.params.get('multipart-manifest') == 'get')):
# copy source etag so that copied content is verified, unless:
# - not a 200 OK response: source etag may not match the actual
# content, for example with a 206 Partial Content response to a
# ranged request
# - SLO manifest: etag cannot be specified in manifest PUT; SLO
# generates its own etag value which may differ from source
# - SLO: etag in SLO response is not hash of actual content
# - DLO: etag in DLO response is not hash of actual content
sink_req.headers['Etag'] = source_resp.etag
else:
# since we're not copying the source etag, make sure that any
# container update override values are not copied.
remove_items(
sink_req.headers, lambda k: k.startswith(
'X-Object-Sysmeta-Container-Update-Override-'))
# We no longer need these headers
sink_req.headers.pop('X-Copy-From', None)
sink_req.headers.pop('X-Copy-From-Account', None)
# If the copy request does not explicitly override content-type,
# use the one present in the source object.
if not req.headers.get('content-type'):
sink_req.headers['Content-Type'] = \
source_resp.headers['Content-Type']
# Create response headers for PUT response
resp_headers = self._create_response_headers(source_path, source_resp,
sink_req)
put_resp = ssc_ctx.send_put_req(sink_req, resp_headers, start_response)
close_if_possible(source_resp.app_iter)
return put_resp
def handle_PUT(self, req, start_response):
if req.content_length:
return HTTPBadRequest(body='Copy requests require a zero byte '
'body', request=req,
content_type='text/plain')(req.environ,
start_response)
# Form the path of source object to be fetched
ver, acct, _rest = req.split_path(2, 3, True)
src_account_name = req.headers.get('X-Copy-From-Account')
if src_account_name:
src_account_name = check_account_format(req, src_account_name)
else:
src_account_name = acct
src_container_name, src_obj_name = _check_copy_from_header(req)
source_path = '/%s/%s/%s/%s' % (ver, src_account_name,
src_container_name, src_obj_name)
if req.environ.get('swift.orig_req_method', req.method) != 'POST':
self.logger.info("Copying object from %s to %s" %
(source_path, req.path))
# GET the source object, bail out on error
ssc_ctx = ServerSideCopyWebContext(self.app, self.logger)
source_resp = self._get_source_object(ssc_ctx, source_path, req)
if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
return source_resp(source_resp.environ, start_response)
# Create a new Request object based on the original request instance.
# This will preserve original request environ including headers.
sink_req = Request.blank(req.path_info, environ=req.environ)
def is_object_sysmeta(k):
return is_sys_meta('object', k)
if 'swift.post_as_copy' in sink_req.environ:
# Post-as-copy: ignore new sysmeta, copy existing sysmeta
remove_items(sink_req.headers, is_object_sysmeta)
copy_header_subset(source_resp, sink_req, is_object_sysmeta)
elif config_true_value(req.headers.get('x-fresh-metadata', 'false')):
# x-fresh-metadata only applies to copy, not post-as-copy: ignore
# existing user metadata, update existing sysmeta with new
copy_header_subset(source_resp, sink_req, is_object_sysmeta)
copy_header_subset(req, sink_req, is_object_sysmeta)
else:
# First copy existing sysmeta, user meta and other headers from the
# source to the sink, apart from headers that are conditionally
# copied below and timestamps.
exclude_headers = ('x-static-large-object', 'x-object-manifest',
'etag', 'content-type', 'x-timestamp',
'x-backend-timestamp')
copy_header_subset(source_resp, sink_req,
lambda k: k.lower() not in exclude_headers)
# now update with original req headers
sink_req.headers.update(req.headers)
params = sink_req.params
if params.get('multipart-manifest') == 'get':
if 'X-Static-Large-Object' in source_resp.headers:
params['multipart-manifest'] = 'put'
if 'X-Object-Manifest' in source_resp.headers:
del params['multipart-manifest']
if 'swift.post_as_copy' not in sink_req.environ:
sink_req.headers['X-Object-Manifest'] = \
source_resp.headers['X-Object-Manifest']
sink_req.params = params
# Set swift.source, data source, content length and etag
# for the PUT request
sink_req.environ['swift.source'] = 'SSC'
sink_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter)
sink_req.content_length = source_resp.content_length
if (source_resp.status_int == HTTP_OK and
'X-Static-Large-Object' not in source_resp.headers and
('X-Object-Manifest' not in source_resp.headers or
req.params.get('multipart-manifest') == 'get')):
# copy source etag so that copied content is verified, unless:
# - not a 200 OK response: source etag may not match the actual
# content, for example with a 206 Partial Content response to a
# ranged request
# - SLO manifest: etag cannot be specified in manifest PUT; SLO
# generates its own etag value which may differ from source
# - SLO: etag in SLO response is not hash of actual content
# - DLO: etag in DLO response is not hash of actual content
sink_req.headers['Etag'] = source_resp.etag
else:
# since we're not copying the source etag, make sure that any
# container update override values are not copied.
remove_items(sink_req.headers, lambda k: k.startswith(
'X-Object-Sysmeta-Container-Update-Override-'))
# We no longer need these headers
sink_req.headers.pop('X-Copy-From', None)
sink_req.headers.pop('X-Copy-From-Account', None)
# If the copy request does not explicitly override content-type,
# use the one present in the source object.
if not req.headers.get('content-type'):
sink_req.headers['Content-Type'] = \
source_resp.headers['Content-Type']
# Create response headers for PUT response
resp_headers = self._create_response_headers(source_path,
source_resp, sink_req)
put_resp = ssc_ctx.send_put_req(sink_req, resp_headers, start_response)
close_if_possible(source_resp.app_iter)
return put_resp
def PUT(self, req):
"""HTTP PUT request handler."""
if req.if_none_match is not None and '*' not in req.if_none_match:
# Sending an etag with if-none-match isn't currently supported
return HTTPBadRequest(request=req,
content_type='text/plain',
body='If-None-Match only supports *')
container_info = self.container_info(self.account_name,
self.container_name, req)
policy_index = req.headers.get('X-Backend-Storage-Policy-Index',
container_info['storage_policy'])
obj_ring = self.app.get_object_ring(policy_index)
# pass the policy index to storage nodes via req header
req.headers['X-Backend-Storage-Policy-Index'] = policy_index
container_partition = container_info['partition']
containers = container_info['nodes']
req.acl = container_info['write_acl']
req.environ['swift_sync_key'] = container_info['sync_key']
object_versions = container_info['versions']
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
return aresp
if not containers:
return HTTPNotFound(request=req)
# Sometimes the 'content-type' header exists, but is set to None.
content_type_manually_set = True
detect_content_type = \
config_true_value(req.headers.get('x-detect-content-type'))
if detect_content_type or not req.headers.get('content-type'):
guessed_type, _junk = mimetypes.guess_type(req.path_info)
req.headers['Content-Type'] = guessed_type or \
'application/octet-stream'
if detect_content_type:
req.headers.pop('x-detect-content-type')
else:
content_type_manually_set = False
error_response = check_object_creation(req, self.object_name) or \
check_content_type(req)
if error_response:
return error_response
partition, nodes = obj_ring.get_nodes(self.account_name,
self.container_name,
self.object_name)
#################################### CHANGED_CODE ############################################################
# Change the nodes list to contain only one dictionary item instead of the original 3 returned by the ring.
d = dict()
# d[partition] = nodes[1:]
# f.write(str(d)+"\n")
# f.close()
print("===Original Nodes===")
print(nodes)
temp_nodes = []
flag = 0
f = open("/home/hduser/swift/swift/proxy/controllers/spindowndevices",
"r")
sdlist = f.read().split("\n")
print("===Spun down devices===:", sdlist)
f.close()
upnodes = [item for item in nodes if item['device'] not in sdlist]
downnodes = [item for item in nodes if item['device'] in sdlist]
temp_nodes = upnodes
if (len(downnodes) > 0):
d = ast.literal_eval(
open("/home/hduser/swift/swift/proxy/controllers/nodes.txt",
"r").read())
# d_temp=pickle.load("/home/hduser/swift/proxy/controllers/nodes.p","rb")
# print("===Current dict===:",d)
for item in downnodes:
if (partition in d):
d[partition].append(item)
# print("===Modified dict===:",d)
else:
d[partition] = [item]
# print("===Modified dict===:",d)
# pickle.dump(d,open("/home/hduser/nodes.p","wb"))
# print("Before writing:",d)
fo = open("/home/hduser/swift/swift/proxy/controllers/nodes.txt", "w")
fo.write(str(d) + "\n")
fo.close()
# pickle.dump(d,open("/home/hduser/swift/swift/proxy/controllers/nodes.p","wb"))
## Old method, IGNORE
# for item in nodes:
# device = item['device']
# if(device not in sdlist):
# # if(os.path.ismount("path"))
# temp_nodes.append(item)
# flag = 1
# break
# else:
# pickle.dump(d,open("/home/hduser/nodes.p","wb"))
# # d = pickle.load(open("/home/hduser/nodes.p","rb"))
# import ast
# d = ast.literal_eval(open("/home/hduser/nodes.txt","r").read())
# print("===Current dict===:",d)
# if(partition in d):
# print("In IF")
# d[partition].append(item)
# print("===Modified dict===:",d)
# else:
# print("In ELSE")
# d[partition] = [item]
# print("===Modified dict===:",d)
# pickle.dump(d,open("/home/hduser/nodes.p","wb"))
# fo = open("/home/hduser/nodes.txt","w")
# fo.write(str(d)+"\n")
# Code to spin up a device if none are running already.
if (len(upnodes) == 0):
dev = nodes[0]['device']
print("===ALL NODES DOWN===")
print("===Mounting device===", dev)
os.system("mount /dev/" + str(dev))
print('===In controller PUT===:')
print("===Partition===", partition)
nodes = temp_nodes
print('===In controller PUT===:')
print("===Partition===", partition)
nodes = temp_nodes
print("===Nodes===:", nodes)
check_ssd()
############################################ CHANGED_CODE ########################################################
# do a HEAD request for checking object versions
if object_versions and not req.environ.get('swift_versioned_copy'):
# make sure proxy-server uses the right policy index
_headers = {
'X-Backend-Storage-Policy-Index': policy_index,
'X-Newest': 'True'
}
hreq = Request.blank(req.path_info,
headers=_headers,
environ={'REQUEST_METHOD': 'HEAD'})
hresp = self.GETorHEAD_base(hreq, _('Object'), obj_ring, partition,
hreq.swift_entity_path)
# Used by container sync feature
if 'x-timestamp' in req.headers:
try:
req_timestamp = Timestamp(req.headers['X-Timestamp'])
except ValueError:
return HTTPBadRequest(
request=req,
content_type='text/plain',
body='X-Timestamp should be a UNIX timestamp float value; '
'was %r' % req.headers['x-timestamp'])
req.headers['X-Timestamp'] = req_timestamp.internal
else:
req.headers['X-Timestamp'] = Timestamp(time.time()).internal
if object_versions and not req.environ.get('swift_versioned_copy'):
is_manifest = 'X-Object-Manifest' in req.headers or \
'X-Object-Manifest' in hresp.headers
if hresp.status_int != HTTP_NOT_FOUND and not is_manifest:
# This is a version manifest and needs to be handled
# differently. First copy the existing data to a new object,
# then write the data from this request to the version manifest
# object.
lcontainer = object_versions.split('/')[0]
prefix_len = '%03x' % len(self.object_name)
lprefix = prefix_len + self.object_name + '/'
ts_source = hresp.environ.get('swift_x_timestamp')
if ts_source is None:
ts_source = time.mktime(
time.strptime(hresp.headers['last-modified'],
'%a, %d %b %Y %H:%M:%S GMT'))
new_ts = Timestamp(ts_source).internal
vers_obj_name = lprefix + new_ts
copy_headers = {
'Destination': '%s/%s' % (lcontainer, vers_obj_name)
}
copy_environ = {
'REQUEST_METHOD': 'COPY',
'swift_versioned_copy': True
}
copy_req = Request.blank(req.path_info,
headers=copy_headers,
environ=copy_environ)
copy_resp = self.COPY(copy_req)
if is_client_error(copy_resp.status_int):
# missing container or bad permissions
return HTTPPreconditionFailed(request=req)
elif not is_success(copy_resp.status_int):
# could not copy the data, bail
return HTTPServiceUnavailable(request=req)
reader = req.environ['wsgi.input'].read
data_source = iter(lambda: reader(self.app.client_chunk_size), '')
source_header = req.headers.get('X-Copy-From')
source_resp = None
if source_header:
if req.environ.get('swift.orig_req_method', req.method) != 'POST':
req.environ.setdefault('swift.log_info', []).append(
'x-copy-from:%s' % source_header)
ver, acct, _rest = req.split_path(2, 3, True)
src_account_name = req.headers.get('X-Copy-From-Account', None)
if src_account_name:
src_account_name = check_account_format(req, src_account_name)
else:
src_account_name = acct
src_container_name, src_obj_name = check_copy_from_header(req)
source_header = '/%s/%s/%s/%s' % (ver, src_account_name,
src_container_name, src_obj_name)
source_req = req.copy_get()
# make sure the source request uses it's container_info
source_req.headers.pop('X-Backend-Storage-Policy-Index', None)
source_req.path_info = source_header
source_req.headers['X-Newest'] = 'true'
orig_obj_name = self.object_name
orig_container_name = self.container_name
orig_account_name = self.account_name
self.object_name = src_obj_name
self.container_name = src_container_name
self.account_name = src_account_name
sink_req = Request.blank(req.path_info,
environ=req.environ,
headers=req.headers)
source_resp = self.GET(source_req)
# This gives middlewares a way to change the source; for example,
# this lets you COPY a SLO manifest and have the new object be the
# concatenation of the segments (like what a GET request gives
# the client), not a copy of the manifest file.
hook = req.environ.get(
'swift.copy_hook',
(lambda source_req, source_resp, sink_req: source_resp))
source_resp = hook(source_req, source_resp, sink_req)
if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
return source_resp
self.object_name = orig_obj_name
self.container_name = orig_container_name
self.account_name = orig_account_name
data_source = iter(source_resp.app_iter)
sink_req.content_length = source_resp.content_length
if sink_req.content_length is None:
# This indicates a transfer-encoding: chunked source object,
# which currently only happens because there are more than
# CONTAINER_LISTING_LIMIT segments in a segmented object. In
# this case, we're going to refuse to do the server-side copy.
return HTTPRequestEntityTooLarge(request=req)
if sink_req.content_length > constraints.MAX_FILE_SIZE:
return HTTPRequestEntityTooLarge(request=req)
sink_req.etag = source_resp.etag
# we no longer need the X-Copy-From header
del sink_req.headers['X-Copy-From']
if 'X-Copy-From-Account' in sink_req.headers:
del sink_req.headers['X-Copy-From-Account']
if not content_type_manually_set:
sink_req.headers['Content-Type'] = \
source_resp.headers['Content-Type']
if config_true_value(
sink_req.headers.get('x-fresh-metadata', 'false')):
# post-as-copy: ignore new sysmeta, copy existing sysmeta
condition = lambda k: is_sys_meta('object', k)
remove_items(sink_req.headers, condition)
copy_header_subset(source_resp, sink_req, condition)
else:
# copy/update existing sysmeta and user meta
copy_headers_into(source_resp, sink_req)
copy_headers_into(req, sink_req)
# copy over x-static-large-object for POSTs and manifest copies
if 'X-Static-Large-Object' in source_resp.headers and \
req.params.get('multipart-manifest') == 'get':
sink_req.headers['X-Static-Large-Object'] = \
source_resp.headers['X-Static-Large-Object']
req = sink_req
req, delete_at_container, delete_at_part, \
delete_at_nodes = self._config_obj_expiration(req)
node_iter = GreenthreadSafeIterator(
self.iter_nodes_local_first(obj_ring, partition))
pile = GreenPile(len(nodes))
te = req.headers.get('transfer-encoding', '')
chunked = ('chunked' in te)
outgoing_headers = self._backend_requests(
req, len(nodes), container_partition, containers,
delete_at_container, delete_at_part, delete_at_nodes)
for nheaders in outgoing_headers:
# RFC2616:8.2.3 disallows 100-continue without a body
if (req.content_length > 0) or chunked:
nheaders['Expect'] = '100-continue'
################################# CHANGED_CODE ###################################################################
# Replaced node_iter by nodes in the following line to make sure that a new list with different order isnt used.
# Change from node_iter to nodes to make sure it writes to the same device.
# Without this, it gets a new list of nodes from the ring in a different order and connects to the first one.
pile.spawn(self._connect_put_node, nodes, partition,
req.swift_entity_path, nheaders,
self.app.logger.thread_locals)
################################# CHANGED_CODE ###################################################################
conns = [conn for conn in pile if conn]
min_conns = quorum_size(len(nodes))
if req.if_none_match is not None and '*' in req.if_none_match:
statuses = [conn.resp.status for conn in conns if conn.resp]
if HTTP_PRECONDITION_FAILED in statuses:
# If we find any copy of the file, it shouldn't be uploaded
self.app.logger.debug(
_('Object PUT returning 412, %(statuses)r'),
{'statuses': statuses})
return HTTPPreconditionFailed(request=req)
if any(conn for conn in conns
if conn.resp and conn.resp.status == HTTP_CONFLICT):
timestamps = [
HeaderKeyDict(
conn.resp.getheaders()).get('X-Backend-Timestamp')
for conn in conns if conn.resp
]
self.app.logger.debug(
_('Object PUT returning 202 for 409: '
'%(req_timestamp)s <= %(timestamps)r'), {
'req_timestamp': req.timestamp.internal,
'timestamps': ', '.join(timestamps)
})
return HTTPAccepted(request=req)
if len(conns) < min_conns:
self.app.logger.error(
_('Object PUT returning 503, %(conns)s/%(nodes)s '
'required connections'), {
'conns': len(conns),
'nodes': min_conns
})
return HTTPServiceUnavailable(request=req)
bytes_transferred = 0
try:
with ContextPool(len(nodes)) as pool:
for conn in conns:
conn.failed = False
conn.queue = Queue(self.app.put_queue_depth)
pool.spawn(self._send_file, conn, req.path)
while True:
with ChunkReadTimeout(self.app.client_timeout):
try:
chunk = next(data_source)
except StopIteration:
if chunked:
for conn in conns:
conn.queue.put('0\r\n\r\n')
break
bytes_transferred += len(chunk)
if bytes_transferred > constraints.MAX_FILE_SIZE:
return HTTPRequestEntityTooLarge(request=req)
for conn in list(conns):
if not conn.failed:
conn.queue.put('%x\r\n%s\r\n' %
(len(chunk),
chunk) if chunked else chunk)
else:
conns.remove(conn)
if len(conns) < min_conns:
self.app.logger.error(
_('Object PUT exceptions during'
' send, %(conns)s/%(nodes)s required connections'
), {
'conns': len(conns),
'nodes': min_conns
})
return HTTPServiceUnavailable(request=req)
for conn in conns:
if conn.queue.unfinished_tasks:
conn.queue.join()
conns = [conn for conn in conns if not conn.failed]
except ChunkReadTimeout as err:
self.app.logger.warn(_('ERROR Client read timeout (%ss)'),
err.seconds)
self.app.logger.increment('client_timeouts')
return HTTPRequestTimeout(request=req)
except (Exception, Timeout):
self.app.logger.exception(
_('ERROR Exception causing client disconnect'))
return HTTPClientDisconnect(request=req)
if req.content_length and bytes_transferred < req.content_length:
req.client_disconnect = True
self.app.logger.warn(
_('Client disconnected without sending enough data'))
self.app.logger.increment('client_disconnects')
return HTTPClientDisconnect(request=req)
statuses, reasons, bodies, etags = self._get_put_responses(
req, conns, nodes)
if len(etags) > 1:
self.app.logger.error(
_('Object servers returned %s mismatched etags'), len(etags))
return HTTPServerError(request=req)
etag = etags.pop() if len(etags) else None
resp = self.best_response(req,
statuses,
reasons,
bodies,
_('Object PUT'),
etag=etag)
if source_header:
acct, path = source_header.split('/', 3)[2:4]
resp.headers['X-Copied-From-Account'] = quote(acct)
resp.headers['X-Copied-From'] = quote(path)
if 'last-modified' in source_resp.headers:
resp.headers['X-Copied-From-Last-Modified'] = \
source_resp.headers['last-modified']
copy_headers_into(req, resp)
resp.last_modified = math.ceil(
float(Timestamp(req.headers['X-Timestamp'])))
return resp
def handle_PUT(self, req, start_response):
if req.content_length:
return HTTPBadRequest(
body="Copy requests require a zero byte " "body", request=req, content_type="text/plain"
)(req.environ, start_response)
# Form the path of source object to be fetched
ver, acct, _rest = req.split_path(2, 3, True)
src_account_name = req.headers.get("X-Copy-From-Account")
if src_account_name:
src_account_name = check_account_format(req, src_account_name)
else:
src_account_name = acct
src_container_name, src_obj_name = _check_copy_from_header(req)
source_path = "/%s/%s/%s/%s" % (ver, src_account_name, src_container_name, src_obj_name)
if req.environ.get("swift.orig_req_method", req.method) != "POST":
self.logger.info("Copying object from %s to %s" % (source_path, req.path))
# GET the source object, bail out on error
ssc_ctx = ServerSideCopyWebContext(self.app, self.logger)
source_resp = self._get_source_object(ssc_ctx, source_path, req)
if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
close_if_possible(source_resp.app_iter)
return source_resp(source_resp.environ, start_response)
# Create a new Request object based on the original req instance.
# This will preserve env and headers.
sink_req = Request.blank(req.path_info, environ=req.environ, headers=req.headers)
params = sink_req.params
if params.get("multipart-manifest") == "get":
if "X-Static-Large-Object" in source_resp.headers:
params["multipart-manifest"] = "put"
if "X-Object-Manifest" in source_resp.headers:
del params["multipart-manifest"]
sink_req.headers["X-Object-Manifest"] = source_resp.headers["X-Object-Manifest"]
sink_req.params = params
# Set data source, content length and etag for the PUT request
sink_req.environ["wsgi.input"] = FileLikeIter(source_resp.app_iter)
sink_req.content_length = source_resp.content_length
if (
source_resp.status_int == HTTP_OK
and "X-Static-Large-Object" not in source_resp.headers
and ("X-Object-Manifest" not in source_resp.headers or req.params.get("multipart-manifest") == "get")
):
# copy source etag so that copied content is verified, unless:
# - not a 200 OK response: source etag may not match the actual
# content, for example with a 206 Partial Content response to a
# ranged request
# - SLO manifest: etag cannot be specified in manifest PUT; SLO
# generates its own etag value which may differ from source
# - SLO: etag in SLO response is not hash of actual content
# - DLO: etag in DLO response is not hash of actual content
sink_req.headers["Etag"] = source_resp.etag
else:
# since we're not copying the source etag, make sure that any
# container update override values are not copied.
remove_items(source_resp.headers, lambda k: k.startswith("X-Object-Sysmeta-Container-Update-Override-"))
# We no longer need these headers
sink_req.headers.pop("X-Copy-From", None)
sink_req.headers.pop("X-Copy-From-Account", None)
# If the copy request does not explicitly override content-type,
# use the one present in the source object.
if not req.headers.get("content-type"):
sink_req.headers["Content-Type"] = source_resp.headers["Content-Type"]
fresh_meta_flag = config_true_value(sink_req.headers.get("x-fresh-metadata", "false"))
if fresh_meta_flag or "swift.post_as_copy" in sink_req.environ:
# Post-as-copy: ignore new sysmeta, copy existing sysmeta
condition = lambda k: is_sys_meta("object", k)
remove_items(sink_req.headers, condition)
copy_header_subset(source_resp, sink_req, condition)
else:
# Copy/update existing sysmeta, transient-sysmeta and user meta
_copy_headers(source_resp.headers, sink_req.headers)
# Copy/update new metadata provided in request if any
_copy_headers(req.headers, sink_req.headers)
# Create response headers for PUT response
resp_headers = self._create_response_headers(source_path, source_resp, sink_req)
put_resp = ssc_ctx.send_put_req(sink_req, resp_headers, start_response)
close_if_possible(source_resp.app_iter)
return put_resp
def __call__(self, req):
try:
(version, account, container, obj) = req.split_path(3, 4, True)
except ValueError:
return self.app
# verify new quota headers are properly formatted
if not obj and req.method in ('PUT', 'POST'):
val = req.headers.get('X-Container-Meta-Quota-Bytes')
if val and not val.isdigit():
return HTTPBadRequest(body='Invalid bytes quota.')
val = req.headers.get('X-Container-Meta-Quota-Count')
if val and not val.isdigit():
return HTTPBadRequest(body='Invalid count quota.')
# check user uploads against quotas
elif obj and req.method in ('PUT', 'COPY'):
container_info = None
if req.method == 'PUT':
container_info = get_container_info(
req.environ, self.app, swift_source='CQ')
if req.method == 'COPY' and 'Destination' in req.headers:
dest_account = account
if 'Destination-Account' in req.headers:
dest_account = req.headers.get('Destination-Account')
dest_account = check_account_format(req, dest_account)
dest_container, dest_object = check_destination_header(req)
path_info = req.environ['PATH_INFO']
req.environ['PATH_INFO'] = "/%s/%s/%s/%s" % (
version, dest_account, dest_container, dest_object)
try:
container_info = get_container_info(
req.environ, self.app, swift_source='CQ')
finally:
req.environ['PATH_INFO'] = path_info
if not container_info or not is_success(container_info['status']):
# this will hopefully 404 later
return self.app
if 'quota-bytes' in container_info.get('meta', {}) and \
'bytes' in container_info and \
container_info['meta']['quota-bytes'].isdigit():
content_length = (req.content_length or 0)
if 'x-copy-from' in req.headers or req.method == 'COPY':
if 'x-copy-from' in req.headers:
container, obj = check_copy_from_header(req)
path = '/%s/%s/%s/%s' % (version, account,
container, obj)
object_info = get_object_info(req.environ, self.app, path)
if not object_info or not object_info['length']:
content_length = 0
else:
content_length = int(object_info['length'])
new_size = int(container_info['bytes']) + content_length
if int(container_info['meta']['quota-bytes']) < new_size:
return self.bad_response(req, container_info)
if 'quota-count' in container_info.get('meta', {}) and \
'object_count' in container_info and \
container_info['meta']['quota-count'].isdigit():
new_count = int(container_info['object_count']) + 1
if int(container_info['meta']['quota-count']) < new_count:
return self.bad_response(req, container_info)
return self.app
def _handle_copy_request(self, req):
"""
This method handles copying objects based on values set in the headers
'X-Copy-From' and 'X-Copy-From-Account'
This method was added as part of the refactoring of the PUT method and
the functionality is expected to be moved to middleware
"""
if req.environ.get('swift.orig_req_method', req.method) != 'POST':
req.environ.setdefault('swift.log_info', []).append(
'x-copy-from:%s' % req.headers['X-Copy-From'])
ver, acct, _rest = req.split_path(2, 3, True)
src_account_name = req.headers.get('X-Copy-From-Account', None)
if src_account_name:
src_account_name = check_account_format(req, src_account_name)
else:
src_account_name = acct
src_container_name, src_obj_name = check_copy_from_header(req)
source_header = '/%s/%s/%s/%s' % (
ver, src_account_name, src_container_name, src_obj_name)
source_req = req.copy_get()
# make sure the source request uses it's container_info
source_req.headers.pop('X-Backend-Storage-Policy-Index', None)
source_req.path_info = source_header
source_req.headers['X-Newest'] = 'true'
orig_obj_name = self.object_name
orig_container_name = self.container_name
orig_account_name = self.account_name
sink_req = Request.blank(req.path_info,
environ=req.environ, headers=req.headers)
self.object_name = src_obj_name
self.container_name = src_container_name
self.account_name = src_account_name
source_resp = self.GET(source_req)
# This gives middlewares a way to change the source; for example,
# this lets you COPY a SLO manifest and have the new object be the
# concatenation of the segments (like what a GET request gives
# the client), not a copy of the manifest file.
hook = req.environ.get(
'swift.copy_hook',
(lambda source_req, source_resp, sink_req: source_resp))
source_resp = hook(source_req, source_resp, sink_req)
# reset names
self.object_name = orig_obj_name
self.container_name = orig_container_name
self.account_name = orig_account_name
if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
# this is a bit of ugly code, but I'm willing to live with it
# until copy request handling moves to middleware
return source_resp, None, None, None
if source_resp.content_length is None:
# This indicates a transfer-encoding: chunked source object,
# which currently only happens because there are more than
# CONTAINER_LISTING_LIMIT segments in a segmented object. In
# this case, we're going to refuse to do the server-side copy.
raise HTTPRequestEntityTooLarge(request=req)
if source_resp.content_length > constraints.MAX_FILE_SIZE:
raise HTTPRequestEntityTooLarge(request=req)
data_source = iter(source_resp.app_iter)
sink_req.content_length = source_resp.content_length
sink_req.etag = source_resp.etag
# we no longer need the X-Copy-From header
del sink_req.headers['X-Copy-From']
if 'X-Copy-From-Account' in sink_req.headers:
del sink_req.headers['X-Copy-From-Account']
if not req.content_type_manually_set:
sink_req.headers['Content-Type'] = \
source_resp.headers['Content-Type']
if config_true_value(
sink_req.headers.get('x-fresh-metadata', 'false')):
# post-as-copy: ignore new sysmeta, copy existing sysmeta
condition = lambda k: is_sys_meta('object', k)
remove_items(sink_req.headers, condition)
copy_header_subset(source_resp, sink_req, condition)
else:
# copy/update existing sysmeta and user meta
copy_headers_into(source_resp, sink_req)
copy_headers_into(req, sink_req)
# copy over x-static-large-object for POSTs and manifest copies
if 'X-Static-Large-Object' in source_resp.headers and \
req.params.get('multipart-manifest') == 'get':
sink_req.headers['X-Static-Large-Object'] = \
source_resp.headers['X-Static-Large-Object']
req = sink_req
def update_response(req, resp):
acct, path = source_resp.environ['PATH_INFO'].split('/', 3)[2:4]
resp.headers['X-Copied-From-Account'] = quote(acct)
resp.headers['X-Copied-From'] = quote(path)
if 'last-modified' in source_resp.headers:
resp.headers['X-Copied-From-Last-Modified'] = \
source_resp.headers['last-modified']
copy_headers_into(req, resp)
return resp
# this is a bit of ugly code, but I'm willing to live with it
# until copy request handling moves to middleware
return None, req, data_source, update_response
def handle_PUT(self, req, start_response):
if req.content_length:
return HTTPBadRequest(body='Copy requests require a zero byte '
'body', request=req,
content_type='text/plain')(req.environ,
start_response)
# Form the path of source object to be fetched
ver, acct, _rest = req.split_path(2, 3, True)
src_account_name = req.headers.get('X-Copy-From-Account')
if src_account_name:
src_account_name = check_account_format(req, src_account_name)
else:
src_account_name = acct
src_container_name, src_obj_name = _check_copy_from_header(req)
source_path = '/%s/%s/%s/%s' % (ver, src_account_name,
src_container_name, src_obj_name)
if req.environ.get('swift.orig_req_method', req.method) != 'POST':
self.logger.info("Copying object from %s to %s" %
(source_path, req.path))
# GET the source object, bail out on error
ssc_ctx = ServerSideCopyWebContext(self.app, self.logger)
source_resp = self._get_source_object(ssc_ctx, source_path, req)
if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
close_if_possible(source_resp.app_iter)
return source_resp(source_resp.environ, start_response)
# Create a new Request object based on the original req instance.
# This will preserve env and headers.
sink_req = Request.blank(req.path_info,
environ=req.environ, headers=req.headers)
params = sink_req.params
if params.get('multipart-manifest') == 'get':
if 'X-Static-Large-Object' in source_resp.headers:
params['multipart-manifest'] = 'put'
if 'X-Object-Manifest' in source_resp.headers:
del params['multipart-manifest']
sink_req.headers['X-Object-Manifest'] = \
source_resp.headers['X-Object-Manifest']
sink_req.params = params
# Set data source, content length and etag for the PUT request
sink_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter)
sink_req.content_length = source_resp.content_length
sink_req.etag = source_resp.etag
# We no longer need these headers
sink_req.headers.pop('X-Copy-From', None)
sink_req.headers.pop('X-Copy-From-Account', None)
# If the copy request does not explicitly override content-type,
# use the one present in the source object.
if not req.headers.get('content-type'):
sink_req.headers['Content-Type'] = \
source_resp.headers['Content-Type']
fresh_meta_flag = config_true_value(
sink_req.headers.get('x-fresh-metadata', 'false'))
if fresh_meta_flag or 'swift.post_as_copy' in sink_req.environ:
# Post-as-copy: ignore new sysmeta, copy existing sysmeta
condition = lambda k: is_sys_meta('object', k)
remove_items(sink_req.headers, condition)
copy_header_subset(source_resp, sink_req, condition)
else:
# Copy/update existing sysmeta and user meta
_copy_headers_into(source_resp, sink_req)
# Copy/update new metadata provided in request if any
_copy_headers_into(req, sink_req)
# Create response headers for PUT response
resp_headers = self._create_response_headers(source_path,
source_resp, sink_req)
put_resp = ssc_ctx.send_put_req(sink_req, resp_headers, start_response)
close_if_possible(source_resp.app_iter)
return put_resp
def _handle_copy_request(self, req):
"""
This method handles copying objects based on values set in the headers
'X-Copy-From' and 'X-Copy-From-Account'
This method was added as part of the refactoring of the PUT method and
the functionality is expected to be moved to middleware
"""
if req.environ.get('swift.orig_req_method', req.method) != 'POST':
req.environ.setdefault('swift.log_info', []).append(
'x-copy-from:%s' % req.headers['X-Copy-From'])
ver, acct, _rest = req.split_path(2, 3, True)
src_account_name = req.headers.get('X-Copy-From-Account', None)
if src_account_name:
src_account_name = check_account_format(req, src_account_name)
else:
src_account_name = acct
src_container_name, src_obj_name = check_copy_from_header(req)
source_header = '/%s/%s/%s/%s' % (ver, src_account_name,
src_container_name, src_obj_name)
source_req = req.copy_get()
# make sure the source request uses it's container_info
source_req.headers.pop('X-Backend-Storage-Policy-Index', None)
source_req.path_info = source_header
source_req.headers['X-Newest'] = 'true'
orig_obj_name = self.object_name
orig_container_name = self.container_name
orig_account_name = self.account_name
sink_req = Request.blank(req.path_info,
environ=req.environ,
headers=req.headers)
self.object_name = src_obj_name
self.container_name = src_container_name
self.account_name = src_account_name
source_resp = self.GET(source_req)
# This gives middlewares a way to change the source; for example,
# this lets you COPY a SLO manifest and have the new object be the
# concatenation of the segments (like what a GET request gives
# the client), not a copy of the manifest file.
hook = req.environ.get(
'swift.copy_hook',
(lambda source_req, source_resp, sink_req: source_resp))
source_resp = hook(source_req, source_resp, sink_req)
# reset names
self.object_name = orig_obj_name
self.container_name = orig_container_name
self.account_name = orig_account_name
if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
# this is a bit of ugly code, but I'm willing to live with it
# until copy request handling moves to middleware
return source_resp, None, None, None
if source_resp.content_length is None:
# This indicates a transfer-encoding: chunked source object,
# which currently only happens because there are more than
# CONTAINER_LISTING_LIMIT segments in a segmented object. In
# this case, we're going to refuse to do the server-side copy.
raise HTTPRequestEntityTooLarge(request=req)
if source_resp.content_length > constraints.MAX_FILE_SIZE:
raise HTTPRequestEntityTooLarge(request=req)
data_source = iter(source_resp.app_iter)
sink_req.content_length = source_resp.content_length
sink_req.etag = source_resp.etag
# we no longer need the X-Copy-From header
del sink_req.headers['X-Copy-From']
if 'X-Copy-From-Account' in sink_req.headers:
del sink_req.headers['X-Copy-From-Account']
if not req.content_type_manually_set:
sink_req.headers['Content-Type'] = \
source_resp.headers['Content-Type']
if config_true_value(sink_req.headers.get('x-fresh-metadata',
'false')):
# post-as-copy: ignore new sysmeta, copy existing sysmeta
condition = lambda k: is_sys_meta('object', k)
remove_items(sink_req.headers, condition)
copy_header_subset(source_resp, sink_req, condition)
else:
# copy/update existing sysmeta and user meta
copy_headers_into(source_resp, sink_req)
copy_headers_into(req, sink_req)
# copy over x-static-large-object for POSTs and manifest copies
if 'X-Static-Large-Object' in source_resp.headers and \
req.params.get('multipart-manifest') == 'get':
sink_req.headers['X-Static-Large-Object'] = \
source_resp.headers['X-Static-Large-Object']
req = sink_req
def update_response(req, resp):
acct, path = source_resp.environ['PATH_INFO'].split('/', 3)[2:4]
resp.headers['X-Copied-From-Account'] = quote(acct)
resp.headers['X-Copied-From'] = quote(path)
if 'last-modified' in source_resp.headers:
resp.headers['X-Copied-From-Last-Modified'] = \
source_resp.headers['last-modified']
copy_headers_into(req, resp)
return resp
# this is a bit of ugly code, but I'm willing to live with it
# until copy request handling moves to middleware
return None, req, data_source, update_response