def PUT(self, req):
"""Handle HTTP PUT request."""
drive, part, account, container = split_and_validate_path(req, 3, 4)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
if container: # put account container
pending_timeout = None
if 'x-trans-id' in req.headers:
pending_timeout = 3
broker = self._get_account_broker(drive, part, account,
pending_timeout=pending_timeout)
if account.startswith(self.auto_create_account_prefix) and \
not os.path.exists(broker.db_file):
try:
broker.initialize(normalize_timestamp(
req.headers.get('x-timestamp') or time.time()))
except DatabaseAlreadyExists:
pass
if req.headers.get('x-account-override-deleted', 'no').lower() != \
'yes' and broker.is_deleted():
return HTTPNotFound(request=req)
broker.put_container(container, req.headers['x-put-timestamp'],
req.headers['x-delete-timestamp'],
req.headers['x-object-count'],
req.headers['x-bytes-used'])
if req.headers['x-delete-timestamp'] > \
req.headers['x-put-timestamp']:
return HTTPNoContent(request=req)
else:
return HTTPCreated(request=req)
else: # put account
broker = self._get_account_broker(drive, part, account)
timestamp = normalize_timestamp(req.headers['x-timestamp'])
if not os.path.exists(broker.db_file):
try:
broker.initialize(timestamp)
created = True
except DatabaseAlreadyExists:
created = False
elif broker.is_status_deleted():
return self._deleted_response(broker, req, HTTPForbidden,
body='Recently deleted')
else:
created = broker.is_deleted()
broker.update_put_timestamp(timestamp)
if broker.is_deleted():
return HTTPConflict(request=req)
metadata = {}
metadata.update((key, (value, timestamp))
for key, value in req.headers.iteritems()
if key.lower().startswith('x-account-meta-'))
if metadata:
broker.update_metadata(metadata)
if created:
return HTTPCreated(request=req)
else:
return HTTPAccepted(request=req)
def PUT(self, req):
"""Handle HTTP PUT request."""
drive, part, account, container, obj = split_and_validate_path(
req, 4, 5, True)
if 'x-timestamp' not in req.headers or \
not check_float(req.headers['x-timestamp']):
return HTTPBadRequest(body='Missing timestamp', request=req,
content_type='text/plain')
if 'x-container-sync-to' in req.headers:
err, sync_to, realm, realm_key = validate_sync_to(
req.headers['x-container-sync-to'], self.allowed_sync_hosts,
self.realms_conf)
if err:
return HTTPBadRequest(err)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
timestamp = normalize_timestamp(req.headers['x-timestamp'])
broker = self._get_container_broker(drive, part, account, container)
if obj: # put container object
if account.startswith(self.auto_create_account_prefix) and \
not os.path.exists(broker.db_file):
try:
broker.initialize(timestamp)
except DatabaseAlreadyExists:
pass
if not os.path.exists(broker.db_file):
return HTTPNotFound()
broker.put_object(obj, timestamp, int(req.headers['x-size']),
req.headers['x-content-type'],
req.headers['x-etag'])
return HTTPCreated(request=req)
else: # put container
created = self._update_or_create(req, broker, timestamp)
metadata = {}
metadata.update(
(key, (value, timestamp))
for key, value in req.headers.iteritems()
if key.lower() in self.save_headers or
is_sys_or_user_meta('container', key))
if metadata:
if 'X-Container-Sync-To' in metadata:
if 'X-Container-Sync-To' not in broker.metadata or \
metadata['X-Container-Sync-To'][0] != \
broker.metadata['X-Container-Sync-To'][0]:
broker.set_x_container_sync_points(-1, -1)
broker.update_metadata(metadata)
resp = self.account_update(req, account, container, broker)
if resp:
return resp
if created:
return HTTPCreated(request=req)
else:
return HTTPAccepted(request=req)
def __call__(self, env, start_response):
# note: no read from wsgi.input
req = Request(env)
env['swift.callback.update_footers'](req.headers)
call_headers.append(req.headers)
resp = HTTPCreated(req=req, headers={'Etag': 'response etag'})
return resp(env, start_response)
def remove_old_chunks(self, env, chunks_diff, container):
error = False
self.app.logger.info('StackSync API: remove old chunks: container: %s',
str(container))
for chunk_name in chunks_diff:
env_aux = env.copy()
new_path = "/v1/" + env[
'stacksync_user_account'] + "/" + container + "/" + str(
chunk_name)
del env_aux['HTTP_STACKSYNC_API']
seg_req = make_pre_authed_request(env_aux,
method='DELETE',
path=new_path,
agent=str(container))
seg_resp = seg_req.get_response(self.app)
if not is_valid_status(seg_resp.status_int):
self.app.logger.error(
'StackSync API: upload_file_chunks: error deleting old chunk %s',
str(chunk_name))
error = True
break
if error:
self.app.logger.error(
'StackSync API: upload_file_chunks: status: %s description: Error uploading chunks to storage backend',
seg_resp.status)
response = create_error_response(
500, "Error uploading chunks to storage backend")
else:
response = HTTPCreated()
return response
def __call__(self, env, start_response):
if env['REQUEST_METHOD'] == 'GET':
if self.status == 200:
start_response(Response().status,
[('Content-Type', 'text/xml')])
json_pattern = [
'"name":%s', '"last_modified":%s', '"hash":%s',
'"bytes":%s'
]
json_pattern = '{' + ','.join(json_pattern) + '}'
json_out = []
for b in self.objects:
name = simplejson.dumps(b[0])
time = simplejson.dumps(b[1])
json_out.append(json_pattern % (name, time, b[2], b[3]))
account_list = '[' + ','.join(json_out) + ']'
return account_list
elif self.status == 401:
start_response(HTTPUnauthorized().status, [])
elif self.status == 403:
start_response(HTTPForbidden().status, [])
elif self.status == 404:
start_response(HTTPNotFound().status, [])
else:
start_response(HTTPBadRequest().status, [])
elif env['REQUEST_METHOD'] == 'PUT':
if self.status == 201:
start_response(HTTPCreated().status, [])
elif self.status == 401:
start_response(HTTPUnauthorized().status, [])
elif self.status == 403:
start_response(HTTPForbidden().status, [])
elif self.status == 202:
start_response(HTTPAccepted().status, [])
else:
start_response(HTTPBadRequest().status, [])
elif env['REQUEST_METHOD'] == 'POST':
if self.status == 204:
start_response(HTTPNoContent().status, [])
else:
start_response(HTTPBadRequest().status, [])
elif env['REQUEST_METHOD'] == 'DELETE':
if self.status == 204:
start_response(HTTPNoContent().status, [])
elif self.status == 401:
start_response(HTTPUnauthorized().status, [])
elif self.status == 403:
start_response(HTTPForbidden().status, [])
elif self.status == 404:
start_response(HTTPNotFound().status, [])
elif self.status == 409:
start_response(HTTPConflict().status, [])
else:
start_response(HTTPBadRequest().status, [])
return []
def get_container_create_resp(self, req, headers):
properties, system = self.properties_from_headers(headers)
# TODO container update metadata
oio_headers = {'X-oio-req-id': self.trans_id}
created = self.app.storage.container_create(
self.account_name, self.container_name,
properties=properties, system=system,
headers=oio_headers)
if created:
return HTTPCreated(request=req)
else:
return HTTPNoContent(request=req)
def __call__(self, env, start_response):
req = Request(env)
if env['REQUEST_METHOD'] == 'GET' or env['REQUEST_METHOD'] == 'HEAD':
if self.status == 200:
if 'HTTP_RANGE' in env:
resp = Response(request=req,
body=self.object_body,
conditional_response=True)
return resp(env, start_response)
start_response(
Response(request=req).status,
self.response_headers.items())
if env['REQUEST_METHOD'] == 'GET':
return self.object_body
elif self.status == 401:
start_response(HTTPUnauthorized(request=req).status, [])
elif self.status == 403:
start_response(HTTPForbidden(request=req).status, [])
elif self.status == 404:
start_response(HTTPNotFound(request=req).status, [])
else:
start_response(HTTPBadRequest(request=req).status, [])
elif env['REQUEST_METHOD'] == 'PUT':
if self.status == 201:
start_response(
HTTPCreated(request=req).status,
[('etag', self.response_headers['etag'])])
elif self.status == 401:
start_response(HTTPUnauthorized(request=req).status, [])
elif self.status == 403:
start_response(HTTPForbidden(request=req).status, [])
elif self.status == 404:
start_response(HTTPNotFound(request=req).status, [])
elif self.status == 413:
start_response(
HTTPRequestEntityTooLarge(request=req).status, [])
else:
start_response(HTTPBadRequest(request=req).status, [])
elif env['REQUEST_METHOD'] == 'DELETE':
if self.status == 204:
start_response(HTTPNoContent(request=req).status, [])
elif self.status == 401:
start_response(HTTPUnauthorized(request=req).status, [])
elif self.status == 403:
start_response(HTTPForbidden(request=req).status, [])
elif self.status == 404:
start_response(HTTPNotFound(request=req).status, [])
else:
start_response(HTTPBadRequest(request=req).status, [])
return []
def get_account_put_resp(self, req, headers):
created = self.app.storage.account_create(self.account_name)
metadata = {}
metadata.update((key, value)
for key, value in req.headers.items()
if is_sys_or_user_meta('account', key))
if metadata:
self.app.storage.account_update(self.account_name, metadata)
if created:
resp = HTTPCreated(request=req)
else:
resp = HTTPAccepted(request=req)
return resp
def handle_put_access_key(self, req, access_key):
"""Create auth details of access key.
Required headers:
- `x-s3auth-secret-key`: secret key to store
- `x-s3auth-account`: account to store
- `x-s3auth-admin-key`: admin key
"""
secret_key = req.headers.get('x-s3auth-secret-key')
account = req.headers.get('x-s3auth-account')
if not (secret_key and account):
return HTTPBadRequest(
body='x-s3auth-secret-key and x-s3auth-account '
'headers required',
request=req)
self._set_details(req, access_key, secret_key, account)
return HTTPCreated(request=req)
def get_account_put_resp(self, req, headers):
oio_headers = {'X-oio-req-id': self.trans_id}
created = self.app.storage.account_create(
self.account_name, headers=oio_headers)
metadata = {}
metadata.update((key, value)
for key, value in req.headers.items()
if is_sys_or_user_meta('account', key))
if metadata:
self.app.storage.account_set_properties(
self.account_name, metadata, headers=oio_headers)
if created:
resp = HTTPCreated(request=req)
else:
resp = HTTPAccepted(request=req)
return resp
def upload_file_chunks(self, env, chunked_file, container):
error = False
self.app.logger.info(
'StackSync API: upload_file_chunks: container: %s', str(container))
upload_chunks = []
for i in range(len(chunked_file.chunks)):
chunk_name = chunked_file.name_list[i - 1]
chunk_content = chunked_file.chunks[i - 1]
env_aux = env.copy()
new_path = "/v1/" + env[
'stacksync_user_account'] + "/" + container + "/" + chunk_name
del env_aux['HTTP_STACKSYNC_API']
seg_req = make_pre_authed_request(env_aux,
method='PUT',
path=new_path,
body=chunk_content,
agent=str(container))
seg_resp = seg_req.get_response(self.app)
if not is_valid_status(seg_resp.status_int):
self.app.logger.error(
'StackSync API: upload_file_chunks: error uploading chunk %s',
chunk_name)
error = True
break
upload_chunks.append(chunk_name)
if error:
self.app.logger.error(
'StackSync API: upload_file_chunks: status: %s description: Error uploading chunks to storage backend',
seg_resp.status)
response = create_error_response(
500, "Error uploading chunks to storage backend")
self.remove_chunks(env, upload_chunks, container)
else:
response = HTTPCreated()
return response
def get_container_create_resp(self, req, headers):
properties, system = self.properties_from_headers(headers)
# Save the name of the S3 bucket in a container property.
# This will be used when aggregating container statistics
# to make bucket statistics.
bname = self.container_name
if bname.endswith(MULTIUPLOAD_SUFFIX):
bname = bname[:-len(MULTIUPLOAD_SUFFIX)]
system[BUCKET_NAME_PROP] = bname
# TODO container update metadata
oio_headers = {REQID_HEADER: self.trans_id}
oio_cache = req.environ.get('oio.cache')
perfdata = req.environ.get('oio.perfdata')
created = self.app.storage.container_create(
self.account_name, self.container_name,
properties=properties, system=system,
headers=oio_headers, cache=oio_cache, perfdata=perfdata)
if created:
return HTTPCreated(request=req)
else:
return HTTPNoContent(request=req)
def PUT(self, req):
"""Handle HTTP PUT request."""
drive, part, account, container = split_and_validate_path(req, 3, 4)
try:
check_drive(self.root, drive, self.mount_check)
except ValueError:
return HTTPInsufficientStorage(drive=drive, request=req)
if not self.check_free_space(drive):
return HTTPInsufficientStorage(drive=drive, request=req)
if container: # put account container
if 'x-timestamp' not in req.headers:
timestamp = Timestamp.now()
else:
timestamp = valid_timestamp(req)
pending_timeout = None
container_policy_index = \
req.headers.get('X-Backend-Storage-Policy-Index', 0)
if 'x-trans-id' in req.headers:
pending_timeout = 3
broker = self._get_account_broker(drive,
part,
account,
pending_timeout=pending_timeout)
if account.startswith(self.auto_create_account_prefix) and \
not os.path.exists(broker.db_file):
try:
broker.initialize(timestamp.internal)
except DatabaseAlreadyExists:
pass
if req.headers.get('x-account-override-deleted', 'no').lower() != \
'yes' and broker.is_deleted():
return HTTPNotFound(request=req)
broker.put_container(container, req.headers['x-put-timestamp'],
req.headers['x-delete-timestamp'],
req.headers['x-object-count'],
req.headers['x-bytes-used'],
container_policy_index)
if req.headers['x-delete-timestamp'] > \
req.headers['x-put-timestamp']:
return HTTPNoContent(request=req)
else:
return HTTPCreated(request=req)
else: # put account
timestamp = valid_timestamp(req)
broker = self._get_account_broker(drive, part, account)
if not os.path.exists(broker.db_file):
try:
broker.initialize(timestamp.internal)
created = True
except DatabaseAlreadyExists:
created = False
elif broker.is_status_deleted():
return self._deleted_response(broker,
req,
HTTPForbidden,
body='Recently deleted')
else:
created = broker.is_deleted()
broker.update_put_timestamp(timestamp.internal)
if broker.is_deleted():
return HTTPConflict(request=req)
self._update_metadata(req, broker, timestamp)
if created:
return HTTPCreated(request=req)
else:
return HTTPAccepted(request=req)
broker.initialize(normalize_timestamp(
req.headers.get('x-timestamp') or time.time()))
if req.headers.get('x-account-override-deleted', 'no').lower() != \
'yes' and broker.is_deleted():
self.logger.timing_since('PUT.timing', start_time)
return HTTPNotFound(request=req)
broker.put_container(container, req.headers['x-put-timestamp'],
req.headers['x-delete-timestamp'],
req.headers['x-object-count'],
req.headers['x-bytes-used'])
self.logger.timing_since('PUT.timing', start_time)
if req.headers['x-delete-timestamp'] > \
req.headers['x-put-timestamp']:
return HTTPNoContent(request=req)
else:
return HTTPCreated(request=req)
else: # put account
timestamp = normalize_timestamp(req.headers['x-timestamp'])
if not os.path.exists(broker.db_file):
broker.initialize(timestamp)
created = True
elif broker.is_status_deleted():
self.logger.timing_since('PUT.timing', start_time)
return HTTPForbidden(request=req, body='Recently deleted')
else:
created = broker.is_deleted()
broker.update_put_timestamp(timestamp)
if broker.is_deleted():
self.logger.increment('PUT.errors')
return HTTPConflict(request=req)
metadata = {}
def _link_object(self, req):
_, container, obj = req.headers['Oio-Copy-From'].split('/', 2)
from_account = req.headers.get('X-Copy-From-Account',
self.account_name)
self.app.logger.info("LINK (%s,%s,%s) TO (%s,%s,%s)", from_account,
self.container_name, self.object_name,
self.account_name, container, obj)
storage = self.app.storage
if req.headers.get('Range'):
raise Exception("Fast Copy with Range is unsupported")
ranges = ranges_from_http_header(req.headers.get('Range'))
if len(ranges) != 1:
raise HTTPInternalServerError(
request=req, body="mutiple ranges unsupported")
ranges = ranges[0]
else:
ranges = None
headers = self._prepare_headers(req)
metadata = self.load_object_metadata(headers)
oio_headers = {'X-oio-req-id': self.trans_id}
# FIXME(FVE): use object_show, cache in req.environ
props = storage.object_get_properties(from_account, container, obj)
if props['properties'].get(SLO, None):
raise Exception("Fast Copy with SLO is unsupported")
if ranges is None:
raise HTTPInternalServerError(request=req,
body="LINK a MPU requires range")
self.app.logger.debug("LINK, original object is a SLO")
# retrieve manifest
_, data = storage.object_fetch(from_account, container, obj)
manifest = json.loads("".join(data))
offset = 0
# identify segment to copy
for entry in manifest:
if (ranges[0] == offset
and ranges[1] + 1 == offset + entry['bytes']):
_, container, obj = entry['name'].split('/', 2)
checksum = entry['hash']
self.app.logger.info("LINK SLO (%s,%s,%s) TO (%s,%s,%s)",
from_account, self.container_name,
self.object_name, self.account_name,
container, obj)
break
offset += entry['bytes']
else:
raise HTTPInternalServerError(request=req,
body="no segment matching range")
else:
checksum = props['hash']
if ranges:
raise HTTPInternalServerError(
request=req, body="no range supported with single object")
try:
# TODO check return code (values ?)
storage.object_fastcopy(from_account,
container,
obj,
self.account_name,
self.container_name,
self.object_name,
headers=oio_headers,
properties=metadata,
properties_directive='REPLACE')
# TODO(FVE): this exception catching block has to be refactored
# TODO check which ones are ok or make non sense
except exceptions.Conflict:
raise HTTPConflict(request=req)
except exceptions.PreconditionFailed:
raise HTTPPreconditionFailed(request=req)
except exceptions.SourceReadError:
req.client_disconnect = True
self.app.logger.warning(
_('Client disconnected without sending last chunk'))
self.app.logger.increment('client_disconnects')
raise HTTPClientDisconnect(request=req)
except exceptions.EtagMismatch:
return HTTPUnprocessableEntity(request=req)
except (exceptions.ServiceBusy, exceptions.OioTimeout):
raise
except (exceptions.NoSuchContainer, exceptions.NotFound):
raise HTTPNotFound(request=req)
except exceptions.ClientException as err:
# 481 = CODE_POLICY_NOT_SATISFIABLE
if err.status == 481:
raise exceptions.ServiceBusy()
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
except Exception:
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
resp = HTTPCreated(request=req, etag=checksum)
return resp
create_obj_req = Request.blank(destination, new_env)
resp = create_obj_req.get_response(self.app)
if resp.status_int // 100 == 2:
success_count += 1
else:
if resp.status_int == HTTP_UNAUTHORIZED:
return HTTPUnauthorized(request=req)
failed_files.append([
quote(destination[:MAX_PATH_LENGTH]), resp.status
])
resp_body = get_response_body(
out_content_type, {'Number Files Created': success_count},
failed_files)
if success_count and not failed_files:
return HTTPCreated(resp_body, content_type=out_content_type)
if failed_files:
return HTTPBadGateway(resp_body, content_type=out_content_type)
return HTTPBadRequest('Invalid Tar File: No Valid Files')
except tarfile.TarError, tar_error:
return HTTPBadRequest('Invalid Tar File: %s' % tar_error)
@wsgify
def __call__(self, req):
extract_type = req.params.get('extract-archive')
if extract_type is not None and req.method == 'PUT':
archive_type = {
'tar': '',
'tar.gz': 'gz',
'tar.bz2': 'bz2'
def _store_object(self, req, data_source, headers):
kwargs = {}
content_type = req.headers.get('content-type', 'octet/stream')
policy = None
container_info = self.container_info(self.account_name,
self.container_name, req)
if 'X-Oio-Storage-Policy' in req.headers:
policy = req.headers.get('X-Oio-Storage-Policy')
if not self.app.POLICIES.get_by_name(policy):
raise HTTPBadRequest(
"invalid policy '%s', must be in %s" %
(policy, self.app.POLICIES.by_name.keys()))
else:
try:
policy_index = int(
req.headers.get('X-Backend-Storage-Policy-Index',
container_info['storage_policy']))
except TypeError:
policy_index = 0
if policy_index != 0:
policy = self.app.POLICIES.get_by_index(policy_index).name
else:
content_length = int(req.headers.get('content-length', 0))
policy = self._get_auto_policy_from_size(content_length)
ct_props = {'properties': {}, 'system': {}}
metadata = self.load_object_metadata(headers)
oio_headers = {REQID_HEADER: self.trans_id}
oio_cache = req.environ.get('oio.cache')
perfdata = req.environ.get('swift.perfdata')
# only send headers if needed
if SUPPORT_VERSIONING and headers.get(FORCEVERSIONING_HEADER):
oio_headers[FORCEVERSIONING_HEADER] = \
headers.get(FORCEVERSIONING_HEADER)
if req.environ.get('oio.force-version'):
# In a case of MPU, it contains version of the UploadId
# to be able to include version-id of MPU in S3 reponse
kwargs['version'] = req.environ.get('oio.force-version')
# In case a shard is being created, save the name of the S3 bucket
# in a container property. This will be used when aggregating
# container statistics to make bucket statistics.
if BUCKET_NAME_HEADER in headers:
bname = headers[BUCKET_NAME_HEADER]
# FIXME(FVE): the segments container is not part of another bucket!
# We should not have to strip this here.
if bname and bname.endswith(MULTIUPLOAD_SUFFIX):
bname = bname[:-len(MULTIUPLOAD_SUFFIX)]
ct_props['system'][BUCKET_NAME_PROP] = bname
try:
_chunks, _size, checksum, _meta = self._object_create(
self.account_name,
self.container_name,
obj_name=self.object_name,
file_or_path=data_source,
mime_type=content_type,
policy=policy,
headers=oio_headers,
etag=req.headers.get('etag', '').strip('"'),
properties=metadata,
container_properties=ct_props,
cache=oio_cache,
perfdata=perfdata,
**kwargs)
# TODO(FVE): when oio-sds supports it, do that in a callback
# passed to object_create (or whatever upload method supports it)
footer_md = self.load_object_metadata(self._get_footers(req))
if footer_md:
self.app.storage.object_set_properties(self.account_name,
self.container_name,
self.object_name,
version=_meta.get(
'version', None),
properties=footer_md,
headers=oio_headers,
cache=oio_cache,
perfdata=perfdata)
except exceptions.Conflict:
raise HTTPConflict(request=req)
except exceptions.PreconditionFailed:
raise HTTPPreconditionFailed(request=req)
except SourceReadTimeout as err:
self.app.logger.warning(_('ERROR Client read timeout (%s)'), err)
self.app.logger.increment('client_timeouts')
raise HTTPRequestTimeout(request=req)
except exceptions.SourceReadError:
req.client_disconnect = True
self.app.logger.warning(
_('Client disconnected without sending last chunk'))
self.app.logger.increment('client_disconnects')
raise HTTPClientDisconnect(request=req)
except exceptions.EtagMismatch:
return HTTPUnprocessableEntity(request=req)
except (exceptions.ServiceBusy, exceptions.OioTimeout,
exceptions.DeadlineReached):
raise
except exceptions.NoSuchContainer:
raise HTTPNotFound(request=req)
except exceptions.ClientException as err:
# 481 = CODE_POLICY_NOT_SATISFIABLE
if err.status == 481:
raise exceptions.ServiceBusy()
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
except Exception:
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
last_modified = int(_meta.get('mtime', math.ceil(time.time())))
# FIXME(FVE): if \x10 character in object name, decode version
# number and set it in the response headers, instead of the oio
# version number.
version_id = _meta.get('version', 'null')
resp = HTTPCreated(request=req,
etag=checksum,
last_modified=last_modified,
headers={'x-object-sysmeta-version-id': version_id})
return resp
def PUT(self, req):
"""HTTP PUT request handler."""
if req.if_none_match is not None and '*' not in req.if_none_match:
# Sending an etag with if-none-match isn't currently supported
return HTTPBadRequest(request=req, content_type='text/plain',
body='If-None-Match only supports *')
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
return aresp
# Sometimes the 'content-type' header exists, but is set to None.
content_type_manually_set = True
detect_content_type = \
config_true_value(req.headers.get('x-detect-content-type'))
if detect_content_type or not req.headers.get('content-type'):
guessed_type, _junk = mimetypes.guess_type(req.path_info)
req.headers['Content-Type'] = guessed_type or \
'application/octet-stream'
if detect_content_type:
req.headers.pop('x-detect-content-type')
else:
content_type_manually_set = False
error_response = check_object_creation(req, self.object_name) or \
check_content_type(req)
if error_response:
return error_response
req.headers['X-Timestamp'] = Timestamp(time.time()).internal
stream = req.environ['wsgi.input']
source_header = req.headers.get('X-Copy-From')
source_resp = None
if source_header:
if req.environ.get('swift.orig_req_method', req.method) != 'POST':
req.environ.setdefault('swift.log_info', []).append(
'x-copy-from:%s' % source_header)
ver, acct, _rest = req.split_path(2, 3, True)
src_account_name = req.headers.get('X-Copy-From-Account', None)
if src_account_name:
src_account_name = check_account_format(req, src_account_name)
else:
src_account_name = acct
src_container_name, src_obj_name = check_copy_from_header(req)
source_header = '/%s/%s/%s/%s' % (ver, src_account_name,
src_container_name, src_obj_name)
source_req = req.copy_get()
# make sure the source request uses it's container_info
source_req.headers.pop('X-Backend-Storage-Policy-Index', None)
source_req.path_info = source_header
source_req.headers['X-Newest'] = 'true'
orig_obj_name = self.object_name
orig_container_name = self.container_name
orig_account_name = self.account_name
self.object_name = src_obj_name
self.container_name = src_container_name
self.account_name = src_account_name
sink_req = Request.blank(req.path_info,
environ=req.environ, headers=req.headers)
source_resp = self.GET(source_req)
# This gives middlewares a way to change the source; for example,
# this lets you COPY a SLO manifest and have the new object be the
# concatenation of the segments (like what a GET request gives
# the client), not a copy of the manifest file.
hook = req.environ.get(
'swift.copy_hook',
(lambda source_req, source_resp, sink_req: source_resp))
source_resp = hook(source_req, source_resp, sink_req)
if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
return source_resp
self.object_name = orig_obj_name
self.container_name = orig_container_name
self.account_name = orig_account_name
stream = IterO(source_resp.app_iter)
sink_req.content_length = source_resp.content_length
if sink_req.content_length is None:
# This indicates a transfer-encoding: chunked source object,
# which currently only happens because there are more than
# CONTAINER_LISTING_LIMIT segments in a segmented object. In
# this case, we're going to refuse to do the server-side copy.
return HTTPRequestEntityTooLarge(request=req)
if sink_req.content_length > constraints.MAX_FILE_SIZE:
return HTTPRequestEntityTooLarge(request=req)
sink_req.etag = source_resp.etag
# we no longer need the X-Copy-From header
del sink_req.headers['X-Copy-From']
if 'X-Copy-From-Account' in sink_req.headers:
del sink_req.headers['X-Copy-From-Account']
if not content_type_manually_set:
sink_req.headers['Content-Type'] = \
source_resp.headers['Content-Type']
if config_true_value(
sink_req.headers.get('x-fresh-metadata', 'false')):
# post-as-copy: ignore new sysmeta, copy existing sysmeta
condition = lambda k: is_sys_meta('object', k)
remove_items(sink_req.headers, condition)
copy_header_subset(source_resp, sink_req, condition)
else:
# copy/update existing sysmeta and user meta
copy_headers_into(source_resp, sink_req)
copy_headers_into(req, sink_req)
# copy over x-static-large-object for POSTs and manifest copies
if 'X-Static-Large-Object' in source_resp.headers and \
req.params.get('multipart-manifest') == 'get':
sink_req.headers['X-Static-Large-Object'] = \
source_resp.headers['X-Static-Large-Object']
req = sink_req
content_length = req.content_length
content_type = req.headers.get('content-type', 'octet/stream')
storage = self.app.storage
if content_length is None:
content_length = 0
try:
chunks, size, checksum = storage.object_create(self.account_name, self.container_name,
obj_name=self.object_name,
file_or_path=stream,
content_length=content_length,
content_type=content_type)
except exceptions.NoSuchContainer:
return HTTPNotFound(request=req)
except exceptions.ClientReadTimeout:
return HTTPRequestTimeout(request=req)
resp = HTTPCreated(request=req, etag=checksum)
if source_header:
acct, path = source_header.split('/', 3)[2:4]
resp.headers['X-Copied-From-Account'] = quote(acct)
resp.headers['X-Copied-From'] = quote(path)
if 'last-modified' in source_resp.headers:
resp.headers['X-Copied-From-Last-Modified'] = \
source_resp.headers['last-modified']
copy_headers_into(req, resp)
resp.last_modified = math.ceil(
float(Timestamp(req.headers['X-Timestamp'])))
return resp
def PUT(self, request):
"""Handle HTTP PUT requests for the Swift Object Server."""
device, partition, account, container, obj, policy_idx = \
get_name_and_placement(request, 5, 5, True)
req_timestamp = valid_timestamp(request)
error_response = check_object_creation(request, obj)
if error_response:
return error_response
new_delete_at = int(request.headers.get('X-Delete-At') or 0)
if new_delete_at and new_delete_at < time.time():
return HTTPBadRequest(body='X-Delete-At in past',
request=request,
content_type='text/plain')
try:
fsize = request.message_length()
except ValueError as e:
return HTTPBadRequest(body=str(e),
request=request,
content_type='text/plain')
try:
disk_file = self.get_diskfile(device,
partition,
account,
container,
obj,
policy_idx=policy_idx)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
orig_metadata = disk_file.read_metadata()
except DiskFileXattrNotSupported:
return HTTPInsufficientStorage(drive=device, request=request)
except (DiskFileNotExist, DiskFileQuarantined):
orig_metadata = {}
# Checks for If-None-Match
if request.if_none_match is not None and orig_metadata:
if '*' in request.if_none_match:
# File exists already so return 412
return HTTPPreconditionFailed(request=request)
if orig_metadata.get('ETag') in request.if_none_match:
# The current ETag matches, so return 412
return HTTPPreconditionFailed(request=request)
orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0))
if orig_timestamp >= req_timestamp:
return HTTPConflict(
request=request,
headers={'X-Backend-Timestamp': orig_timestamp.internal})
orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0)
upload_expiration = time.time() + self.max_upload_time
etag = md5()
elapsed_time = 0
try:
with disk_file.create(size=fsize) as writer:
upload_size = 0
def timeout_reader():
with ChunkReadTimeout(self.client_timeout):
return request.environ['wsgi.input'].read(
self.network_chunk_size)
try:
for chunk in iter(lambda: timeout_reader(), ''):
start_time = time.time()
if start_time > upload_expiration:
self.logger.increment('PUT.timeouts')
return HTTPRequestTimeout(request=request)
etag.update(chunk)
upload_size = writer.write(chunk)
elapsed_time += time.time() - start_time
except ChunkReadTimeout:
return HTTPRequestTimeout(request=request)
if upload_size:
self.logger.transfer_rate('PUT.' + device + '.timing',
elapsed_time, upload_size)
if fsize is not None and fsize != upload_size:
return HTTPClientDisconnect(request=request)
etag = etag.hexdigest()
if 'etag' in request.headers and \
request.headers['etag'].lower() != etag:
return HTTPUnprocessableEntity(request=request)
metadata = {
'X-Timestamp': request.timestamp.internal,
'Content-Type': request.headers['content-type'],
'ETag': etag,
'Content-Length': str(upload_size),
}
metadata.update(val for val in request.headers.iteritems()
if is_sys_or_user_meta('object', val[0]))
headers_to_copy = (request.headers.get(
'X-Backend-Replication-Headers', '').split() +
list(self.allowed_headers))
for header_key in headers_to_copy:
if header_key in request.headers:
header_caps = header_key.title()
metadata[header_caps] = request.headers[header_key]
writer.put(metadata)
except (DiskFileXattrNotSupported, DiskFileNoSpace):
return HTTPInsufficientStorage(drive=device, request=request)
if orig_delete_at != new_delete_at:
if new_delete_at:
self.delete_at_update('PUT', new_delete_at, account, container,
obj, request, device, policy_idx)
if orig_delete_at:
self.delete_at_update('DELETE', orig_delete_at, account,
container, obj, request, device,
policy_idx)
self.container_update(
'PUT', account, container, obj, request,
HeaderKeyDict({
'x-size': metadata['Content-Length'],
'x-content-type': metadata['Content-Type'],
'x-timestamp': metadata['X-Timestamp'],
'x-etag': metadata['ETag']
}), device, policy_idx)
return HTTPCreated(request=request, etag=etag)
def PUT(self, req):
"""Handle HTTP PUT request."""
drive, part, account, container, obj = split_and_validate_path(
req, 4, 5, True)
req_timestamp = valid_timestamp(req)
if 'x-container-sync-to' in req.headers:
err, sync_to, realm, realm_key = validate_sync_to(
req.headers['x-container-sync-to'], self.allowed_sync_hosts,
self.realms_conf)
if err:
return HTTPBadRequest(err)
if self.mount_check and not check_mount(self.root, drive):
return HTTPInsufficientStorage(drive=drive, request=req)
requested_policy_index = self.get_and_validate_policy_index(req)
broker = self._get_container_broker(drive, part, account, container)
if obj: # put container object
# obj put expects the policy_index header, default is for
# legacy support during upgrade.
obj_policy_index = requested_policy_index or 0
if account.startswith(self.auto_create_account_prefix) and \
not os.path.exists(broker.db_file):
try:
broker.initialize(req_timestamp.internal, obj_policy_index)
except DatabaseAlreadyExists:
pass
if not os.path.exists(broker.db_file):
return HTTPNotFound()
broker.put_object(obj, req_timestamp.internal,
int(req.headers['x-size']),
req.headers['x-content-type'],
req.headers['x-etag'], 0, obj_policy_index,
req.headers.get('x-content-type-timestamp'),
req.headers.get('x-meta-timestamp'))
return HTTPCreated(request=req)
else: # put container
if requested_policy_index is None:
# use the default index sent by the proxy if available
new_container_policy = req.headers.get(
'X-Backend-Storage-Policy-Default', int(POLICIES.default))
else:
new_container_policy = requested_policy_index
created = self._update_or_create(req, broker,
req_timestamp.internal,
new_container_policy,
requested_policy_index)
metadata = {}
metadata.update((key, (value, req_timestamp.internal))
for key, value in req.headers.items()
if key.lower() in self.save_headers
or is_sys_or_user_meta('container', key))
if 'X-Container-Sync-To' in metadata:
if 'X-Container-Sync-To' not in broker.metadata or \
metadata['X-Container-Sync-To'][0] != \
broker.metadata['X-Container-Sync-To'][0]:
broker.set_x_container_sync_points(-1, -1)
broker.update_metadata(metadata, validate_metadata=True)
if metadata:
self._update_sync_store(broker, 'PUT')
resp = self.account_update(req, account, container, broker)
if resp:
return resp
if created:
return HTTPCreated(request=req,
headers={
'x-backend-storage-policy-index':
broker.storage_policy_index
})
else:
return HTTPAccepted(request=req,
headers={
'x-backend-storage-policy-index':
broker.storage_policy_index
})
def handle_extract_iter(self, req, compress_type,
out_content_type='text/plain'):
"""
A generator that can be assigned to a swob Response's app_iter which,
when iterated over, will extract and PUT the objects pulled from the
request body. Will occasionally yield whitespace while request is being
processed. When the request is completed will yield a response body
that can be parsed to determine success. See above documentation for
details.
:params req: a swob Request
:params compress_type: specifying the compression type of the tar.
Accepts '', 'gz', or 'bz2'
"""
resp_dict = {'Response Status': HTTPCreated().status,
'Response Body': '', 'Number Files Created': 0}
failed_files = []
last_yield = time()
separator = ''
containers_accessed = set()
try:
if not out_content_type:
raise HTTPNotAcceptable(request=req)
if out_content_type.endswith('/xml'):
yield '<?xml version="1.0" encoding="UTF-8"?>\n'
if req.content_length is None and \
req.headers.get('transfer-encoding',
'').lower() != 'chunked':
raise HTTPLengthRequired(request=req)
try:
vrs, account, extract_base = req.split_path(2, 3, True)
except ValueError:
raise HTTPNotFound(request=req)
extract_base = extract_base or ''
extract_base = extract_base.rstrip('/')
tar = tarfile.open(mode='r|' + compress_type,
fileobj=req.body_file)
failed_response_type = HTTPBadRequest
req.environ['eventlet.minimum_write_chunk_size'] = 0
containers_created = 0
while True:
if last_yield + self.yield_frequency < time():
separator = '\r\n\r\n'
last_yield = time()
yield ' '
tar_info = tar.next()
if tar_info is None or \
len(failed_files) >= self.max_failed_extractions:
break
if tar_info.isfile():
obj_path = tar_info.name
if obj_path.startswith('./'):
obj_path = obj_path[2:]
obj_path = obj_path.lstrip('/')
if extract_base:
obj_path = extract_base + '/' + obj_path
if '/' not in obj_path:
continue # ignore base level file
destination = '/'.join(
['', vrs, account, obj_path])
container = obj_path.split('/', 1)[0]
if not check_utf8(destination):
failed_files.append(
[quote(obj_path[:MAX_PATH_LENGTH]),
HTTPPreconditionFailed().status])
continue
if tar_info.size > MAX_FILE_SIZE:
failed_files.append([
quote(obj_path[:MAX_PATH_LENGTH]),
HTTPRequestEntityTooLarge().status])
continue
container_failure = None
if container not in containers_accessed:
cont_path = '/'.join(['', vrs, account, container])
try:
if self.create_container(req, cont_path):
containers_created += 1
if containers_created > self.max_containers:
raise HTTPBadRequest(
'More than %d containers to create '
'from tar.' % self.max_containers)
except CreateContainerError as err:
# the object PUT to this container still may
# succeed if acls are set
container_failure = [
quote(cont_path[:MAX_PATH_LENGTH]),
err.status]
if err.status_int == HTTP_UNAUTHORIZED:
raise HTTPUnauthorized(request=req)
except ValueError:
failed_files.append([
quote(obj_path[:MAX_PATH_LENGTH]),
HTTPBadRequest().status])
continue
tar_file = tar.extractfile(tar_info)
new_env = req.environ.copy()
new_env['REQUEST_METHOD'] = 'PUT'
new_env['wsgi.input'] = tar_file
new_env['PATH_INFO'] = destination
new_env['CONTENT_LENGTH'] = tar_info.size
new_env['swift.source'] = 'EA'
new_env['HTTP_USER_AGENT'] = \
'%s BulkExpand' % req.environ.get('HTTP_USER_AGENT')
create_obj_req = Request.blank(destination, new_env)
resp = create_obj_req.get_response(self.app)
containers_accessed.add(container)
if resp.is_success:
resp_dict['Number Files Created'] += 1
else:
if container_failure:
failed_files.append(container_failure)
if resp.status_int == HTTP_UNAUTHORIZED:
failed_files.append([
quote(obj_path[:MAX_PATH_LENGTH]),
HTTPUnauthorized().status])
raise HTTPUnauthorized(request=req)
if resp.status_int // 100 == 5:
failed_response_type = HTTPBadGateway
failed_files.append([
quote(obj_path[:MAX_PATH_LENGTH]), resp.status])
if failed_files:
resp_dict['Response Status'] = failed_response_type().status
elif not resp_dict['Number Files Created']:
resp_dict['Response Status'] = HTTPBadRequest().status
resp_dict['Response Body'] = 'Invalid Tar File: No Valid Files'
except HTTPException as err:
resp_dict['Response Status'] = err.status
resp_dict['Response Body'] = err.body
except (tarfile.TarError, zlib.error) as tar_error:
resp_dict['Response Status'] = HTTPBadRequest().status
resp_dict['Response Body'] = 'Invalid Tar File: %s' % tar_error
except Exception:
self.logger.exception('Error in extract archive.')
resp_dict['Response Status'] = HTTPServerError().status
yield separator + get_response_body(
out_content_type, resp_dict, failed_files)
self.container_update(
"PUT",
account,
container,
obj,
request.headers,
{
"x-size": file.metadata["Content-Length"],
"x-content-type": file.metadata["Content-Type"],
"x-timestamp": file.metadata["X-Timestamp"],
"x-etag": file.metadata["ETag"],
"x-trans-id": request.headers.get("x-trans-id", "-"),
},
device,
)
resp = HTTPCreated(request=request, etag=etag)
return resp
@public
@timing_stats
def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
try:
device, partition, account, container, obj = split_path(unquote(request.path), 5, 5, True)
validate_device_partition(device, partition)
except ValueError, err:
return HTTPBadRequest(body=str(err), request=request, content_type="text/plain")
if self.mount_check and not check_mount(self.devices, device):
return HTTPInsufficientStorage(drive=device, request=request)
file = DiskFile(
self.devices,
def handle_extract_iter(self,
req,
compress_type,
out_content_type='text/plain'):
"""
A generator that can be assigned to a swob Response's app_iter which,
when iterated over, will extract and PUT the objects pulled from the
request body. Will occasionally yield whitespace while request is being
processed. When the request is completed will yield a response body
that can be parsed to determine success. See above documentation for
details.
:params req: a swob Request
:params compress_type: specifying the compression type of the tar.
Accepts '', 'gz', or 'bz2'
"""
resp_dict = {
'Response Status': HTTPCreated().status,
'Response Body': '',
'Number Files Created': 0
}
failed_files = []
last_yield = time()
separator = ''
containers_accessed = set()
try:
if not out_content_type:
raise HTTPNotAcceptable(request=req)
if out_content_type.endswith('/xml'):
yield '<?xml version="1.0" encoding="UTF-8"?>\n'
if req.content_length is None and \
req.headers.get('transfer-encoding',
'').lower() != 'chunked':
raise HTTPLengthRequired(request=req)
try:
vrs, account, extract_base = req.split_path(2, 3, True)
except ValueError:
raise HTTPNotFound(request=req)
extract_base = extract_base or ''
extract_base = extract_base.rstrip('/')
tar = tarfile.open(mode='r|' + compress_type,
fileobj=req.body_file)
failed_response_type = HTTPBadRequest
req.environ['eventlet.minimum_write_chunk_size'] = 0
containers_created = 0
while True:
if last_yield + self.yield_frequency < time():
separator = '\r\n\r\n'
last_yield = time()
yield ' '
tar_info = tar.next()
if tar_info is None or \
len(failed_files) >= self.max_failed_extractions:
break
if tar_info.isfile():
obj_path = tar_info.name
if obj_path.startswith('./'):
obj_path = obj_path[2:]
obj_path = obj_path.lstrip('/')
if extract_base:
obj_path = extract_base + '/' + obj_path
if '/' not in obj_path:
continue # ignore base level file
destination = '/'.join(['', vrs, account, obj_path])
container = obj_path.split('/', 1)[0]
if not check_utf8(destination):
failed_files.append([
quote(obj_path[:MAX_PATH_LENGTH]),
HTTPPreconditionFailed().status
])
continue
if tar_info.size > MAX_FILE_SIZE:
failed_files.append([
quote(obj_path[:MAX_PATH_LENGTH]),
HTTPRequestEntityTooLarge().status
])
continue
container_failure = None
if container not in containers_accessed:
cont_path = '/'.join(['', vrs, account, container])
try:
if self.create_container(req, cont_path):
containers_created += 1
if containers_created > self.max_containers:
raise HTTPBadRequest(
'More than %d containers to create '
'from tar.' % self.max_containers)
except CreateContainerError, err:
# the object PUT to this container still may
# succeed if acls are set
container_failure = [
quote(cont_path[:MAX_PATH_LENGTH]), err.status
]
if err.status_int == HTTP_UNAUTHORIZED:
raise HTTPUnauthorized(request=req)
except ValueError:
failed_files.append([
quote(obj_path[:MAX_PATH_LENGTH]),
HTTPBadRequest().status
])
continue
def _store_object(self, req, data_source, headers):
content_type = req.headers.get('content-type', 'octet/stream')
storage = self.app.storage
policy = None
container_info = self.container_info(self.account_name,
self.container_name, req)
if 'X-Oio-Storage-Policy' in req.headers:
policy = req.headers.get('X-Oio-Storage-Policy')
if not self.app.POLICIES.get_by_name(policy):
raise HTTPBadRequest(
"invalid policy '%s', must be in %s" %
(policy, self.app.POLICIES.by_name.keys()))
else:
try:
policy_index = int(
req.headers.get('X-Backend-Storage-Policy-Index',
container_info['storage_policy']))
except TypeError:
policy_index = 0
if policy_index != 0:
policy = self.app.POLICIES.get_by_index(policy_index).name
else:
content_length = int(req.headers.get('content-length', 0))
policy = self._get_auto_policy_from_size(content_length)
metadata = self.load_object_metadata(headers)
oio_headers = {'X-oio-req-id': self.trans_id}
try:
# FIXME(FVE): use 'properties' instead of 'metadata'
# as soon as we require oio>=4.2.0
_chunks, _size, checksum = storage.object_create(
self.account_name,
self.container_name,
obj_name=self.object_name,
file_or_path=data_source,
mime_type=content_type,
policy=policy,
headers=oio_headers,
etag=req.headers.get('etag', '').strip('"'),
metadata=metadata)
# TODO(FVE): when oio-sds supports it, do that in a callback
# passed to object_create (or whatever upload method supports it)
footer_md = self.load_object_metadata(self._get_footers(req))
if footer_md:
storage.object_set_properties(self.account_name,
self.container_name,
self.object_name,
properties=footer_md)
except exceptions.Conflict:
raise HTTPConflict(request=req)
except exceptions.PreconditionFailed:
raise HTTPPreconditionFailed(request=req)
except (SourceReadTimeout, GreenSourceReadTimeout) as err:
self.app.logger.warning(_('ERROR Client read timeout (%s)'), err)
self.app.logger.increment('client_timeouts')
raise HTTPRequestTimeout(request=req)
except exceptions.SourceReadError:
req.client_disconnect = True
self.app.logger.warning(
_('Client disconnected without sending last chunk'))
self.app.logger.increment('client_disconnects')
raise HTTPClientDisconnect(request=req)
except exceptions.EtagMismatch:
return HTTPUnprocessableEntity(request=req)
except (exceptions.ServiceBusy, exceptions.OioTimeout):
raise
# TODO(FVE): exceptions.NotFound is never raised from oio
# Remove when dependency is oio>=4.2.0
except (exceptions.NoSuchContainer, exceptions.NotFound):
raise HTTPNotFound(request=req)
except exceptions.ClientException as err:
# 481 = CODE_POLICY_NOT_SATISFIABLE
if err.status == 481:
raise exceptions.ServiceBusy()
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
except Exception:
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
# TODO(FVE): use the timestamp of the object.
# Unfortunately the oio-sds API does not return the actual ctime.
resp = HTTPCreated(request=req,
etag=checksum,
last_modified=int(time.time()))
return resp
request, device)
if old_delete_at:
self.delete_at_update(
'DELETE', old_delete_at, account, container, obj,
request, device)
if not orig_timestamp or \
orig_timestamp < request.headers['x-timestamp']:
self.container_update(
'PUT', account, container, obj, request,
HeaderKeyDict({
'x-size': disk_file.metadata['Content-Length'],
'x-content-type': disk_file.metadata['Content-Type'],
'x-timestamp': disk_file.metadata['X-Timestamp'],
'x-etag': disk_file.metadata['ETag']}),
device)
resp = HTTPCreated(request=request, etag=etag)
return resp
@public
@timing_stats()
def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
try:
device, partition, account, container, obj = \
split_path(unquote(request.path), 5, 5, True)
validate_device_partition(device, partition)
except ValueError, err:
return HTTPBadRequest(body=str(err), request=request,
content_type='text/plain')
if self.mount_check and not check_mount(self.devices, device):
return HTTPInsufficientStorage(drive=device, request=request)
self.delete_at_update('DELETE', old_delete_at, account,
container, obj, request.headers,
device)
file.put(fd, metadata)
file.unlinkold(metadata['X-Timestamp'])
if not orig_timestamp or \
orig_timestamp < request.headers['x-timestamp']:
self.container_update(
'PUT', account, container, obj, request.headers, {
'x-size': file.metadata['Content-Length'],
'x-content-type': file.metadata['Content-Type'],
'x-timestamp': file.metadata['X-Timestamp'],
'x-etag': file.metadata['ETag'],
'x-trans-id': request.headers.get('x-trans-id', '-')
}, device)
resp = HTTPCreated(request=request, etag=etag)
return resp
@public
@timing_stats()
def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
try:
device, partition, account, container, obj = \
split_path(unquote(request.path), 5, 5, True)
validate_device_partition(device, partition)
except ValueError, err:
return HTTPBadRequest(body=str(err),
request=request,
content_type='text/plain')
if self.mount_check and not check_mount(self.devices, device):
def PUT(self, request):
"""Handle HTTP PUT requests for the Swift Object Server."""
device, partition, account, container, obj = \
split_and_validate_path(request, 5, 5, True)
if 'x-timestamp' not in request.headers or \
not check_float(request.headers['x-timestamp']):
return HTTPBadRequest(body='Missing timestamp',
request=request,
content_type='text/plain')
error_response = check_object_creation(request, obj)
if error_response:
return error_response
new_delete_at = int(request.headers.get('X-Delete-At') or 0)
if new_delete_at and new_delete_at < time.time():
return HTTPBadRequest(body='X-Delete-At in past',
request=request,
content_type='text/plain')
try:
fsize = request.message_length()
except ValueError as e:
return HTTPBadRequest(body=str(e),
request=request,
content_type='text/plain')
try:
disk_file = self._diskfile(device, partition, account, container,
obj)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
old_delete_at = int(disk_file.metadata.get('X-Delete-At') or 0)
orig_timestamp = disk_file.metadata.get('X-Timestamp')
if orig_timestamp and orig_timestamp >= request.headers['x-timestamp']:
return HTTPConflict(request=request)
upload_expiration = time.time() + self.max_upload_time
etag = md5()
elapsed_time = 0
try:
with disk_file.create(size=fsize) as writer:
reader = request.environ['wsgi.input'].read
for chunk in iter(lambda: reader(self.network_chunk_size), ''):
start_time = time.time()
if start_time > upload_expiration:
self.logger.increment('PUT.timeouts')
return HTTPRequestTimeout(request=request)
etag.update(chunk)
writer.write(chunk)
sleep()
elapsed_time += time.time() - start_time
upload_size = writer.upload_size
if upload_size:
self.logger.transfer_rate('PUT.' + device + '.timing',
elapsed_time, upload_size)
if fsize is not None and fsize != upload_size:
return HTTPClientDisconnect(request=request)
etag = etag.hexdigest()
if 'etag' in request.headers and \
request.headers['etag'].lower() != etag:
return HTTPUnprocessableEntity(request=request)
metadata = {
'X-Timestamp': request.headers['x-timestamp'],
'Content-Type': request.headers['content-type'],
'ETag': etag,
'Content-Length': str(upload_size),
}
metadata.update(val for val in request.headers.iteritems()
if val[0].lower().startswith('x-object-meta-')
and len(val[0]) > 14)
for header_key in self.allowed_headers:
if header_key in request.headers:
header_caps = header_key.title()
metadata[header_caps] = request.headers[header_key]
writer.put(metadata)
except DiskFileNoSpace:
return HTTPInsufficientStorage(drive=device, request=request)
if old_delete_at != new_delete_at:
if new_delete_at:
self.delete_at_update('PUT', new_delete_at, account, container,
obj, request, device)
if old_delete_at:
self.delete_at_update('DELETE', old_delete_at, account,
container, obj, request, device)
if not orig_timestamp or \
orig_timestamp < request.headers['x-timestamp']:
self.container_update(
'PUT', account, container, obj, request,
HeaderKeyDict({
'x-size':
disk_file.metadata['Content-Length'],
'x-content-type':
disk_file.metadata['Content-Type'],
'x-timestamp':
disk_file.metadata['X-Timestamp'],
'x-etag':
disk_file.metadata['ETag']
}), device)
resp = HTTPCreated(request=request, etag=etag)
return resp
request.headers, device)
file.put(fd, metadata)
file.unlinkold(metadata['X-Timestamp'])
if not orig_timestamp or \
orig_timestamp < request.headers['x-timestamp']:
#FIXME(vvechkanov): maybe Original-Content-Length
# should be used here
self.container_update(
'PUT', account, container, obj, request.headers,
{'x-size': file.metadata['Content-Length'],
'x-content-type': file.metadata['Content-Type'],
'x-timestamp': file.metadata['X-Timestamp'],
'x-etag': file.metadata['Original-Etag'],
'x-trans-id': request.headers.get('x-trans-id', '-')},
device)
resp = HTTPCreated(request=request, etag=etag_orig)
self.logger.timing_since('PUT.timing', start_time)
return resp
@public
@timing_stats()
def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
try:
device, partition, account, container, obj = \
split_path(unquote(request.path), 5, 5, True)
validate_device_partition(device, partition)
except ValueError, err:
return HTTPBadRequest(body=str(err), request=request,
content_type='text/plain')
if self.mount_check and not check_mount(self.devices, device):
def PUT(self, req):
"""Handle HTTP PUT request."""
drive, part, account, container, obj = get_obj_name_and_placement(req)
req_timestamp = valid_timestamp(req)
if 'x-container-sync-to' in req.headers:
err, sync_to, realm, realm_key = validate_sync_to(
req.headers['x-container-sync-to'], self.allowed_sync_hosts,
self.realms_conf)
if err:
return HTTPBadRequest(err)
try:
check_drive(self.root, drive, self.mount_check)
except ValueError:
return HTTPInsufficientStorage(drive=drive, request=req)
if not self.check_free_space(drive):
return HTTPInsufficientStorage(drive=drive, request=req)
requested_policy_index = self.get_and_validate_policy_index(req)
broker = self._get_container_broker(drive, part, account, container)
if obj: # put container object
# obj put expects the policy_index header, default is for
# legacy support during upgrade.
obj_policy_index = requested_policy_index or 0
self._maybe_autocreate(broker, req_timestamp, account,
obj_policy_index)
# redirect if a shard exists for this object name
response = self._redirect_to_shard(req, broker, obj)
if response:
return response
broker.put_object(
obj, req_timestamp.internal, int(req.headers['x-size']),
wsgi_to_str(req.headers['x-content-type']),
wsgi_to_str(req.headers['x-etag']), 0, obj_policy_index,
wsgi_to_str(req.headers.get('x-content-type-timestamp')),
wsgi_to_str(req.headers.get('x-meta-timestamp')))
return HTTPCreated(request=req)
record_type = req.headers.get('x-backend-record-type', '').lower()
if record_type == RECORD_TYPE_SHARD:
try:
# validate incoming data...
shard_ranges = [
ShardRange.from_dict(sr) for sr in json.loads(req.body)
]
except (ValueError, KeyError, TypeError) as err:
return HTTPBadRequest('Invalid body: %r' % err)
created = self._maybe_autocreate(broker, req_timestamp, account,
requested_policy_index)
self._update_metadata(req, broker, req_timestamp, 'PUT')
if shard_ranges:
# TODO: consider writing the shard ranges into the pending
# file, but if so ensure an all-or-none semantic for the write
broker.merge_shard_ranges(shard_ranges)
else: # put container
if requested_policy_index is None:
# use the default index sent by the proxy if available
new_container_policy = req.headers.get(
'X-Backend-Storage-Policy-Default', int(POLICIES.default))
else:
new_container_policy = requested_policy_index
created = self._update_or_create(req, broker,
req_timestamp.internal,
new_container_policy,
requested_policy_index)
self._update_metadata(req, broker, req_timestamp, 'PUT')
resp = self.account_update(req, account, container, broker)
if resp:
return resp
if created:
return HTTPCreated(request=req,
headers={
'x-backend-storage-policy-index':
broker.storage_policy_index
})
else:
return HTTPAccepted(request=req,
headers={
'x-backend-storage-policy-index':
broker.storage_policy_index
})
def PUT(self, req):
"""Handle HTTP PUT request."""
drive, part, account, container = split_and_validate_path(req, 3, 4)
try:
check_drive(self.root, drive, self.mount_check)
except ValueError:
return HTTPInsufficientStorage(drive=drive, request=req)
if not self.check_free_space(drive):
return HTTPInsufficientStorage(drive=drive, request=req)
if container: # put account container
if 'x-timestamp' not in req.headers:
timestamp = Timestamp.now()
else:
timestamp = valid_timestamp(req)
pending_timeout = None
container_policy_index = \
req.headers.get('X-Backend-Storage-Policy-Index', 0)
if 'x-trans-id' in req.headers:
pending_timeout = 3
broker = self._get_account_broker(drive,
part,
account,
pending_timeout=pending_timeout)
if account.startswith(self.auto_create_account_prefix) and \
not os.path.exists(broker.db_file):
try:
broker.initialize(timestamp.internal)
except DatabaseAlreadyExists:
pass
if req.headers.get('x-account-override-deleted', 'no').lower() != \
'yes' and broker.is_deleted():
return HTTPNotFound(request=req)
broker.put_container(container, req.headers['x-put-timestamp'],
req.headers['x-delete-timestamp'],
req.headers['x-object-count'],
req.headers['x-bytes-used'],
container_policy_index)
if req.headers['x-delete-timestamp'] > \
req.headers['x-put-timestamp']:
return HTTPNoContent(request=req)
else:
return HTTPCreated(request=req)
else: # put account
timestamp = valid_timestamp(req)
broker = self._get_account_broker(drive, part, account)
if not os.path.exists(broker.db_file):
try:
broker.initialize(timestamp.internal)
created = True
except DatabaseAlreadyExists:
created = False
elif broker.is_status_deleted():
return self._deleted_response(broker,
req,
HTTPForbidden,
body='Recently deleted')
else:
created = broker.is_deleted()
broker.update_put_timestamp(timestamp.internal)
if broker.is_deleted():
return HTTPConflict(request=req)
metadata = {}
if six.PY2:
metadata.update((key, (value, timestamp.internal))
for key, value in req.headers.items()
if is_sys_or_user_meta('account', key))
else:
for key, value in req.headers.items():
if is_sys_or_user_meta('account', key):
# Cast to native strings, so that json inside
# updata_metadata eats the data.
try:
value = value.encode('latin-1').decode('utf-8')
except UnicodeDecodeError:
raise HTTPBadRequest(
'Metadata must be valid UTF-8')
metadata[key] = (value, timestamp.internal)
if metadata:
broker.update_metadata(metadata, validate_metadata=True)
if created:
return HTTPCreated(request=req)
else:
return HTTPAccepted(request=req)
def _link_object(self, req):
_, container, obj = req.headers['Oio-Copy-From'].split('/', 2)
from_account = req.headers.get('X-Copy-From-Account',
self.account_name)
self.app.logger.info(
"Creating link from %s/%s/%s to %s/%s/%s",
# Existing
from_account,
container,
obj,
# New
self.account_name,
self.container_name,
self.object_name)
storage = self.app.storage
if req.headers.get('Range'):
raise Exception("Fast Copy with Range is unsupported")
ranges = ranges_from_http_header(req.headers.get('Range'))
if len(ranges) != 1:
raise HTTPInternalServerError(
request=req, body="mutiple ranges unsupported")
ranges = ranges[0]
else:
ranges = None
headers = self._prepare_headers(req)
metadata = self.load_object_metadata(headers)
oio_headers = {REQID_HEADER: self.trans_id}
oio_cache = req.environ.get('oio.cache')
perfdata = req.environ.get('swift.perfdata')
# FIXME(FVE): use object_show, cache in req.environ
version = obj_version_from_env(req.environ)
props = storage.object_get_properties(from_account,
container,
obj,
headers=oio_headers,
version=version,
cache=oio_cache,
perfdata=perfdata)
if props['properties'].get(SLO, None):
raise Exception("Fast Copy with SLO is unsupported")
else:
if ranges:
raise HTTPInternalServerError(
request=req, body="no range supported with single object")
try:
# TODO check return code (values ?)
link_meta = storage.object_link(from_account,
container,
obj,
self.account_name,
self.container_name,
self.object_name,
headers=oio_headers,
properties=metadata,
properties_directive='REPLACE',
target_version=version,
cache=oio_cache,
perfdata=perfdata)
# TODO(FVE): this exception catching block has to be refactored
# TODO check which ones are ok or make non sense
except exceptions.Conflict:
raise HTTPConflict(request=req)
except exceptions.PreconditionFailed:
raise HTTPPreconditionFailed(request=req)
except exceptions.SourceReadError:
req.client_disconnect = True
self.app.logger.warning(
_('Client disconnected without sending last chunk'))
self.app.logger.increment('client_disconnects')
raise HTTPClientDisconnect(request=req)
except exceptions.EtagMismatch:
return HTTPUnprocessableEntity(request=req)
except (exceptions.ServiceBusy, exceptions.OioTimeout,
exceptions.DeadlineReached):
raise
except (exceptions.NoSuchContainer, exceptions.NotFound):
raise HTTPNotFound(request=req)
except exceptions.ClientException as err:
# 481 = CODE_POLICY_NOT_SATISFIABLE
if err.status == 481:
raise exceptions.ServiceBusy()
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
except Exception:
self.app.logger.exception(
_('ERROR Exception transferring data %s'), {'path': req.path})
raise HTTPInternalServerError(request=req)
resp = HTTPCreated(request=req, etag=link_meta['hash'])
return resp
def PUT(self, request):
"""Handle HTTP PUT requests for the Swift Object Server."""
device, partition, account, container, obj, policy = \
get_name_and_placement(request, 5, 5, True)
req_timestamp = valid_timestamp(request)
error_response = check_object_creation(request, obj)
if error_response:
return error_response
new_delete_at = int(request.headers.get('X-Delete-At') or 0)
if new_delete_at and new_delete_at < time.time():
return HTTPBadRequest(body='X-Delete-At in past', request=request,
content_type='text/plain')
try:
fsize = request.message_length()
except ValueError as e:
return HTTPBadRequest(body=str(e), request=request,
content_type='text/plain')
# In case of multipart-MIME put, the proxy sends a chunked request,
# but may let us know the real content length so we can verify that
# we have enough disk space to hold the object.
if fsize is None:
fsize = request.headers.get('X-Backend-Obj-Content-Length')
if fsize is not None:
try:
fsize = int(fsize)
except ValueError as e:
return HTTPBadRequest(body=str(e), request=request,
content_type='text/plain')
# SSYNC will include Frag-Index header for subrequests to primary
# nodes; handoff nodes should 409 subrequests to over-write an
# existing data fragment until they offloaded the existing fragment
frag_index = request.headers.get('X-Backend-Ssync-Frag-Index')
try:
disk_file = self.get_diskfile(
device, partition, account, container, obj,
policy=policy, frag_index=frag_index)
except DiskFileDeviceUnavailable:
return HTTPInsufficientStorage(drive=device, request=request)
try:
orig_metadata = disk_file.read_metadata()
except DiskFileXattrNotSupported:
return HTTPInsufficientStorage(drive=device, request=request)
except (DiskFileNotExist, DiskFileQuarantined):
orig_metadata = {}
# Checks for If-None-Match
if request.if_none_match is not None and orig_metadata:
if '*' in request.if_none_match:
# File exists already so return 412
return HTTPPreconditionFailed(request=request)
if orig_metadata.get('ETag') in request.if_none_match:
# The current ETag matches, so return 412
return HTTPPreconditionFailed(request=request)
orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0))
if orig_timestamp >= req_timestamp:
return HTTPConflict(
request=request,
headers={'X-Backend-Timestamp': orig_timestamp.internal})
orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0)
upload_expiration = time.time() + self.max_upload_time
etag = md5()
elapsed_time = 0
try:
with disk_file.create(size=fsize) as writer:
upload_size = 0
# If the proxy wants to send us object metadata after the
# object body, it sets some headers. We have to tell the
# proxy, in the 100 Continue response, that we're able to
# parse a multipart MIME document and extract the object and
# metadata from it. If we don't, then the proxy won't
# actually send the footer metadata.
have_metadata_footer = False
use_multiphase_commit = False
mime_documents_iter = iter([])
obj_input = request.environ['wsgi.input']
hundred_continue_headers = []
if config_true_value(
request.headers.get(
'X-Backend-Obj-Multiphase-Commit')):
use_multiphase_commit = True
hundred_continue_headers.append(
('X-Obj-Multiphase-Commit', 'yes'))
if config_true_value(
request.headers.get('X-Backend-Obj-Metadata-Footer')):
have_metadata_footer = True
hundred_continue_headers.append(
('X-Obj-Metadata-Footer', 'yes'))
if have_metadata_footer or use_multiphase_commit:
obj_input.set_hundred_continue_response_headers(
hundred_continue_headers)
mime_boundary = request.headers.get(
'X-Backend-Obj-Multipart-Mime-Boundary')
if not mime_boundary:
return HTTPBadRequest("no MIME boundary")
try:
with ChunkReadTimeout(self.client_timeout):
mime_documents_iter = iter_mime_headers_and_bodies(
request.environ['wsgi.input'],
mime_boundary, self.network_chunk_size)
_junk_hdrs, obj_input = next(mime_documents_iter)
except ChunkReadTimeout:
return HTTPRequestTimeout(request=request)
timeout_reader = self._make_timeout_reader(obj_input)
try:
for chunk in iter(timeout_reader, ''):
start_time = time.time()
if start_time > upload_expiration:
self.logger.increment('PUT.timeouts')
return HTTPRequestTimeout(request=request)
etag.update(chunk)
upload_size = writer.write(chunk)
elapsed_time += time.time() - start_time
except ChunkReadTimeout:
return HTTPRequestTimeout(request=request)
if upload_size:
self.logger.transfer_rate(
'PUT.' + device + '.timing', elapsed_time,
upload_size)
if fsize is not None and fsize != upload_size:
return HTTPClientDisconnect(request=request)
footer_meta = {}
if have_metadata_footer:
footer_meta = self._read_metadata_footer(
mime_documents_iter)
request_etag = (footer_meta.get('etag') or
request.headers.get('etag', '')).lower()
etag = etag.hexdigest()
if request_etag and request_etag != etag:
return HTTPUnprocessableEntity(request=request)
metadata = {
'X-Timestamp': request.timestamp.internal,
'Content-Type': request.headers['content-type'],
'ETag': etag,
'Content-Length': str(upload_size),
}
metadata.update(val for val in request.headers.items()
if is_sys_or_user_meta('object', val[0]))
metadata.update(val for val in footer_meta.items()
if is_sys_or_user_meta('object', val[0]))
headers_to_copy = (
request.headers.get(
'X-Backend-Replication-Headers', '').split() +
list(self.allowed_headers))
for header_key in headers_to_copy:
if header_key in request.headers:
header_caps = header_key.title()
metadata[header_caps] = request.headers[header_key]
writer.put(metadata)
# if the PUT requires a two-phase commit (a data and a commit
# phase) send the proxy server another 100-continue response
# to indicate that we are finished writing object data
if use_multiphase_commit:
request.environ['wsgi.input'].\
send_hundred_continue_response()
if not self._read_put_commit_message(mime_documents_iter):
return HTTPServerError(request=request)
# got 2nd phase confirmation, write a timestamp.durable
# state file to indicate a successful PUT
writer.commit(request.timestamp)
# Drain any remaining MIME docs from the socket. There
# shouldn't be any, but we must read the whole request body.
try:
while True:
with ChunkReadTimeout(self.client_timeout):
_junk_hdrs, _junk_body = next(mime_documents_iter)
drain(_junk_body, self.network_chunk_size,
self.client_timeout)
except ChunkReadTimeout:
raise HTTPClientDisconnect()
except StopIteration:
pass
except (DiskFileXattrNotSupported, DiskFileNoSpace):
return HTTPInsufficientStorage(drive=device, request=request)
if orig_delete_at != new_delete_at:
if new_delete_at:
self.delete_at_update(
'PUT', new_delete_at, account, container, obj, request,
device, policy)
if orig_delete_at:
self.delete_at_update(
'DELETE', orig_delete_at, account, container, obj,
request, device, policy)
update_headers = HeaderKeyDict({
'x-size': metadata['Content-Length'],
'x-content-type': metadata['Content-Type'],
'x-timestamp': metadata['X-Timestamp'],
'x-etag': metadata['ETag']})
# apply any container update header overrides sent with request
self._check_container_override(update_headers, request.headers)
self._check_container_override(update_headers, footer_meta)
self.container_update(
'PUT', account, container, obj, request,
update_headers,
device, policy)
return HTTPCreated(request=request, etag=etag)
self.delete_at_update(
'DELETE', old_delete_at, account, container, obj,
request.headers, device)
file.put(fd, tmppath, metadata)
file.unlinkold(metadata['X-Timestamp'])
if not orig_timestamp or \
orig_timestamp < request.headers['x-timestamp']:
self.container_update(
'PUT', account, container, obj, request.headers,
{'x-size': file.metadata['Content-Length'],
'x-content-type': file.metadata['Content-Type'],
'x-timestamp': file.metadata['X-Timestamp'],
'x-etag': file.metadata['ETag'],
'x-trans-id': request.headers.get('x-trans-id', '-')},
device)
resp = HTTPCreated(request=request, etag=etag)
self.logger.timing_since('PUT.timing', start_time)
return resp
@public
def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
start_time = time.time()
try:
device, partition, account, container, obj = \
split_path(unquote(request.path), 5, 5, True)
validate_device_partition(device, partition)
except ValueError, err:
self.logger.increment('GET.errors')
return HTTPBadRequest(body=str(err), request=request,
content_type='text/plain')
"PUT",
account,
container,
obj,
request,
HeaderKeyDict(
{
"x-size": disk_file.metadata["Content-Length"],
"x-content-type": disk_file.metadata["Content-Type"],
"x-timestamp": disk_file.metadata["X-Timestamp"],
"x-etag": disk_file.metadata["ETag"],
}
),
device,
)
resp = HTTPCreated(request=request, etag=etag)
return resp
@public
@timing_stats()
def GET(self, request):
"""Handle HTTP GET requests for the Swift Object Server."""
try:
device, partition, account, container, obj = split_path(unquote(request.path), 5, 5, True)
validate_device_partition(device, partition)
except ValueError, err:
return HTTPBadRequest(body=str(err), request=request, content_type="text/plain")
if self.mount_check and not check_mount(self.devices, device):
return HTTPInsufficientStorage(drive=device, request=request)
disk_file = DiskFile(
self.devices,
def handle_extract_iter(self,
req,
compress_type,
out_content_type='text/plain'):
"""
A generator that can be assigned to a swob Response's app_iter which,
when iterated over, will extract and PUT the objects pulled from the
request body. Will occasionally yield whitespace while request is being
processed. When the request is completed will yield a response body
that can be parsed to determine success. See above documentation for
details.
:params req: a swob Request
:params compress_type: specifying the compression type of the tar.
Accepts '', 'gz', or 'bz2'
"""
resp_dict = {
'Response Status': HTTPCreated().status,
'Response Body': '',
'Number Files Created': 0
}
failed_files = []
last_yield = time()
if out_content_type and out_content_type.endswith('/xml'):
to_yield = b'<?xml version="1.0" encoding="UTF-8"?>\n'
else:
to_yield = b' '
separator = b''
containers_accessed = set()
req.environ['eventlet.minimum_write_chunk_size'] = 0
try:
if not out_content_type:
raise HTTPNotAcceptable(request=req)
if req.content_length is None and \
req.headers.get('transfer-encoding',
'').lower() != 'chunked':
raise HTTPLengthRequired(request=req)
try:
vrs, account, extract_base = req.split_path(2, 3, True)
except ValueError:
raise HTTPNotFound(request=req)
extract_base = extract_base or ''
extract_base = extract_base.rstrip('/')
tar = tarfile.open(mode='r|' + compress_type,
fileobj=req.body_file)
failed_response_type = HTTPBadRequest
containers_created = 0
while True:
if last_yield + self.yield_frequency < time():
last_yield = time()
yield to_yield
to_yield, separator = b' ', b'\r\n\r\n'
tar_info = tar.next()
if tar_info is None or \
len(failed_files) >= self.max_failed_extractions:
break
if tar_info.isfile():
obj_path = tar_info.name
if not six.PY2:
obj_path = obj_path.encode('utf-8', 'surrogateescape')
obj_path = bytes_to_wsgi(obj_path)
if obj_path.startswith('./'):
obj_path = obj_path[2:]
obj_path = obj_path.lstrip('/')
if extract_base:
obj_path = extract_base + '/' + obj_path
if '/' not in obj_path:
continue # ignore base level file
destination = '/'.join(['', vrs, account, obj_path])
container = obj_path.split('/', 1)[0]
if not constraints.check_utf8(wsgi_to_str(destination)):
failed_files.append([
wsgi_quote(obj_path[:self.max_path_length]),
HTTPPreconditionFailed().status
])
continue
if tar_info.size > constraints.MAX_FILE_SIZE:
failed_files.append([
wsgi_quote(obj_path[:self.max_path_length]),
HTTPRequestEntityTooLarge().status
])
continue
container_failure = None
if container not in containers_accessed:
cont_path = '/'.join(['', vrs, account, container])
try:
if self.create_container(req, cont_path):
containers_created += 1
if containers_created > self.max_containers:
raise HTTPBadRequest(
'More than %d containers to create '
'from tar.' % self.max_containers)
except CreateContainerError as err:
# the object PUT to this container still may
# succeed if acls are set
container_failure = [
wsgi_quote(cont_path[:self.max_path_length]),
err.status
]
if err.status_int == HTTP_UNAUTHORIZED:
raise HTTPUnauthorized(request=req)
except ValueError:
failed_files.append([
wsgi_quote(obj_path[:self.max_path_length]),
HTTPBadRequest().status
])
continue
tar_file = tar.extractfile(tar_info)
create_headers = {
'Content-Length': tar_info.size,
'X-Auth-Token': req.headers.get('X-Auth-Token'),
}
# Copy some whitelisted headers to the subrequest
for k, v in req.headers.items():
if ((k.lower() in ('x-delete-at', 'x-delete-after'))
or is_user_meta('object', k)):
create_headers[k] = v
create_obj_req = make_subrequest(
req.environ,
method='PUT',
path=wsgi_quote(destination),
headers=create_headers,
agent='%(orig)s BulkExpand',
swift_source='EA')
create_obj_req.environ['wsgi.input'] = tar_file
for pax_key, pax_value in tar_info.pax_headers.items():
header_name = pax_key_to_swift_header(pax_key)
if header_name:
# Both pax_key and pax_value are unicode
# strings; the key is already UTF-8 encoded, but
# we still have to encode the value.
create_obj_req.headers[header_name] = \
pax_value.encode("utf-8")
resp = create_obj_req.get_response(self.app)
containers_accessed.add(container)
if resp.is_success:
resp_dict['Number Files Created'] += 1
else:
if container_failure:
failed_files.append(container_failure)
if resp.status_int == HTTP_UNAUTHORIZED:
failed_files.append([
wsgi_quote(obj_path[:self.max_path_length]),
HTTPUnauthorized().status
])
raise HTTPUnauthorized(request=req)
if resp.status_int // 100 == 5:
failed_response_type = HTTPBadGateway
failed_files.append([
wsgi_quote(obj_path[:self.max_path_length]),
resp.status
])
if failed_files:
resp_dict['Response Status'] = failed_response_type().status
elif not resp_dict['Number Files Created']:
resp_dict['Response Status'] = HTTPBadRequest().status
resp_dict['Response Body'] = 'Invalid Tar File: No Valid Files'
except HTTPException as err:
resp_dict['Response Status'] = err.status
resp_dict['Response Body'] = err.body.decode('utf-8')
except (tarfile.TarError, zlib.error) as tar_error:
resp_dict['Response Status'] = HTTPBadRequest().status
resp_dict['Response Body'] = 'Invalid Tar File: %s' % tar_error
except Exception:
self.logger.exception('Error in extract archive.')
resp_dict['Response Status'] = HTTPServerError().status
yield separator + get_response_body(out_content_type, resp_dict,
failed_files, 'extract')
'Content-Type': listing_content_type,
'Content-Length': 0
},
agent='SwiftOrigin',
swift_source='SOS').get_response(self.app)
if cdn_list_resp.status_int // 100 != 2:
raise OriginDbFailure('Could not PUT/POST to cdn listing in '
'origin db: %s %s' %
(cdn_list_path, cdn_list_resp.status_int))
# PUTs and POSTs have the headers as HEAD
cdn_url_headers = self.get_cdn_urls(hsh, 'HEAD')
if req.method == 'POST':
resp = HTTPAccepted(request=req, headers=cdn_url_headers)
else:
resp = HTTPCreated(request=req, headers=cdn_url_headers)
resp.extra_log_data = listing_content_type
return resp
def handle_request(self, env, req):
"""
This handles requests from a user to activate cdn access for their
containers, list them, etc.
"""
if 'swift.authorize' in req.environ:
aresp = req.environ['swift.authorize'](req)
if aresp:
return aresp
try:
if req.method in ('PUT', 'POST'):
return self.origin_db_puts_posts(env, req)
