def test_invocation_flow(client):
client.ping.return_value = SBusResponse(True, 'OK')
client.stop_daemon.return_value = SBusResponse(True, 'OK')
client.start_daemon.return_value = SBusResponse(True, 'OK')
sresp = self.gateway.invocation_flow(st_req, extra_sources)
eventlet.sleep(0.1)
file_like = FileLikeIter(sresp.data_iter)
self.assertEqual(b'something', file_like.read())
def open_object_stream(self):
status, self._headers, body = self._swift.get_object(
self._account,
self._container,
self._key,
headers=self.swift_req_hdrs)
if status != 200:
raise RuntimeError('Failed to get the object')
self._bytes_read = 0
self._swift_stream = body
self._iter = FileLikeIter(body)
self._s3_headers = convert_to_s3_headers(self._headers)
def test_PUT_multiseg_good_client_etag(self):
body_key = os.urandom(32)
chunks = ['some', 'chunks', 'of data']
body = ''.join(chunks)
env = {
'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
'wsgi.input': FileLikeIter(chunks)
}
hdrs = {
'content-type': 'text/plain',
'content-length': str(len(body)),
'Etag': md5hex(body)
}
req = Request.blank('/v1/a/c/o', environ=env, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
with mock.patch(
'swift.common.middleware.crypto.crypto_utils.'
'Crypto.create_random_key', lambda *args: body_key):
resp = req.get_response(self.encrypter)
self.assertEqual('201 Created', resp.status)
# verify object is encrypted by getting direct from the app
get_req = Request.blank('/v1/a/c/o', environ={'REQUEST_METHOD': 'GET'})
self.assertEqual(encrypt(body, body_key, FAKE_IV),
get_req.get_response(self.app).body)
def multipart_response_iter(self, resp, boundary, body_key, crypto_meta):
"""
Decrypts a multipart mime doc response body.
:param resp: application response
:param boundary: multipart boundary string
:param body_key: decryption key for the response body
:param crypto_meta: crypto_meta for the response body
:return: generator for decrypted response body
"""
with closing_if_possible(resp):
parts_iter = multipart_byteranges_to_document_iters(
FileLikeIter(resp), boundary)
for first_byte, last_byte, length, headers, body in parts_iter:
yield "--" + boundary + "\r\n"
for header_pair in headers:
yield "%s: %s\r\n" % header_pair
yield "\r\n"
decrypt_ctxt = self.crypto.create_decryption_ctxt(
body_key, crypto_meta['iv'], first_byte)
for chunk in iter(lambda: body.read(DECRYPT_CHUNK_SIZE), ''):
yield decrypt_ctxt.update(chunk)
yield "\r\n"
yield "--" + boundary + "--"
def _check_large_objects(self, aws_bucket, container, key, client):
local_meta = client.get_object_metadata(self.config['account'],
container, key)
remote_resp = self.provider.head_object(key)
if 'x-object-manifest' in remote_resp.headers and\
'x-object-manifest' in local_meta:
if remote_resp.headers['x-object-manifest'] !=\
local_meta['x-object-manifest']:
self.errors.put(
(container, key,
'Dynamic Large objects with differing manifests: '
'%s %s' % (remote_resp.headers['x-object-manifest'],
local_meta['x-object-manifest'])))
# TODO: once swiftclient supports query_string on HEAD requests, we
# would be able to compare the ETag of the manifest object itself.
return
if 'x-static-large-object' in remote_resp.headers and\
'x-static-large-object' in local_meta:
# We have to GET the manifests and cannot rely on the ETag, as
# these are not guaranteed to be in stable order from Swift. Once
# that issue is fixed in Swift, we can compare ETags.
status, headers, local_manifest = client.get_object(
self.config['account'], container, key, {})
remote_manifest = self.provider.get_manifest(key,
bucket=aws_bucket)
if json.load(FileLikeIter(local_manifest)) != remote_manifest:
self.errors.put((aws_bucket, key,
'Matching date, but differing SLO manifests'))
return
self.errors.put(
(aws_bucket, key,
'Mismatching ETag for regular objects with the same date'))
def handle_put_copy_response(self, app_iter):
self._remove_storlet_headers(self.request.headers)
if 'CONTENT_LENGTH' in self.request.environ:
self.request.environ.pop('CONTENT_LENGTH')
self.request.headers['Transfer-Encoding'] = 'chunked'
self.request.environ['wsgi.input'] = FileLikeIter(app_iter)
return self.request.get_response(self.app)
def _put_versioned_obj(self, req, put_path_info, source_resp):
# Create a new Request object to PUT to the versions container, copying
# all headers from the source object apart from x-timestamp.
put_req = make_pre_authed_request(
req.environ, path=put_path_info, method='PUT',
swift_source='VW')
copy_header_subset(source_resp, put_req,
lambda k: k.lower() != 'x-timestamp')
put_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter)
return put_req.get_response(self.app)
def _migrate_object(self, aws_bucket, container, key):
args = {'bucket': aws_bucket}
if self.config.get('protocol', 's3') == 'swift':
args['resp_chunk_size'] = 65536
if (aws_bucket, container, key) in self._manifests:
# Special handling for the DLO manifests
args['query_string'] = 'multipart-manifest=get'
resp = self.provider.get_object(key, **args)
if 'x-object-manifest' not in resp.headers:
self.logger.warning('DLO object changed before upload: %s/%s' %
(aws_bucket, key))
resp.body.close()
return
self._upload_object(
UploadObjectWork(
container, key, FileLikeIter(resp.body),
convert_to_local_headers(resp.headers.items(),
remove_timestamp=False),
aws_bucket))
return
resp = self.provider.get_object(key, **args)
if resp.status != 200:
resp.body.close()
raise MigrationError('Failed to GET "%s/%s": %s' %
(aws_bucket, key, resp.body))
put_headers = convert_to_local_headers(resp.headers.items(),
remove_timestamp=False)
if 'x-object-manifest' in resp.headers:
self.logger.warning('Migrating Dynamic Large Object "%s/%s" -- '
'results may not be consistent' %
(container, key))
resp.body.close()
self._migrate_dlo(aws_bucket, container, key, put_headers)
elif 'x-static-large-object' in resp.headers:
# We have to move the segments and then move the manifest file
resp.body.close()
self._migrate_slo(aws_bucket, container, key, put_headers)
else:
work = UploadObjectWork(container, key, FileLikeIter(resp.body),
put_headers, aws_bucket)
self._upload_object(work)
def _migrate_slo(self, aws_bucket, slo_container, key, headers):
manifest = self.provider.get_manifest(key, aws_bucket)
if not manifest:
raise MigrationError('Failed to fetch the manifest for "%s/%s"' %
(aws_bucket, key))
for entry in manifest:
container, segment_key = entry['name'][1:].split('/', 1)
meta = None
with self.ic_pool.item() as ic:
try:
meta = ic.get_object_metadata(self.config['account'],
container, segment_key)
except UnexpectedResponse as e:
if e.resp.status_int != 404:
self.errors.put(
(container, segment_key, sys.exc_info()))
continue
if meta:
resp = self.provider.head_object(segment_key, container)
if resp.status != 200:
raise MigrationError('Failed to HEAD "%s/%s"' %
(container, segment_key))
src_meta = resp.headers
if self.config.get('protocol', 's3') != 'swift':
src_meta = convert_to_swift_headers(src_meta)
ret = cmp_meta(meta, src_meta)
if ret == EQUAL:
continue
if ret == TIME_DIFF:
# TODO: update metadata
self.logger.warning('Object metadata changed for "%s/%s"' %
(container, segment_key))
continue
work = MigrateObjectWork(container, container, segment_key)
try:
self.object_queue.put(work, block=False)
except eventlet.queue.Full:
self._migrate_object(work.aws_bucket, work.container,
segment_key)
manifest_blob = json.dumps(manifest)
headers['Content-Length'] = str(len(manifest_blob))
# The SLO middleware is not in the pipeline. The ETag we provide should
# be for the manifest *JSON content*, rather than the hash of hashes
# that the SLO middleware can validate.
headers['etag'] = hashlib.md5(manifest_blob).hexdigest()
work = UploadObjectWork(slo_container, key,
FileLikeIter(manifest_blob), headers,
slo_container)
try:
self.object_queue.put(work, block=False)
except eventlet.queue.Full:
self._upload_object(work)
def test_PUT_multiseg_bad_client_etag(self):
chunks = [b'some', b'chunks', b'of data']
body = b''.join(chunks)
env = {'REQUEST_METHOD': 'PUT',
CRYPTO_KEY_CALLBACK: fetch_crypto_keys,
'wsgi.input': FileLikeIter(chunks)}
hdrs = {'content-type': 'text/plain',
'content-length': str(len(body)),
'Etag': 'badclientetag'}
req = Request.blank('/v1/a/c/o', environ=env, headers=hdrs)
self.app.register('PUT', '/v1/a/c/o', HTTPCreated, {})
resp = req.get_response(self.encrypter)
self.assertEqual('422 Unprocessable Entity', resp.status)
def _upload_slo(self, name, swift_headers, internal_client):
status, headers, body = internal_client.get_object(
self.account, self.container, name, headers=swift_headers)
if status != 200:
body.close()
raise RuntimeError('Failed to get the manifest')
manifest = json.load(FileLikeIter(body))
body.close()
self.logger.debug("JSON manifest: %s" % str(manifest))
work_queue = eventlet.queue.Queue(self.SLO_QUEUE_SIZE)
worker_pool = eventlet.greenpool.GreenPool(self.SLO_WORKERS)
workers = []
for _ in range(0, self.SLO_WORKERS):
workers.append(
worker_pool.spawn(self._upload_slo_worker, swift_headers,
work_queue, internal_client))
for segment in manifest:
work_queue.put(segment)
work_queue.join()
for _ in range(0, self.SLO_WORKERS):
work_queue.put(None)
errors = []
for thread in workers:
errors += thread.wait()
# TODO: errors list contains the failed segments. We should retry
# them on failure.
if errors:
raise RuntimeError('Failed to upload an SLO %s' % name)
# we need to mutate the container in the manifest
container = self.remote_container + '_segments'
new_manifest = []
for segment in manifest:
_, obj = segment['name'].split('/', 2)[1:]
new_manifest.append(
dict(path='/%s/%s' % (container, obj),
etag=segment['hash'],
size_bytes=segment['bytes']))
self.logger.debug(json.dumps(new_manifest))
# Upload the manifest itself
with self.client_pool.get_client() as swift_client:
swift_client.put_object(self.remote_container,
name,
json.dumps(new_manifest),
headers=self._get_user_headers(headers),
query_string='multipart-manifest=put')
def _put_versioned_obj(self, req, put_path_info, source_resp):
# Create a new Request object to PUT to the container, copying
# all headers from the source object apart from x-timestamp.
put_req = make_pre_authed_request(
req.environ, path=wsgi_quote(put_path_info), method='PUT',
swift_source='VW')
copy_header_subset(source_resp, put_req,
lambda k: k.lower() != 'x-timestamp')
slo_size = put_req.headers.get('X-Object-Sysmeta-Slo-Size')
if slo_size:
put_req.headers['Content-Type'] += '; swift_bytes=' + slo_size
put_req.environ['swift.content_type_overridden'] = True
put_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter)
put_resp = put_req.get_response(self.app)
close_if_possible(source_resp.app_iter)
return put_resp
def _migrate_object(self, container, key):
args = {'bucket': container, 'native': True}
if self.config.get('protocol', 's3') == 'swift':
args['resp_chunk_size'] = 65536
resp = self.provider.get_object(key, **args)
if resp.status != 200:
raise MigrationError('Failed to GET %s/%s: %s' % (
container, key, resp.body))
put_headers = convert_to_local_headers(
resp.headers.items(), remove_timestamp=False)
if 'x-object-manifest' in resp.headers:
self.logger.warning('Skipping Dynamic Large Object %s/%s' % (
container, key))
resp.body.close()
return
if 'x-static-large-object' in resp.headers:
# We have to move the segments and then move the manifest file
resp.body.close()
self._migrate_slo(container, key, put_headers)
else:
self._upload_object(
container, key, FileLikeIter(resp.body), put_headers)
def _migrate_slo(self, slo_container, key, headers):
manifest = self.provider.get_manifest(key, slo_container)
if not manifest:
raise MigrationError('Failed to fetch the manifest for %s/%s' % (
slo_container, key))
for entry in manifest:
container, segment_key = entry['name'][1:].split('/', 1)
meta = None
with self.ic_pool.item() as ic:
try:
meta = ic.get_object_metadata(
self.config['account'], container, segment_key)
except UnexpectedResponse as e:
if e.resp.status_int != 404:
self.errors.put((container, segment_key,
sys.exc_info()))
continue
if meta:
resp = self.provider.head_object(
segment_key, container, native=True)
if resp.status != 200:
raise MigrationError('Failed to HEAD %s/%s' % (
container, segment_key))
src_meta = resp.headers
if self.config.get('protocol', 's3') != 'swift':
src_meta = convert_to_swift_headers(src_meta)
ret = cmp_meta(meta, src_meta)
if ret == EQUAL:
continue
if ret == TIME_DIFF:
# TODO: update metadata
self.logger.warning('Object metadata changed for %s/%s' % (
container, segment_key))
continue
self.object_queue.put((container, segment_key))
manifest_blob = json.dumps(manifest)
headers['Content-Length'] = len(manifest_blob)
self.object_queue.put((
slo_container, key, FileLikeIter(manifest_blob), headers))
def handle_PUT(self, req, start_response):
if req.content_length:
return HTTPBadRequest(body='Copy requests require a zero byte '
'body',
request=req,
content_type='text/plain')(req.environ,
start_response)
# Form the path of source object to be fetched
ver, acct, _rest = req.split_path(2, 3, True)
src_account_name = req.headers.get('X-Copy-From-Account')
if src_account_name:
src_account_name = check_account_format(req, src_account_name)
else:
src_account_name = acct
src_container_name, src_obj_name = _check_copy_from_header(req)
source_path = '/%s/%s/%s/%s' % (ver, src_account_name,
src_container_name, src_obj_name)
if req.environ.get('swift.orig_req_method', req.method) != 'POST':
self.logger.info("Copying object from %s to %s" %
(source_path, req.path))
# GET the source object, bail out on error
ssc_ctx = ServerSideCopyWebContext(self.app, self.logger)
source_resp = self._get_source_object(ssc_ctx, source_path, req)
if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
return source_resp(source_resp.environ, start_response)
# Create a new Request object based on the original request instance.
# This will preserve original request environ including headers.
sink_req = Request.blank(req.path_info, environ=req.environ)
def is_object_sysmeta(k):
return is_sys_meta('object', k)
if config_true_value(req.headers.get('x-fresh-metadata', 'false')):
# x-fresh-metadata only applies to copy, not post-as-copy: ignore
# existing user metadata, update existing sysmeta with new
copy_header_subset(source_resp, sink_req, is_object_sysmeta)
copy_header_subset(req, sink_req, is_object_sysmeta)
else:
# First copy existing sysmeta, user meta and other headers from the
# source to the sink, apart from headers that are conditionally
# copied below and timestamps.
exclude_headers = ('x-static-large-object', 'x-object-manifest',
'etag', 'content-type', 'x-timestamp',
'x-backend-timestamp')
copy_header_subset(source_resp, sink_req,
lambda k: k.lower() not in exclude_headers)
# now update with original req headers
sink_req.headers.update(req.headers)
params = sink_req.params
if params.get('multipart-manifest') == 'get':
if 'X-Static-Large-Object' in source_resp.headers:
params['multipart-manifest'] = 'put'
if 'X-Object-Manifest' in source_resp.headers:
del params['multipart-manifest']
sink_req.headers['X-Object-Manifest'] = \
source_resp.headers['X-Object-Manifest']
sink_req.params = params
# Set swift.source, data source, content length and etag
# for the PUT request
sink_req.environ['swift.source'] = 'SSC'
sink_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter)
sink_req.content_length = source_resp.content_length
if (source_resp.status_int == HTTP_OK
and 'X-Static-Large-Object' not in source_resp.headers
and ('X-Object-Manifest' not in source_resp.headers
or req.params.get('multipart-manifest') == 'get')):
# copy source etag so that copied content is verified, unless:
# - not a 200 OK response: source etag may not match the actual
# content, for example with a 206 Partial Content response to a
# ranged request
# - SLO manifest: etag cannot be specified in manifest PUT; SLO
# generates its own etag value which may differ from source
# - SLO: etag in SLO response is not hash of actual content
# - DLO: etag in DLO response is not hash of actual content
sink_req.headers['Etag'] = source_resp.etag
else:
# since we're not copying the source etag, make sure that any
# container update override values are not copied.
remove_items(
sink_req.headers, lambda k: k.startswith(
'X-Object-Sysmeta-Container-Update-Override-'))
# We no longer need these headers
sink_req.headers.pop('X-Copy-From', None)
sink_req.headers.pop('X-Copy-From-Account', None)
# If the copy request does not explicitly override content-type,
# use the one present in the source object.
if not req.headers.get('content-type'):
sink_req.headers['Content-Type'] = \
source_resp.headers['Content-Type']
# Create response headers for PUT response
resp_headers = self._create_response_headers(source_path, source_resp,
sink_req)
put_resp = ssc_ctx.send_put_req(sink_req, resp_headers, start_response)
close_if_possible(source_resp.app_iter)
return put_resp
raise Exception(
_('Unknown exception trying to GET: %(node)r '
'%(account)r %(container)r %(object)r'),
{'node': node, 'part': part,
'account': info['account'],
'container': info['container'],
'object': row['name']})
for key in ('date', 'last-modified'):
if key in headers:
del headers[key]
if 'etag' in headers:
headers['etag'] = headers['etag'].strip('"')
headers['x-timestamp'] = row['created_at']
headers['x-container-sync-key'] = sync_key
put_object(sync_to, name=row['name'], headers=headers,
contents=FileLikeIter(body),
proxy=self.proxy)
self.container_puts += 1
self.logger.increment('puts')
self.logger.timing_since('puts.timing', start_time)
except ClientException, err:
if err.http_status == HTTP_UNAUTHORIZED:
self.logger.info(
_('Unauth %(sync_from)r => %(sync_to)r'),
{'sync_from': '%s/%s' %
(quote(info['account']), quote(info['container'])),
'sync_to': sync_to})
elif err.http_status == HTTP_NOT_FOUND:
self.logger.info(
_('Not found %(sync_from)r => %(sync_to)r \
- object %(obj_name)r'),
def _make_req(node,
part,
method,
path,
headers,
stype,
conn_timeout=5,
response_timeout=15,
send_timeout=15,
contents=None,
content_length=None,
chunk_size=65535):
"""
Make request to backend storage node.
(i.e. 'Account', 'Container', 'Object')
:param node: a node dict from a ring
:param part: an integer, the partition number
:param method: a string, the HTTP method (e.g. 'PUT', 'DELETE', etc)
:param path: a string, the request path
:param headers: a dict, header name => value
:param stype: a string, describing the type of service
:param conn_timeout: timeout while waiting for connection; default is 5
seconds
:param response_timeout: timeout while waiting for response; default is 15
seconds
:param send_timeout: timeout for sending request body; default is 15
seconds
:param contents: an iterable or string to read object data from
:param content_length: value to send as content-length header
:param chunk_size: if defined, chunk size of data to send
:returns: an HTTPResponse object
:raises DirectClientException: if the response status is not 2xx
:raises eventlet.Timeout: if either conn_timeout or response_timeout is
exceeded
"""
if contents is not None:
if content_length is not None:
headers['Content-Length'] = str(content_length)
else:
for n, v in headers.items():
if n.lower() == 'content-length':
content_length = int(v)
if not contents:
headers['Content-Length'] = '0'
if isinstance(contents, six.string_types):
contents = [contents]
if content_length is None:
headers['Transfer-Encoding'] = 'chunked'
with Timeout(conn_timeout):
conn = http_connect(node['ip'],
node['port'],
node['device'],
part,
method,
path,
headers=headers)
if contents is not None:
contents_f = FileLikeIter(contents)
with Timeout(send_timeout):
if content_length is None:
chunk = contents_f.read(chunk_size)
while chunk:
conn.send(b'%x\r\n%s\r\n' % (len(chunk), chunk))
chunk = contents_f.read(chunk_size)
conn.send(b'0\r\n\r\n')
else:
left = content_length
while left > 0:
size = chunk_size
if size > left:
size = left
chunk = contents_f.read(size)
if not chunk:
break
conn.send(chunk)
left -= len(chunk)
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()
if not is_success(resp.status):
raise DirectClientException(stype, method, node, part, path, resp)
return resp
class FileWrapper(object):
def __init__(self, swift_client, account, container, key, headers={}):
self._swift = swift_client
self._account = account
self._container = container
self._key = key
self.swift_req_hdrs = headers
self._bytes_read = 0
self.open_object_stream()
def open_object_stream(self):
status, self._headers, body = self._swift.get_object(
self._account,
self._container,
self._key,
headers=self.swift_req_hdrs)
if status != 200:
raise RuntimeError('Failed to get the object')
self._bytes_read = 0
self._swift_stream = body
self._iter = FileLikeIter(body)
self._s3_headers = convert_to_s3_headers(self._headers)
def tell(self):
return self._bytes_read
def seek(self, pos, flag=0):
if pos != 0:
raise RuntimeError('Arbitrary seeks are not supported')
if self._bytes_read == 0:
return
self._swift_stream.close()
self.open_object_stream()
def reset(self, *args, **kwargs):
self.seek(0)
def read(self, size=-1):
if self._bytes_read == self.__len__():
return ''
data = self._iter.read(size)
self._bytes_read += len(data)
# TODO: we do not need to read an extra byte after
# https://review.openstack.org/#/c/363199/ is released
if self._bytes_read == self.__len__():
self._iter.read(1)
self._swift_stream.close()
return data
def __len__(self):
if 'Content-Length' not in self._headers:
raise RuntimeError('Length is not implemented')
return int(self._headers['Content-Length'])
def __iter__(self):
return self._iter
def get_s3_headers(self):
return self._s3_headers
def get_headers(self):
return self._headers
def close(self):
self._swift_stream.close()
def test_invocation_flow():
sresp = self.gateway.invocation_flow(st_req, extra_sources)
eventlet.sleep(0.1)
file_like = FileLikeIter(sresp.data_iter)
self.assertEqual('something', file_like.read())
def test_invocation_flow():
sresp = self.gateway.invocation_flow(st_req, extra_sources)
eventlet.sleep(0.1)
file_like = FileLikeIter(sresp.data_iter)
self.assertEqual('something', file_like.read())
def container_sync_row(self, row, sync_to, user_key, broker, info, realm,
realm_key):
"""
Sends the update the row indicates to the sync_to container.
:param row: The updated row in the local database triggering the sync
update.
:param sync_to: The URL to the remote container.
:param user_key: The X-Container-Sync-Key to use when sending requests
to the other container.
:param broker: The local container database broker.
:param info: The get_info result from the local container database
broker.
:param realm: The realm from self.realms_conf, if there is one.
If None, fallback to using the older allowed_sync_hosts
way of syncing.
:param realm_key: The realm key from self.realms_conf, if there
is one. If None, fallback to using the older
allowed_sync_hosts way of syncing.
:returns: True on success
"""
try:
start_time = time()
if row['deleted']:
try:
headers = {'x-timestamp': row['created_at']}
if realm and realm_key:
nonce = uuid.uuid4().hex
path = urlparse(sync_to).path + '/' + quote(
row['name'])
sig = self.realms_conf.get_sig('DELETE', path,
headers['x-timestamp'],
nonce, realm_key,
user_key)
headers['x-container-sync-auth'] = '%s %s %s' % (
realm, nonce, sig)
else:
headers['x-container-sync-key'] = user_key
delete_object(sync_to,
name=row['name'],
headers=headers,
proxy=self.select_http_proxy(),
logger=self.logger,
timeout=self.conn_timeout)
except ClientException as err:
if err.http_status != HTTP_NOT_FOUND:
raise
self.container_deletes += 1
self.logger.increment('deletes')
self.logger.timing_since('deletes.timing', start_time)
else:
part, nodes = \
self.get_object_ring(info['storage_policy_index']). \
get_nodes(info['account'], info['container'],
row['name'])
shuffle(nodes)
exc = None
looking_for_timestamp = Timestamp(row['created_at'])
timestamp = -1
headers = body = None
# look up for the newest one
headers_out = {
'X-Newest':
True,
'X-Backend-Storage-Policy-Index':
str(info['storage_policy_index'])
}
try:
source_obj_status, source_obj_info, source_obj_iter = \
self.swift.get_object(info['account'],
info['container'], row['name'],
headers=headers_out,
acceptable_statuses=(2, 4))
except (Exception, UnexpectedResponse, Timeout) as err:
source_obj_info = {}
source_obj_iter = None
exc = err
timestamp = Timestamp(source_obj_info.get('x-timestamp', 0))
headers = source_obj_info
body = source_obj_iter
if timestamp < looking_for_timestamp:
if exc:
raise exc
raise Exception(
_('Unknown exception trying to GET: '
'%(account)r %(container)r %(object)r'), {
'account': info['account'],
'container': info['container'],
'object': row['name']
})
for key in ('date', 'last-modified'):
if key in headers:
del headers[key]
if 'etag' in headers:
headers['etag'] = headers['etag'].strip('"')
if 'content-type' in headers:
headers['content-type'] = clean_content_type(
headers['content-type'])
headers['x-timestamp'] = row['created_at']
if realm and realm_key:
nonce = uuid.uuid4().hex
path = urlparse(sync_to).path + '/' + quote(row['name'])
sig = self.realms_conf.get_sig('PUT', path,
headers['x-timestamp'],
nonce, realm_key, user_key)
headers['x-container-sync-auth'] = '%s %s %s' % (
realm, nonce, sig)
else:
headers['x-container-sync-key'] = user_key
put_object(sync_to,
name=row['name'],
headers=headers,
contents=FileLikeIter(body),
proxy=self.select_http_proxy(),
logger=self.logger,
timeout=self.conn_timeout)
self.container_puts += 1
self.logger.increment('puts')
self.logger.timing_since('puts.timing', start_time)
except ClientException as err:
if err.http_status == HTTP_UNAUTHORIZED:
self.logger.info(
_('Unauth %(sync_from)r => %(sync_to)r'), {
'sync_from':
'%s/%s' %
(quote(info['account']), quote(info['container'])),
'sync_to':
sync_to
})
elif err.http_status == HTTP_NOT_FOUND:
self.logger.info(
_('Not found %(sync_from)r => %(sync_to)r \
- object %(obj_name)r'), {
'sync_from':
'%s/%s' %
(quote(info['account']), quote(info['container'])),
'sync_to':
sync_to,
'obj_name':
row['name']
})
else:
self.logger.exception(_('ERROR Syncing %(db_file)s %(row)s'), {
'db_file': str(broker),
'row': row
})
self.container_failures += 1
self.logger.increment('failures')
return False
except (Exception, Timeout) as err:
self.logger.exception(_('ERROR Syncing %(db_file)s %(row)s'), {
'db_file': str(broker),
'row': row
})
self.container_failures += 1
self.logger.increment('failures')
return False
return True
def handle_PUT(self, req, start_response):
if req.content_length:
return HTTPBadRequest(body='Copy requests require a zero byte '
'body', request=req,
content_type='text/plain')(req.environ,
start_response)
# Form the path of source object to be fetched
ver, acct, _rest = req.split_path(2, 3, True)
src_account_name = req.headers.get('X-Copy-From-Account')
if src_account_name:
src_account_name = check_account_format(req, src_account_name)
else:
src_account_name = acct
src_container_name, src_obj_name = _check_copy_from_header(req)
source_path = '/%s/%s/%s/%s' % (ver, src_account_name,
src_container_name, src_obj_name)
if req.environ.get('swift.orig_req_method', req.method) != 'POST':
self.logger.info("Copying object from %s to %s" %
(source_path, req.path))
# GET the source object, bail out on error
ssc_ctx = ServerSideCopyWebContext(self.app, self.logger)
source_resp = self._get_source_object(ssc_ctx, source_path, req)
if source_resp.status_int >= HTTP_MULTIPLE_CHOICES:
close_if_possible(source_resp.app_iter)
return source_resp(source_resp.environ, start_response)
# Create a new Request object based on the original req instance.
# This will preserve env and headers.
sink_req = Request.blank(req.path_info,
environ=req.environ, headers=req.headers)
params = sink_req.params
if params.get('multipart-manifest') == 'get':
if 'X-Static-Large-Object' in source_resp.headers:
params['multipart-manifest'] = 'put'
if 'X-Object-Manifest' in source_resp.headers:
del params['multipart-manifest']
sink_req.headers['X-Object-Manifest'] = \
source_resp.headers['X-Object-Manifest']
sink_req.params = params
# Set data source, content length and etag for the PUT request
sink_req.environ['wsgi.input'] = FileLikeIter(source_resp.app_iter)
sink_req.content_length = source_resp.content_length
sink_req.etag = source_resp.etag
# We no longer need these headers
sink_req.headers.pop('X-Copy-From', None)
sink_req.headers.pop('X-Copy-From-Account', None)
# If the copy request does not explicitly override content-type,
# use the one present in the source object.
if not req.headers.get('content-type'):
sink_req.headers['Content-Type'] = \
source_resp.headers['Content-Type']
fresh_meta_flag = config_true_value(
sink_req.headers.get('x-fresh-metadata', 'false'))
if fresh_meta_flag or 'swift.post_as_copy' in sink_req.environ:
# Post-as-copy: ignore new sysmeta, copy existing sysmeta
condition = lambda k: is_sys_meta('object', k)
remove_items(sink_req.headers, condition)
copy_header_subset(source_resp, sink_req, condition)
else:
# Copy/update existing sysmeta and user meta
_copy_headers_into(source_resp, sink_req)
# Copy/update new metadata provided in request if any
_copy_headers_into(req, sink_req)
# Create response headers for PUT response
resp_headers = self._create_response_headers(source_path,
source_resp, sink_req)
put_resp = ssc_ctx.send_put_req(sink_req, resp_headers, start_response)
close_if_possible(source_resp.app_iter)
return put_resp
def direct_put_object(node,
part,
account,
container,
name,
contents,
content_length=None,
etag=None,
content_type=None,
headers=None,
conn_timeout=5,
response_timeout=15,
chunk_size=65535):
"""
Put object directly from the object server.
:param node: node dictionary from the ring
:param part: partition the container is on
:param account: account name
:param container: container name
:param name: object name
:param contents: an iterable or string to read object data from
:param content_length: value to send as content-length header
:param etag: etag of contents
:param content_type: value to send as content-type header
:param headers: additional headers to include in the request
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:param chunk_size: if defined, chunk size of data to send.
:returns: etag from the server response
:raises ClientException: HTTP PUT request failed
"""
path = '/%s/%s/%s' % (account, container, name)
if headers is None:
headers = {}
if etag:
headers['ETag'] = etag.strip('"')
if content_length is not None:
headers['Content-Length'] = str(content_length)
else:
for n, v in headers.items():
if n.lower() == 'content-length':
content_length = int(v)
if content_type is not None:
headers['Content-Type'] = content_type
else:
headers['Content-Type'] = 'application/octet-stream'
if not contents:
headers['Content-Length'] = '0'
if isinstance(contents, basestring):
contents = [contents]
#Incase the caller want to insert an object with specific age
add_ts = 'X-Timestamp' not in headers
if content_length is None:
headers['Transfer-Encoding'] = 'chunked'
with Timeout(conn_timeout):
conn = http_connect(node['ip'],
node['port'],
node['device'],
part,
'PUT',
path,
headers=gen_headers(headers, add_ts))
contents_f = FileLikeIter(contents)
if content_length is None:
chunk = contents_f.read(chunk_size)
while chunk:
conn.send('%x\r\n%s\r\n' % (len(chunk), chunk))
chunk = contents_f.read(chunk_size)
conn.send('0\r\n\r\n')
else:
left = content_length
while left > 0:
size = chunk_size
if size > left:
size = left
chunk = contents_f.read(size)
if not chunk:
break
conn.send(chunk)
left -= len(chunk)
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()
if not is_success(resp.status):
raise DirectClientException('Object', 'PUT', node, part, path, resp)
return resp.getheader('etag').strip('"')
def container_sync_row(self, row, sync_to, sync_key, broker, info):
"""
Sends the update the row indicates to the sync_to container.
:param row: The updated row in the local database triggering the sync
update.
:param sync_to: The URL to the remote container.
:param sync_key: The X-Container-Sync-Key to use when sending requests
to the other container.
:param broker: The local container database broker.
:param info: The get_info result from the local container database
broker.
:returns: True on success
"""
try:
start_time = time()
if row['deleted']:
try:
delete_object(sync_to,
name=row['name'],
headers={
'x-timestamp': row['created_at'],
'x-container-sync-key': sync_key
},
proxy=self.proxy)
except ClientException as err:
if err.http_status != HTTP_NOT_FOUND:
raise
self.container_deletes += 1
self.logger.increment('deletes')
self.logger.timing_since('deletes.timing', start_time)
else:
part, nodes = self.object_ring.get_nodes(
info['account'], info['container'], row['name'])
shuffle(nodes)
exc = None
looking_for_timestamp = float(row['created_at'])
timestamp = -1
headers = body = None
for node in nodes:
try:
these_headers, this_body = direct_get_object(
node,
part,
info['account'],
info['container'],
row['name'],
resp_chunk_size=65536)
this_timestamp = float(these_headers['x-timestamp'])
if this_timestamp > timestamp:
timestamp = this_timestamp
headers = these_headers
body = this_body
except ClientException as err:
# If any errors are not 404, make sure we report the
# non-404 one. We don't want to mistakenly assume the
# object no longer exists just because one says so and
# the others errored for some other reason.
if not exc or exc.http_status == HTTP_NOT_FOUND:
exc = err
except (Exception, Timeout) as err:
exc = err
if timestamp < looking_for_timestamp:
if exc:
raise exc
raise Exception(
_('Unknown exception trying to GET: %(node)r '
'%(account)r %(container)r %(object)r'), {
'node': node,
'part': part,
'account': info['account'],
'container': info['container'],
'object': row['name']
})
for key in ('date', 'last-modified'):
if key in headers:
del headers[key]
if 'etag' in headers:
headers['etag'] = headers['etag'].strip('"')
headers['x-timestamp'] = row['created_at']
headers['x-container-sync-key'] = sync_key
put_object(sync_to,
name=row['name'],
headers=headers,
contents=FileLikeIter(body),
proxy=self.proxy)
self.container_puts += 1
self.logger.increment('puts')
self.logger.timing_since('puts.timing', start_time)
except ClientException as err:
if err.http_status == HTTP_UNAUTHORIZED:
self.logger.info(
_('Unauth %(sync_from)r => %(sync_to)r'), {
'sync_from':
'%s/%s' %
(quote(info['account']), quote(info['container'])),
'sync_to':
sync_to
})
elif err.http_status == HTTP_NOT_FOUND:
self.logger.info(
_('Not found %(sync_from)r => %(sync_to)r \
- object %(obj_name)r'), {
'sync_from':
'%s/%s' %
(quote(info['account']), quote(info['container'])),
'sync_to':
sync_to,
'obj_name':
row['name']
})
else:
self.logger.exception(_('ERROR Syncing %(db_file)s %(row)s'), {
'db_file': str(broker),
'row': row
})
self.container_failures += 1
self.logger.increment('failures')
return False
except (Exception, Timeout) as err:
self.logger.exception(_('ERROR Syncing %(db_file)s %(row)s'), {
'db_file': str(broker),
'row': row
})
self.container_failures += 1
self.logger.increment('failures')
return False
return True
def upload_slo(self, swift_key, storage_policy_index, s3_meta,
internal_client):
# Converts an SLO into a multipart upload. We use the segments as
# is, for the part sizes.
# NOTE: If the SLO segment is < 5MB and is not the last segment, the
# UploadPart call will fail. We need to stitch segments together in
# that case.
#
# For Google Cloud Storage, we will convert the SLO into a single
# object put, assuming the SLO is < 5TB. If the SLO is > 5TB, we have
# to fail the upload. With GCS _compose_, we could support larger
# objects, but defer this work for the time being.
swift_req_hdrs = {
'X-Backend-Storage-Policy-Index': storage_policy_index,
'X-Newest': True
}
status, headers, body = internal_client.get_object(
self.account, self.container, swift_key, headers=swift_req_hdrs)
if status != 200:
body.close()
raise RuntimeError('Failed to get the manifest')
manifest = json.load(FileLikeIter(body))
body.close()
self.logger.debug("JSON manifest: %s" % str(manifest))
s3_key = self.get_s3_name(swift_key)
if not self._validate_slo_manifest(manifest):
# We do not raise an exception here -- we should not retry these
# errors and they will be logged.
# TODO: When we report statistics, we need to account for permanent
# failures.
self.logger.error('Failed to validate the SLO manifest for %s' %
self._full_name(swift_key))
return
if self._google():
if s3_meta:
slo_etag = s3_meta['Metadata'].get(SLO_ETAG_FIELD, None)
if slo_etag == headers['etag']:
if self.is_object_meta_synced(s3_meta, headers):
return
self.update_metadata(swift_key, headers)
return
self._upload_google_slo(manifest, headers, s3_key, swift_req_hdrs,
internal_client)
else:
expected_etag = get_slo_etag(manifest)
if s3_meta and self.check_etag(expected_etag, s3_meta['ETag']):
if self.is_object_meta_synced(s3_meta, headers):
return
elif not self.in_glacier(s3_meta):
self.update_slo_metadata(headers, manifest, s3_key,
swift_req_hdrs, internal_client)
return
self._upload_slo(manifest, headers, s3_key, swift_req_hdrs,
internal_client)
with self.client_pool.get_client() as s3_client:
# We upload the manifest so that we can restore the object in
# Swift and have it match the S3 multipart ETag. To avoid name
# length issues, we hash the object name and append the suffix
params = dict(Bucket=self.aws_bucket,
Key=self.get_manifest_name(s3_key),
Body=json.dumps(manifest),
ContentLength=len(json.dumps(manifest)),
ContentType='application/json')
if self._is_amazon() and self.encryption:
params['ServerSideEncryption'] = 'AES256'
s3_client.put_object(**params)
def container_sync_row(self, row, sync_to, user_key, broker, info, realm,
realm_key):
"""
Sends the update the row indicates to the sync_to container.
Update can be either delete or put.
:param row: The updated row in the local database triggering the sync
update.
:param sync_to: The URL to the remote container.
:param user_key: The X-Container-Sync-Key to use when sending requests
to the other container.
:param broker: The local container database broker.
:param info: The get_info result from the local container database
broker.
:param realm: The realm from self.realms_conf, if there is one.
If None, fallback to using the older allowed_sync_hosts
way of syncing.
:param realm_key: The realm key from self.realms_conf, if there
is one. If None, fallback to using the older
allowed_sync_hosts way of syncing.
:returns: True on success
"""
try:
start_time = time()
# extract last modified time from the created_at value
ts_data, ts_ctype, ts_meta = decode_timestamps(row['created_at'])
if row['deleted']:
# when sync'ing a deleted object, use ts_data - this is the
# timestamp of the source tombstone
try:
headers = {'x-timestamp': ts_data.internal}
self._update_sync_to_headers(row['name'], sync_to,
user_key, realm, realm_key,
'DELETE', headers)
delete_object(sync_to,
name=row['name'],
headers=headers,
proxy=self.select_http_proxy(),
logger=self.logger,
timeout=self.conn_timeout)
except ClientException as err:
if err.http_status != HTTP_NOT_FOUND:
raise
self.container_deletes += 1
self.container_stats['deletes'] += 1
self.logger.increment('deletes')
self.logger.timing_since('deletes.timing', start_time)
else:
# when sync'ing a live object, use ts_meta - this is the time
# at which the source object was last modified by a PUT or POST
if self._object_in_remote_container(row['name'], sync_to,
user_key, realm, realm_key,
ts_meta):
return True
exc = None
# look up for the newest one; the symlink=get query-string has
# no effect unless symlinks are enabled in the internal client
# in which case it ensures that symlink objects retain their
# symlink property when sync'd.
headers_out = {
'X-Newest':
True,
'X-Backend-Storage-Policy-Index':
str(info['storage_policy_index'])
}
try:
source_obj_status, headers, body = \
self.swift.get_object(info['account'],
info['container'], row['name'],
headers=headers_out,
acceptable_statuses=(2, 4),
params={'symlink': 'get'})
except (Exception, UnexpectedResponse, Timeout) as err:
headers = {}
body = None
exc = err
timestamp = Timestamp(headers.get('x-timestamp', 0))
if timestamp < ts_meta:
if exc:
raise exc
raise Exception(
_('Unknown exception trying to GET: '
'%(account)r %(container)r %(object)r'), {
'account': info['account'],
'container': info['container'],
'object': row['name']
})
for key in ('date', 'last-modified'):
if key in headers:
del headers[key]
if 'etag' in headers:
headers['etag'] = normalize_etag(headers['etag'])
if 'content-type' in headers:
headers['content-type'] = clean_content_type(
headers['content-type'])
self._update_sync_to_headers(row['name'], sync_to, user_key,
realm, realm_key, 'PUT', headers)
put_object(sync_to,
name=row['name'],
headers=headers,
contents=FileLikeIter(body),
proxy=self.select_http_proxy(),
logger=self.logger,
timeout=self.conn_timeout)
self.container_puts += 1
self.container_stats['puts'] += 1
self.container_stats['bytes'] += row['size']
self.logger.increment('puts')
self.logger.timing_since('puts.timing', start_time)
except ClientException as err:
if err.http_status == HTTP_UNAUTHORIZED:
self.logger.info(
_('Unauth %(sync_from)r => %(sync_to)r'), {
'sync_from':
'%s/%s' %
(quote(info['account']), quote(info['container'])),
'sync_to':
sync_to
})
elif err.http_status == HTTP_NOT_FOUND:
self.logger.info(
_('Not found %(sync_from)r => %(sync_to)r \
- object %(obj_name)r'), {
'sync_from':
'%s/%s' %
(quote(info['account']), quote(info['container'])),
'sync_to':
sync_to,
'obj_name':
row['name']
})
else:
self.logger.exception(_('ERROR Syncing %(db_file)s %(row)s'), {
'db_file': str(broker),
'row': row
})
self.container_failures += 1
self.logger.increment('failures')
return False
except (Exception, Timeout) as err:
self.logger.exception(_('ERROR Syncing %(db_file)s %(row)s'), {
'db_file': str(broker),
'row': row
})
self.container_failures += 1
self.logger.increment('failures')
return False
return True
def container_sync_row(self, row, sync_to, user_key, broker, info, realm,
realm_key):
"""
Sends the update the row indicates to the sync_to container.
:param row: The updated row in the local database triggering the sync
update.
:param sync_to: The URL to the remote container.
:param user_key: The X-Container-Sync-Key to use when sending requests
to the other container.
:param broker: The local container database broker.
:param info: The get_info result from the local container database
broker.
:param realm: The realm from self.realms_conf, if there is one.
If None, fallback to using the older allowed_sync_hosts
way of syncing.
:param realm_key: The realm key from self.realms_conf, if there
is one. If None, fallback to using the older
allowed_sync_hosts way of syncing.
:returns: True on success
"""
try:
start_time = time()
if row['deleted']:
try:
headers = {'x-timestamp': row['created_at']}
if realm and realm_key:
nonce = uuid.uuid4().hex
path = urlparse(sync_to).path + '/' + quote(
row['name'])
sig = self.realms_conf.get_sig('DELETE', path,
headers['x-timestamp'],
nonce, realm_key,
user_key)
headers['x-container-sync-auth'] = '%s %s %s' % (
realm, nonce, sig)
else:
headers['x-container-sync-key'] = user_key
delete_object(sync_to,
name=row['name'],
headers=headers,
proxy=self.select_http_proxy(),
logger=self.logger)
except ClientException as err:
if err.http_status != HTTP_NOT_FOUND:
raise
self.container_deletes += 1
self.logger.increment('deletes')
self.logger.timing_since('deletes.timing', start_time)
else:
part, nodes = \
self.get_object_ring(info['storage_policy_index']). \
get_nodes(info['account'], info['container'],
row['name'])
shuffle(nodes)
exc = None
looking_for_timestamp = Timestamp(row['created_at'])
timestamp = -1
headers = body = None
headers_out = {
'X-Backend-Storage-Policy-Index':
str(info['storage_policy_index'])
}
for node in nodes:
try:
these_headers, this_body = direct_get_object(
node,
part,
info['account'],
info['container'],
row['name'],
headers=headers_out,
resp_chunk_size=65536)
this_timestamp = Timestamp(
these_headers['x-timestamp'])
if this_timestamp > timestamp:
timestamp = this_timestamp
headers = these_headers
body = this_body
except ClientException as err:
# If any errors are not 404, make sure we report the
# non-404 one. We don't want to mistakenly assume the
# object no longer exists just because one says so and
# the others errored for some other reason.
if not exc or getattr(
exc, 'http_status', HTTP_NOT_FOUND) == \
HTTP_NOT_FOUND:
exc = err
except (Exception, Timeout) as err:
exc = err
if timestamp < looking_for_timestamp:
if exc:
raise exc
raise Exception(
_('Unknown exception trying to GET: %(node)r '
'%(account)r %(container)r %(object)r'), {
'node': node,
'part': part,
'account': info['account'],
'container': info['container'],
'object': row['name']
})
for key in ('date', 'last-modified'):
if key in headers:
del headers[key]
if 'etag' in headers:
headers['etag'] = headers['etag'].strip('"')
if 'content-type' in headers:
headers['content-type'] = clean_content_type(
headers['content-type'])
headers['x-timestamp'] = row['created_at']
if realm and realm_key:
nonce = uuid.uuid4().hex
path = urlparse(sync_to).path + '/' + quote(row['name'])
sig = self.realms_conf.get_sig('PUT', path,
headers['x-timestamp'],
nonce, realm_key, user_key)
headers['x-container-sync-auth'] = '%s %s %s' % (
realm, nonce, sig)
else:
headers['x-container-sync-key'] = user_key
put_object(sync_to,
name=row['name'],
headers=headers,
contents=FileLikeIter(body),
proxy=self.select_http_proxy(),
logger=self.logger)
self.container_puts += 1
self.logger.increment('puts')
self.logger.timing_since('puts.timing', start_time)
except ClientException as err:
if err.http_status == HTTP_UNAUTHORIZED:
self.logger.info(
_('Unauth %(sync_from)r => %(sync_to)r'), {
'sync_from':
'%s/%s' %
(quote(info['account']), quote(info['container'])),
'sync_to':
sync_to
})
elif err.http_status == HTTP_NOT_FOUND:
self.logger.info(
_('Not found %(sync_from)r => %(sync_to)r \
- object %(obj_name)r'), {
'sync_from':
'%s/%s' %
(quote(info['account']), quote(info['container'])),
'sync_to':
sync_to,
'obj_name':
row['name']
})
else:
self.logger.exception(_('ERROR Syncing %(db_file)s %(row)s'), {
'db_file': str(broker),
'row': row
})
self.container_failures += 1
self.logger.increment('failures')
return False
except (Exception, Timeout) as err:
self.logger.exception(_('ERROR Syncing %(db_file)s %(row)s'), {
'db_file': str(broker),
'row': row
})
self.container_failures += 1
self.logger.increment('failures')
return False
return True
def ensure_object_in_right_location(self, q_policy_index, account,
container, obj, q_ts, path,
container_policy_index, source_ts,
source_obj_status, source_obj_info,
source_obj_iter, **kwargs):
"""
Validate source object will satisfy the misplaced object queue entry
and move to destination.
:param q_policy_index: the policy_index for the source object
:param account: the account name of the misplaced object
:param container: the container name of the misplaced object
:param obj: the name of the misplaced object
:param q_ts: the timestamp of the misplaced object
:param path: the full path of the misplaced object for logging
:param container_policy_index: the policy_index of the destination
:param source_ts: the timestamp of the source object
:param source_obj_status: the HTTP status source object request
:param source_obj_info: the HTTP headers of the source object request
:param source_obj_iter: the body iter of the source object request
"""
if source_obj_status // 100 != 2 or source_ts < q_ts:
if q_ts < time.time() - self.reclaim_age:
# it's old and there are no tombstones or anything; give up
self.stats_log('lost_source', '%r (%s) was not available in '
'policy_index %s and has expired',
path,
q_ts.internal,
q_policy_index,
level=logging.CRITICAL)
return True
# the source object is unavailable or older than the queue
# entry; a version that will satisfy the queue entry hopefully
# exists somewhere in the cluster, so wait and try again
self.stats_log('unavailable_source', '%r (%s) in '
'policy_index %s responded %s (%s)',
path,
q_ts.internal,
q_policy_index,
source_obj_status,
source_ts.internal,
level=logging.WARNING)
return False
# optimistically move any source with a timestamp >= q_ts
ts = max(Timestamp(source_ts), q_ts)
# move the object
put_timestamp = slightly_later_timestamp(ts, offset=2)
self.stats_log(
'copy_attempt', '%r (%f) in policy_index %s will be '
'moved to policy_index %s (%s)', path, source_ts, q_policy_index,
container_policy_index, put_timestamp)
headers = source_obj_info.copy()
headers['X-Backend-Storage-Policy-Index'] = container_policy_index
headers['X-Timestamp'] = put_timestamp
try:
self.swift.upload_object(FileLikeIter(source_obj_iter),
account,
container,
obj,
headers=headers)
except UnexpectedResponse as err:
self.stats_log('copy_failed', 'upload %r (%f) from '
'policy_index %s to policy_index %s '
'returned %s',
path,
source_ts,
q_policy_index,
container_policy_index,
err,
level=logging.WARNING)
return False
except: # noqa
self.stats_log('unhandled_error', 'unable to upload %r (%f) '
'from policy_index %s to policy_index %s ',
path,
source_ts,
q_policy_index,
container_policy_index,
level=logging.ERROR,
exc_info=True)
return False
self.stats_log(
'copy_success', '%r (%f) moved from policy_index %s '
'to policy_index %s (%s)', path, source_ts, q_policy_index,
container_policy_index, put_timestamp)
return self.throw_tombstones(account, container, obj, q_ts,
q_policy_index, path)
def _make_req(node, part, method, path, headers, stype,
conn_timeout=5, response_timeout=15, send_timeout=15,
contents=None, content_length=None, chunk_size=65535):
"""
Make request to backend storage node.
(i.e. 'Account', 'Container', 'Object')
:param node: a node dict from a ring
:param part: an integer, the partition number
:param method: a string, the HTTP method (e.g. 'PUT', 'DELETE', etc)
:param path: a string, the request path
:param headers: a dict, header name => value
:param stype: a string, describing the type of service
:param conn_timeout: timeout while waiting for connection; default is 5
seconds
:param response_timeout: timeout while waiting for response; default is 15
seconds
:param send_timeout: timeout for sending request body; default is 15
seconds
:param contents: an iterable or string to read object data from
:param content_length: value to send as content-length header
:param chunk_size: if defined, chunk size of data to send
:returns: an HTTPResponse object
:raises DirectClientException: if the response status is not 2xx
:raises eventlet.Timeout: if either conn_timeout or response_timeout is
exceeded
"""
if contents is not None:
if content_length is not None:
headers['Content-Length'] = str(content_length)
else:
for n, v in headers.items():
if n.lower() == 'content-length':
content_length = int(v)
if not contents:
headers['Content-Length'] = '0'
if isinstance(contents, six.string_types):
contents = [contents]
if content_length is None:
headers['Transfer-Encoding'] = 'chunked'
with Timeout(conn_timeout):
conn = http_connect(node['ip'], node['port'], node['device'], part,
method, path, headers=headers)
if contents is not None:
contents_f = FileLikeIter(contents)
with Timeout(send_timeout):
if content_length is None:
chunk = contents_f.read(chunk_size)
while chunk:
conn.send(b'%x\r\n%s\r\n' % (len(chunk), chunk))
chunk = contents_f.read(chunk_size)
conn.send(b'0\r\n\r\n')
else:
left = content_length
while left > 0:
size = chunk_size
if size > left:
size = left
chunk = contents_f.read(size)
if not chunk:
break
conn.send(chunk)
left -= len(chunk)
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()
if not is_success(resp.status):
raise DirectClientException(stype, method, node, part, path, resp)
return resp
def test_invocation_flow():
sresp = self.gateway.invocation_flow(st_req)
file_like = FileLikeIter(sresp.data_iter)
self.assertEqual('something', file_like.read())
def direct_put_object(node, part, account, container, name, contents,
content_length=None, etag=None, content_type=None,
headers=None, conn_timeout=5, response_timeout=15,
chunk_size=65535):
"""
Put object directly from the object server.
:param node: node dictionary from the ring
:param part: partition the container is on
:param account: account name
:param container: container name
:param name: object name
:param contents: an iterable or string to read object data from
:param content_length: value to send as content-length header
:param etag: etag of contents
:param content_type: value to send as content-type header
:param headers: additional headers to include in the request
:param conn_timeout: timeout in seconds for establishing the connection
:param response_timeout: timeout in seconds for getting the response
:param chunk_size: if defined, chunk size of data to send.
:returns: etag from the server response
:raises ClientException: HTTP PUT request failed
"""
path = '/%s/%s/%s' % (account, container, name)
if headers is None:
headers = {}
if etag:
headers['ETag'] = etag.strip('"')
if content_length is not None:
headers['Content-Length'] = str(content_length)
else:
for n, v in headers.items():
if n.lower() == 'content-length':
content_length = int(v)
if content_type is not None:
headers['Content-Type'] = content_type
else:
headers['Content-Type'] = 'application/octet-stream'
if not contents:
headers['Content-Length'] = '0'
if isinstance(contents, six.string_types):
contents = [contents]
# Incase the caller want to insert an object with specific age
add_ts = 'X-Timestamp' not in headers
if content_length is None:
headers['Transfer-Encoding'] = 'chunked'
with Timeout(conn_timeout):
conn = http_connect(node['ip'], node['port'], node['device'], part,
'PUT', path, headers=gen_headers(headers, add_ts))
contents_f = FileLikeIter(contents)
if content_length is None:
chunk = contents_f.read(chunk_size)
while chunk:
conn.send('%x\r\n%s\r\n' % (len(chunk), chunk))
chunk = contents_f.read(chunk_size)
conn.send('0\r\n\r\n')
else:
left = content_length
while left > 0:
size = chunk_size
if size > left:
size = left
chunk = contents_f.read(size)
if not chunk:
break
conn.send(chunk)
left -= len(chunk)
with Timeout(response_timeout):
resp = conn.getresponse()
resp.read()
if not is_success(resp.status):
raise DirectClientException('Object', 'PUT',
node, part, path, resp)
return resp.getheader('etag').strip('"')
