def HEAD(self, request): """Handle HTTP HEAD requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) try: disk_file = self.get_diskfile( device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request, conditional_response=True) response = Response(request=request, conditional_response=True) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if is_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] ts = metadata['X-Timestamp'] response.last_modified = math.ceil(float(ts)) # Needed for container sync feature response.headers['X-Timestamp'] = ts response.content_length = int(metadata['Content-Length']) try: response.content_encoding = metadata['Content-Encoding'] except KeyError: pass return response
def POST(self, request): """Handle HTTP POST requests for the Swift Object Server.""" device, partition, account, container, obj = split_and_validate_path(request, 5, 5, True) if "x-timestamp" not in request.headers or not check_float(request.headers["x-timestamp"]): return HTTPBadRequest(body="Missing timestamp", request=request, content_type="text/plain") new_delete_at = int(request.headers.get("X-Delete-At") or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body="X-Delete-At in past", request=request, content_type="text/plain") try: disk_file = self.get_diskfile(device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request) orig_timestamp = orig_metadata.get("X-Timestamp", "0") if orig_timestamp >= request.headers["x-timestamp"]: return HTTPConflict(request=request) metadata = {"X-Timestamp": request.headers["x-timestamp"]} metadata.update(val for val in request.headers.iteritems() if is_user_meta("object", val[0])) for header_key in self.allowed_headers: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] orig_delete_at = int(orig_metadata.get("X-Delete-At") or 0) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update("PUT", new_delete_at, account, container, obj, request, device) if orig_delete_at: self.delete_at_update("DELETE", orig_delete_at, account, container, obj, request, device) disk_file.write_metadata(metadata) return HTTPAccepted(request=request)
def encrypt_user_metadata(self, req, keys): """ Encrypt user-metadata header values. Replace each x-object-meta-<key> user metadata header with a corresponding x-object-transient-sysmeta-crypto-meta-<key> header which has the crypto metadata required to decrypt appended to the encrypted value. :param req: a swob Request :param keys: a dict of encryption keys """ prefix = get_object_transient_sysmeta('crypto-meta-') user_meta_headers = [ h for h in req.headers.items() if is_user_meta(self.server_type, h[0]) and h[1] ] crypto_meta = None for name, val in user_meta_headers: short_name = strip_user_meta_prefix(self.server_type, name) new_name = prefix + short_name enc_val, crypto_meta = encrypt_header_val(self.crypto, val, keys[self.server_type]) req.headers[new_name] = append_crypto_meta(enc_val, crypto_meta) req.headers.pop(name) # store a single copy of the crypto meta items that are common to all # encrypted user metadata independently of any such meta that is stored # with the object body because it might change on a POST. This is done # for future-proofing - the meta stored here is not currently used # during decryption. if crypto_meta: meta = dump_crypto_meta({ 'cipher': crypto_meta['cipher'], 'key_id': keys['id'] }) req.headers[get_object_transient_sysmeta('crypto-meta')] = meta
def object_update(self, req, broker, name, timestamp): metadata = json.dumps(dict([val for val in req.headers.iteritems() if is_user_meta('object', val[0])])) broker.put_object(name, timestamp, int(req.headers['x-size']), req.headers['x-content-type'], req.headers['x-etag'], metadata=metadata)
def encrypt_user_metadata(self, req, keys): """ Encrypt user-metadata header values. Replace each x-object-meta-<key> user metadata header with a corresponding x-object-transient-sysmeta-crypto-meta-<key> header which has the crypto metadata required to decrypt appended to the encrypted value. :param req: a swob Request :param keys: a dict of encryption keys """ prefix = get_object_transient_sysmeta('crypto-meta-') user_meta_headers = [h for h in req.headers.items() if is_user_meta(self.server_type, h[0]) and h[1]] crypto_meta = None for name, val in user_meta_headers: short_name = strip_user_meta_prefix(self.server_type, name) new_name = prefix + short_name enc_val, crypto_meta = encrypt_header_val( self.crypto, val, keys[self.server_type]) req.headers[new_name] = append_crypto_meta(enc_val, crypto_meta) req.headers.pop(name) # store a single copy of the crypto meta items that are common to all # encrypted user metadata independently of any such meta that is stored # with the object body because it might change on a POST. This is done # for future-proofing - the meta stored here is not currently used # during decryption. if crypto_meta: meta = dump_crypto_meta({'cipher': crypto_meta['cipher'], 'key_id': keys['id']}) req.headers[get_object_transient_sysmeta('crypto-meta')] = meta
def GET(self, request): """Handle HTTP GET requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) keep_cache = self.keep_cache_private or ( 'X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers) try: disk_file = self.get_diskfile( device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: with disk_file.open(): metadata = disk_file.get_metadata() obj_size = int(metadata['Content-Length']) file_x_ts = metadata['X-Timestamp'] file_x_ts_flt = float(file_x_ts) try: if_unmodified_since = request.if_unmodified_since except (OverflowError, ValueError): # catches timestamps before the epoch return HTTPPreconditionFailed(request=request) file_x_ts_utc = datetime.fromtimestamp(file_x_ts_flt, UTC) if if_unmodified_since and file_x_ts_utc > if_unmodified_since: return HTTPPreconditionFailed(request=request) try: if_modified_since = request.if_modified_since except (OverflowError, ValueError): # catches timestamps before the epoch return HTTPPreconditionFailed(request=request) if if_modified_since and file_x_ts_utc <= if_modified_since: return HTTPNotModified(request=request) keep_cache = (self.keep_cache_private or ('X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers)) response = Response( app_iter=disk_file.reader(keep_cache=keep_cache), request=request, conditional_response=True) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if is_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] response.last_modified = math.ceil(file_x_ts_flt) response.content_length = obj_size try: response.content_encoding = metadata[ 'Content-Encoding'] except KeyError: pass response.headers['X-Timestamp'] = file_x_ts resp = request.get_response(response) except (DiskFileNotExist, DiskFileQuarantined): resp = HTTPNotFound(request=request, conditional_response=True) return resp
def GET(self, request): """Handle HTTP GET requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) keep_cache = self.keep_cache_private or ( 'X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers) try: disk_file = self.get_diskfile(device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: with disk_file.open(): metadata = disk_file.get_metadata() obj_size = int(metadata['Content-Length']) file_x_ts = metadata['X-Timestamp'] file_x_ts_flt = float(file_x_ts) try: if_unmodified_since = request.if_unmodified_since except (OverflowError, ValueError): # catches timestamps before the epoch return HTTPPreconditionFailed(request=request) file_x_ts_utc = datetime.fromtimestamp(file_x_ts_flt, UTC) if if_unmodified_since and file_x_ts_utc > if_unmodified_since: return HTTPPreconditionFailed(request=request) try: if_modified_since = request.if_modified_since except (OverflowError, ValueError): # catches timestamps before the epoch return HTTPPreconditionFailed(request=request) if if_modified_since and file_x_ts_utc <= if_modified_since: return HTTPNotModified(request=request) keep_cache = (self.keep_cache_private or ('X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers)) response = Response( app_iter=disk_file.reader(keep_cache=keep_cache), request=request, conditional_response=True) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if is_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] response.last_modified = math.ceil(file_x_ts_flt) response.content_length = obj_size try: response.content_encoding = metadata['Content-Encoding'] except KeyError: pass response.headers['X-Timestamp'] = file_x_ts resp = request.get_response(response) except (DiskFileNotExist, DiskFileQuarantined): resp = HTTPNotFound(request=request, conditional_response=True) return resp
def POST(self, request): """Handle HTTP POST requests for the Swift on File object server""" device, partition, account, container, obj, policy = \ get_name_and_placement(request, 5, 5, True) req_timestamp = valid_timestamp(request) new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') # Get DiskFile try: disk_file = self.get_diskfile(device, partition, account, container, obj, policy, uid=int(self.hpss_uid), gid=int(self.hpss_gid)) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) # Set class of service if we got it new_cos = request.headers.get('X-HPSS-Class-Of-Service-Id', None) if new_cos: disk_file.set_cos(int(new_cos)) # Set purge lock status if we got it if self.allow_purgelock: purge_lock = request.headers.get('X-HPSS-Purgelock-Status', None) if purge_lock is not None: disk_file.set_purge_lock(purge_lock) # Update metadata from request try: orig_metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request) orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0)) if orig_timestamp >= req_timestamp: backend_headers = {'X-Backend-Timestamp': orig_timestamp.internal} return HTTPConflict(request=request, headers=backend_headers) metadata = {'X-Timestamp': req_timestamp.internal} metadata.update(val for val in request.headers.iteritems() if is_user_meta('object', val[0])) for header_key in self.allowed_headers: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update('PUT', new_delete_at, account, container, obj, request, device, policy) if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device, policy) disk_file.write_metadata(metadata) return HTTPAccepted(request=request)
def copy_headers_into(from_r, to_r): """ Will copy desired headers from from_r to to_r :params from_r: a swob Request or Response :params to_r: a swob Request or Response """ pass_headers = ['x-delete-at'] for k, v in from_r.headers.items(): if is_user_meta('object', k) or k.lower() in pass_headers: to_r.headers[k] = v
def POST(self, request): """Handle HTTP POST requests for the Swift Object Server.""" device, partition, account, container, obj, policy = \ get_name_and_placement(request, 5, 5, True) req_timestamp = valid_timestamp(request) new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') try: # 获取文件对象管理器实例 disk_file = self.get_diskfile( device, partition, account, container, obj, policy=policy) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: # 读取原始元数据 orig_metadata = disk_file.read_metadata() except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) # 对象文件不存在,报错 except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request) orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0)) if orig_timestamp >= req_timestamp: return HTTPConflict( request=request, headers={'X-Backend-Timestamp': orig_timestamp.internal}) metadata = {'X-Timestamp': req_timestamp.internal} # 保存大对象的manifest,为什么??? self._preserve_slo_manifest(metadata, orig_metadata) # 保存请求中用户自定义的元数据到字典中 metadata.update(val for val in request.headers.items() if is_user_meta('object', val[0])) for header_key in self.allowed_headers: if header_key in request.headers: header_caps = header_key.title() # 保存请求中可允许更改的系统元数据 metadata[header_caps] = request.headers[header_key] orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update('PUT', new_delete_at, account, container, obj, request, device, policy) if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device, policy) try: # 将元数据更新到对象的扩展属性 disk_file.write_metadata(metadata) except (DiskFileXattrNotSupported, DiskFileNoSpace): return HTTPInsufficientStorage(drive=device, request=request) return HTTPAccepted(request=request)
def load_object_metadata(self, headers): metadata = {} metadata.update( (k.lower(), v) for k, v in headers.iteritems() if is_user_meta('object', k)) for header_key in self.allowed_headers: if header_key in headers: headers_lower = header_key.lower() metadata[headers_lower] = headers[header_key] print metadata return metadata
def POST(self, req): """HTTP POST request handler.""" if self.app.object_post_as_copy: req.method = 'PUT' req.path_info = '/v1/%s/%s/%s' % ( self.account_name, self.container_name, self.object_name) req.headers['Content-Length'] = 0 req.headers['X-Copy-From'] = quote('/%s/%s' % (self.container_name, self.object_name)) req.headers['X-Fresh-Metadata'] = 'true' req.environ['swift_versioned_copy'] = True if req.environ.get('QUERY_STRING'): req.environ['QUERY_STRING'] += '&multipart-manifest=get' else: req.environ['QUERY_STRING'] = 'multipart-manifest=get' resp = self.PUT(req) # Older editions returned 202 Accepted on object POSTs, so we'll # convert any 201 Created responses to that for compatibility with # picky clients. if resp.status_int != HTTP_CREATED: return resp return HTTPAccepted(request=req) else: error_response = check_metadata(req, 'object') if error_response: return error_response container_info = self.container_info(self.account_name, self.container_name, req) req.acl = container_info['write_acl'] if 'swift.authorize' in req.environ: aresp = req.environ['swift.authorize'](req) if aresp: return aresp storage = self.app.storage metadata = {} metadata.update( ("user.%s" % k, v) for k, v in req.headers.iteritems() if is_user_meta('object', k)) for header_key in self.allowed_headers: if header_key in req.headers: headers_caps = header_key.title() metadata[headers_caps] = req.headers[header_key] try: storage.object_update(self.account_name, self.container_name, self.object_name, metadata, clear=True) except (exceptions.NoSuchObject, exceptions.NoSuchContainer): return HTTPNotFound(request=req) resp = HTTPAccepted(request=req) return resp
def POST(self, request): """Handle HTTP POST requests for the Swift Object Server.""" device, partition, account, container, obj, policy = \ get_name_and_placement(request, 5, 5, True) req_timestamp = valid_timestamp(request) new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') try: disk_file = self.get_diskfile( device, partition, account, container, obj, policy=policy) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request) orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0)) if orig_timestamp >= req_timestamp: return HTTPConflict( request=request, headers={'X-Backend-Timestamp': orig_timestamp.internal}) metadata = {'X-Timestamp': req_timestamp.internal} self._preserve_slo_manifest(metadata, orig_metadata) metadata.update(val for val in request.headers.items() if is_user_meta('object', val[0])) headers_to_copy = ( request.headers.get( 'X-Backend-Replication-Headers', '').split() + list(self.allowed_headers)) for header_key in headers_to_copy: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update('PUT', new_delete_at, account, container, obj, request, device, policy) if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device, policy) try: disk_file.write_metadata(metadata) except (DiskFileXattrNotSupported, DiskFileNoSpace): return HTTPInsufficientStorage(drive=device, request=request) return HTTPAccepted(request=request)
def POST(self, request): """Handle HTTP POST requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = \ get_name_and_placement(request, 5, 5, True) print("----------------------------------------------------: In POST") req_timestamp = valid_timestamp(request) new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') try: disk_file = self.get_diskfile( device, partition, account, container, obj, policy_idx=policy_idx) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except DiskFileXattrNotSupported: return HTTPInsufficientStorage(drive=device, request=request) except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request) orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0)) if orig_timestamp >= req_timestamp: return HTTPConflict( request=request, headers={'X-Backend-Timestamp': orig_timestamp.internal}) metadata = {'X-Timestamp': req_timestamp.internal} metadata.update(val for val in request.headers.iteritems() if is_user_meta('object', val[0])) for header_key in self.allowed_headers: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update('PUT', new_delete_at, account, container, obj, request, device, policy_idx) if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device, policy_idx) try: disk_file.write_metadata(metadata) except (DiskFileXattrNotSupported, DiskFileNoSpace): return HTTPInsufficientStorage(drive=device, request=request) return HTTPAccepted(request=request)
def GET(self, request): """Handle HTTP GET requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = \ get_name_and_placement(request, 5, 5, True) keep_cache = self.keep_cache_private or ( 'X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers) try: disk_file = self.get_diskfile( device, partition, account, container, obj, policy_idx=policy_idx) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: with disk_file.open(): metadata = disk_file.get_metadata() obj_size = int(metadata['Content-Length']) file_x_ts = Timestamp(metadata['X-Timestamp']) keep_cache = (self.keep_cache_private or ('X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers)) response = Response( app_iter=disk_file.reader(keep_cache=keep_cache), request=request, conditional_response=True) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if is_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] response.last_modified = math.ceil(float(file_x_ts)) response.content_length = obj_size try: response.content_encoding = metadata[ 'Content-Encoding'] except KeyError: pass response.headers['X-Timestamp'] = file_x_ts.normal response.headers['X-Backend-Timestamp'] = file_x_ts.internal resp = request.get_response(response) except (DiskFileNotExist, DiskFileQuarantined) as e: headers = {} if hasattr(e, 'timestamp'): headers['X-Backend-Timestamp'] = e.timestamp.internal resp = HTTPNotFound(request=request, headers=headers, conditional_response=True) return resp
def GET(self, request): """Handle HTTP GET requests for the Swift Object Server.""" device, partition, account, container, obj, policy_idx = \ get_name_and_placement(request, 5, 5, True) keep_cache = self.keep_cache_private or ( 'X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers) try: disk_file = self.get_diskfile( device, partition, account, container, obj, policy_idx=policy_idx) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: with disk_file.open(): metadata = disk_file.get_metadata() obj_size = int(metadata['Content-Length']) file_x_ts = metadata['X-Timestamp'] file_x_ts_flt = float(file_x_ts) keep_cache = (self.keep_cache_private or ('X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers)) response = Response( app_iter=disk_file.reader(keep_cache=keep_cache), request=request, conditional_response=True) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if is_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] response.last_modified = math.ceil(file_x_ts_flt) response.content_length = obj_size try: response.content_encoding = metadata[ 'Content-Encoding'] except KeyError: pass response.headers['X-Timestamp'] = file_x_ts resp = request.get_response(response) except (DiskFileNotExist, DiskFileQuarantined) as e: headers = {} if hasattr(e, 'timestamp'): headers['X-Timestamp'] = e.timestamp resp = HTTPNotFound(request=request, headers=headers, conditional_response=True) return resp
def POST(self, request): """Handle HTTP POST requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) #### CHANGED CODE #### logging.info("---------- IN POST -----------") ### END CHANGED CODE #### if 'x-timestamp' not in request.headers or \ not check_float(request.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=request, content_type='text/plain') new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') try: disk_file = self.get_diskfile( device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request) orig_timestamp = orig_metadata.get('X-Timestamp', '0') if orig_timestamp >= request.headers['x-timestamp']: return HTTPConflict(request=request) metadata = {'X-Timestamp': request.headers['x-timestamp']} metadata.update(val for val in request.headers.iteritems() if is_user_meta('object', val[0])) for header_key in self.allowed_headers: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update('PUT', new_delete_at, account, container, obj, request, device) if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device) disk_file.write_metadata(metadata) return HTTPAccepted(request=request)
def POST(self, request): """Handle HTTP POST requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) if 'x-timestamp' not in request.headers or \ not check_float(request.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=request, content_type='text/plain') new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') try: disk_file = self.get_diskfile(device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request) orig_timestamp = orig_metadata.get('X-Timestamp', '0') if orig_timestamp >= request.headers['x-timestamp']: return HTTPConflict(request=request) metadata = {'X-Timestamp': request.headers['x-timestamp']} metadata.update(val for val in request.headers.iteritems() if is_user_meta('object', val[0])) for header_key in self.allowed_headers: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update('PUT', new_delete_at, account, container, obj, request, device) if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device) disk_file.write_metadata(metadata) return HTTPAccepted(request=request)
def _prep_headers_to_info(headers, server_type): """ Helper method that iterates once over a dict of headers, converting all keys to lower case and separating into subsets containing user metadata, system metadata and other headers. """ meta = {} sysmeta = {} other = {} for key, val in dict(headers).iteritems(): lkey = key.lower() if is_user_meta(server_type, lkey): meta[strip_user_meta_prefix(server_type, lkey)] = val elif is_sys_meta(server_type, lkey): sysmeta[strip_sys_meta_prefix(server_type, lkey)] = val else: other[lkey] = val return other, meta, sysmeta
def save_object(self, env): # Restorer 데몬에 의해 호출됨 req = Request(env) self._split_request_path(req) try: disk_file = self.get_diskfile(self.device, self.partition, self.account, self.container, self.obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=self.device, request=Request(env)) ori_meta = disk_file.read_metadata() metadata = {} metadata.update(val for val in req.headers.iteritems() if is_user_meta('object', val[0])) del metadata['X-Object-Meta-S3-Restored'] # Timestamp 값 유지 metadata['X-Timestamp'] = ori_meta['X-Timestamp'] metadata['Content-Type'] = ori_meta['Content-Type'] fsize = req.message_length() etag = md5() try: with disk_file.create(size=fsize) as writer: def timeout_reader(): with ChunkReadTimeout(60): return req.environ['wsgi.input'].read(65536) try: for chunk in iter(lambda: timeout_reader(), ''): etag.update(chunk) writer.write(chunk) except ChunkReadTimeout: return HTTPRequestTimeout(request=req) etag = etag.hexdigest() metadata['ETag'] = etag metadata['Content-Length'] = str(fsize) writer.put(metadata) except DiskFileNoSpace: return HTTPInsufficientStorage(drive=self.device, request=req) return HTTPCreated(request=req, etag=etag)
def set_restoring(self, env): # Lifecycle Middleware 에서 restore 중이라고 object 를 설정할 때 호출됨 req = Request(env) self._split_request_path(req) try: disk_file = self.get_diskfile(self.device, self.partition, self.account, self.container, self.obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=self.device, request=Request(env)) ori_meta = disk_file.read_metadata() metadata = ori_meta metadata.update(val for val in req.headers.iteritems() if is_user_meta('object', val[0])) # Timestamp 값 유지 with disk_file.create(size=0) as writer: writer.put(metadata) return HTTPCreated(request=req, etag=ori_meta['ETag'])
def truncate(self, env): req = Request(env) try: disk_file = self.get_diskfile(self.device, self.partition, self.account, self.container, self.obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=self.device, request=Request(copy(env))) # object flow 상, 임시 데이터를 삭제 후 DiskFileWrite 의 put을 하게 되면, # _finalize_put을 호출하게 된다. 이 때, metadata에 설정된 X-Timestamp 값으로 # object 파일명을 생성하고, 임시 파일로 대체한다. # 따라서 별 다른 truncate를 할 필요가 없다. ori_meta = disk_file.read_metadata() metadata = { 'X-Timestamp': ori_meta['X-Timestamp'], 'Content-Type': ori_meta['Content-Type'], 'ETag': 'd41d8cd98f00b204e9800998ecf8427e', 'Content-Length': 0, 'X-Object-Meta-Glacier': True, 'X-Object-Meta-S3-Content-Length': ori_meta['Content-Length'], 'X-Object-Meta-S3-ETag': ori_meta['ETag'] } # 원본 Object Metatdata 도 저장 metadata.update(val for val in ori_meta.iteritems() if is_user_meta('object', val[0])) # Object Restore 정보가 있으면 해당 정보 지움. # 이 경우는 restored object expiration 임. if 'X-Object-Meta-S3-Restore' in metadata: del metadata['X-Object-Meta-S3-Restore'] with disk_file.create(size=0) as writer: writer.put(metadata) return HTTPCreated(request=req, etag=ori_meta['ETag'])
def print_obj_metadata(metadata, drop_prefixes=False): """ Print out basic info and metadata from object, as returned from :func:`swift.obj.diskfile.read_metadata`. Metadata should include the keys: name, Content-Type, and X-Timestamp. Additional metadata is displayed unmodified. :param metadata: dict of object metadata :param drop_prefixes: if True, strip "X-Object-Meta-", "X-Object-Sysmeta-", and "X-Object-Transient-Sysmeta-" when displaying User Metadata, System Metadata, and Transient System Metadata entries :raises ValueError: """ user_metadata = {} sys_metadata = {} transient_sys_metadata = {} other_metadata = {} if not metadata: raise ValueError('Metadata is None') path = metadata.pop('name', '') content_type = metadata.pop('Content-Type', '') ts = Timestamp(metadata.pop('X-Timestamp', 0)) account = container = obj = obj_hash = None if path: try: account, container, obj = path.split('/', 3)[1:] except ValueError: raise ValueError('Path is invalid for object %r' % path) else: obj_hash = hash_path(account, container, obj) print('Path: %s' % path) print(' Account: %s' % account) print(' Container: %s' % container) print(' Object: %s' % obj) print(' Object hash: %s' % obj_hash) else: print('Path: Not found in metadata') if content_type: print('Content-Type: %s' % content_type) else: print('Content-Type: Not found in metadata') if ts: print('Timestamp: %s (%s)' % (ts.isoformat, ts.internal)) else: print('Timestamp: Not found in metadata') for key, value in metadata.items(): if is_user_meta('Object', key): if drop_prefixes: key = strip_user_meta_prefix('Object', key) user_metadata[key] = value elif is_sys_meta('Object', key): if drop_prefixes: key = strip_sys_meta_prefix('Object', key) sys_metadata[key] = value elif is_object_transient_sysmeta(key): if drop_prefixes: key = strip_object_transient_sysmeta_prefix(key) transient_sys_metadata[key] = value else: other_metadata[key] = value def print_metadata(title, items): print(title) if items: for key, value in sorted(items.items()): print(' %s: %s' % (key, value)) else: print(' No metadata found') print_metadata('System Metadata:', sys_metadata) print_metadata('Transient System Metadata:', transient_sys_metadata) print_metadata('User Metadata:', user_metadata) print_metadata('Other Metadata:', other_metadata)
def __call__(self, env, start_response): method = env['REQUEST_METHOD'] if method not in self.ALLOWED_METHODS: raise HTTPNotImplemented() path = env['PATH_INFO'] _, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4, rest_with_last=True) if env.get('QUERY_STRING'): path += '?' + env['QUERY_STRING'] if 'swift.authorize' in env: resp = env['swift.authorize'](swob.Request(env)) if resp: return resp(env, start_response) req = swob.Request(env) self.swift_sources.append(env.get('swift.source')) self.txn_ids.append(env.get('swift.trans_id')) try: resp_class, raw_headers, body = self._find_response(method, path) headers = HeaderKeyDict(raw_headers) except KeyError: if (env.get('QUERY_STRING') and (method, env['PATH_INFO']) in self._responses): resp_class, raw_headers, body = self._find_response( method, env['PATH_INFO']) headers = HeaderKeyDict(raw_headers) elif method == 'HEAD' and ('GET', path) in self._responses: resp_class, raw_headers, body = self._find_response( 'GET', path) body = None headers = HeaderKeyDict(raw_headers) elif method == 'GET' and obj and path in self.uploaded: resp_class = swob.HTTPOk headers, body = self.uploaded[path] else: raise KeyError("Didn't find %r in allowed responses" % ((method, path), )) # simulate object PUT if method == 'PUT' and obj: put_body = ''.join(iter(env['wsgi.input'].read, '')) if 'swift.callback.update_footers' in env: footers = HeaderKeyDict() env['swift.callback.update_footers'](footers) req.headers.update(footers) etag = md5(put_body).hexdigest() headers.setdefault('Etag', etag) headers.setdefault('Content-Length', len(put_body)) # keep it for subsequent GET requests later self.uploaded[path] = (dict(req.headers), put_body) if "CONTENT_TYPE" in env: self.uploaded[path][0]['Content-Type'] = env["CONTENT_TYPE"] # simulate object POST elif method == 'POST' and obj: metadata, data = self.uploaded.get(path, ({}, None)) # select items to keep from existing... new_metadata = dict((k, v) for k, v in metadata.items() if (not is_user_meta('object', k) and not is_object_transient_sysmeta(k))) # apply from new new_metadata.update( dict((k, v) for k, v in req.headers.items() if ( is_user_meta('object', k) or is_object_transient_sysmeta(k) or k.lower == 'content-type'))) self.uploaded[path] = new_metadata, data # note: tests may assume this copy of req_headers is case insensitive # so we deliberately use a HeaderKeyDict self._calls.append( FakeSwiftCall(method, path, HeaderKeyDict(req.headers))) # Apply conditional etag overrides conditional_etag = resolve_etag_is_at_header(req, headers) # range requests ought to work, hence conditional_response=True if isinstance(body, list): resp = resp_class(req=req, headers=headers, app_iter=body, conditional_response=req.method in ('GET', 'HEAD'), conditional_etag=conditional_etag) else: resp = resp_class(req=req, headers=headers, body=body, conditional_response=req.method in ('GET', 'HEAD'), conditional_etag=conditional_etag) wsgi_iter = resp(env, start_response) self.mark_opened(path) return LeakTrackingIter(wsgi_iter, self.mark_closed, path)
def print_db_info_metadata(db_type, info, metadata, drop_prefixes=False): """ print out data base info/metadata based on its type :param db_type: database type, account or container :param info: dict of data base info :param metadata: dict of data base metadata :param drop_prefixes: if True, strip "X-Account-Meta-", "X-Container-Meta-", "X-Account-Sysmeta-", and "X-Container-Sysmeta-" when displaying User Metadata and System Metadata dicts """ if info is None: raise ValueError('DB info is None') if db_type not in ['container', 'account']: raise ValueError('Wrong DB type') try: account = info['account'] container = None if db_type == 'container': container = info['container'] path = '/%s/%s' % (account, container) else: path = '/%s' % account print('Path: %s' % path) print(' Account: %s' % account) if db_type == 'container': print(' Container: %s' % container) path_hash = hash_path(account, container) if db_type == 'container': print(' Container Hash: %s' % path_hash) else: print(' Account Hash: %s' % path_hash) print('Metadata:') print(' Created at: %s (%s)' % (Timestamp(info['created_at']).isoformat, info['created_at'])) print(' Put Timestamp: %s (%s)' % (Timestamp(info['put_timestamp']).isoformat, info['put_timestamp'])) print(' Delete Timestamp: %s (%s)' % (Timestamp(info['delete_timestamp']).isoformat, info['delete_timestamp'])) print(' Status Timestamp: %s (%s)' % (Timestamp(info['status_changed_at']).isoformat, info['status_changed_at'])) if db_type == 'account': print(' Container Count: %s' % info['container_count']) print(' Object Count: %s' % info['object_count']) print(' Bytes Used: %s' % info['bytes_used']) if db_type == 'container': try: policy_name = POLICIES[info['storage_policy_index']].name except KeyError: policy_name = 'Unknown' print(' Storage Policy: %s (%s)' % ( policy_name, info['storage_policy_index'])) print(' Reported Put Timestamp: %s (%s)' % (Timestamp(info['reported_put_timestamp']).isoformat, info['reported_put_timestamp'])) print(' Reported Delete Timestamp: %s (%s)' % (Timestamp(info['reported_delete_timestamp']).isoformat, info['reported_delete_timestamp'])) print(' Reported Object Count: %s' % info['reported_object_count']) print(' Reported Bytes Used: %s' % info['reported_bytes_used']) print(' Chexor: %s' % info['hash']) print(' UUID: %s' % info['id']) except KeyError as e: raise ValueError('Info is incomplete: %s' % e) meta_prefix = 'x_' + db_type + '_' for key, value in info.items(): if key.lower().startswith(meta_prefix): title = key.replace('_', '-').title() print(' %s: %s' % (title, value)) user_metadata = {} sys_metadata = {} for key, (value, timestamp) in metadata.items(): if is_user_meta(db_type, key): if drop_prefixes: key = strip_user_meta_prefix(db_type, key) user_metadata[key] = value elif is_sys_meta(db_type, key): if drop_prefixes: key = strip_sys_meta_prefix(db_type, key) sys_metadata[key] = value else: title = key.replace('_', '-').title() print(' %s: %s' % (title, value)) if sys_metadata: print(' System Metadata: %s' % sys_metadata) else: print('No system metadata found in db file') if user_metadata: print(' User Metadata: %s' % user_metadata) else: print('No user metadata found in db file') if db_type == 'container': print('Sharding Metadata:') shard_type = 'root' if info['is_root'] else 'shard' print(' Type: %s' % shard_type) print(' State: %s' % info['db_state']) if info.get('shard_ranges'): print('Shard Ranges (%d):' % len(info['shard_ranges'])) for srange in info['shard_ranges']: srange = dict(srange, state_text=srange.state_text) print(' Name: %(name)s' % srange) print(' lower: %(lower)r, upper: %(upper)r' % srange) print(' Object Count: %(object_count)d, Bytes Used: ' '%(bytes_used)d, State: %(state_text)s (%(state)d)' % srange) print(' Created at: %s (%s)' % (Timestamp(srange['timestamp']).isoformat, srange['timestamp'])) print(' Meta Timestamp: %s (%s)' % (Timestamp(srange['meta_timestamp']).isoformat, srange['meta_timestamp']))
def GET(self, request): """Handle HTTP GET requests for the Swift Object Server.""" device, partition, account, container, obj = split_and_validate_path(request, 5, 5, True) keep_cache = self.keep_cache_private or ( "X-Auth-Token" not in request.headers and "X-Storage-Token" not in request.headers ) try: disk_file = self.get_diskfile(device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: with disk_file.open(): metadata = disk_file.get_metadata() obj_size = int(metadata["Content-Length"]) if request.headers.get("if-match") not in (None, "*") and metadata["ETag"] not in request.if_match: return HTTPPreconditionFailed(request=request) if request.headers.get("if-none-match") is not None: if metadata["ETag"] in request.if_none_match: resp = HTTPNotModified(request=request) resp.etag = metadata["ETag"] return resp file_x_ts = metadata["X-Timestamp"] file_x_ts_flt = float(file_x_ts) try: if_unmodified_since = request.if_unmodified_since except (OverflowError, ValueError): # catches timestamps before the epoch return HTTPPreconditionFailed(request=request) file_x_ts_utc = datetime.fromtimestamp(file_x_ts_flt, UTC) if if_unmodified_since and file_x_ts_utc > if_unmodified_since: return HTTPPreconditionFailed(request=request) try: if_modified_since = request.if_modified_since except (OverflowError, ValueError): # catches timestamps before the epoch return HTTPPreconditionFailed(request=request) if if_modified_since and file_x_ts_utc <= if_modified_since: return HTTPNotModified(request=request) keep_cache = self.keep_cache_private or ( "X-Auth-Token" not in request.headers and "X-Storage-Token" not in request.headers ) response = Response( app_iter=disk_file.reader(keep_cache=keep_cache), request=request, conditional_response=True ) response.headers["Content-Type"] = metadata.get("Content-Type", "application/octet-stream") for key, value in metadata.iteritems(): if is_user_meta("object", key) or key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata["ETag"] response.last_modified = math.ceil(file_x_ts_flt) response.content_length = obj_size try: response.content_encoding = metadata["Content-Encoding"] except KeyError: pass response.headers["X-Timestamp"] = file_x_ts resp = request.get_response(response) except DiskFileNotExist: if request.headers.get("if-match") == "*": resp = HTTPPreconditionFailed(request=request) else: resp = HTTPNotFound(request=request) except DiskFileQuarantined: resp = HTTPNotFound(request=request) return resp
def PUT(self, request): """Handle HTTP PUT requests for the Swift Object Server.""" device, partition, account, container, obj = split_and_validate_path(request, 5, 5, True) if "x-timestamp" not in request.headers or not check_float(request.headers["x-timestamp"]): return HTTPBadRequest(body="Missing timestamp", request=request, content_type="text/plain") error_response = check_object_creation(request, obj) if error_response: return error_response new_delete_at = int(request.headers.get("X-Delete-At") or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body="X-Delete-At in past", request=request, content_type="text/plain") try: fsize = request.message_length() except ValueError as e: return HTTPBadRequest(body=str(e), request=request, content_type="text/plain") try: disk_file = self.get_diskfile(device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): orig_metadata = {} orig_timestamp = orig_metadata.get("X-Timestamp") if orig_timestamp and orig_timestamp >= request.headers["x-timestamp"]: return HTTPConflict(request=request) orig_delete_at = int(orig_metadata.get("X-Delete-At") or 0) upload_expiration = time.time() + self.max_upload_time etag = md5() elapsed_time = 0 try: with disk_file.create(size=fsize) as writer: upload_size = 0 reader = request.environ["wsgi.input"].read for chunk in iter(lambda: reader(self.network_chunk_size), ""): start_time = time.time() if start_time > upload_expiration: self.logger.increment("PUT.timeouts") return HTTPRequestTimeout(request=request) etag.update(chunk) upload_size = writer.write(chunk) elapsed_time += time.time() - start_time if upload_size: self.logger.transfer_rate("PUT." + device + ".timing", elapsed_time, upload_size) if fsize is not None and fsize != upload_size: return HTTPClientDisconnect(request=request) etag = etag.hexdigest() if "etag" in request.headers and request.headers["etag"].lower() != etag: return HTTPUnprocessableEntity(request=request) metadata = { "X-Timestamp": request.headers["x-timestamp"], "Content-Type": request.headers["content-type"], "ETag": etag, "Content-Length": str(upload_size), } metadata.update(val for val in request.headers.iteritems() if is_user_meta("object", val[0])) for header_key in request.headers.get("X-Backend-Replication-Headers") or self.allowed_headers: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] writer.put(metadata) except DiskFileNoSpace: return HTTPInsufficientStorage(drive=device, request=request) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update("PUT", new_delete_at, account, container, obj, request, device) if orig_delete_at: self.delete_at_update("DELETE", orig_delete_at, account, container, obj, request, device) self.container_update( "PUT", account, container, obj, request, HeaderKeyDict( { "x-size": metadata["Content-Length"], "x-content-type": metadata["Content-Type"], "x-timestamp": metadata["X-Timestamp"], "x-etag": metadata["ETag"], } ), device, ) return HTTPCreated(request=request, etag=etag)
def print_db_info_metadata(db_type, info, metadata, drop_prefixes=False, verbose=False): """ print out data base info/metadata based on its type :param db_type: database type, account or container :param info: dict of data base info :param metadata: dict of data base metadata :param drop_prefixes: if True, strip "X-Account-Meta-", "X-Container-Meta-", "X-Account-Sysmeta-", and "X-Container-Sysmeta-" when displaying User Metadata and System Metadata dicts """ if info is None: raise ValueError('DB info is None') if db_type not in ['container', 'account']: raise ValueError('Wrong DB type') try: account = info['account'] container = None if db_type == 'container': container = info['container'] path = '/%s/%s' % (account, container) else: path = '/%s' % account print('Path: %s' % path) print(' Account: %s' % account) if db_type == 'container': print(' Container: %s' % container) print(' Deleted: %s' % info['is_deleted']) path_hash = hash_path(account, container) if db_type == 'container': print(' Container Hash: %s' % path_hash) else: print(' Account Hash: %s' % path_hash) print('Metadata:') print(' Created at: %s (%s)' % (Timestamp(info['created_at']).isoformat, info['created_at'])) print(' Put Timestamp: %s (%s)' % (Timestamp(info['put_timestamp']).isoformat, info['put_timestamp'])) print(' Delete Timestamp: %s (%s)' % (Timestamp(info['delete_timestamp']).isoformat, info['delete_timestamp'])) print(' Status Timestamp: %s (%s)' % (Timestamp(info['status_changed_at']).isoformat, info['status_changed_at'])) if db_type == 'account': print(' Container Count: %s' % info['container_count']) print(' Object Count: %s' % info['object_count']) print(' Bytes Used: %s' % info['bytes_used']) if db_type == 'container': try: policy_name = POLICIES[info['storage_policy_index']].name except KeyError: policy_name = 'Unknown' print(' Storage Policy: %s (%s)' % ( policy_name, info['storage_policy_index'])) print(' Reported Put Timestamp: %s (%s)' % (Timestamp(info['reported_put_timestamp']).isoformat, info['reported_put_timestamp'])) print(' Reported Delete Timestamp: %s (%s)' % (Timestamp(info['reported_delete_timestamp']).isoformat, info['reported_delete_timestamp'])) print(' Reported Object Count: %s' % info['reported_object_count']) print(' Reported Bytes Used: %s' % info['reported_bytes_used']) print(' Chexor: %s' % info['hash']) print(' UUID: %s' % info['id']) except KeyError as e: raise ValueError('Info is incomplete: %s' % e) meta_prefix = 'x_' + db_type + '_' for key, value in info.items(): if key.lower().startswith(meta_prefix): title = key.replace('_', '-').title() print(' %s: %s' % (title, value)) user_metadata = {} sys_metadata = {} for key, (value, timestamp) in metadata.items(): if is_user_meta(db_type, key): if drop_prefixes: key = strip_user_meta_prefix(db_type, key) user_metadata[key] = value elif is_sys_meta(db_type, key): if drop_prefixes: key = strip_sys_meta_prefix(db_type, key) sys_metadata[key] = value else: title = key.replace('_', '-').title() print(' %s: %s' % (title, value)) if sys_metadata: print(' System Metadata: %s' % sys_metadata) else: print('No system metadata found in db file') if user_metadata: print(' User Metadata: %s' % user_metadata) else: print('No user metadata found in db file') if db_type == 'container': print('Sharding Metadata:') shard_type = 'root' if info['is_root'] else 'shard' print(' Type: %s' % shard_type) print(' State: %s' % info['db_state']) if info.get('shard_ranges'): num_shards = len(info['shard_ranges']) print('Shard Ranges (%d):' % num_shards) count_by_state = defaultdict(int) for srange in info['shard_ranges']: count_by_state[(srange.state, srange.state_text)] += 1 print(' States:') for key_state, count in sorted(count_by_state.items()): key, state = key_state print(' %9s: %s' % (state, count)) if verbose: for srange in info['shard_ranges']: srange = dict(srange, state_text=srange.state_text) print(' Name: %(name)s' % srange) print(' lower: %(lower)r, upper: %(upper)r' % srange) print(' Object Count: %(object_count)d, Bytes Used: ' '%(bytes_used)d, State: %(state_text)s (%(state)d)' % srange) print(' Created at: %s (%s)' % (Timestamp(srange['timestamp']).isoformat, srange['timestamp'])) print(' Meta Timestamp: %s (%s)' % (Timestamp(srange['meta_timestamp']).isoformat, srange['meta_timestamp'])) else: print('(Use -v/--verbose to show more Shard Ranges details)')
def PUT(self, request): """Handle HTTP PUT requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) if 'x-timestamp' not in request.headers or \ not check_float(request.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=request, content_type='text/plain') error_response = check_object_creation(request, obj) if error_response: return error_response new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') try: fsize = request.message_length() except ValueError as e: return HTTPBadRequest(body=str(e), request=request, content_type='text/plain') try: disk_file = self.get_diskfile( device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): orig_metadata = {} # Checks for If-None-Match if request.if_none_match is not None and orig_metadata: if '*' in request.if_none_match: # File exists already so return 412 return HTTPPreconditionFailed(request=request) if orig_metadata.get('ETag') in request.if_none_match: # The current ETag matches, so return 412 return HTTPPreconditionFailed(request=request) orig_timestamp = orig_metadata.get('X-Timestamp') if orig_timestamp and orig_timestamp >= request.headers['x-timestamp']: return HTTPConflict(request=request) orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) upload_expiration = time.time() + self.max_upload_time etag = md5() elapsed_time = 0 try: with disk_file.create(size=fsize) as writer: upload_size = 0 def timeout_reader(): with ChunkReadTimeout(self.client_timeout): return request.environ['wsgi.input'].read( self.network_chunk_size) try: for chunk in iter(lambda: timeout_reader(), ''): start_time = time.time() if start_time > upload_expiration: self.logger.increment('PUT.timeouts') return HTTPRequestTimeout(request=request) etag.update(chunk) upload_size = writer.write(chunk) elapsed_time += time.time() - start_time except ChunkReadTimeout: return HTTPRequestTimeout(request=request) if upload_size: self.logger.transfer_rate( 'PUT.' + device + '.timing', elapsed_time, upload_size) if fsize is not None and fsize != upload_size: return HTTPClientDisconnect(request=request) etag = etag.hexdigest() if 'etag' in request.headers and \ request.headers['etag'].lower() != etag: return HTTPUnprocessableEntity(request=request) metadata = { 'X-Timestamp': request.headers['x-timestamp'], 'Content-Type': request.headers['content-type'], 'ETag': etag, 'Content-Length': str(upload_size), } metadata.update(val for val in request.headers.iteritems() if is_user_meta('object', val[0])) for header_key in ( request.headers.get('X-Backend-Replication-Headers') or self.allowed_headers): if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] writer.put(metadata) except DiskFileNoSpace: return HTTPInsufficientStorage(drive=device, request=request) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update( 'PUT', new_delete_at, account, container, obj, request, device) if orig_delete_at: self.delete_at_update( 'DELETE', orig_delete_at, account, container, obj, request, device) self.container_update( 'PUT', account, container, obj, request, HeaderKeyDict({ 'x-size': metadata['Content-Length'], 'x-content-type': metadata['Content-Type'], 'x-timestamp': metadata['X-Timestamp'], 'x-etag': metadata['ETag']}), device) return HTTPCreated(request=request, etag=etag)
def print_obj_metadata(metadata, drop_prefixes=False): """ Print out basic info and metadata from object, as returned from :func:`swift.obj.diskfile.read_metadata`. Metadata should include the keys: name, Content-Type, and X-Timestamp. Additional metadata is displayed unmodified. :param metadata: dict of object metadata :param drop_prefixes: if True, strip "X-Object-Meta-", "X-Object-Sysmeta-", and "X-Object-Transient-Sysmeta-" when displaying User Metadata, System Metadata, and Transient System Metadata entries :raises ValueError: """ user_metadata = {} sys_metadata = {} transient_sys_metadata = {} other_metadata = {} if not metadata: raise ValueError('Metadata is None') path = metadata.pop('name', '') content_type = metadata.pop('Content-Type', '') ts = Timestamp(metadata.pop('X-Timestamp', 0)) account = container = obj = obj_hash = None if path: try: account, container, obj = path.split('/', 3)[1:] except ValueError: raise ValueError('Path is invalid for object %r' % path) else: obj_hash = hash_path(account, container, obj) print('Path: %s' % path) print(' Account: %s' % account) print(' Container: %s' % container) print(' Object: %s' % obj) print(' Object hash: %s' % obj_hash) else: print('Path: Not found in metadata') if content_type: print('Content-Type: %s' % content_type) else: print('Content-Type: Not found in metadata') if ts: print('Timestamp: %s (%s)' % (ts.isoformat, ts.internal)) else: print('Timestamp: Not found in metadata') for key, value in metadata.items(): if is_user_meta('Object', key): if drop_prefixes: key = strip_user_meta_prefix('Object', key) user_metadata[key] = value elif is_sys_meta('Object', key): if drop_prefixes: key = strip_sys_meta_prefix('Object', key) sys_metadata[key] = value elif is_object_transient_sysmeta(key): if drop_prefixes: key = strip_object_transient_sysmeta_prefix(key) transient_sys_metadata[key] = value else: other_metadata[key] = value def print_metadata(title, items): print(title) if items: for key, value in sorted(items.items()): print(' %s: %s' % (key, value)) else: print(' No metadata found') print_metadata('System Metadata:', sys_metadata) print_metadata('Transient System Metadata:', transient_sys_metadata) print_metadata('User Metadata:', user_metadata) print_metadata('Other Metadata:', other_metadata) for label, meta in [ ('Data crypto details', sys_metadata.get('X-Object-Sysmeta-Crypto-Body-Meta')), ('Metadata crypto details', transient_sys_metadata.get('X-Object-Transient-Sysmeta-Crypto-Meta')), ]: if meta is None: continue print('%s: %s' % ( label, json.dumps(load_crypto_meta(meta, b64decode=False), indent=2, sort_keys=True, separators=(',', ': '))))
def GET(self, request): __CBAC__ = True """Handle HTTP GET requests for the Swift Object Server.""" my_debug('testing', '@GET()') my_debug('#request headers', request.headers) #print my_debug device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) keep_cache = self.keep_cache_private or ( 'X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers) try: disk_file = self.get_diskfile( device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) my_debug("locals()",locals()) my_debug("original Diskfile", disk_file) try: with disk_file.open(): metadata = disk_file.get_metadata() my_debug("metadata", metadata) obj_size = int(metadata['Content-Length']) file_x_ts = metadata['X-Timestamp'] #json_policy = metadata['JSON-Policy'] file_x_ts_flt = float(file_x_ts) keep_cache = (self.keep_cache_private or ('X-Auth-Token' not in request.headers and 'X-Storage-Token' not in request.headers)) ''' if policy file is present & user_label is present apply JSONAC Additional thing to have: 1. Check if policy from metadata is valid json 2. Check if user_clearance is present is headers. Read keystone info. 3. Pass JSONPath as headers 4. ''' if __CBAC__: my_debug("----CBAC Starts here-----", "***********") disk_file_iter = disk_file.reader(keep_cache=keep_cache) original_data = "" for chunk in iter(disk_file_iter): original_data += chunk userRoles = request.headers['X-Roles'] json_policy = metadata['X-Object-Meta-Jsonpolicy'] \ if metadata.has_key('X-Object-Meta-Jsonpolicy') else None user_labels = metadata['X-Object-Meta-Userlabels'] \ if metadata.has_key('X-Object-Meta-Userlabels') else None object_labels = metadata['X-Object-Meta-Objectlabels'] \ if metadata.has_key('X-Object-Meta-Objectlabels') else None object_labelling = metadata['X-Object-Meta-Jsonlabelling'] \ if metadata.has_key('X-Object-Meta-Jsonlabelling') else None if json_policy and user_labels and object_labels : cbac_policy = {} cbac_policy['user_labels'] = json.loads(user_labels) cbac_policy['object_labels'] = json.loads(object_labels) cbac_policy['policy'] = json.loads(json_policy) #user_clearance = ['manager'] user_clearance = userRoles jsonpath = "/" filtered_content = "" my_debug("json_policy is", json_policy) my_debug("original data is ", original_data) #try: if json_policy : if json_policy and user_clearance and jsonpath and object_labelling: '''filtered_content = ContentFilter(content_str=original_data, labeling_policy_str=object_labelling, user_clearance=user_clearance, query=jsonpath, cbac_policy=cbac_policy).apply()''' filtered_content = "testing" my_debug("#filtered content is ", filtered_content) #else: #my_debug("#content_filter not working", True) #except Exception as e: #else: # my_debug("Exception with content filtering {}".format("e"), True) '''end of content -filter ''' tmp_file = self.tmp_disk_file( request=request, device=device, partition=partition, container=container, obj="tmp", data=filtered_content, account=account) tmp_file = self.get_diskfile( device, partition, account, container, "tmp") my_debug("tmp_file is", tmp_file) try: with tmp_file.open(): tmp_metadata = tmp_file.get_metadata() my_debug("with tmp_file.open()", tmp_file.reader(keep_cache=keep_cache)) response = Response( app_iter=tmp_file.reader(keep_cache=keep_cache), request=request, conditional_response=True) response.headers['Content-Type'] = tmp_metadata.get( 'Content-Type', 'application/octet-stream') response.content_length = len(filtered_content) for key, value in tmp_metadata.iteritems(): if is_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = tmp_metadata['ETag'] #response.last_modified = math.ceil(file_x_ts_flt) #response.content_length = obj_size my_debug("get_response", "test") resp = request.get_response(response) my_debug( "response", resp ) my_debug("rsponse.__dict__", resp.__dict__) except (DiskFileNotExist, DiskFileQuarantined): my_debug("tmp file is not found", "test") resp = HTTPNotFound(request=request, conditional_response=True) my_debug("final resp object", resp) return resp disk_file.open() response = Response( app_iter=disk_file.reader(keep_cache=keep_cache), # instead of app_iter which reads from the file, content is given in body #body="Yes, you are smart!\n", request=request, conditional_response=True) response.headers['Content-Type'] = metadata.get( 'Content-Type', 'application/octet-stream') for key, value in metadata.iteritems(): if is_user_meta('object', key) or \ key.lower() in self.allowed_headers: response.headers[key] = value response.etag = metadata['ETag'] response.last_modified = math.ceil(file_x_ts_flt) response.content_length = obj_size #response.content_length = response.headers['Content-Length'] try: response.content_encoding = metadata[ 'Content-Encoding'] except KeyError: pass response.headers['X-Timestamp'] = file_x_ts resp = request.get_response(response) except (DiskFileNotExist, DiskFileQuarantined): resp = HTTPNotFound(request=request, conditional_response=True) my_debug("resp object without cbac", resp.__dict__) return resp
def POST(self, request): """Handle HTTP POST requests for the Swift on File object server""" device, partition, account, container, obj, policy = \ get_name_and_placement(request, 5, 5, True) req_timestamp = valid_timestamp(request) new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') # Get DiskFile try: disk_file = self.get_diskfile(device, partition, account, container, obj, policy) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) # Set Purgelock status if we got it purgelock = request.headers.get('X-HPSS-Purgelock-Status') if purgelock: try: hpssfs.ioctl(disk_file._fd, hpssfs.HPSSFS_PURGE_LOCK, int(purgelock)) except IOError as err: raise SwiftOnFileSystemIOError( err.errno, '%s, xattr.getxattr("%s", ...)' % (err.strerror, disk_file._fd)) # Set class of service if we got it cos = request.headers.get('X-HPSS-Class-Of-Service-ID') if cos: try: xattr.setxattr(disk_file._fd, 'system.hpss.cos', int(cos)) except IOError as err: raise SwiftOnFileSystemIOError( err.errno, '%s, xattr.setxattr("%s", ...)' % (err.strerror, disk_file._fd)) # Update metadata from request try: orig_metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): return HTTPNotFound(request=request) orig_timestamp = Timestamp(orig_metadata.get('X-Timestamp', 0)) if orig_timestamp >= req_timestamp: return HTTPConflict(request=request, headers={ 'X-Backend-Timestamp': orig_timestamp.internal }) metadata = {'X-Timestamp': req_timestamp.internal} metadata.update(val for val in request.headers.iteritems() if is_user_meta('object', val[0])) for header_key in self.allowed_headers: if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update('PUT', new_delete_at, account, container, obj, request, device, policy) if orig_delete_at: self.delete_at_update('DELETE', orig_delete_at, account, container, obj, request, device, policy) disk_file.write_metadata(metadata) return HTTPAccepted(request=request)
def print_obj_metadata(metadata): """ Print out basic info and metadata from object, as returned from :func:`swift.obj.diskfile.read_metadata`. Metadata should include the keys: name, Content-Type, and X-Timestamp. Additional metadata is displayed unmodified. :param metadata: dict of object metadata :raises: ValueError """ user_metadata = {} sys_metadata = {} other_metadata = {} if not metadata: raise ValueError('Metadata is None') path = metadata.pop('name', '') content_type = metadata.pop('Content-Type', '') ts = Timestamp(metadata.pop('X-Timestamp', 0)) account = container = obj = obj_hash = None if path: try: account, container, obj = path.split('/', 3)[1:] except ValueError: raise ValueError('Path is invalid for object %r' % path) else: obj_hash = hash_path(account, container, obj) print 'Path: %s' % path print ' Account: %s' % account print ' Container: %s' % container print ' Object: %s' % obj print ' Object hash: %s' % obj_hash else: print 'Path: Not found in metadata' if content_type: print 'Content-Type: %s' % content_type else: print 'Content-Type: Not found in metadata' if ts: print ('Timestamp: %s (%s)' % (ts.isoformat, ts.internal)) else: print 'Timestamp: Not found in metadata' for key, value in metadata.iteritems(): if is_user_meta('Object', key): user_metadata[key] = value elif is_sys_meta('Object', key): sys_metadata[key] = value else: other_metadata[key] = value def print_metadata(title, items): print title if items: for meta_key in sorted(items): print ' %s: %s' % (meta_key, items[meta_key]) else: print ' No metadata found' print_metadata('System Metadata:', sys_metadata) print_metadata('User Metadata:', user_metadata) print_metadata('Other Metadata:', other_metadata)
def test_is_user_meta(self): m_type = 'meta' for st in server_types: self.assertTrue(rh.is_user_meta(st, 'x-%s-%s-foo' % (st, m_type))) self.assertFalse(rh.is_user_meta(st, 'x-%s-%s-' % (st, m_type))) self.assertFalse(rh.is_user_meta(st, 'x-%s-%sfoo' % (st, m_type)))
def print_db_info_metadata(db_type, info, metadata): """ print out data base info/metadata based on its type :param db_type: database type, account or container :param info: dict of data base info :param metadata: dict of data base metadata """ if info is None: raise ValueError('DB info is None') if db_type not in ['container', 'account']: raise ValueError('Wrong DB type') try: account = info['account'] container = None if db_type == 'container': container = info['container'] path = '/%s/%s' % (account, container) else: path = '/%s' % account print 'Path: %s' % path print ' Account: %s' % account if db_type == 'container': print ' Container: %s' % container path_hash = hash_path(account, container) if db_type == 'container': print ' Container Hash: %s' % path_hash else: print ' Account Hash: %s' % path_hash print 'Metadata:' print (' Created at: %s (%s)' % (Timestamp(info['created_at']).isoformat, info['created_at'])) print (' Put Timestamp: %s (%s)' % (Timestamp(info['put_timestamp']).isoformat, info['put_timestamp'])) print (' Delete Timestamp: %s (%s)' % (Timestamp(info['delete_timestamp']).isoformat, info['delete_timestamp'])) print (' Status Timestamp: %s (%s)' % (Timestamp(info['status_changed_at']).isoformat, info['status_changed_at'])) if db_type == 'account': print ' Container Count: %s' % info['container_count'] print ' Object Count: %s' % info['object_count'] print ' Bytes Used: %s' % info['bytes_used'] if db_type == 'container': try: policy_name = POLICIES[info['storage_policy_index']].name except KeyError: policy_name = 'Unknown' print (' Storage Policy: %s (%s)' % ( policy_name, info['storage_policy_index'])) print (' Reported Put Timestamp: %s (%s)' % (Timestamp(info['reported_put_timestamp']).isoformat, info['reported_put_timestamp'])) print (' Reported Delete Timestamp: %s (%s)' % (Timestamp(info['reported_delete_timestamp']).isoformat, info['reported_delete_timestamp'])) print ' Reported Object Count: %s' % info['reported_object_count'] print ' Reported Bytes Used: %s' % info['reported_bytes_used'] print ' Chexor: %s' % info['hash'] print ' UUID: %s' % info['id'] except KeyError as e: raise ValueError('Info is incomplete: %s' % e) meta_prefix = 'x_' + db_type + '_' for key, value in info.iteritems(): if key.lower().startswith(meta_prefix): title = key.replace('_', '-').title() print ' %s: %s' % (title, value) user_metadata = {} sys_metadata = {} for key, (value, timestamp) in metadata.iteritems(): if is_user_meta(db_type, key): user_metadata[strip_user_meta_prefix(db_type, key)] = value elif is_sys_meta(db_type, key): sys_metadata[strip_sys_meta_prefix(db_type, key)] = value else: title = key.replace('_', '-').title() print ' %s: %s' % (title, value) if sys_metadata: print ' System Metadata: %s' % sys_metadata else: print 'No system metadata found in db file' if user_metadata: print ' User Metadata: %s' % user_metadata else: print 'No user metadata found in db file'
def __call__(self, env, start_response): method = env["REQUEST_METHOD"] if method not in self.ALLOWED_METHODS: raise HTTPNotImplemented() path = env["PATH_INFO"] _, acc, cont, obj = split_path(env["PATH_INFO"], 0, 4, rest_with_last=True) if env.get("QUERY_STRING"): path += "?" + env["QUERY_STRING"] if "swift.authorize" in env: resp = env["swift.authorize"](swob.Request(env)) if resp: return resp(env, start_response) req = swob.Request(env) self.swift_sources.append(env.get("swift.source")) self.txn_ids.append(env.get("swift.trans_id")) try: resp_class, raw_headers, body = self._find_response(method, path) headers = HeaderKeyDict(raw_headers) except KeyError: if env.get("QUERY_STRING") and (method, env["PATH_INFO"]) in self._responses: resp_class, raw_headers, body = self._find_response(method, env["PATH_INFO"]) headers = HeaderKeyDict(raw_headers) elif method == "HEAD" and ("GET", path) in self._responses: resp_class, raw_headers, body = self._find_response("GET", path) body = None headers = HeaderKeyDict(raw_headers) elif method == "GET" and obj and path in self.uploaded: resp_class = swob.HTTPOk headers, body = self.uploaded[path] else: raise KeyError("Didn't find %r in allowed responses" % ((method, path),)) # simulate object PUT if method == "PUT" and obj: put_body = "".join(iter(env["wsgi.input"].read, "")) if "swift.callback.update_footers" in env: footers = HeaderKeyDict() env["swift.callback.update_footers"](footers) req.headers.update(footers) etag = md5(put_body).hexdigest() headers.setdefault("Etag", etag) headers.setdefault("Content-Length", len(put_body)) # keep it for subsequent GET requests later self.uploaded[path] = (dict(req.headers), put_body) if "CONTENT_TYPE" in env: self.uploaded[path][0]["Content-Type"] = env["CONTENT_TYPE"] # simulate object POST elif method == "POST" and obj: metadata, data = self.uploaded.get(path, ({}, None)) # select items to keep from existing... new_metadata = dict( (k, v) for k, v in metadata.items() if (not is_user_meta("object", k) and not is_object_transient_sysmeta(k)) ) # apply from new new_metadata.update( dict( (k, v) for k, v in req.headers.items() if (is_user_meta("object", k) or is_object_transient_sysmeta(k) or k.lower == "content-type") ) ) self.uploaded[path] = new_metadata, data # note: tests may assume this copy of req_headers is case insensitive # so we deliberately use a HeaderKeyDict self._calls.append((method, path, HeaderKeyDict(req.headers))) # range requests ought to work, hence conditional_response=True if isinstance(body, list): resp = resp_class( req=req, headers=headers, app_iter=body, conditional_response=req.method in ("GET", "HEAD") ) else: resp = resp_class(req=req, headers=headers, body=body, conditional_response=req.method in ("GET", "HEAD")) wsgi_iter = resp(env, start_response) self.mark_opened(path) return LeakTrackingIter(wsgi_iter, self, path)
def print_obj_metadata(metadata): """ Print out basic info and metadata from object, as returned from :func:`swift.obj.diskfile.read_metadata`. Metadata should include the keys: name, Content-Type, and X-Timestamp. Additional metadata is displayed unmodified. :param metadata: dict of object metadata :raises: ValueError """ user_metadata = {} sys_metadata = {} transient_sys_metadata = {} other_metadata = {} if not metadata: raise ValueError('Metadata is None') path = metadata.pop('name', '') content_type = metadata.pop('Content-Type', '') ts = Timestamp(metadata.pop('X-Timestamp', 0)) account = container = obj = obj_hash = None if path: try: account, container, obj = path.split('/', 3)[1:] except ValueError: raise ValueError('Path is invalid for object %r' % path) else: obj_hash = hash_path(account, container, obj) print('Path: %s' % path) print(' Account: %s' % account) print(' Container: %s' % container) print(' Object: %s' % obj) print(' Object hash: %s' % obj_hash) else: print('Path: Not found in metadata') if content_type: print('Content-Type: %s' % content_type) else: print('Content-Type: Not found in metadata') if ts: print('Timestamp: %s (%s)' % (ts.isoformat, ts.internal)) else: print('Timestamp: Not found in metadata') for key, value in metadata.items(): if is_user_meta('Object', key): user_metadata[key] = value elif is_sys_meta('Object', key): sys_metadata[key] = value elif is_object_transient_sysmeta(key): transient_sys_metadata[key] = value else: other_metadata[key] = value def print_metadata(title, items): print(title) if items: for meta_key in sorted(items): print(' %s: %s' % (meta_key, items[meta_key])) else: print(' No metadata found') print_metadata('System Metadata:', sys_metadata) print_metadata('Transient System Metadata:', transient_sys_metadata) print_metadata('User Metadata:', user_metadata) print_metadata('Other Metadata:', other_metadata)
def PUT(self, request): """Handle HTTP PUT requests for the Swift Object Server.""" device, partition, account, container, obj = \ split_and_validate_path(request, 5, 5, True) my_debug("local variables in PUT()",locals()) my_debug("request.path", request.path) if 'x-timestamp' not in request.headers or \ not check_float(request.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=request, content_type='text/plain') error_response = check_object_creation(request, obj) if error_response: return error_response new_delete_at = int(request.headers.get('X-Delete-At') or 0) if new_delete_at and new_delete_at < time.time(): return HTTPBadRequest(body='X-Delete-At in past', request=request, content_type='text/plain') try: fsize = request.message_length() except ValueError as e: return HTTPBadRequest(body=str(e), request=request, content_type='text/plain') try: disk_file = self.get_diskfile( device, partition, account, container, obj) except DiskFileDeviceUnavailable: return HTTPInsufficientStorage(drive=device, request=request) try: orig_metadata = disk_file.read_metadata() except (DiskFileNotExist, DiskFileQuarantined): orig_metadata = {} # Checks for If-None-Match if request.if_none_match is not None and orig_metadata: if '*' in request.if_none_match: # File exists already so return 412 return HTTPPreconditionFailed(request=request) if orig_metadata.get('ETag') in request.if_none_match: # The current ETag matches, so return 412 return HTTPPreconditionFailed(request=request) orig_timestamp = orig_metadata.get('X-Timestamp') if orig_timestamp and orig_timestamp >= request.headers['x-timestamp']: return HTTPConflict(request=request) orig_delete_at = int(orig_metadata.get('X-Delete-At') or 0) upload_expiration = time.time() + self.max_upload_time etag = md5() elapsed_time = 0 try: with disk_file.create(size=fsize) as writer: upload_size = 0 def timeout_reader(): with ChunkReadTimeout(self.client_timeout): return request.environ['wsgi.input'].read( self.network_chunk_size) try: for chunk in iter(lambda: timeout_reader(), ''): start_time = time.time() if start_time > upload_expiration: self.logger.increment('PUT.timeouts') return HTTPRequestTimeout(request=request) etag.update(chunk) upload_size = writer.write(chunk) elapsed_time += time.time() - start_time except ChunkReadTimeout: return HTTPRequestTimeout(request=request) if upload_size: self.logger.transfer_rate( 'PUT.' + device + '.timing', elapsed_time, upload_size) if fsize is not None and fsize != upload_size: return HTTPClientDisconnect(request=request) etag = etag.hexdigest() if 'etag' in request.headers and \ request.headers['etag'].lower() != etag: return HTTPUnprocessableEntity(request=request) metadata = { 'X-Timestamp': request.headers['x-timestamp'], 'Content-Type': request.headers['content-type'], 'ETag': etag, 'Content-Length': str(upload_size), } metadata.update(val for val in request.headers.iteritems() if is_user_meta('object', val[0])) for header_key in ( request.headers.get('X-Backend-Replication-Headers') or self.allowed_headers): if header_key in request.headers: header_caps = header_key.title() metadata[header_caps] = request.headers[header_key] writer.put(metadata) except DiskFileNoSpace: return HTTPInsufficientStorage(drive=device, request=request) if orig_delete_at != new_delete_at: if new_delete_at: self.delete_at_update( 'PUT', new_delete_at, account, container, obj, request, device) if orig_delete_at: self.delete_at_update( 'DELETE', orig_delete_at, account, container, obj, request, device) self.container_update( 'PUT', account, container, obj, request, HeaderKeyDict({ 'x-size': metadata['Content-Length'], 'x-content-type': metadata['Content-Type'], 'x-timestamp': metadata['X-Timestamp'], 'x-etag': metadata['ETag']}), device) return HTTPCreated(request=request, etag=etag)
def handle_extract_iter(self, req, compress_type, out_content_type='text/plain'): """ A generator that can be assigned to a swob Response's app_iter which, when iterated over, will extract and PUT the objects pulled from the request body. Will occasionally yield whitespace while request is being processed. When the request is completed will yield a response body that can be parsed to determine success. See above documentation for details. :params req: a swob Request :params compress_type: specifying the compression type of the tar. Accepts '', 'gz', or 'bz2' """ resp_dict = { 'Response Status': HTTPCreated().status, 'Response Body': '', 'Number Files Created': 0 } failed_files = [] last_yield = time() if out_content_type and out_content_type.endswith('/xml'): to_yield = b'<?xml version="1.0" encoding="UTF-8"?>\n' else: to_yield = b' ' separator = b'' containers_accessed = set() req.environ['eventlet.minimum_write_chunk_size'] = 0 try: if not out_content_type: raise HTTPNotAcceptable(request=req) if req.content_length is None and \ req.headers.get('transfer-encoding', '').lower() != 'chunked': raise HTTPLengthRequired(request=req) try: vrs, account, extract_base = req.split_path(2, 3, True) except ValueError: raise HTTPNotFound(request=req) extract_base = extract_base or '' extract_base = extract_base.rstrip('/') tar = tarfile.open(mode='r|' + compress_type, fileobj=req.body_file) failed_response_type = HTTPBadRequest containers_created = 0 while True: if last_yield + self.yield_frequency < time(): last_yield = time() yield to_yield to_yield, separator = b' ', b'\r\n\r\n' tar_info = tar.next() if tar_info is None or \ len(failed_files) >= self.max_failed_extractions: break if tar_info.isfile(): obj_path = tar_info.name if not six.PY2: obj_path = obj_path.encode('utf-8', 'surrogateescape') obj_path = bytes_to_wsgi(obj_path) if obj_path.startswith('./'): obj_path = obj_path[2:] obj_path = obj_path.lstrip('/') if extract_base: obj_path = extract_base + '/' + obj_path if '/' not in obj_path: continue # ignore base level file destination = '/'.join(['', vrs, account, obj_path]) container = obj_path.split('/', 1)[0] if not constraints.check_utf8(wsgi_to_str(destination)): failed_files.append([ wsgi_quote(obj_path[:self.max_path_length]), HTTPPreconditionFailed().status ]) continue if tar_info.size > constraints.MAX_FILE_SIZE: failed_files.append([ wsgi_quote(obj_path[:self.max_path_length]), HTTPRequestEntityTooLarge().status ]) continue container_failure = None if container not in containers_accessed: cont_path = '/'.join(['', vrs, account, container]) try: if self.create_container(req, cont_path): containers_created += 1 if containers_created > self.max_containers: raise HTTPBadRequest( 'More than %d containers to create ' 'from tar.' % self.max_containers) except CreateContainerError as err: # the object PUT to this container still may # succeed if acls are set container_failure = [ wsgi_quote(cont_path[:self.max_path_length]), err.status ] if err.status_int == HTTP_UNAUTHORIZED: raise HTTPUnauthorized(request=req) except ValueError: failed_files.append([ wsgi_quote(obj_path[:self.max_path_length]), HTTPBadRequest().status ]) continue tar_file = tar.extractfile(tar_info) create_headers = { 'Content-Length': tar_info.size, 'X-Auth-Token': req.headers.get('X-Auth-Token'), } # Copy some whitelisted headers to the subrequest for k, v in req.headers.items(): if ((k.lower() in ('x-delete-at', 'x-delete-after')) or is_user_meta('object', k)): create_headers[k] = v create_obj_req = make_subrequest( req.environ, method='PUT', path=wsgi_quote(destination), headers=create_headers, agent='%(orig)s BulkExpand', swift_source='EA') create_obj_req.environ['wsgi.input'] = tar_file for pax_key, pax_value in tar_info.pax_headers.items(): header_name = pax_key_to_swift_header(pax_key) if header_name: # Both pax_key and pax_value are unicode # strings; the key is already UTF-8 encoded, but # we still have to encode the value. create_obj_req.headers[header_name] = \ pax_value.encode("utf-8") resp = create_obj_req.get_response(self.app) containers_accessed.add(container) if resp.is_success: resp_dict['Number Files Created'] += 1 else: if container_failure: failed_files.append(container_failure) if resp.status_int == HTTP_UNAUTHORIZED: failed_files.append([ wsgi_quote(obj_path[:self.max_path_length]), HTTPUnauthorized().status ]) raise HTTPUnauthorized(request=req) if resp.status_int // 100 == 5: failed_response_type = HTTPBadGateway failed_files.append([ wsgi_quote(obj_path[:self.max_path_length]), resp.status ]) if failed_files: resp_dict['Response Status'] = failed_response_type().status elif not resp_dict['Number Files Created']: resp_dict['Response Status'] = HTTPBadRequest().status resp_dict['Response Body'] = 'Invalid Tar File: No Valid Files' except HTTPException as err: resp_dict['Response Status'] = err.status resp_dict['Response Body'] = err.body.decode('utf-8') except (tarfile.TarError, zlib.error) as tar_error: resp_dict['Response Status'] = HTTPBadRequest().status resp_dict['Response Body'] = 'Invalid Tar File: %s' % tar_error except Exception: self.logger.exception('Error in extract archive.') resp_dict['Response Status'] = HTTPServerError().status yield separator + get_response_body(out_content_type, resp_dict, failed_files, 'extract')
def print_db_info_metadata(db_type, info, metadata): """ print out data base info/metadata based on its type :param db_type: database type, account or container :param info: dict of data base info :param metadata: dict of data base metadata """ if info is None: raise ValueError("DB info is None") if db_type not in ["container", "account"]: raise ValueError("Wrong DB type") try: account = info["account"] container = None if db_type == "container": container = info["container"] path = "/%s/%s" % (account, container) else: path = "/%s" % account print "Path: %s" % path print " Account: %s" % account if db_type == "container": print " Container: %s" % container path_hash = hash_path(account, container) if db_type == "container": print " Container Hash: %s" % path_hash else: print " Account Hash: %s" % path_hash print "Metadata:" print (" Created at: %s (%s)" % (datetime.utcfromtimestamp(float(info["created_at"])), info["created_at"])) print ( " Put Timestamp: %s (%s)" % (datetime.utcfromtimestamp(float(info["put_timestamp"])), info["put_timestamp"]) ) print ( " Delete Timestamp: %s (%s)" % (datetime.utcfromtimestamp(float(info["delete_timestamp"])), info["delete_timestamp"]) ) if db_type == "account": print " Container Count: %s" % info["container_count"] print " Object Count: %s" % info["object_count"] print " Bytes Used: %s" % info["bytes_used"] if db_type == "container": print ( " Reported Put Timestamp: %s (%s)" % (datetime.utcfromtimestamp(float(info["reported_put_timestamp"])), info["reported_put_timestamp"]) ) print ( " Reported Delete Timestamp: %s (%s)" % ( datetime.utcfromtimestamp(float(info["reported_delete_timestamp"])), info["reported_delete_timestamp"], ) ) print " Reported Object Count: %s" % info["reported_object_count"] print " Reported Bytes Used: %s" % info["reported_bytes_used"] print " Chexor: %s" % info["hash"] print " UUID: %s" % info["id"] except KeyError: raise ValueError("Info is incomplete") meta_prefix = "x_" + db_type + "_" for key, value in info.iteritems(): if key.lower().startswith(meta_prefix): title = key.replace("_", "-").title() print " %s: %s" % (title, value) user_metadata = {} sys_metadata = {} for key, (value, timestamp) in metadata.iteritems(): if is_user_meta(db_type, key): user_metadata[strip_user_meta_prefix(db_type, key)] = value elif is_sys_meta(db_type, key): sys_metadata[strip_sys_meta_prefix(db_type, key)] = value else: title = key.replace("_", "-").title() print " %s: %s" % (title, value) if sys_metadata: print " System Metadata: %s" % sys_metadata else: print "No system metadata found in db file" if user_metadata: print " User Metadata: %s" % user_metadata else: print "No user metadata found in db file"
def __call__(self, env, start_response): method = env['REQUEST_METHOD'] if method not in self.ALLOWED_METHODS: raise HTTPNotImplemented() path = env['PATH_INFO'] _, acc, cont, obj = split_path(env['PATH_INFO'], 0, 4, rest_with_last=True) if env.get('QUERY_STRING'): path += '?' + env['QUERY_STRING'] if 'swift.authorize' in env: resp = env['swift.authorize'](swob.Request(env)) if resp: return resp(env, start_response) req = swob.Request(env) self.swift_sources.append(env.get('swift.source')) self.txn_ids.append(env.get('swift.trans_id')) try: resp_class, raw_headers, body = self._find_response(method, path) headers = HeaderKeyDict(raw_headers) except KeyError: if (env.get('QUERY_STRING') and (method, env['PATH_INFO']) in self._responses): resp_class, raw_headers, body = self._find_response( method, env['PATH_INFO']) headers = HeaderKeyDict(raw_headers) elif method == 'HEAD' and ('GET', path) in self._responses: resp_class, raw_headers, body = self._find_response( 'GET', path) body = None headers = HeaderKeyDict(raw_headers) elif method == 'GET' and obj and path in self.uploaded: resp_class = swob.HTTPOk headers, body = self.uploaded[path] else: raise KeyError("Didn't find %r in allowed responses" % ( (method, path),)) # simulate object PUT if method == 'PUT' and obj: put_body = ''.join(iter(env['wsgi.input'].read, '')) if 'swift.callback.update_footers' in env: footers = HeaderKeyDict() env['swift.callback.update_footers'](footers) req.headers.update(footers) etag = md5(put_body).hexdigest() headers.setdefault('Etag', etag) headers.setdefault('Content-Length', len(put_body)) # keep it for subsequent GET requests later self.uploaded[path] = (dict(req.headers), put_body) if "CONTENT_TYPE" in env: self.uploaded[path][0]['Content-Type'] = env["CONTENT_TYPE"] # simulate object POST elif method == 'POST' and obj: metadata, data = self.uploaded.get(path, ({}, None)) # select items to keep from existing... new_metadata = dict( (k, v) for k, v in metadata.items() if (not is_user_meta('object', k) and not is_object_transient_sysmeta(k))) # apply from new new_metadata.update( dict((k, v) for k, v in req.headers.items() if (is_user_meta('object', k) or is_object_transient_sysmeta(k) or k.lower == 'content-type'))) self.uploaded[path] = new_metadata, data # note: tests may assume this copy of req_headers is case insensitive # so we deliberately use a HeaderKeyDict self._calls.append( FakeSwiftCall(method, path, HeaderKeyDict(req.headers))) backend_etag_header = req.headers.get('X-Backend-Etag-Is-At') conditional_etag = None if backend_etag_header and backend_etag_header in headers: # Apply conditional etag overrides conditional_etag = headers[backend_etag_header] # range requests ought to work, hence conditional_response=True if isinstance(body, list): resp = resp_class( req=req, headers=headers, app_iter=body, conditional_response=req.method in ('GET', 'HEAD'), conditional_etag=conditional_etag) else: resp = resp_class( req=req, headers=headers, body=body, conditional_response=req.method in ('GET', 'HEAD'), conditional_etag=conditional_etag) wsgi_iter = resp(env, start_response) self.mark_opened(path) return LeakTrackingIter(wsgi_iter, self, path)
def test_is_user_meta(self): m_type = 'meta' for st in server_types: self.assertTrue(is_user_meta(st, 'x-%s-%s-foo' % (st, m_type))) self.assertFalse(is_user_meta(st, 'x-%s-%s-' % (st, m_type))) self.assertFalse(is_user_meta(st, 'x-%s-%sfoo' % (st, m_type)))