def HEAD(self, req): """Handle HTTP HEAD request.""" drive, part, account, container, obj = split_and_validate_path( req, 4, 5, True) out_content_type = listing_formats.get_listing_content_type(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container, pending_timeout=0.1, stale_reads_ok=True) info, is_deleted = broker.get_info_is_deleted() headers = gen_resp_headers(info, is_deleted=is_deleted) if is_deleted: return HTTPNotFound(request=req, headers=headers) headers.update( (str_to_wsgi(key), str_to_wsgi(value)) for key, (value, timestamp) in broker.metadata.items() if value != '' and (key.lower() in self.save_headers or is_sys_or_user_meta('container', key))) headers['Content-Type'] = out_content_type resp = HTTPNoContent(request=req, headers=headers, charset='utf-8') resp.last_modified = math.ceil(float(headers['X-PUT-Timestamp'])) return resp
def account_listing_response(account, req, response_content_type, broker=None, limit='', marker='', end_marker='', prefix='', delimiter='', reverse=False): """ This is an exact copy of swift.account.utis.account_listing_response() except for one difference i.e this method passes response_content_type to broker.list_containers_iter() method. """ if broker is None: broker = FakeAccountBroker() resp_headers = get_response_headers(broker) account_list = broker.list_containers_iter(limit, marker, end_marker, prefix, delimiter, response_content_type, reverse) if response_content_type == 'application/json': data = [] for (name, object_count, bytes_used, put_tstamp, is_subdir) in account_list: if is_subdir: data.append({'subdir': name}) else: data.append({'name': name, 'count': object_count, 'bytes': bytes_used, 'last_modified': Timestamp(put_tstamp).isoformat}) account_list = json.dumps(data) elif response_content_type.endswith('/xml'): output_list = ['<?xml version="1.0" encoding="UTF-8"?>', '<account name=%s>' % saxutils.quoteattr(account)] for (name, object_count, bytes_used, put_tstamp, is_subdir) in account_list: if is_subdir: output_list.append( '<subdir name=%s />' % saxutils.quoteattr(name)) else: item = '<container><name>%s</name><count>%s</count>' \ '<bytes>%s</bytes><last_modified>%s</last_modified> \ </container>' % \ (saxutils.escape(name), object_count, bytes_used, Timestamp(put_tstamp).isoformat) output_list.append(item) output_list.append('</account>') account_list = '\n'.join(output_list) else: if not account_list: resp = HTTPNoContent(request=req, headers=resp_headers) resp.content_type = response_content_type resp.charset = 'utf-8' return resp account_list = '\n'.join(r[0] for r in account_list) + '\n' ret = HTTPOk(body=account_list, request=req, headers=resp_headers) ret.content_type = response_content_type ret.charset = 'utf-8' return ret
def account_listing_response(account, req, response_content_type, broker=None, limit='', marker='', end_marker='', prefix='', delimiter=''): if broker is None: broker = FakeAccountBroker() info = broker.get_info() resp_headers = { 'X-Account-Container-Count': info['container_count'], 'X-Account-Object-Count': info['object_count'], 'X-Account-Bytes-Used': info['bytes_used'], 'X-Timestamp': info['created_at'], 'X-PUT-Timestamp': info['put_timestamp']} resp_headers.update((key, value) for key, (value, timestamp) in broker.metadata.iteritems() if value != '') account_list = broker.list_containers_iter(limit, marker, end_marker, prefix, delimiter) if response_content_type == 'application/json': data = [] for (name, object_count, bytes_used, is_subdir) in account_list: if is_subdir: data.append({'subdir': name}) else: data.append({'name': name, 'count': object_count, 'bytes': bytes_used}) account_list = json.dumps(data) elif response_content_type.endswith('/xml'): output_list = ['<?xml version="1.0" encoding="UTF-8"?>', '<account name=%s>' % saxutils.quoteattr(account)] for (name, object_count, bytes_used, is_subdir) in account_list: if is_subdir: output_list.append( '<subdir name=%s />' % saxutils.quoteattr(name)) else: item = '<container><name>%s</name><count>%s</count>' \ '<bytes>%s</bytes></container>' % \ (saxutils.escape(name), object_count, bytes_used) output_list.append(item) output_list.append('</account>') account_list = '\n'.join(output_list) else: if not account_list: resp = HTTPNoContent(request=req, headers=resp_headers) resp.content_type = response_content_type resp.charset = 'utf-8' return resp account_list = '\n'.join(r[0] for r in account_list) + '\n' ret = HTTPOk(body=account_list, request=req, headers=resp_headers) ret.content_type = response_content_type ret.charset = 'utf-8' return ret
def account_listing_response(account, req, response_content_type, info=None, listing=None): if info is None: now = Timestamp(time.time()).internal info = {'containers': 0, 'objects': 0, 'bytes': 0, 'metadata': {}, 'ctime': now} if listing is None: listing = [] resp_headers = get_response_headers(info) if response_content_type == 'application/json': data = [] for (name, object_count, bytes_used, is_subdir) in listing: if is_subdir: data.append({'subdir': name}) else: data.append({'name': name, 'count': object_count, 'bytes': bytes_used}) account_list = json.dumps(data) elif response_content_type.endswith('/xml'): output_list = ['<?xml version="1.0" encoding="UTF-8"?>', '<account name=%s>' % saxutils.quoteattr(account)] for (name, object_count, bytes_used, is_subdir) in listing: if is_subdir: output_list.append( '<subdir name=%s />' % saxutils.quoteattr(name)) else: item = '<container><name>%s</name><count>%s</count>' \ '<bytes>%s</bytes></container>' % \ (saxutils.escape(name), object_count, bytes_used) output_list.append(item) output_list.append('</account>') account_list = '\n'.join(output_list) else: if not listing: resp = HTTPNoContent(request=req, headers=resp_headers) resp.content_type = response_content_type resp.charset = 'utf-8' return resp account_list = '\n'.join(r[0] for r in listing) + '\n' ret = HTTPOk(body=account_list, request=req, headers=resp_headers) ret.content_type = response_content_type ret.charset = 'utf-8' return ret
def account_listing_response(account, req, response_content_type, broker=None, limit='', marker='', end_marker='', prefix='', delimiter='', reverse=False): if broker is None: broker = FakeAccountBroker() resp_headers = get_response_headers(broker) account_list = broker.list_containers_iter(limit, marker, end_marker, prefix, delimiter, reverse) if response_content_type == 'application/json': data = [] for (name, object_count, bytes_used, is_subdir) in account_list: if is_subdir: data.append({'subdir': name}) else: data.append({'name': name, 'count': object_count, 'bytes': bytes_used}) account_list = json.dumps(data) elif response_content_type.endswith('/xml'): output_list = ['<?xml version="1.0" encoding="UTF-8"?>', '<account name=%s>' % saxutils.quoteattr(account)] for (name, object_count, bytes_used, is_subdir) in account_list: if is_subdir: output_list.append( '<subdir name=%s />' % saxutils.quoteattr(name)) else: item = '<container><name>%s</name><count>%s</count>' \ '<bytes>%s</bytes></container>' % \ (saxutils.escape(name), object_count, bytes_used) output_list.append(item) output_list.append('</account>') account_list = '\n'.join(output_list) else: if not account_list: resp = HTTPNoContent(request=req, headers=resp_headers) resp.content_type = response_content_type resp.charset = 'utf-8' return resp account_list = '\n'.join(r[0] for r in account_list) + '\n' ret = HTTPOk(body=account_list, request=req, headers=resp_headers) ret.content_type = response_content_type ret.charset = 'utf-8' return ret
def PUT(self, req): """Handle HTTP PUT request.""" drive, part, account, container = get_container_name_and_placement(req) try: check_drive(self.root, drive, self.mount_check) except ValueError: return HTTPInsufficientStorage(drive=drive, request=req) if not self.check_free_space(drive): return HTTPInsufficientStorage(drive=drive, request=req) if container: # put account container if 'x-timestamp' not in req.headers: timestamp = Timestamp.now() else: timestamp = valid_timestamp(req) pending_timeout = None container_policy_index = \ req.headers.get('X-Backend-Storage-Policy-Index', 0) if 'x-trans-id' in req.headers: pending_timeout = 3 broker = self._get_account_broker(drive, part, account, pending_timeout=pending_timeout) if account.startswith(self.auto_create_account_prefix) and \ not os.path.exists(broker.db_file): try: broker.initialize(timestamp.internal) except DatabaseAlreadyExists: pass if req.headers.get('x-account-override-deleted', 'no').lower() != \ 'yes' and broker.is_deleted(): return HTTPNotFound(request=req) broker.put_container(container, req.headers['x-put-timestamp'], req.headers['x-delete-timestamp'], req.headers['x-object-count'], req.headers['x-bytes-used'], container_policy_index) if req.headers['x-delete-timestamp'] > \ req.headers['x-put-timestamp']: return HTTPNoContent(request=req) else: return HTTPCreated(request=req) else: # put account timestamp = valid_timestamp(req) broker = self._get_account_broker(drive, part, account) if not os.path.exists(broker.db_file): try: broker.initialize(timestamp.internal) created = True except DatabaseAlreadyExists: created = False elif broker.is_status_deleted(): return self._deleted_response(broker, req, HTTPForbidden, body='Recently deleted') else: created = broker.is_deleted() broker.update_put_timestamp(timestamp.internal) if broker.is_deleted(): return HTTPConflict(request=req) self._update_metadata(req, broker, timestamp) if created: return HTTPCreated(request=req) else: return HTTPAccepted(request=req)
def GET(self, req): """Handle HTTP GET request.""" #logging.info("...account server get...") drive, part, account, container, obj = split_and_validate_path( req, 4, 5, True) path = get_param(req, 'path') prefix = get_param(req, 'prefix') delimiter = get_param(req, 'delimiter') if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254): # delimiters can be made more flexible later return HTTPPreconditionFailed(body='Bad delimiter') marker = get_param(req, 'marker', '') end_marker = get_param(req, 'end_marker') limit = CONTAINER_LISTING_LIMIT given_limit = get_param(req, 'limit') if given_limit and given_limit.isdigit(): limit = int(given_limit) if limit > CONTAINER_LISTING_LIMIT: return HTTPPreconditionFailed(request=req, body='Maximum limit is %d' % CONTAINER_LISTING_LIMIT) out_content_type = get_listing_content_type(req) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container, pending_timeout=0.1, stale_reads_ok=True) if broker.is_deleted(): return HTTPNotFound(request=req) info = broker.get_info() resp_headers = { 'X-Container-Object-Count': info['object_count'], 'X-Container-Bytes-Used': info['bytes_used'], 'X-Timestamp': info['created_at'], 'X-PUT-Timestamp': info['put_timestamp'], } for key, (value, timestamp) in broker.metadata.iteritems(): if value and (key.lower() in self.save_headers or key.lower().startswith('x-container-meta-')): resp_headers[key] = value ret = Response(request=req, headers=resp_headers, content_type=out_content_type, charset='utf-8') container_list = broker.list_objects_iter(limit, marker, end_marker, prefix, delimiter, path) if out_content_type == 'application/json': ret.body = json.dumps( [self.update_data_record(record) for record in container_list]) elif out_content_type.endswith('/xml'): doc = Element('container', name=container.decode('utf-8')) for obj in container_list: record = self.update_data_record(obj) if 'subdir' in record: name = record['subdir'].decode('utf-8') sub = SubElement(doc, 'subdir', name=name) SubElement(sub, 'name').text = name else: obj_element = SubElement(doc, 'object') for field in [ "name", "hash", "bytes", "content_type", "last_modified" ]: SubElement(obj_element, field).text = str( record.pop(field)).decode('utf-8') for field in sorted(record): SubElement(obj_element, field).text = str( record[field]).decode('utf-8') ret.body = tostring(doc, encoding='UTF-8').replace( "<?xml version='1.0' encoding='UTF-8'?>", '<?xml version="1.0" encoding="UTF-8"?>', 1) else: if not container_list: return HTTPNoContent(request=req, headers=resp_headers) ret.body = '\n'.join(rec[0] for rec in container_list) + '\n' return ret
class ContainerController(object): """WSGI Controller for the container server.""" # Ensure these are all lowercase save_headers = [ 'x-container-read', 'x-container-write', 'x-container-sync-key', 'x-container-sync-to' ] def __init__(self, conf): self.logger = get_logger(conf, log_route='container-server') self.root = conf.get('devices', '/srv/node/') self.mount_check = config_true_value(conf.get('mount_check', 'true')) self.node_timeout = int(conf.get('node_timeout', 3)) self.conn_timeout = float(conf.get('conn_timeout', 0.5)) self.allowed_sync_hosts = [ h.strip() for h in conf.get('allowed_sync_hosts', '127.0.0.1').split(',') if h.strip() ] self.replicator_rpc = ReplicatorRpc(self.root, DATADIR, ContainerBroker, self.mount_check, logger=self.logger) self.auto_create_account_prefix = \ conf.get('auto_create_account_prefix') or '.' if config_true_value(conf.get('allow_versions', 'f')): self.save_headers.append('x-versions-location') swift.common.db.DB_PREALLOCATION = \ config_true_value(conf.get('db_preallocation', 'f')) def _get_container_broker(self, drive, part, account, container): """ Get a DB broker for the container. :param drive: drive that holds the container :param part: partition the container is in :param account: account name :param container: container name :returns: ContainerBroker object """ hsh = hash_path(account, container) db_dir = storage_directory(DATADIR, part, hsh) db_path = os.path.join(self.root, drive, db_dir, hsh + '.db') return ContainerBroker(db_path, account=account, container=container, logger=self.logger) def account_update(self, req, account, container, broker): """ Update the account server(s) with latest container info. :param req: swob.Request object :param account: account name :param container: container name :param broker: container DB broker object :returns: if all the account requests return a 404 error code, HTTPNotFound response object, otherwise None. """ account_hosts = [ h.strip() for h in req.headers.get('X-Account-Host', '').split(',') ] account_devices = [ d.strip() for d in req.headers.get('X-Account-Device', '').split(',') ] account_partition = req.headers.get('X-Account-Partition', '') if len(account_hosts) != len(account_devices): # This shouldn't happen unless there's a bug in the proxy, # but if there is, we want to know about it. self.logger.error( _('ERROR Account update failed: different ' 'numbers of hosts and devices in request: ' '"%s" vs "%s"' % (req.headers.get('X-Account-Host', ''), req.headers.get('X-Account-Device', '')))) return if account_partition: updates = zip(account_hosts, account_devices) else: updates = [] account_404s = 0 for account_host, account_device in updates: account_ip, account_port = account_host.rsplit(':', 1) new_path = '/' + '/'.join([account, container]) info = broker.get_info() account_headers = { 'x-put-timestamp': info['put_timestamp'], 'x-delete-timestamp': info['delete_timestamp'], 'x-object-count': info['object_count'], 'x-bytes-used': info['bytes_used'], 'x-trans-id': req.headers.get('x-trans-id', '-') } if req.headers.get('x-account-override-deleted', 'no').lower() == \ 'yes': account_headers['x-account-override-deleted'] = 'yes' try: with ConnectionTimeout(self.conn_timeout): conn = http_connect(account_ip, account_port, account_device, account_partition, 'PUT', new_path, account_headers) with Timeout(self.node_timeout): account_response = conn.getresponse() account_response.read() if account_response.status == HTTP_NOT_FOUND: account_404s += 1 elif not is_success(account_response.status): self.logger.error( _('ERROR Account update failed ' 'with %(ip)s:%(port)s/%(device)s (will retry ' 'later): Response %(status)s %(reason)s'), { 'ip': account_ip, 'port': account_port, 'device': account_device, 'status': account_response.status, 'reason': account_response.reason }) except (Exception, Timeout): self.logger.exception( _('ERROR account update failed with ' '%(ip)s:%(port)s/%(device)s (will retry later)'), { 'ip': account_ip, 'port': account_port, 'device': account_device }) if updates and account_404s == len(updates): return HTTPNotFound(req=req) else: return None @public @timing_stats() def DELETE(self, req): """Handle HTTP DELETE request.""" try: drive, part, account, container, obj = req.split_path(4, 5, True) validate_device_partition(drive, part) except ValueError, err: return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) if 'x-timestamp' not in req.headers or \ not check_float(req.headers['x-timestamp']): return HTTPBadRequest(body='Missing timestamp', request=req, content_type='text/plain') if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) broker = self._get_container_broker(drive, part, account, container) if account.startswith(self.auto_create_account_prefix) and obj and \ not os.path.exists(broker.db_file): broker.initialize( normalize_timestamp( req.headers.get('x-timestamp') or time.time())) if not os.path.exists(broker.db_file): return HTTPNotFound() if obj: # delete object broker.delete_object(obj, req.headers.get('x-timestamp')) return HTTPNoContent(request=req) else: # delete container if not broker.empty(): return HTTPConflict(request=req) existed = float(broker.get_info()['put_timestamp']) and \ not broker.is_deleted() broker.delete_db(req.headers['X-Timestamp']) if not broker.is_deleted(): return HTTPConflict(request=req) resp = self.account_update(req, account, container, broker) if resp: return resp if existed: return HTTPNoContent(request=req) return HTTPNotFound()
'X-Timestamp': info['created_at'], 'X-PUT-Timestamp': info['put_timestamp'], } headers.update( (key, value) for key, (value, timestamp) in broker.metadata.iteritems() if value != '' and (key.lower() in self.save_headers or key.lower().startswith('x-container-meta-'))) if get_param(req, 'format'): req.accept = FORMAT2CONTENT_TYPE.get( get_param(req, 'format').lower(), FORMAT2CONTENT_TYPE['plain']) headers['Content-Type'] = req.accept.best_match( ['text/plain', 'application/json', 'application/xml', 'text/xml']) if not headers['Content-Type']: return HTTPNotAcceptable(request=req) return HTTPNoContent(request=req, headers=headers, charset='utf-8') def derive_content_type_metadata(self, content_type, size): """ Will check the last parameter and if it starts with 'swift_bytes=' will strip it off. Returns either the passed in content_type and size or the content_type without the swift_bytes param and its value as the new size. :params content_type: Content Type from db :params size: # bytes from db, an int :returns: tuple: content_type, size """ if ';' in content_type: new_content_type, param = content_type.rsplit(';', 1) if param.lstrip().startswith('swift_bytes='): key, value = param.split('=')
return Response(body=response_body, headers=resp_headers) except OriginDbNotFound: if list_format == 'xml': empty_xml = '\n'.join([ '<?xml version="1.0" encoding="UTF-8"?>', '<account name="%s">' % account, '</account>' ]) return HTTPOk(request=req, headers={'Content-Type': 'application/xml'}, body=empty_xml) elif list_format == 'json': return HTTPOk(request=req, headers={'Content-Type': 'application/json'}, body=json.dumps([])) return HTTPNoContent(request=req) def origin_db_delete(self, env, req): """ Handles DELETEs in the Origin database. This is not really a delete- it will remove the object from the container listing and set cdn_enabled=false and a deleted flag in the .hash_* obj that says that the obj is deleted. This way the container won't show up in the listings, HEAD to the object will return 404s but behind the scenes lookups to the object will be able to determine the account and container from a container_hash. """ try: vsn, account, container = split_path(req.path, 3, 3) except ValueError: return HTTPBadRequest(
def account_listing_response(account, req, response_content_type, info=None, listing=None, s3_buckets_only=False): now = time.time() if info is None: info = { 'containers': 0, 'objects': 0, 'bytes': 0, 'metadata': {}, 'ctime': Timestamp(now).internal } if listing is None: listing = [] elif listing and len(listing[0]) < 5: # oio-sds < 4.2 does not return mtime listing = [x + [now] for x in listing] resp_headers = get_response_headers(info) if response_content_type == 'application/json': data = [] for (name, object_count, bytes_used, is_subdir, mtime) in listing: if is_subdir: if not s3_buckets_only: data.append({'subdir': name}) else: data.append({ 'name': name, 'count': object_count, 'bytes': bytes_used, 'last_modified': Timestamp(mtime).isoformat }) account_list = json.dumps(data) elif response_content_type.endswith('/xml'): output_list = [ '<?xml version="1.0" encoding="UTF-8"?>', '<account name=%s>' % saxutils.quoteattr(account) ] for (name, object_count, bytes_used, is_subdir, mtime) in listing: if is_subdir: if not s3_buckets_only: output_list.append('<subdir name=%s />' % saxutils.quoteattr(name)) else: item = '<container><name>%s</name><count>%s</count>' \ '<bytes>%s</bytes><last_modified>%s</last_modified>' \ '</container>' % \ (saxutils.escape(name), object_count, bytes_used, Timestamp(mtime).isoformat) output_list.append(item) output_list.append('</account>') account_list = '\n'.join(output_list) else: if not listing: resp = HTTPNoContent(request=req, headers=resp_headers) resp.content_type = response_content_type resp.charset = 'utf-8' return resp account_list = '\n'.join(r[0] for r in listing) + '\n' ret = HTTPOk(body=account_list, request=req, headers=resp_headers) ret.content_type = response_content_type ret.charset = 'utf-8' return ret
def get_account_delete_resp(self, req, headers): # TODO perform delete return HTTPNoContent(request=req)
def PUT(self, req): """Handle HTTP PUT request.""" drive, part, account, container = split_and_validate_path(req, 3, 4) if self.mount_check and not check_mount(self.root, drive): return HTTPInsufficientStorage(drive=drive, request=req) if container: # put account container if 'x-timestamp' not in req.headers: timestamp = Timestamp(time.time()) else: timestamp = valid_timestamp(req) pending_timeout = None container_policy_index = \ req.headers.get('X-Backend-Storage-Policy-Index', 0) if 'x-trans-id' in req.headers: pending_timeout = 3 broker = self._get_account_broker(drive, part, account, pending_timeout=pending_timeout) if account.startswith(self.auto_create_account_prefix) and \ not os.path.exists(broker.db_file): try: broker.initialize(timestamp.internal) except DatabaseAlreadyExists: pass if req.headers.get('x-account-override-deleted', 'no').lower() != \ 'yes' and broker.is_deleted(): return HTTPNotFound(request=req) broker.put_container(container, req.headers['x-put-timestamp'], req.headers['x-delete-timestamp'], req.headers['x-object-count'], req.headers['x-bytes-used'], container_policy_index) if req.headers['x-delete-timestamp'] > \ req.headers['x-put-timestamp']: return HTTPNoContent(request=req) else: return HTTPCreated(request=req) else: # put account timestamp = valid_timestamp(req) broker = self._get_account_broker(drive, part, account) if not os.path.exists(broker.db_file): try: broker.initialize(timestamp.internal) created = True except DatabaseAlreadyExists: created = False elif broker.is_status_deleted(): return self._deleted_response(broker, req, HTTPForbidden, body='Recently deleted') else: created = broker.is_deleted() broker.update_put_timestamp(timestamp.internal) if broker.is_deleted(): return HTTPConflict(request=req) metadata = {} metadata.update((key, (value, timestamp.internal)) for key, value in req.headers.iteritems() if is_sys_or_user_meta('account', key)) if metadata: broker.update_metadata(metadata, validate_metadata=True) if created: return HTTPCreated(request=req) else: return HTTPAccepted(request=req)
if broker.is_deleted(): self.logger.timing_since('HEAD.timing', start_time) return HTTPNotFound(request=req) info = broker.get_info() headers = { 'X-Container-Object-Count': info['object_count'], 'X-Container-Bytes-Used': info['bytes_used'], 'X-Timestamp': info['created_at'], 'X-PUT-Timestamp': info['put_timestamp'], } headers.update((key, value) for key, (value, timestamp) in broker.metadata.iteritems() if value != '' and (key.lower() in self.save_headers or key.lower().startswith('x-container-meta-'))) self.logger.timing_since('HEAD.timing', start_time) return HTTPNoContent(request=req, headers=headers) @public def GET(self, req): """Handle HTTP GET request.""" start_time = time.time() try: drive, part, account, container, obj = split_path( unquote(req.path), 4, 5, True) validate_device_partition(drive, part) except ValueError, err: self.logger.increment('GET.errors') return HTTPBadRequest(body=str(err), content_type='text/plain', request=req) if self.mount_check and not check_mount(self.root, drive): self.logger.increment('GET.errors')