Example #1
0
 def test_is_sys_or_user_meta(self):
     m_types = ['sysmeta', 'meta']
     for mt in m_types:
         for st in server_types:
             self.assertTrue(is_sys_or_user_meta(st, 'x-%s-%s-foo'
                                                 % (st, mt)))
             self.assertFalse(is_sys_or_user_meta(st, 'x-%s-%s-'
                                                  % (st, mt)))
             self.assertFalse(is_sys_or_user_meta(st, 'x-%s-%sfoo'
                                                  % (st, mt)))
Example #2
0
 def test_is_sys_or_user_meta(self):
     m_types = ['sysmeta', 'meta']
     for mt in m_types:
         for st in server_types:
             self.assertTrue(
                 rh.is_sys_or_user_meta(st, 'x-%s-%s-foo' % (st, mt)))
             self.assertFalse(
                 rh.is_sys_or_user_meta(st, 'x-%s-%s-' % (st, mt)))
             self.assertFalse(
                 rh.is_sys_or_user_meta(st, 'x-%s-%sfoo' % (st, mt)))
Example #3
0
 def POST(self, req):
     """Handle HTTP POST request."""
     drive, part, account, container = split_and_validate_path(req, 4)
     req_timestamp = valid_timestamp(req)
     if 'x-container-sync-to' in req.headers:
         err, sync_to, realm, realm_key = validate_sync_to(
             req.headers['x-container-sync-to'], self.allowed_sync_hosts,
             self.realms_conf)
         if err:
             return HTTPBadRequest(err)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_container_broker(drive, part, account, container)
     if broker.is_deleted():
         return HTTPNotFound(request=req)
     metadata = {}
     metadata.update(
         (key, (value, req_timestamp.internal))
         for key, value in req.headers.iteritems()
         if key.lower() in self.save_headers or
         is_sys_or_user_meta('container', key))
     if metadata:
         if 'X-Container-Sync-To' in metadata:
             if 'X-Container-Sync-To' not in broker.metadata or \
                     metadata['X-Container-Sync-To'][0] != \
                     broker.metadata['X-Container-Sync-To'][0]:
                 broker.set_x_container_sync_points(-1, -1)
         broker.update_metadata(metadata, validate_metadata=True)
     return HTTPNoContent(request=req)
Example #4
0
 def get_metadata_resp_headers(self, meta):
     headers = {}
     system = meta.get('system') or {}
     # sys.m2.ctime is microseconds
     ctime = float(system.get('sys.m2.ctime', 0)) / 1000000.0
     headers.update({
         'X-Container-Object-Count':
         system.get('sys.m2.objects', 0),
         'X-Container-Bytes-Used':
         system.get('sys.m2.usage', 0),
         'X-Timestamp':
         Timestamp(ctime).normal,
         # FIXME: save modification time somewhere
         'X-PUT-Timestamp':
         Timestamp(ctime).normal,
     })
     for (k, v) in meta['properties'].items():
         if v and (k.lower() in self.pass_through_headers
                   or is_sys_or_user_meta('container', k)):
             headers[k] = v
     # HACK: oio-sds always sets version numbers, so let some middlewares
     # think that versioning is always enabled.
     if SYSMETA_VERSIONS_CONT not in headers and 'sys.user.name' in system:
         try:
             v_con = get_reserved_name('versions', system['sys.user.name'])
             headers[SYSMETA_VERSIONS_CONT] = v_con
         except ValueError:
             # sys.user.name contains reserved characters
             # -> this is probably a versioning container.
             pass
     return headers
Example #5
0
 def HEAD(self, request):
     """Handle HTTP HEAD requests for the Swift Object Server."""
     device, partition, account, container, obj, policy_idx = get_name_and_placement(request, 5, 5, True)
     try:
         disk_file = self.get_diskfile(device, partition, account, container, obj, policy_idx=policy_idx)
     except DiskFileDeviceUnavailable:
         return HTTPInsufficientStorage(drive=device, request=request)
     try:
         metadata = disk_file.read_metadata()
     except DiskFileXattrNotSupported:
         return HTTPInsufficientStorage(drive=device, request=request)
     except (DiskFileNotExist, DiskFileQuarantined) as e:
         headers = {}
         if hasattr(e, "timestamp"):
             headers["X-Backend-Timestamp"] = e.timestamp.internal
         return HTTPNotFound(request=request, headers=headers, conditional_response=True)
     response = Response(request=request, conditional_response=True)
     response.headers["Content-Type"] = metadata.get("Content-Type", "application/octet-stream")
     for key, value in metadata.iteritems():
         if is_sys_or_user_meta("object", key) or key.lower() in self.allowed_headers:
             response.headers[key] = value
     response.etag = metadata["ETag"]
     ts = Timestamp(metadata["X-Timestamp"])
     response.last_modified = math.ceil(float(ts))
     # Needed for container sync feature
     response.headers["X-Timestamp"] = ts.normal
     response.headers["X-Backend-Timestamp"] = ts.internal
     response.content_length = int(metadata["Content-Length"])
     try:
         response.content_encoding = metadata["Content-Encoding"]
     except KeyError:
         pass
     return response
Example #6
0
 def HEAD(self, req):
     """Handle HTTP HEAD request."""
     drive, part, account, container, obj = split_and_validate_path(
         req, 4, 5, True)
     out_content_type = get_listing_content_type(req)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_container_broker(drive,
                                         part,
                                         account,
                                         container,
                                         pending_timeout=0.1,
                                         stale_reads_ok=True)
     if broker.is_deleted():
         return HTTPNotFound(request=req)
     info = broker.get_info()
     headers = {
         'X-Container-Object-Count': info['object_count'],
         'X-Container-Bytes-Used': info['bytes_used'],
         'X-Timestamp': info['created_at'],
         'X-PUT-Timestamp': info['put_timestamp'],
     }
     headers.update(
         (key, value)
         for key, (value, timestamp) in broker.metadata.iteritems()
         if value != '' and (key.lower() in self.save_headers
                             or is_sys_or_user_meta('container', key)))
     headers['Content-Type'] = out_content_type
     return HTTPNoContent(request=req, headers=headers, charset='utf-8')
Example #7
0
 def HEAD(self, req):
     """Handle HTTP HEAD request."""
     drive, part, account, container, obj = split_and_validate_path(
         req, 4, 5, True)
     out_content_type = listing_formats.get_listing_content_type(req)
     try:
         check_drive(self.root, drive, self.mount_check)
     except ValueError:
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_container_broker(drive, part, account, container,
                                         pending_timeout=0.1,
                                         stale_reads_ok=True)
     info, is_deleted = broker.get_info_is_deleted()
     headers = gen_resp_headers(info, is_deleted=is_deleted)
     if is_deleted:
         return HTTPNotFound(request=req, headers=headers)
     headers.update(
         (str_to_wsgi(key), str_to_wsgi(value))
         for key, (value, timestamp) in broker.metadata.items()
         if value != '' and (key.lower() in self.save_headers or
                             is_sys_or_user_meta('container', key)))
     headers['Content-Type'] = out_content_type
     resp = HTTPNoContent(request=req, headers=headers, charset='utf-8')
     resp.last_modified = math.ceil(float(headers['X-PUT-Timestamp']))
     return resp
Example #8
0
 def _update_metadata(self, req, broker, req_timestamp):
     metadata = {
         wsgi_to_str(key): (wsgi_to_str(value), req_timestamp.internal)
         for key, value in req.headers.items()
         if is_sys_or_user_meta('account', key)}
     if metadata:
         broker.update_metadata(metadata, validate_metadata=True)
Example #9
0
 def HEAD(self, req):
     """Handle HTTP HEAD request."""
     drive, part, account, container, obj = split_and_validate_path(
         req, 4, 5, True)
     out_content_type = get_listing_content_type(req)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_container_broker(drive, part, account, container,
                                         pending_timeout=0.1,
                                         stale_reads_ok=True)
     if broker.is_deleted():
         return HTTPNotFound(request=req)
     info = broker.get_info()
     headers = {
         'X-Container-Object-Count': info['object_count'],
         'X-Container-Bytes-Used': info['bytes_used'],
         'X-Timestamp': info['created_at'],
         'X-PUT-Timestamp': info['put_timestamp'],
     }
     headers.update(
         (key, value)
         for key, (value, timestamp) in broker.metadata.iteritems()
         if value != '' and (key.lower() in self.save_headers or
                             is_sys_or_user_meta('container', key)))
     headers['Content-Type'] = out_content_type
     return HTTPNoContent(request=req, headers=headers, charset='utf-8')
Example #10
0
    def create_listing(self, req, out_content_type, info, resp_headers,
                       metadata, container_list, container):
        for key, (value, _timestamp) in metadata.items():
            if value and (key.lower() in self.save_headers
                          or is_sys_or_user_meta('container', key)):
                resp_headers[str_to_wsgi(key)] = str_to_wsgi(value)
        listing = [
            self.update_data_record(record) for record in container_list
        ]
        if out_content_type.endswith('/xml'):
            body = listing_formats.container_to_xml(listing, container)
        elif out_content_type.endswith('/json'):
            body = json.dumps(listing).encode('ascii')
        else:
            body = listing_formats.listing_to_text(listing)

        ret = Response(request=req,
                       headers=resp_headers,
                       body=body,
                       content_type=out_content_type,
                       charset='utf-8')
        ret.last_modified = math.ceil(float(resp_headers['X-PUT-Timestamp']))
        if not ret.body:
            ret.status_int = HTTP_NO_CONTENT
        return ret
Example #11
0
 def HEAD(self, req):
     """Handle HTTP HEAD request."""
     drive, part, account, container, obj = split_and_validate_path(
         req, 4, 5, True)
     out_content_type = get_listing_content_type(req)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_container_broker(drive,
                                         part,
                                         account,
                                         container,
                                         pending_timeout=0.1,
                                         stale_reads_ok=True)
     info, is_deleted = broker.get_info_is_deleted()
     headers = gen_resp_headers(info, is_deleted=is_deleted)
     if is_deleted:
         return HTTPNotFound(request=req, headers=headers)
     headers.update(
         (key, value)
         for key, (value, timestamp) in broker.metadata.items()
         if value != '' and (key.lower() in self.save_headers
                             or is_sys_or_user_meta('container', key)))
     headers['Content-Type'] = out_content_type
     resp = HTTPNoContent(request=req, headers=headers, charset='utf-8')
     resp.last_modified = math.ceil(float(headers['X-PUT-Timestamp']))
     return resp
Example #12
0
 def HEAD(self, req):
     """Handle HTTP HEAD request."""
     drive, part, account, container, obj = get_obj_name_and_placement(req)
     out_content_type = listing_formats.get_listing_content_type(req)
     try:
         check_drive(self.root, drive, self.mount_check)
     except ValueError:
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_container_broker(drive,
                                         part,
                                         account,
                                         container,
                                         pending_timeout=0.1,
                                         stale_reads_ok=True)
     info, is_deleted = broker.get_info_is_deleted()
     headers = gen_resp_headers(info, is_deleted=is_deleted)
     if is_deleted:
         return HTTPNotFound(request=req, headers=headers)
     headers.update(
         (str_to_wsgi(key), str_to_wsgi(value))
         for key, (value, timestamp) in broker.metadata.items()
         if value != '' and (key.lower() in self.save_headers
                             or is_sys_or_user_meta('container', key)))
     headers['Content-Type'] = out_content_type
     resp = HTTPNoContent(request=req, headers=headers, charset='utf-8')
     resp.last_modified = math.ceil(float(headers['X-PUT-Timestamp']))
     return resp
Example #13
0
    def make_object_response(self, req, metadata, stream=None):
        conditional_etag = None
        if 'X-Backend-Etag-Is-At' in req.headers:
            conditional_etag = metadata.get(
                req.headers['X-Backend-Etag-Is-At'])

        resp = Response(request=req, conditional_response=True,
                        conditional_etag=conditional_etag)

        resp.headers['Content-Type'] = metadata.get(
            'mime-type', 'application/octet-stream')
        for k, v in metadata.iteritems():
            if k.startswith("user."):
                meta = k[5:]
                if is_sys_or_user_meta('object', meta) or \
                                meta.lower() in self.allowed_headers:
                    resp.headers[meta] = v
        resp.etag = metadata['hash'].lower()
        ts = Timestamp(metadata['ctime'])
        resp.last_modified = math.ceil(float(ts))
        if stream:
            resp.app_iter = stream
        resp.content_length = int(metadata['length'])
        try:
            resp.content_encoding = metadata['encoding']
        except KeyError:
            pass
        return resp
Example #14
0
 def POST(self, req):
     """Handle HTTP POST request."""
     drive, part, account, container = split_and_validate_path(req, 4)
     if 'x-timestamp' not in req.headers or \
             not check_float(req.headers['x-timestamp']):
         return HTTPBadRequest(body='Missing or bad timestamp',
                               request=req, content_type='text/plain')
     if 'x-container-sync-to' in req.headers:
         err = validate_sync_to(req.headers['x-container-sync-to'],
                                self.allowed_sync_hosts)
         if err:
             return HTTPBadRequest(err)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_container_broker(drive, part, account, container)
     if broker.is_deleted():
         return HTTPNotFound(request=req)
     timestamp = normalize_timestamp(req.headers['x-timestamp'])
     metadata = {}
     metadata.update(
         (key, (value, timestamp)) for key, value in req.headers.iteritems()
         if key.lower() in self.save_headers or
         is_sys_or_user_meta('container', key))
     if metadata:
         if 'X-Container-Sync-To' in metadata:
             if 'X-Container-Sync-To' not in broker.metadata or \
                     metadata['X-Container-Sync-To'][0] != \
                     broker.metadata['X-Container-Sync-To'][0]:
                 broker.set_x_container_sync_points(-1, -1)
         broker.update_metadata(metadata)
     return HTTPNoContent(request=req)
Example #15
0
    def POST(self, req):
        """HTTP POST request handler."""
        error_response = \
            self.clean_acls(req) or check_metadata(req, 'container')
        if error_response:
            return error_response
        if not req.environ.get('swift_owner'):
            for key in self.app.swift_owner_headers:
                req.headers.pop(key, None)

        headers = self.generate_request_headers(req, transfer=True)
        clear_info_cache(self.app, req.environ,
                         self.account_name, self.container_name)

        storage = self.app.storage
        metadata = {}
        metadata.update(("user.%s" % k, v) for k, v in req.headers.iteritems()
                        if k.lower() in self.pass_through_headers or
                        is_sys_or_user_meta('container', k))

        try:
            storage.container_update(self.account_name, self.container_name,
                                     metadata, headers=headers)
            resp = HTTPNoContent(request=req)
        except exceptions.NoSuchContainer:
            resp = self.PUT(req)
        return resp
Example #16
0
 def POST(self, req):
     """Handle HTTP POST request."""
     drive, part, account, container = split_and_validate_path(req, 4)
     req_timestamp = valid_timestamp(req)
     if 'x-container-sync-to' in req.headers:
         err, sync_to, realm, realm_key = validate_sync_to(
             req.headers['x-container-sync-to'], self.allowed_sync_hosts,
             self.realms_conf)
         if err:
             return HTTPBadRequest(err)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_container_broker(drive, part, account, container)
     if broker.is_deleted():
         return HTTPNotFound(request=req)
     broker.update_put_timestamp(req_timestamp.internal)
     metadata = {}
     metadata.update((key, (value, req_timestamp.internal))
                     for key, value in req.headers.items()
                     if key.lower() in self.save_headers
                     or is_sys_or_user_meta('container', key))
     if metadata:
         if 'X-Container-Sync-To' in metadata:
             if 'X-Container-Sync-To' not in broker.metadata or \
                     metadata['X-Container-Sync-To'][0] != \
                     broker.metadata['X-Container-Sync-To'][0]:
                 broker.set_x_container_sync_points(-1, -1)
         broker.update_metadata(metadata, validate_metadata=True)
         self._update_sync_store(broker, 'POST')
     return HTTPNoContent(request=req)
Example #17
0
 def POST(self, req):
     """Handle HTTP POST request."""
     drive, part, account, container = split_and_validate_path(req, 4)
     if 'x-timestamp' not in req.headers or \
             not check_float(req.headers['x-timestamp']):
         return HTTPBadRequest(body='Missing or bad timestamp',
                               request=req,
                               content_type='text/plain')
     if 'x-container-sync-to' in req.headers:
         err = validate_sync_to(req.headers['x-container-sync-to'],
                                self.allowed_sync_hosts)
         if err:
             return HTTPBadRequest(err)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_container_broker(drive, part, account, container)
     if broker.is_deleted():
         return HTTPNotFound(request=req)
     timestamp = normalize_timestamp(req.headers['x-timestamp'])
     metadata = {}
     metadata.update((key, (value, timestamp))
                     for key, value in req.headers.iteritems()
                     if key.lower() in self.save_headers
                     or is_sys_or_user_meta('container', key))
     if metadata:
         if 'X-Container-Sync-To' in metadata:
             if 'X-Container-Sync-To' not in broker.metadata or \
                     metadata['X-Container-Sync-To'][0] != \
                     broker.metadata['X-Container-Sync-To'][0]:
                 broker.set_x_container_sync_points(-1, -1)
         broker.update_metadata(metadata)
     return HTTPNoContent(request=req)
Example #18
0
    def HEAD(self, request):
        """Handle HTTP HEAD requests for the Swift on File object server"""
        device, partition, account, container, obj, policy = \
            get_name_and_placement(request, 5, 5, True)

        # Get DiskFile
        try:
            disk_file = self.get_diskfile(device, partition, account, container,
                                          obj, policy=policy)

        except DiskFileDeviceUnavailable:
            return HTTPInsufficientStorage(drive=device, request=request)

        # Read DiskFile metadata
        try:
            disk_file.open()
            metadata = disk_file.get_metadata()
        except (DiskFileNotExist, DiskFileQuarantined) as e:
            headers = {}
            if hasattr(e, 'timestamp'):
                headers['X-Backend-Timestamp'] = e.timestamp.internal
            return HTTPNotFound(request=request, headers=headers,
                                conditional_respose=True)

        # Create and populate our response
        response = Response(request=request, conditional_response=True)
        response.headers['Content-Type'] = \
            metadata.get('Content-Type', 'application/octet-stream')
        for key, value in metadata.iteritems():
            if is_sys_or_user_meta('object', key) or key.lower() in \
                    self.allowed_headers:
                response.headers[key] = value
        response.etag = metadata['ETag']
        ts = Timestamp(metadata['X-Timestamp'])
        response.last_modified = math.ceil(float(ts))
        # Needed for container sync feature
        response.headers['X-Timestamp'] = ts.normal
        response.headers['X-Backend-Timestamp'] = ts.internal
        response.content_length = int(metadata['Content-Length'])
        try:
            response.content_encoding = metadata['Content-Encoding']
        except KeyError:
            pass

        # (HPSS) Inject HPSS xattr metadata into headers
        want_hpss_metadata = request.headers.get('X-HPSS-Get-Metadata',
                                                 False)
        if config_true_value(want_hpss_metadata):
            try:
                hpss_headers = disk_file.read_hpss_system_metadata()
                response.headers.update(hpss_headers)
            except SwiftOnFileSystemIOError:
                return HTTPServiceUnavailable(request=request)

        if 'X-Object-Sysmeta-Update-Container' in response.headers:
            self._sof_container_update(request, response)
            response.headers.pop('X-Object-Sysmeta-Update-Container')

        return response
Example #19
0
 def GET(self, request):
     """Handle HTTP GET requests for the Swift Object Server."""
     device, partition, account, container, obj, policy = \
         get_name_and_placement(request, 5, 5, True)
     keep_cache = self.keep_cache_private or (
         'X-Auth-Token' not in request.headers
         and 'X-Storage-Token' not in request.headers)
     try:
         disk_file = self.get_diskfile(device,
                                       partition,
                                       account,
                                       container,
                                       obj,
                                       policy=policy)
     except DiskFileDeviceUnavailable:
         return HTTPInsufficientStorage(drive=device, request=request)
     try:
         with disk_file.open():
             metadata = disk_file.get_metadata()
             obj_size = int(metadata['Content-Length'])
             file_x_ts = Timestamp(metadata['X-Timestamp'])
             keep_cache = (self.keep_cache_private or
                           ('X-Auth-Token' not in request.headers
                            and 'X-Storage-Token' not in request.headers))
             conditional_etag = None
             if 'X-Backend-Etag-Is-At' in request.headers:
                 conditional_etag = metadata.get(
                     request.headers['X-Backend-Etag-Is-At'])
             response = Response(
                 app_iter=disk_file.reader(keep_cache=keep_cache),
                 request=request,
                 conditional_response=True,
                 conditional_etag=conditional_etag)
             response.headers['Content-Type'] = metadata.get(
                 'Content-Type', 'application/octet-stream')
             for key, value in metadata.iteritems():
                 if is_sys_or_user_meta('object', key) or \
                         key.lower() in self.allowed_headers:
                     response.headers[key] = value
             response.etag = metadata['ETag']
             response.last_modified = math.ceil(float(file_x_ts))
             response.content_length = obj_size
             try:
                 response.content_encoding = metadata['Content-Encoding']
             except KeyError:
                 pass
             response.headers['X-Timestamp'] = file_x_ts.normal
             response.headers['X-Backend-Timestamp'] = file_x_ts.internal
             resp = request.get_response(response)
     except DiskFileXattrNotSupported:
         return HTTPInsufficientStorage(drive=device, request=request)
     except (DiskFileNotExist, DiskFileQuarantined) as e:
         headers = {}
         if hasattr(e, 'timestamp'):
             headers['X-Backend-Timestamp'] = e.timestamp.internal
         resp = HTTPNotFound(request=request,
                             headers=headers,
                             conditional_response=True)
     return resp
Example #20
0
    def GET(self, request):
        """Handle HTTP GET requests for the Swift Object Server."""
        device, partition, account, container, obj, policy = \
            get_name_and_placement(request, 5, 5, True)
	print 'request',request
	print 'device, partition, account, container,obj, policy', device,partition,account,container,obj,policy
        keep_cache = self.keep_cache_private or (
            'X-Auth-Token' not in request.headers and
            'X-Storage-Token' not in request.headers)
        try:
            disk_file = self.get_diskfile(
                device, partition, account, container, obj,
                policy=policy)
	    print 'disk_file',disk_file 
        except DiskFileDeviceUnavailable:
            return HTTPInsufficientStorage(drive=device, request=request)
        try:
            with disk_file.open():
                metadata = disk_file.get_metadata()
                obj_size = int(metadata['Content-Length'])
                file_x_ts = Timestamp(metadata['X-Timestamp'])
                keep_cache = (self.keep_cache_private or
                              ('X-Auth-Token' not in request.headers and
                               'X-Storage-Token' not in request.headers))
                conditional_etag = None
                if 'X-Backend-Etag-Is-At' in request.headers:
                    conditional_etag = metadata.get(
                        request.headers['X-Backend-Etag-Is-At'])
                response = Response(
                    app_iter=disk_file.reader(keep_cache=keep_cache),
                    request=request, conditional_response=True,
                    conditional_etag=conditional_etag)
                response.headers['Content-Type'] = metadata.get(
                    'Content-Type', 'application/octet-stream')
                for key, value in metadata.iteritems():
                    if is_sys_or_user_meta('object', key) or \
                            key.lower() in self.allowed_headers:
                        response.headers[key] = value
                response.etag = metadata['ETag']
                response.last_modified = math.ceil(float(file_x_ts))
                response.content_length = obj_size
                try:
                    response.content_encoding = metadata[
                        'Content-Encoding']
                except KeyError:
                    pass
                response.headers['X-Timestamp'] = file_x_ts.normal
                response.headers['X-Backend-Timestamp'] = file_x_ts.internal
                resp = request.get_response(response)
        except DiskFileXattrNotSupported:
            return HTTPInsufficientStorage(drive=device, request=request)
        except (DiskFileNotExist, DiskFileQuarantined) as e:
            headers = {}
            if hasattr(e, 'timestamp'):
                headers['X-Backend-Timestamp'] = e.timestamp.internal
            resp = HTTPNotFound(request=request, headers=headers,
                                conditional_response=True)
	print 'resp',resp
	return resp
Example #21
0
 def PUT(self, req):
     """Handle HTTP PUT request."""
     drive, part, account, container = split_and_validate_path(req, 3, 4)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     if container:   # put account container
         pending_timeout = None
         container_policy_index = req.headers.get(POLICY_INDEX, 0)
         if 'x-trans-id' in req.headers:
             pending_timeout = 3
         broker = self._get_account_broker(drive, part, account,
                                           pending_timeout=pending_timeout)
         if account.startswith(self.auto_create_account_prefix) and \
                 not os.path.exists(broker.db_file):
             try:
                 broker.initialize(normalize_timestamp(
                     req.headers.get('x-timestamp') or time.time()))
             except DatabaseAlreadyExists:
                 pass
         if req.headers.get('x-account-override-deleted', 'no').lower() != \
                 'yes' and broker.is_deleted():
             return HTTPNotFound(request=req)
         broker.put_container(container, req.headers['x-put-timestamp'],
                              req.headers['x-delete-timestamp'],
                              req.headers['x-object-count'],
                              req.headers['x-bytes-used'],
                              container_policy_index)
         if req.headers['x-delete-timestamp'] > \
                 req.headers['x-put-timestamp']:
             return HTTPNoContent(request=req)
         else:
             return HTTPCreated(request=req)
     else:   # put account
         broker = self._get_account_broker(drive, part, account)
         timestamp = normalize_timestamp(req.headers['x-timestamp'])
         if not os.path.exists(broker.db_file):
             try:
                 broker.initialize(timestamp)
                 created = True
             except DatabaseAlreadyExists:
                 created = False
         elif broker.is_status_deleted():
             return self._deleted_response(broker, req, HTTPForbidden,
                                           body='Recently deleted')
         else:
             created = broker.is_deleted()
             broker.update_put_timestamp(timestamp)
             if broker.is_deleted():
                 return HTTPConflict(request=req)
         metadata = {}
         metadata.update((key, (value, timestamp))
                         for key, value in req.headers.iteritems()
                         if is_sys_or_user_meta('account', key))
         if metadata:
             broker.update_metadata(metadata)
         if created:
             return HTTPCreated(request=req)
         else:
             return HTTPAccepted(request=req)
Example #22
0
 def load_object_metadata(self, headers):
     metadata = {}
     metadata.update((k.lower(), v) for k, v in headers.iteritems()
                     if is_sys_or_user_meta('object', k))
     for header_key in self.allowed_headers:
         if header_key in headers:
             headers_lower = header_key.lower()
             metadata[headers_lower] = headers[header_key]
     return metadata
Example #23
0
def _copy_headers(src, dest):
    """
    Will copy desired headers from src to dest.

    :params src: an instance of collections.Mapping
    :params dest: an instance of collections.Mapping
    """
    for k, v in src.items():
        if is_sys_or_user_meta("object", k) or is_object_transient_sysmeta(k) or k.lower() == "x-delete-at":
            dest[k] = v
def _copy_headers_into(from_r, to_r):
    """
    Will copy desired headers from from_r to to_r
    :params from_r: a swob Request or Response
    :params to_r: a swob Request or Response
    """
    pass_headers = ['x-delete-at']
    for k, v in from_r.headers.items():
        if is_sys_or_user_meta('object', k) or k.lower() in pass_headers:
            to_r.headers[k] = v
Example #25
0
 def load_object_metadata(self, headers):
     metadata = {}
     metadata.update(
         (k.lower(), v) for k, v in headers.iteritems()
         if is_sys_or_user_meta('object', k) or
         is_object_transient_sysmeta(k))
     for header_key in self.allowed_headers:
         if header_key in headers:
             headers_lower = header_key.lower()
             metadata[headers_lower] = headers[header_key]
     return metadata
Example #26
0
def _copy_headers(src, dest):
    """
    Will copy desired headers from src to dest.

    :params src: an instance of collections.Mapping
    :params dest: an instance of collections.Mapping
    """
    for k, v in src.items():
        if (is_sys_or_user_meta('object', k) or is_object_transient_sysmeta(k)
                or k.lower() == 'x-delete-at'):
            dest[k] = v
Example #27
0
 def PUT(self, req):
     """Handle HTTP PUT request."""
     drive, part, account, container, obj = split_and_validate_path(
         req, 4, 5, True)
     if 'x-timestamp' not in req.headers or \
             not check_float(req.headers['x-timestamp']):
         return HTTPBadRequest(body='Missing timestamp', request=req,
                               content_type='text/plain')
     if 'x-container-sync-to' in req.headers:
         err, sync_to, realm, realm_key = validate_sync_to(
             req.headers['x-container-sync-to'], self.allowed_sync_hosts,
             self.realms_conf)
         if err:
             return HTTPBadRequest(err)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     timestamp = normalize_timestamp(req.headers['x-timestamp'])
     broker = self._get_container_broker(drive, part, account, container)
     if obj:     # put container object
         if account.startswith(self.auto_create_account_prefix) and \
                 not os.path.exists(broker.db_file):
             try:
                 broker.initialize(timestamp)
             except DatabaseAlreadyExists:
                 pass
         if not os.path.exists(broker.db_file):
             return HTTPNotFound()
         broker.put_object(obj, timestamp, int(req.headers['x-size']),
                           req.headers['x-content-type'],
                           req.headers['x-etag'])
         return HTTPCreated(request=req)
     else:   # put container
         created = self._update_or_create(req, broker, timestamp)
         metadata = {}
         metadata.update(
             (key, (value, timestamp))
             for key, value in req.headers.iteritems()
             if key.lower() in self.save_headers or
             is_sys_or_user_meta('container', key))
         if metadata:
             if 'X-Container-Sync-To' in metadata:
                 if 'X-Container-Sync-To' not in broker.metadata or \
                         metadata['X-Container-Sync-To'][0] != \
                         broker.metadata['X-Container-Sync-To'][0]:
                     broker.set_x_container_sync_points(-1, -1)
             broker.update_metadata(metadata)
         resp = self.account_update(req, account, container, broker)
         if resp:
             return resp
         if created:
             return HTTPCreated(request=req)
         else:
             return HTTPAccepted(request=req)
Example #28
0
 def _update_metadata(self, req, broker, req_timestamp, method):
     metadata = {}
     metadata.update((key, (value, req_timestamp.internal))
                     for key, value in req.headers.items()
                     if key.lower() in self.save_headers
                     or is_sys_or_user_meta('container', key))
     if metadata:
         if 'X-Container-Sync-To' in metadata:
             if 'X-Container-Sync-To' not in broker.metadata or \
                     metadata['X-Container-Sync-To'][0] != \
                     broker.metadata['X-Container-Sync-To'][0]:
                 broker.set_x_container_sync_points(-1, -1)
         broker.update_metadata(metadata, validate_metadata=True)
         self._update_sync_store(broker, method)
 def _create_response_headers(self, source_path, source_resp, sink_req):
     resp_headers = dict()
     acct, path = source_path.split('/', 3)[2:4]
     resp_headers['X-Copied-From-Account'] = quote(acct)
     resp_headers['X-Copied-From'] = quote(path)
     if 'last-modified' in source_resp.headers:
             resp_headers['X-Copied-From-Last-Modified'] = \
                 source_resp.headers['last-modified']
     # Existing sys and user meta of source object is added to response
     # headers in addition to the new ones.
     for k, v in sink_req.headers.items():
         if is_sys_or_user_meta('object', k) or k.lower() == 'x-delete-at':
             resp_headers[k] = v
     return resp_headers
Example #30
0
 def HEAD(self, request):
     """Handle HTTP HEAD requests for the Swift Object Server."""
     device, partition, account, container, obj, policy = \
         get_name_and_placement(request, 5, 5, True)
     try:
         disk_file = self.get_diskfile(device,
                                       partition,
                                       account,
                                       container,
                                       obj,
                                       policy=policy)
     except DiskFileDeviceUnavailable:
         return HTTPInsufficientStorage(drive=device, request=request)
     try:
         metadata = disk_file.read_metadata()
     except DiskFileXattrNotSupported:
         return HTTPInsufficientStorage(drive=device, request=request)
     except (DiskFileNotExist, DiskFileQuarantined) as e:
         headers = {}
         if hasattr(e, 'timestamp'):
             headers['X-Backend-Timestamp'] = e.timestamp.internal
         return HTTPNotFound(request=request,
                             headers=headers,
                             conditional_response=True)
     conditional_etag = None
     if 'X-Backend-Etag-Is-At' in request.headers:
         conditional_etag = metadata.get(
             request.headers['X-Backend-Etag-Is-At'])
     response = Response(request=request,
                         conditional_response=True,
                         conditional_etag=conditional_etag)
     response.headers['Content-Type'] = metadata.get(
         'Content-Type', 'application/octet-stream')
     for key, value in metadata.iteritems():
         if is_sys_or_user_meta('object', key) or \
                 key.lower() in self.allowed_headers:
             response.headers[key] = value
     response.etag = metadata['ETag']
     ts = Timestamp(metadata['X-Timestamp'])
     response.last_modified = math.ceil(float(ts))
     # Needed for container sync feature
     response.headers['X-Timestamp'] = ts.normal
     response.headers['X-Backend-Timestamp'] = ts.internal
     response.content_length = int(metadata['Content-Length'])
     try:
         response.content_encoding = metadata['Content-Encoding']
     except KeyError:
         pass
     return response
Example #31
0
 def create_listing(self, req, out_content_type, info, metadata,
                    container_list, container):
     list_meta = get_param(req, 'list_meta', 'f').lower() in TRUE_VALUES
     resp_headers = {
         'X-Container-Object-Count': info['object_count'],
         'X-Container-Bytes-Used': info['bytes_used'],
         'X-Timestamp': info['created_at'],
         'X-PUT-Timestamp': info['put_timestamp'],
     }
     for key, (value, timestamp) in metadata.iteritems():
         if value and (key.lower() in self.save_headers or
                       is_sys_or_user_meta('container', key)):
             resp_headers[key] = value
     ret = Response(request=req, headers=resp_headers,
                    content_type=out_content_type, charset='utf-8')
     if out_content_type == 'application/json':
         ret.body = json.dumps([self.update_data_record(record, list_meta)
                                for record in container_list])
     elif out_content_type.endswith('/xml'):
         doc = Element('container', name=container.decode('utf-8'))
         for obj in container_list:
             record = self.update_data_record(obj, list_meta)
             if 'subdir' in record:
                 name = record['subdir'].decode('utf-8')
                 sub = SubElement(doc, 'subdir', name=name)
                 SubElement(sub, 'name').text = name
             else:
                 obj_element = SubElement(doc, 'object')
                 for field in ["name", "hash", "bytes", "content_type",
                               "last_modified"]:
                     SubElement(obj_element, field).text = str(
                         record.pop(field)).decode('utf-8')
                 for field in sorted(record):
                     if list_meta and field == 'metadata':
                         meta = SubElement(obj_element, field)
                         for k, v in record[field].iteritems():
                             SubElement(meta, k).text = str(
                                 v.decode('utf-8'))
                     else:
                         SubElement(obj_element, field).text = str(
                             record[field]).decode('utf-8')
         ret.body = tostring(doc, encoding='UTF-8').replace(
             "<?xml version='1.0' encoding='UTF-8'?>",
             '<?xml version="1.0" encoding="UTF-8"?>', 1)
     else:
         if not container_list:
             return HTTPNoContent(request=req, headers=resp_headers)
         ret.body = '\n'.join(rec[0] for rec in container_list) + '\n'
     return ret
Example #32
0
 def _update_metadata(self, req, broker, req_timestamp, method):
     metadata = {}
     metadata.update(
         (wsgi_to_str(key), (wsgi_to_str(value), req_timestamp.internal))
         for key, value in req.headers.items()
         if key.lower() in self.save_headers or
         is_sys_or_user_meta('container', key))
     if metadata:
         if 'X-Container-Sync-To' in metadata:
             if 'X-Container-Sync-To' not in broker.metadata or \
                     metadata['X-Container-Sync-To'][0] != \
                     broker.metadata['X-Container-Sync-To'][0]:
                 broker.set_x_container_sync_points(-1, -1)
         broker.update_metadata(metadata, validate_metadata=True)
         self._update_sync_store(broker, method)
Example #33
0
    def get_account_put_resp(self, req, headers):
        created = self.app.storage.account_create(self.account_name)
        metadata = {}
        metadata.update((key, value)
                        for key, value in req.headers.items()
                        if is_sys_or_user_meta('account', key))

        if metadata:
            self.app.storage.account_update(self.account_name, metadata)

        if created:
            resp = HTTPCreated(request=req)
        else:
            resp = HTTPAccepted(request=req)
        return resp
Example #34
0
    def get_account_put_resp(self, req):
        headers = self.generate_request_headers(req, transfer=True)
        created = self.app.storage.account_create(self.account_name,
                                                  headers=headers)
        metadata = {}
        metadata.update((k, v) for k, v in req.headers.iteritems()
                        if is_sys_or_user_meta('account', k))

        if metadata:
            self.app.storage.account_update(self.account_name, metadata)

        if created:
            resp = HTTPCreated(request=req)
        else:
            resp = HTTPAccepted(request=req)
        return resp
Example #35
0
 def POST(self, req):
     """Handle HTTP POST request."""
     drive, part, account = split_and_validate_path(req, 3)
     req_timestamp = valid_timestamp(req)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_account_broker(drive, part, account)
     if broker.is_deleted():
         return self._deleted_response(broker, req, HTTPNotFound)
     metadata = {}
     metadata.update((key, (value, req_timestamp.internal))
                     for key, value in req.headers.items()
                     if is_sys_or_user_meta('account', key))
     if metadata:
         broker.update_metadata(metadata, validate_metadata=True)
     return HTTPNoContent(request=req)
Example #36
0
 def POST(self, req):
     """Handle HTTP POST request."""
     drive, part, account = split_and_validate_path(req, 3)
     req_timestamp = valid_timestamp(req)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_account_broker(drive, part, account)
     if broker.is_deleted():
         return self._deleted_response(broker, req, HTTPNotFound)
     metadata = {}
     metadata.update((key, (value, req_timestamp.internal))
                     for key, value in req.headers.iteritems()
                     if is_sys_or_user_meta('account', key))
     if metadata:
         broker.update_metadata(metadata, validate_metadata=True)
     return HTTPNoContent(request=req)
Example #37
0
    def properties_from_headers(self, headers):
        metadata = {
            k: v
            for k, v in headers.items()
            if k.lower() in self.pass_through_headers or
            is_sys_or_user_meta('container', k)
        }

        system = dict()
        # This header enables versioning
        ver_loc = headers.get('X-Container-Sysmeta-Versions-Location')
        if ver_loc is not None:
            # When suspending versioning, header has empty string value
            ver_val = "-1" if ver_loc else "1"
            system['sys.m2.policy.version'] = ver_val

        return metadata, system
Example #38
0
 def get_metadata_resp_headers(self, meta):
     headers = {}
     system = meta.get('system') or {}
     # sys.m2.ctime is microseconds
     ctime = float(system.get('sys.m2.ctime', 0)) / 1000000.0
     headers.update({
         'X-Container-Object-Count': system.get('sys.m2.objects', 0),
         'X-Container-Bytes-Used': system.get('sys.m2.usage', 0),
         'X-Timestamp': Timestamp(ctime).normal,
         # FIXME: save modification time somewhere
         'X-PUT-Timestamp': Timestamp(ctime).normal,
     })
     for (k, v) in meta['properties'].iteritems():
         if v and (k.lower() in self.pass_through_headers or
                   is_sys_or_user_meta('container', k)):
             headers[k] = v
     return headers
Example #39
0
 def get_metadata_resp_headers(self, meta):
     headers = {}
     system = meta.get('system') or {}
     # sys.m2.ctime is microseconds
     ctime = float(system.get('sys.m2.ctime', 0)) / 1000000.0
     headers.update({
         'X-Container-Object-Count': system.get('sys.m2.objects', 0),
         'X-Container-Bytes-Used': system.get('sys.m2.usage', 0),
         'X-Timestamp': Timestamp(ctime).normal,
         # FIXME: save modification time somewhere
         'X-PUT-Timestamp': Timestamp(ctime).normal,
     })
     for (k, v) in meta['properties'].iteritems():
         if v and (k.lower() in self.pass_through_headers or
                   is_sys_or_user_meta('container', k)):
             headers[k] = v
     return headers
Example #40
0
    def properties_from_headers(self, headers):
        metadata = {
            k: v
            for k, v in headers.items()
            if k.lower() in self.pass_through_headers
            or is_sys_or_user_meta('container', k)
        }

        system = dict()
        # This header enables versioning
        ver_loc = headers.get('X-Container-Sysmeta-Versions-Location')
        if ver_loc is not None:
            # When suspending versioning, header has empty string value
            ver_val = "-1" if ver_loc else "1"
            system['sys.m2.policy.version'] = ver_val

        return metadata, system
Example #41
0
 def POST(self, req):
     """Handle HTTP POST request."""
     drive, part, account = split_and_validate_path(req, 3)
     if "x-timestamp" not in req.headers or not check_float(req.headers["x-timestamp"]):
         return HTTPBadRequest(body="Missing or bad timestamp", request=req, content_type="text/plain")
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_account_broker(drive, part, account)
     if broker.is_deleted():
         return self._deleted_response(broker, req, HTTPNotFound)
     timestamp = normalize_timestamp(req.headers["x-timestamp"])
     metadata = {}
     metadata.update(
         (key, (value, timestamp)) for key, value in req.headers.iteritems() if is_sys_or_user_meta("account", key)
     )
     if metadata:
         broker.update_metadata(metadata)
     return HTTPNoContent(request=req)
Example #42
0
    def get_account_put_resp(self, req, headers):
        oio_headers = {'X-oio-req-id': self.trans_id}
        created = self.app.storage.account_create(
            self.account_name, headers=oio_headers)
        metadata = {}
        metadata.update((key, value)
                        for key, value in req.headers.items()
                        if is_sys_or_user_meta('account', key))

        if metadata:
            self.app.storage.account_set_properties(
                self.account_name, metadata, headers=oio_headers)

        if created:
            resp = HTTPCreated(request=req)
        else:
            resp = HTTPAccepted(request=req)
        return resp
Example #43
0
 def GET(self, request):
     """Handle HTTP GET requests for the Swift Object Server."""
     device, partition, account, container, obj, policy_idx = get_name_and_placement(request, 5, 5, True)
     keep_cache = self.keep_cache_private or (
         "X-Auth-Token" not in request.headers and "X-Storage-Token" not in request.headers
     )
     try:
         disk_file = self.get_diskfile(device, partition, account, container, obj, policy_idx=policy_idx)
     except DiskFileDeviceUnavailable:
         return HTTPInsufficientStorage(drive=device, request=request)
     try:
         with disk_file.open():
             metadata = disk_file.get_metadata()
             obj_size = int(metadata["Content-Length"])
             file_x_ts = Timestamp(metadata["X-Timestamp"])
             keep_cache = self.keep_cache_private or (
                 "X-Auth-Token" not in request.headers and "X-Storage-Token" not in request.headers
             )
             response = Response(
                 app_iter=disk_file.reader(keep_cache=keep_cache), request=request, conditional_response=True
             )
             response.headers["Content-Type"] = metadata.get("Content-Type", "application/octet-stream")
             for key, value in metadata.iteritems():
                 if is_sys_or_user_meta("object", key) or key.lower() in self.allowed_headers:
                     response.headers[key] = value
             response.etag = metadata["ETag"]
             response.last_modified = math.ceil(float(file_x_ts))
             response.content_length = obj_size
             try:
                 response.content_encoding = metadata["Content-Encoding"]
             except KeyError:
                 pass
             response.headers["X-Timestamp"] = file_x_ts.normal
             response.headers["X-Backend-Timestamp"] = file_x_ts.internal
             resp = request.get_response(response)
     except DiskFileXattrNotSupported:
         return HTTPInsufficientStorage(drive=device, request=request)
     except (DiskFileNotExist, DiskFileQuarantined) as e:
         headers = {}
         if hasattr(e, "timestamp"):
             headers["X-Backend-Timestamp"] = e.timestamp.internal
         resp = HTTPNotFound(request=request, headers=headers, conditional_response=True)
     return resp
Example #44
0
 def get_account_post_resp(self, req, headers):
     metadata = {}
     metadata.update((key, value)
                     for key, value in req.headers.items()
                     if is_sys_or_user_meta('account', key))
     try:
         self.app.storage.account_update(self.account_name, metadata)
         return HTTPNoContent(request=req)
     except (exceptions.NotFound, exceptions.NoSuchAccount):
         if self.app.account_autocreate:
             self.autocreate_account(req, self.account_name)
             if metadata:
                 self.app.storage.account_update(
                         self.account_name, metadata, headers=headers)
             resp = HTTPNoContent(request=req)
         else:
             resp = HTTPNotFound(request=req)
     self.add_acls_from_sys_metadata(resp)
     return resp
 def POST(self, req):
     """Handle HTTP POST request."""
     drive, part, account = split_and_validate_path(req, 3)
     #无用
     req_timestamp = valid_timestamp(req)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     #获取account的数据库对象,用于存放account的数据以及元数据,获取文件对象
     broker = self._get_account_broker(drive, part, account)
     if broker.is_deleted():
         return self._deleted_response(broker, req, HTTPNotFound)
     metadata = {}
     #生成元数据字典,系统元数据或是用户元数据
     metadata.update((key, (value, req_timestamp.internal))
                     for key, value in req.headers.items()
                     if is_sys_or_user_meta('account', key))
     if metadata:
         #如果存在需要更新的元数据,则更新元数据到数据库中,保存到文件
         broker.update_metadata(metadata, validate_metadata=True)
     return HTTPNoContent(request=req)
Example #46
0
    def get_account_post_resp(self, req):
        headers = self.generate_request_headers(req, transfer=True)

        metadata = {}
        metadata.update((k, v) for k, v in req.headers.iteritems()
                        if is_sys_or_user_meta('account', k))
        try:
            self.app.storage.account_update(self.account_name, metadata,
                                            headers=headers)
            return HTTPNoContent(request=req)
        except exceptions.NotFound:
            if self.app.account_autocreate:
                self.autocreate_account(req, self.account_name)
                if metadata:
                    self.app.storage.account_update(self.account_name,
                                                    metadata, headers=headers)
                resp = HTTPNoContent(request=req)
            else:
                resp = HTTPNotFound(request=req)
        return resp
Example #47
0
    def transfer_headers(self, src_headers, dst_headers):
        """
        Transfer legal headers from an original client request to dictionary
        that will be used as headers by the backend request

        :param src_headers: A dictionary of the original client request headers
        :param dst_headers: A dictionary of the backend request headers
        """
        st = self.server_type.lower()

        x_remove = 'x-remove-%s-meta-' % st
        dst_headers.update((k.lower().replace('-remove', '', 1), '')
                           for k in src_headers
                           if k.lower().startswith(x_remove) or
                           k.lower() in self._x_remove_headers())

        dst_headers.update((k.lower(), v)
                           for k, v in src_headers.iteritems()
                           if k.lower() in self.pass_through_headers or
                           is_sys_or_user_meta(st, k))
Example #48
0
 def create_listing(self, req, out_content_type, resp_headers,
                    metadata, result_list, container):
     container_list = result_list['objects']
     for p in result_list.get('prefixes', []):
         record = {'name': p,
                   'subdir': True}
         container_list.append(record)
     for (k, v) in metadata.iteritems():
         if v and (k.lower() in self.pass_through_headers or
                       is_sys_or_user_meta('container', k)):
             resp_headers[k] = v
     ret = Response(request=req, headers=resp_headers,
                    content_type=out_content_type, charset='utf-8')
     if out_content_type == 'application/json':
         ret.body = json.dumps([self.update_data_record(record)
                                for record in container_list])
     elif out_content_type.endswith('/xml'):
         doc = Element('container', name=container.decode('utf-8'))
         for obj in container_list:
             record = self.update_data_record(obj)
             if 'subdir' in record:
                 name = record['subdir'].decode('utf-8')
                 sub = SubElement(doc, 'subdir', name=name)
                 SubElement(sub, 'name').text = name
             else:
                 obj_element = SubElement(doc, 'object')
                 for field in ["name", "hash", "bytes", "content_type",
                               "last_modified"]:
                     SubElement(obj_element, field).text = str(
                         record.pop(field)).decode('utf-8')
                 for field in sorted(record):
                     SubElement(obj_element, field).text = str(
                         record[field]).decode('utf-8')
         ret.body = tostring(doc, encoding='UTF-8').replace(
             "<?xml version='1.0' encoding='UTF-8'?>",
             '<?xml version="1.0" encoding="UTF-8"?>', 1)
     else:
         if not container_list:
             return HTTPNoContent(request=req, headers=resp_headers)
         ret.body = '\n'.join(rec['name'] for rec in container_list) + '\n'
     return ret
Example #49
0
 def create_listing(self, req, out_content_type, info, resp_headers,
                    metadata, container_list, container):
     for key, (value, timestamp) in metadata.items():
         if value and (key.lower() in self.save_headers
                       or is_sys_or_user_meta('container', key)):
             resp_headers[key] = value
     ret = Response(request=req,
                    headers=resp_headers,
                    content_type=out_content_type,
                    charset='utf-8')
     if out_content_type == 'application/json':
         ret.body = json.dumps(
             [self.update_data_record(record) for record in container_list])
     elif out_content_type.endswith('/xml'):
         doc = Element('container', name=container.decode('utf-8'))
         for obj in container_list:
             record = self.update_data_record(obj)
             if 'subdir' in record:
                 name = record['subdir'].decode('utf-8')
                 sub = SubElement(doc, 'subdir', name=name)
                 SubElement(sub, 'name').text = name
             else:
                 obj_element = SubElement(doc, 'object')
                 for field in [
                         "name", "hash", "bytes", "content_type",
                         "last_modified"
                 ]:
                     SubElement(obj_element, field).text = str(
                         record.pop(field)).decode('utf-8')
                 for field in sorted(record):
                     SubElement(obj_element, field).text = str(
                         record[field]).decode('utf-8')
         ret.body = tostring(doc, encoding='UTF-8').replace(
             "<?xml version='1.0' encoding='UTF-8'?>",
             '<?xml version="1.0" encoding="UTF-8"?>', 1)
     else:
         if not container_list:
             return HTTPNoContent(request=req, headers=resp_headers)
         ret.body = '\n'.join(rec[0] for rec in container_list) + '\n'
     ret.last_modified = math.ceil(float(resp_headers['X-PUT-Timestamp']))
     return ret
Example #50
0
 def HEAD(self, req):
     """Handle HTTP HEAD request."""
     drive, part, account, container, obj = split_and_validate_path(
         req, 4, 5, True)
     out_content_type = get_listing_content_type(req)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_container_broker(drive, part, account, container,
                                         pending_timeout=0.1,
                                         stale_reads_ok=True)
     info, is_deleted = broker.get_info_is_deleted()
     headers = gen_resp_headers(info, is_deleted=is_deleted)
     if is_deleted:
         return HTTPNotFound(request=req, headers=headers)
     headers.update(
         (key, value)
         for key, (value, timestamp) in broker.metadata.iteritems()
         if value != '' and (key.lower() in self.save_headers or
                             is_sys_or_user_meta('container', key)))
     headers['Content-Type'] = out_content_type
     return HTTPNoContent(request=req, headers=headers, charset='utf-8')
Example #51
0
    def container_crawl(self, path):
        """
        Crawls the given container path.

        :param path: the path to an container db
        """
        metaDict = {}
        try:
            broker = ContainerBroker(path)
            if not broker.is_deleted():
                #reportedTime = broker.get_info()['put_timestamp']
                #if normalize_timestamp(self.crawled_time)
                #< reportedTime < normalize_timestamp(start_time):
                metaDict = broker.get_info()
                metaDict.update(
                    (key, value)
                    for key, (value, timestamp) in broker.metadata.iteritems()
                    if value != '' and is_sys_or_user_meta('container', key))
        except (Exception, Timeout):
            self.logger.increment('failures')
        return metaDict
Example #52
0
 def POST(self, req):
     """Handle HTTP POST request."""
     drive, part, account = split_and_validate_path(req, 3)
     if 'x-timestamp' not in req.headers or \
             not check_float(req.headers['x-timestamp']):
         return HTTPBadRequest(body='Missing or bad timestamp',
                               request=req,
                               content_type='text/plain')
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_account_broker(drive, part, account)
     if broker.is_deleted():
         return self._deleted_response(broker, req, HTTPNotFound)
     timestamp = normalize_timestamp(req.headers['x-timestamp'])
     metadata = {}
     metadata.update((key, (value, timestamp))
                     for key, value in req.headers.iteritems()
                     if is_sys_or_user_meta('account', key))
     if metadata:
         broker.update_metadata(metadata)
     return HTTPNoContent(request=req)
Example #53
0
    def create_listing(self, req, out_content_type, info, resp_headers,
                       metadata, container_list, container):
        for key, (value, timestamp) in metadata.items():
            if value and (key.lower() in self.save_headers or
                          is_sys_or_user_meta('container', key)):
                resp_headers[str_to_wsgi(key)] = str_to_wsgi(value)
        listing = [self.update_data_record(record)
                   for record in container_list]
        if out_content_type.endswith('/xml'):
            body = listing_formats.container_to_xml(listing, container)
        elif out_content_type.endswith('/json'):
            body = json.dumps(listing).encode('ascii')
        else:
            body = listing_formats.listing_to_text(listing)

        ret = Response(request=req, headers=resp_headers, body=body,
                       content_type=out_content_type, charset='utf-8')
        ret.last_modified = math.ceil(float(resp_headers['X-PUT-Timestamp']))
        if not ret.body:
            ret.status_int = HTTP_NO_CONTENT
        return ret
Example #54
0
    def make_object_response(self, req, metadata, stream=None):
        conditional_etag = None
        if 'X-Backend-Etag-Is-At' in req.headers:
            conditional_etag = metadata.get(
                req.headers['X-Backend-Etag-Is-At'])

        resp = Response(request=req,
                        conditional_response=True,
                        conditional_etag=conditional_etag)

        if config_true_value(metadata['deleted']):
            resp.headers['Content-Type'] = DELETE_MARKER_CONTENT_TYPE
        else:
            resp.headers['Content-Type'] = metadata.get(
                'mime_type', 'application/octet-stream')
        properties = metadata.get('properties')
        if properties:
            for k, v in properties.items():
                if is_sys_or_user_meta('object', k) or \
                        is_object_transient_sysmeta(k) or \
                        k.lower() in self.allowed_headers:
                    resp.headers[str(k)] = v
        hash_ = metadata.get('hash')
        if hash_ is not None:
            hash_ = hash_.lower()
        resp.headers['etag'] = hash_
        resp.headers['x-object-sysmeta-version-id'] = metadata['version']
        resp.last_modified = int(metadata['mtime'])
        if stream:
            # Whether we are bothered with ranges or not, we wrap the
            # stream in order to handle exceptions.
            resp.app_iter = StreamRangeIterator(req, stream)

        length_ = metadata.get('length')
        if length_ is not None:
            length_ = int(length_)
        resp.content_length = length_
        resp.content_encoding = metadata.get('encoding')
        resp.accept_ranges = 'bytes'
        return resp
Example #55
0
    def make_object_response(self, req, metadata, stream=None, ranges=None):
        conditional_etag = None
        if 'X-Backend-Etag-Is-At' in req.headers:
            conditional_etag = metadata.get(
                req.headers['X-Backend-Etag-Is-At'])

        resp = Response(request=req,
                        conditional_response=True,
                        conditional_etag=conditional_etag)

        if config_true_value(metadata['deleted']):
            resp.headers['Content-Type'] = DELETE_MARKER_CONTENT_TYPE
        else:
            resp.headers['Content-Type'] = metadata.get(
                'mime_type', 'application/octet-stream')
        properties = metadata.get('properties')
        if properties:
            for k, v in properties.iteritems():
                if is_sys_or_user_meta('object', k) or \
                        is_object_transient_sysmeta(k) or \
                        k.lower() in self.allowed_headers:
                    resp.headers[str(k)] = v
        resp.headers['etag'] = metadata['hash'].lower()
        resp.headers['x-object-sysmeta-version-id'] = metadata['version']
        ts = Timestamp(metadata['ctime'])
        resp.last_modified = math.ceil(float(ts))
        if stream:
            if ranges:
                resp.app_iter = StreamRangeIterator(stream)
            else:
                resp.app_iter = stream

        resp.content_length = int(metadata['length'])
        try:
            resp.content_encoding = metadata['encoding']
        except KeyError:
            pass
        resp.accept_ranges = 'bytes'
        return resp
Example #56
0
    def properties_from_headers(self, headers):
        metadata = {
            k: v
            for k, v in headers.items()
            if k.lower() in self.pass_through_headers
            or is_sys_or_user_meta('container', k)
        }

        system = dict()
        # This headers enable versioning.
        # First the legacy one.
        ver_loc = headers.get('X-Container-Sysmeta-Versions-Location')
        if ver_loc is not None:
            # When suspending versioning, header has empty string value
            ver_val = "-1" if ver_loc else "1"
            system['sys.m2.policy.version'] = ver_val
        # Then the new one.
        vers_enabled = headers.get(CLIENT_VERSIONS_ENABLED)
        if vers_enabled is not None:
            ver_val = "-1" if config_true_value(vers_enabled) else "1"
            system['sys.m2.policy.version'] = ver_val

        return metadata, system