Пример #1
0
    def setup_objects(self):
        self.objects = (('rose', '2011-01-05T02:19:14.275290', 0, 303),
                        ('viola', '2011-01-05T02:19:14.275290', 0, 3909),
                        ('lily', '2011-01-05T02:19:14.275290', 0, 3909),
                        ('with space', '2011-01-05T02:19:14.275290', 0, 390),
                        ('with%20space', '2011-01-05T02:19:14.275290', 0, 390))

        objects = map(
            lambda item: {'name': str(item[0]), 'last_modified': str(item[1]),
                          'hash': str(item[2]), 'bytes': str(item[3])},
            list(self.objects))
        object_list = json.dumps(objects)

        self.prefixes = ['rose', 'viola', 'lily']
        object_list_subdir = []
        for p in self.prefixes:
            object_list_subdir.append({"subdir": p})

        self.swift.register('HEAD', '/v1/AUTH_test/junk', swob.HTTPNoContent,
                            {}, None)
        self.swift.register('HEAD', '/v1/AUTH_test/nojunk', swob.HTTPNotFound,
                            {}, None)
        self.swift.register('GET', '/v1/AUTH_test/junk', swob.HTTPOk, {},
                            object_list)
        self.swift.register('GET', '/v1/AUTH_test/junk_subdir', swob.HTTPOk,
                            {}, json.dumps(object_list_subdir))
Пример #2
0
 def test_v2_obj_response(self):
     req = Request.blank('/endpoints/v2/a/c/o1')
     resp = req.get_response(self.list_endpoints)
     expected = {
         'endpoints': ["http://10.1.1.1:6000/sdb1/1/a/c/o1",
                       "http://10.1.2.2:6000/sdd1/1/a/c/o1"],
         'headers': {'X-Backend-Storage-Policy-Index': "0"},
     }
     self.assertEqual(resp.body, json.dumps(expected))
     for policy in POLICIES:
         patch_path = 'swift.common.middleware.list_endpoints' \
             '.get_container_info'
         mock_get_container_info = lambda *args, **kwargs: \
             {'storage_policy': int(policy)}
         with mock.patch(patch_path, mock_get_container_info):
             resp = req.get_response(self.list_endpoints)
         part, nodes = policy.object_ring.get_nodes('a', 'c', 'o1')
         [node.update({'part': part}) for node in nodes]
         path = 'http://%(ip)s:%(port)s/%(device)s/%(part)s/a/c/o1'
         expected = {
             'headers': {
                 'X-Backend-Storage-Policy-Index': str(int(policy))},
             'endpoints': [path % node for node in nodes],
         }
         self.assertEqual(resp.body, json.dumps(expected))
Пример #3
0
    def test_parse_input(self):
        self.assertRaises(HTTPException, slo.parse_input, "some non json")
        data = json.dumps([{"path": "/cont/object", "etag": "etagoftheobjecitsegment", "size_bytes": 100}])
        self.assertEquals("/cont/object", slo.parse_input(data)[0]["path"])

        bad_data = json.dumps([{"path": "/cont/object", "size_bytes": 100}])
        self.assertRaises(HTTPException, slo.parse_input, bad_data)
Пример #4
0
    def test_parse_input(self):
        self.assertRaises(HTTPException, slo.parse_input, 'some non json')
        data = json.dumps(
            [{'path': '/cont/object', 'etag': 'etagoftheobjecitsegment',
              'size_bytes': 100}])
        self.assertEquals('/cont/object',
                          slo.parse_input(data)[0]['path'])

        bad_data = json.dumps([{'path': '/cont/object', 'size_bytes': 100}])
        self.assertRaises(HTTPException, slo.parse_input, bad_data)
Пример #5
0
    def getAll(self):
        """
        Dump everything, used for debugging.
        """
        obj_data = [record.oRecordData for record in self.conn.query("SELECT FROM Metadata")]

        return ''.join([
            json.dumps(obj_data), "\n\n", json.dumps(con_data), "\n\n",
            json.dumps(acc_data)
        ])
Пример #6
0
def get_response_body(data_format, data_dict, error_list):
    """
    Returns a properly formatted response body according to format. Handles
    json and xml, otherwise will return text/plain. Note: xml response does not
    include xml declaration.
    :params data_format: resulting format
    :params data_dict: generated data about results.
    :params error_list: list of quoted filenames that failed
    """
    if data_format == "application/json":
        data_dict["Errors"] = error_list
        return json.dumps(data_dict)
    if data_format and data_format.endswith("/xml"):
        output = "<delete>\n"
        for key in sorted(data_dict.keys()):
            xml_key = key.replace(" ", "_").lower()
            output += "<%s>%s</%s>\n" % (xml_key, data_dict[key], xml_key)
        output += "<errors>\n"
        output += "\n".join(
            [
                "<object>" "<name>%s</name><status>%s</status>" "</object>" % (saxutils.escape(name), status)
                for name, status in error_list
            ]
        )
        output += "</errors>\n</delete>\n"
        return output

    output = ""
    for key in sorted(data_dict.keys()):
        output += "%s: %s\n" % (key, data_dict[key])
    output += "Errors:\n"
    output += "\n".join(["%s, %s" % (name, status) for name, status in error_list])
    return output
Пример #7
0
    def verify_request(self,req,env):
        
        try:
            version, account, container, obj = split_path(req.path_info,
                minsegs=1, maxsegs=4, rest_with_last=True)
        except ValueError:
            self.logger.increment('errors')
            return jresponse('-1','not found',req,404)
        
        token = env.get('HTTP_X_AUTH_TOKEN', env.get('HTTP_X_STORAGE_TOKEN'))
        verify_flag = False
        if token :
        
            user_info = self.get_cache_user_info(env, token)
            if user_info:
                
                tenant = 'AUTH_' + user_info.replace('@','').replace('.','')
                if account != tenant:
                    self.logger.increment('unauthorized')
                    verify_flag = False

                verify_flag = True
            else:
               
                self.logger.increment('unauthorized')
                verify_flag = False
            
        else:
            self.logger.increment('unauthorized')
            verify_flag = False
        
        oauth_data_list = json.dumps({'verify_flag':str(verify_flag).lower()})
        return Response(body=oauth_data_list,request=req)
Пример #8
0
    def _reclaim(self, conn, timestamp):
        """
        Removes any empty metadata values older than the timestamp using the
        given database connection. This function will not call commit on the
        conn, but will instead return True if the database needs committing.
        This function was created as a worker to limit transactions and commits
        from other related functions.

        :param conn: Database connection to reclaim metadata within.
        :param timestamp: Empty metadata items last updated before this
                          timestamp will be removed.
        :returns: True if conn.commit() should be called
        """
        try:
            md = conn.execute('SELECT metadata FROM %s_stat' %
                              self.db_type).fetchone()[0]
            if md:
                md = json.loads(md)
                keys_to_delete = []
                for key, (value, value_timestamp) in md.iteritems():
                    if value == '' and value_timestamp < timestamp:
                        keys_to_delete.append(key)
                if keys_to_delete:
                    for key in keys_to_delete:
                        del md[key]
                    conn.execute('UPDATE %s_stat SET metadata = ?' %
                                 self.db_type, (json.dumps(md),))
                    return True
        except sqlite3.OperationalError as err:
            if 'no such column: metadata' not in str(err):
                raise
        return False
Пример #9
0
    def test_handle_multipart_put_check_data_bad(self):
        bad_data = json.dumps(
            [
                {"path": "/checktest/a_1", "etag": "a", "size_bytes": "2"},
                {"path": "/checktest/badreq", "etag": "a", "size_bytes": "1"},
                {"path": "/checktest/b_2", "etag": "not-b", "size_bytes": "2"},
                {"path": "/checktest/slob", "etag": "not-slob", "size_bytes": "2"},
            ]
        )
        req = Request.blank(
            "/v1/AUTH_test/checktest/man?multipart-manifest=put",
            environ={"REQUEST_METHOD": "PUT"},
            headers={"Accept": "application/json"},
            body=bad_data,
        )

        status, headers, body = self.call_slo(req)
        self.assertEquals(self.app.call_count, 4)
        errors = json.loads(body)["Errors"]
        self.assertEquals(len(errors), 5)
        self.assertEquals(errors[0][0], "/checktest/a_1")
        self.assertEquals(errors[0][1], "Size Mismatch")
        self.assertEquals(errors[1][0], "/checktest/badreq")
        self.assertEquals(errors[1][1], "400 Bad Request")
        self.assertEquals(errors[2][0], "/checktest/b_2")
        self.assertEquals(errors[2][1], "Etag Mismatch")
        self.assertEquals(errors[3][0], "/checktest/slob")
        self.assertEquals(errors[3][1], "Size Mismatch")
        self.assertEquals(errors[4][0], "/checktest/slob")
        self.assertEquals(errors[4][1], "Etag Mismatch")
Пример #10
0
 def get_response_body(self, data_format, data_dict, error_list):
     """
     Returns a properly formatted response body according to format.
     :params data_format: resulting format
     :params data_dict: generated data about results.
     :params error_list: list of quoted filenames that failed
     """
     if data_format == "text/plain":
         output = ""
         for key in sorted(data_dict.keys()):
             output += "%s: %s\n" % (key, data_dict[key])
         output += "Errors:\n"
         output += "\n".join(["%s, %s" % (name, status) for name, status in error_list])
         return output
     if data_format == "application/json":
         data_dict["Errors"] = error_list
         return json.dumps(data_dict)
     if data_format.endswith("/xml"):
         output = '<?xml version="1.0" encoding="UTF-8"?>\n<delete>\n'
         for key in sorted(data_dict.keys()):
             xml_key = key.replace(" ", "_").lower()
             output += "<%s>%s</%s>\n" % (xml_key, data_dict[key], xml_key)
         output += "<errors>\n"
         output += "\n".join(
             [
                 "<object>" "<name>%s</name><status>%s</status>" "</object>" % (saxutils.escape(name), status)
                 for name, status in error_list
             ]
         )
         output += "</errors>\n</delete>\n"
         return output
     raise HTTPNotAcceptable("Invalid output type")
Пример #11
0
def get_response_body(data_format, data_dict, error_list):
    """
    Returns a properly formatted response body according to format.
    :params data_format: resulting format
    :params data_dict: generated data about results.
    :params error_list: list of quoted filenames that failed
    """
    if data_format == 'text/plain':
        output = ''
        for key in sorted(data_dict.keys()):
            output += '%s: %s\n' % (key, data_dict[key])
        output += 'Errors:\n'
        output += '\n'.join(
            ['%s, %s' % (name, status)
             for name, status in error_list])
        return output
    if data_format == 'application/json':
        data_dict['Errors'] = error_list
        return json.dumps(data_dict)
    if data_format.endswith('/xml'):
        output = '<?xml version="1.0" encoding="UTF-8"?>\n<delete>\n'
        for key in sorted(data_dict.keys()):
            xml_key = key.replace(' ', '_').lower()
            output += '<%s>%s</%s>\n' % (xml_key, data_dict[key], xml_key)
        output += '<errors>\n'
        output += '\n'.join(
            ['<object>'
             '<name>%s</name><status>%s</status>'
             '</object>' % (saxutils.escape(name), status) for
             name, status in error_list])
        output += '</errors>\n</delete>\n'
        return output
    raise HTTPNotAcceptable('Invalid output type')
Пример #12
0
 def test_container_usage_error_key(self):
     self.conf['quota'] = json.dumps(
         {
             'container_count': {
                 'default': 5,
                 'L10': 10
             },
             'object_count': {
                 'default': 5,
                 'L10': 10
             },
             'container_usage': {
                 'default': 5,
                 'L10': 10
             }
         }
     )
     qa = quota.filter_factory(self.conf)(FakeApp())
     req = Request.blank('/v1/a2/c/obj')
     req.method = 'PUT'
     req.environ['CONTENT_LENGTH'] = 1024
     # no memcached
     resp = req.get_response(qa)
     self.assertEquals(resp.status_int, 201)
     # no cache
     req.environ['swift.cache'] = FakeMemcache()
     resp = req.get_response(qa)
     self.assertEquals(resp.status_int, 201)
     # cached
     resp = req.get_response(qa)
     self.assertEquals(resp.status_int, 201)
Пример #13
0
 def test_handle_multipart_put_check_data_bad(self):
     bad_data = json.dumps(
         [
             {"path": "/c/a_1", "etag": "a", "size_bytes": "1"},
             {"path": "/c/a_2", "etag": "a", "size_bytes": "1"},
             {"path": "/d/b_2", "etag": "b", "size_bytes": "2"},
         ]
     )
     req = Request.blank(
         "/test_good/A/c/man?multipart-manifest=put",
         environ={"REQUEST_METHOD": "PUT"},
         headers={"Accept": "application/json"},
         body=bad_data,
     )
     try:
         self.slo.handle_multipart_put(req)
     except HTTPException, e:
         self.assertEquals(self.app.calls, 3)
         data = json.loads(e.body)
         errors = data["Errors"]
         self.assertEquals(errors[0][0], "/test_good/A/c/a_1")
         self.assertEquals(errors[0][1], "Size Mismatch")
         self.assertEquals(errors[2][1], "400 Bad Request")
         self.assertEquals(errors[-1][0], "/test_good/A/d/b_2")
         self.assertEquals(errors[-1][1], "Etag Mismatch")
Пример #14
0
 def _handle_sync_request(self, broker, remote_info):
     """
     Update metadata, timestamps, sync points.
     """
     with self.debug_timing('info'):
         try:
             info = self._get_synced_replication_info(broker, remote_info)
         except (Exception, Timeout) as e:
             if 'no such table' in str(e):
                 self.logger.error(_("Quarantining DB %s"), broker)
                 quarantine_db(broker.db_file, broker.db_type)
                 return HTTPNotFound()
             raise
     if remote_info['metadata']:
         with self.debug_timing('update_metadata'):
             broker.update_metadata(remote_info['metadata'])
     sync_timestamps = ('created_at', 'put_timestamp', 'delete_timestamp')
     if any(info[ts] != remote_info[ts] for ts in sync_timestamps):
         with self.debug_timing('merge_timestamps'):
             broker.merge_timestamps(*(remote_info[ts] for ts in
                                       sync_timestamps))
     with self.debug_timing('get_sync'):
         info['point'] = broker.get_sync(remote_info['id'])
     if remote_info['hash'] == info['hash'] and \
             info['point'] < remote_info['point']:
         with self.debug_timing('merge_syncs'):
             translate = {
                 'remote_id': 'id',
                 'sync_point': 'point',
             }
             data = dict((k, remote_info[v]) for k, v in translate.items())
             broker.merge_syncs([data])
             info['point'] = remote_info['point']
     return Response(json.dumps(info))
Пример #15
0
 def object_update(self, req, broker, name, timestamp):
     metadata = json.dumps(dict([val for val in req.headers.iteritems()
                                 if is_user_meta('object', val[0])]))
     broker.put_object(name, timestamp, int(req.headers['x-size']),
                       req.headers['x-content-type'],
                       req.headers['x-etag'],
                       metadata=metadata)
Пример #16
0
def get_response_body(data_format, data_dict, error_list):
    """
    Returns a properly formatted response body according to format. Handles
    json and xml, otherwise will return text/plain. Note: xml response does not
    include xml declaration.
    :params data_format: resulting format
    :params data_dict: generated data about results.
    :params error_list: list of quoted filenames that failed
    """
    if data_format == 'application/json':
        data_dict['Errors'] = error_list
        return json.dumps(data_dict)
    if data_format and data_format.endswith('/xml'):
        output = '<delete>\n'
        for key in sorted(data_dict):
            xml_key = key.replace(' ', '_').lower()
            output += '<%s>%s</%s>\n' % (xml_key, data_dict[key], xml_key)
        output += '<errors>\n'
        output += '\n'.join(
            ['<object>'
             '<name>%s</name><status>%s</status>'
             '</object>' % (saxutils.escape(name), status) for
             name, status in error_list])
        output += '</errors>\n</delete>\n'
        return output

    output = ''
    for key in sorted(data_dict):
        output += '%s: %s\n' % (key, data_dict[key])
    output += 'Errors:\n'
    output += '\n'.join(
        ['%s, %s' % (name, status)
         for name, status in error_list])
    return output
Пример #17
0
 def test_handle_multipart_put_check_data_bad(self):
     bad_data = json.dumps(
         [{'path': '/c/a_1', 'etag': 'a', 'size_bytes': '1'},
          {'path': '/c/a_2', 'etag': 'a', 'size_bytes': '1'},
          {'path': '/d/b_2', 'etag': 'b', 'size_bytes': '2'},
          {'path': '/d/slob', 'etag': 'a', 'size_bytes': '2'}])
     req = Request.blank(
         '/test_good/A/c/man?multipart-manifest=put',
         environ={'REQUEST_METHOD': 'PUT'},
         headers={'Accept': 'application/json'},
         body=bad_data)
     try:
         self.slo.handle_multipart_put(req, fake_start_response)
     except HTTPException as e:
         self.assertEquals(self.app.calls, 4)
         data = json.loads(e.body)
         errors = data['Errors']
         self.assertEquals(errors[0][0], '/c/a_1')
         self.assertEquals(errors[0][1], 'Size Mismatch')
         self.assertEquals(errors[2][0], '/c/a_2')
         self.assertEquals(errors[2][1], '400 Bad Request')
         self.assertEquals(errors[4][0], '/d/b_2')
         self.assertEquals(errors[4][1], 'Etag Mismatch')
         self.assertEquals(errors[-1][0], '/d/slob')
         self.assertEquals(errors[-1][1], 'Etag Mismatch')
     else:
         self.assert_(False)
Пример #18
0
def jresponse(status,msg,req,status_int,headers=None,statusstr='',param=None):
    
    msg = msg.lower()
    data = {'status':str(status),'msg':str(msg)}
    
    if param:
        data.update(param)
        
    container_list = json.dumps(data)
    if headers:
        ret = Response(body=container_list, request=req,headers=headers)
    else:
        ret = Response(body=container_list, request=req)
        
    ret.content_type = 'application/json'
    ret.charset = 'utf-8'
    ret.status_int = status_int
    if statusstr:
        ret.status = statusstr
        
    if status != '0' and req.method == 'PUT': 
        pass
        # syslog.syslog(syslog.LOG_ERR,'jresponse:   status: ' + str(status_int)+'  path: '+str(req.path)+'  msg: '+str(msg) + '  tx_id:  '+ req.environ.get('swift.trans_id',''))
        # print 'jresponse:   status: ' + str(status_int)+'  path: '+str(req.path)+'  msg: '+str(msg) + '  tx_id:  '+ req.environ.get('swift.trans_id','') + '  method:  '+str(req.method) + '  headers:  '+str(req.headers) + '  params:  '+str(req.GET)
        # traceback.print_stack() 
        # syslog.syslog(syslog.LOG_ERR,'jresponse:   '+str(traceback.extract_stack() ))     
        # syslog.syslog(syslog.LOG_ERR,'jresponse:   '+str(traceback.print_stack() ))
    return ret
Пример #19
0
    def set_multi(self, mapping, server_key, serialize=True, timeout=0):
        """
        Sets multiple key/value pairs in memcache.

        :param mapping: dictonary of keys and values to be set in memcache
        :param servery_key: key to use in determining which server in the ring
                            is used
        :param serialize: if True, value is serialized with JSON before sending
                          to memcache, or with pickle if configured to use
                          pickle instead of JSON (to avoid cache poisoning)
        :param timeout: ttl for memcache
        """
        server_key = md5hash(server_key)
        timeout = sanitize_timeout(timeout)
        msg = ''
        for key, value in mapping.iteritems():
            key = md5hash(key)
            flags = 0
            if serialize and self._allow_pickle:
                value = pickle.dumps(value, PICKLE_PROTOCOL)
                flags |= PICKLE_FLAG
            elif serialize:
                value = json.dumps(value)
                flags |= JSON_FLAG
            msg += ('set %s %d %d %s noreply\r\n%s\r\n' %
                    (key, flags, timeout, len(value), value))
        for (server, fp, sock) in self._get_conns(server_key):
            try:
                sock.sendall(msg)
                self._return_conn(server, fp, sock)
                return
            except Exception, e:
                self._exception_occurred(server, e)
Пример #20
0
    def set(self, key, value, serialize=True, timeout=0):
        """
        Set a key/value pair in memcache

        :param key: key
        :param value: value
        :param serialize: if True, value is serialized with JSON before sending
                          to memcache, or with pickle if configured to use
                          pickle instead of JSON (to avoid cache poisoning)
        :param timeout: ttl in memcache
        """
        key = md5hash(key)
        timeout = sanitize_timeout(timeout)
        flags = 0
        if serialize and self._allow_pickle:
            value = pickle.dumps(value, PICKLE_PROTOCOL)
            flags |= PICKLE_FLAG
        elif serialize:
            value = json.dumps(value)
            flags |= JSON_FLAG
        for (server, fp, sock) in self._get_conns(key):
            try:
                sock.sendall('set %s %d %d %s noreply\r\n%s\r\n' %
                             (key, flags, timeout, len(value), value))
                self._return_conn(server, fp, sock)
                return
            except Exception, e:
                self._exception_occurred(server, e)
Пример #21
0
 def uploadObj(self, a, c, o):
     metaListO = [self.getTestObjDict(a, c, o)]
     reqO = Request.blank(
         '/', environ={'REQUEST_METHOD': 'PUT',
         'HTTP_X_TIMESTAMP': '0'}, headers={'user-agent': 'object_crawler'},
         body=json.dumps(metaListO))
     respO = reqO.get_response(self.controller)
     self.assert_(respO.status.startswith('204'))
Пример #22
0
    def getAll(self):
        """
        Dump everything
        """
        self.conn.row_factory = dict_factory
        cur = self.conn.cursor()
        cur.execute("SELECT * FROM object_metadata")
        obj_data = cur.fetchall()
        cur.execute("SELECT * FROM container_metadata")
        con_data = cur.fetchall()
        cur.execute("SELECT * FROM account_metadata")
        acc_data = cur.fetchall()

        return ''.join([
            json.dumps(obj_data), "\n\n", json.dumps(con_data), "\n\n",
            json.dumps(acc_data)
        ])
Пример #23
0
def generate_s3acl_environ(account, swift, owner):

    def gen_grant(permission):
        # generate Grant with a grantee named by "permission"
        account_name = '%s:%s' % (account, permission.lower())
        return Grant(User(account_name), permission)

    grants = map(gen_grant, PERMISSIONS)
    container_headers = _gen_test_headers(owner, grants)
    object_headers = _gen_test_headers(owner, grants, 'object')
    object_body = 'hello'
    object_headers['Content-Length'] = len(object_body)

    # TEST method is used to resolve a tenant name
    swift.register('TEST', '/v1/AUTH_test', swob.HTTPMethodNotAllowed,
                   {}, None)
    swift.register('TEST', '/v1/AUTH_X', swob.HTTPMethodNotAllowed,
                   {}, None)

    # for bucket
    swift.register('HEAD', '/v1/AUTH_test/bucket', swob.HTTPNoContent,
                   container_headers, None)
    swift.register('HEAD', '/v1/AUTH_test/bucket+segments', swob.HTTPNoContent,
                   container_headers, None)
    swift.register('PUT', '/v1/AUTH_test/bucket',
                   swob.HTTPCreated, {}, None)
    swift.register('GET', '/v1/AUTH_test/bucket', swob.HTTPNoContent,
                   container_headers, json.dumps([]))
    swift.register('POST', '/v1/AUTH_test/bucket',
                   swob.HTTPNoContent, {}, None)
    swift.register('DELETE', '/v1/AUTH_test/bucket',
                   swob.HTTPNoContent, {}, None)

    # necessary for canned-acl tests
    public_headers = _gen_test_headers(owner, [Grant(AllUsers(), 'READ')])
    swift.register('GET', '/v1/AUTH_test/public', swob.HTTPNoContent,
                   public_headers, json.dumps([]))
    authenticated_headers = _gen_test_headers(
        owner, [Grant(AuthenticatedUsers(), 'READ')], 'bucket')
    swift.register('GET', '/v1/AUTH_test/authenticated',
                   swob.HTTPNoContent, authenticated_headers,
                   json.dumps([]))

    # for object
    swift.register('HEAD', '/v1/AUTH_test/bucket/object', swob.HTTPOk,
                   object_headers, None)
Пример #24
0
def create_bucket_list_json(buckets):
    """
    Create a json from bucket list
    :param buckets: a list of tuples (or lists) consist of elements orderd as
                    name, count, bytes
    """
    bucket_list = [{'name': item[0], 'count': item[1], 'bytes': item[2]}
                   for item in buckets]
    return json.dumps(bucket_list)
Пример #25
0
 def uploadAcc(self, a):
     metaListA = [self.getTestAccDict(a)]
     reqA = Request.blank(
         '/', environ={'REQUEST_METHOD': 'PUT',
         'HTTP_X_TIMESTAMP': '0'},
         headers={'user-agent': 'account_crawler'},
         body=json.dumps(metaListA))
     respA = reqA.get_response(self.controller)
     self.assert_(respA.status.startswith('204'))
Пример #26
0
 def uploadCon(self, a, c):
     metaListC = [self.getTestConDict(a, c)]
     reqC = Request.blank(
         '/', environ={'REQUEST_METHOD': 'PUT',
         'HTTP_X_TIMESTAMP': '0'},
         headers={'user-agent': 'container_crawler'},
         body=json.dumps(metaListC))
     respC = reqC.get_response(self.controller)
     self.assert_(respC.status.startswith('204'))
Пример #27
0
    def setup_objects(self):
        self.objects = (('rose', '2011-01-05T02:19:14.275290', 0, 303),
                        ('viola', '2011-01-05T02:19:14.275290', '0', 3909),
                        ('lily', '2011-01-05T02:19:14.275290', '0', '3909'),
                        ('mu', '2011-01-05T02:19:14.275290',
                         'md5-of-the-manifest; s3_etag=0', '3909'),
                        ('with space', '2011-01-05T02:19:14.275290', 0, 390),
                        ('with%20space', '2011-01-05T02:19:14.275290', 0, 390))

        objects = [
            {'name': str(item[0]), 'last_modified': str(item[1]),
             'hash': str(item[2]), 'bytes': str(item[3])}
            for item in self.objects]
        object_list = json.dumps(objects)

        self.prefixes = ['rose', 'viola', 'lily']
        object_list_subdir = [{"subdir": p} for p in self.prefixes]

        self.swift.register('DELETE', '/v1/AUTH_test/bucket+segments',
                            swob.HTTPNoContent, {}, json.dumps([]))
        for name, _, _, _ in self.objects:
            self.swift.register(
                'DELETE', '/v1/AUTH_test/bucket+segments/' + name,
                swob.HTTPNoContent, {}, json.dumps([]))
        self.swift.register(
            'GET',
            '/v1/AUTH_test/bucket+segments?format=json&marker=with%2520space',
            swob.HTTPOk,
            {'Content-Type': 'application/json; charset=utf-8'},
            json.dumps([]))
        self.swift.register(
            'GET', '/v1/AUTH_test/bucket+segments?format=json&marker=',
            swob.HTTPOk, {'Content-Type': 'application/json'}, object_list)
        self.swift.register(
            'HEAD', '/v1/AUTH_test/junk', swob.HTTPNoContent, {}, None)
        self.swift.register(
            'HEAD', '/v1/AUTH_test/nojunk', swob.HTTPNotFound, {}, None)
        self.swift.register(
            'GET', '/v1/AUTH_test/junk', swob.HTTPOk,
            {'Content-Type': 'application/json'}, object_list)
        self.swift.register(
            'GET',
            '/v1/AUTH_test/junk?delimiter=a&format=json&limit=3&marker=viola',
            swob.HTTPOk,
            {'Content-Type': 'application/json; charset=utf-8'},
            json.dumps(objects[2:]))
        self.swift.register(
            'GET', '/v1/AUTH_test/junk-subdir', swob.HTTPOk,
            {'Content-Type': 'application/json; charset=utf-8'},
            json.dumps(object_list_subdir))
        self.swift.register(
            'GET',
            '/v1/AUTH_test/subdirs?delimiter=/&format=json&limit=3',
            swob.HTTPOk, {}, json.dumps([
                {'subdir': 'nothing/'},
                {'subdir': 'but/'},
                {'subdir': 'subdirs/'},
            ]))
Пример #28
0
def account_listing_response(account, req, response_content_type, broker=None,
                             limit='', marker='', end_marker='', prefix='',
                             delimiter='', reverse=False):
    """
    This is an exact copy of swift.account.utis.account_listing_response()
    except for one difference i.e this method passes response_content_type
    to broker.list_containers_iter() method.
    """
    if broker is None:
        broker = FakeAccountBroker()

    resp_headers = get_response_headers(broker)

    account_list = broker.list_containers_iter(limit, marker, end_marker,
                                               prefix, delimiter,
                                               response_content_type, reverse)
    if response_content_type == 'application/json':
        data = []
        for (name, object_count, bytes_used, put_tstamp,
             is_subdir) in account_list:
            if is_subdir:
                data.append({'subdir': name})
            else:
                data.append({'name': name, 'count': object_count,
                             'bytes': bytes_used,
                             'last_modified': Timestamp(put_tstamp).isoformat})
        account_list = json.dumps(data)
    elif response_content_type.endswith('/xml'):
        output_list = ['<?xml version="1.0" encoding="UTF-8"?>',
                       '<account name=%s>' % saxutils.quoteattr(account)]
        for (name, object_count, bytes_used, put_tstamp,
             is_subdir) in account_list:
            if is_subdir:
                output_list.append(
                    '<subdir name=%s />' % saxutils.quoteattr(name))
            else:
                item = '<container><name>%s</name><count>%s</count>' \
                       '<bytes>%s</bytes><last_modified>%s</last_modified> \
                        </container>' % \
                       (saxutils.escape(name), object_count, bytes_used,
                        Timestamp(put_tstamp).isoformat)
                output_list.append(item)
        output_list.append('</account>')
        account_list = '\n'.join(output_list)
    else:
        if not account_list:
            resp = HTTPNoContent(request=req, headers=resp_headers)
            resp.content_type = response_content_type
            resp.charset = 'utf-8'
            return resp
        account_list = '\n'.join(r[0] for r in account_list) + '\n'
    ret = HTTPOk(body=account_list, request=req, headers=resp_headers)
    ret.content_type = response_content_type
    ret.charset = 'utf-8'
    return ret
Пример #29
0
    def GETorHEAD(self, req):
        """Handler for HTTP GET/HEAD requests."""
        """
        Handles requests to /info
        Should return a WSGI-style callable (such as swob.Response).

        :param req: swob.Request object
        """
        if not self.expose_info:
            return HTTPForbidden(request=req)

        admin_request = False
        sig = req.params.get('swiftinfo_sig', '')
        expires = req.params.get('swiftinfo_expires', '')

        if sig != '' or expires != '':
            admin_request = True
            if not self.admin_key:
                return HTTPForbidden(request=req)
            try:
                expires = int(expires)
            except ValueError:
                return HTTPUnauthorized(request=req)
            if expires < time():
                return HTTPUnauthorized(request=req)

            valid_sigs = []
            for method in self.allowed_hmac_methods[req.method]:
                valid_sigs.append(get_hmac(method,
                                           '/info',
                                           expires,
                                           self.admin_key))

            # While it's true that any() will short-circuit, this doesn't
            # affect the timing-attack resistance since the only way this will
            # short-circuit is when a valid signature is passed in.
            is_valid_hmac = any(streq_const_time(valid_sig, sig)
                                for valid_sig in valid_sigs)
            if not is_valid_hmac:
                return HTTPUnauthorized(request=req)

        headers = {}
        if 'Origin' in req.headers:
            headers['Access-Control-Allow-Origin'] = req.headers['Origin']
            headers['Access-Control-Expose-Headers'] = ', '.join(
                ['x-trans-id'])

        #json.dumps(dict)可以将字典形式的dict对象转换为json格式的对象
        info = json.dumps(get_swift_info(
            admin=admin_request, disallowed_sections=self.disallowed_sections))

        return HTTPOk(request=req,
                      headers=headers,
                      body=info,
                      content_type='application/json; charset=UTF-8')
Пример #30
0
    def __call__(self, env, start_response):
        request = Request(env)

        if not request.path.startswith(self.endpoints_path):
            return self.app(env, start_response)

        if request.method != 'GET':
            return HTTPMethodNotAllowed(
                req=request, headers={"Allow": "GET"})(env, start_response)

        try:
            clean_path = request.path[len(self.endpoints_path) - 1:]
            account, container, obj = \
                split_path(clean_path, 1, 3, True)
        except ValueError:
            return HTTPBadRequest('No account specified')(env, start_response)

        if account is not None:
            account = unquote(account)
        if container is not None:
            container = unquote(container)
        if obj is not None:
            obj = unquote(obj)

        if obj is not None:
            partition, nodes = self.object_ring.get_nodes(
                account, container, obj)
            endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \
                                '{account}/{container}/{obj}'
        elif container is not None:
            partition, nodes = self.container_ring.get_nodes(
                account, container)
            endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \
                                '{account}/{container}'
        else:
            partition, nodes = self.account_ring.get_nodes(
                account)
            endpoint_template = 'http://{ip}:{port}/{device}/{partition}/' + \
                                '{account}'

        endpoints = []
        for node in nodes:
            endpoint = endpoint_template.format(
                ip=node['ip'],
                port=node['port'],
                device=node['device'],
                partition=partition,
                account=quote(account),
                container=quote(container or ''),
                obj=quote(obj or ''))
            endpoints.append(endpoint)

        return Response(json.dumps(endpoints),
                        content_type='application/json')(env, start_response)
Пример #31
0
    def setup_objects(self):
        self.objects = (('lily', '2011-01-05T02:19:14.275290', '0', '3909'),
                        ('rose', '2011-01-05T02:19:14.275290', 0, 303),
                        ('viola', '2011-01-05T02:19:14.275290', '0', 3909),
                        (u'lily-\u062a', '2011-01-05T02:19:14.275290', 0,
                         390), ('mu', '2011-01-05T02:19:14.275290',
                                'md5-of-the-manifest; s3_etag=0', '3909'),
                        ('with space', '2011-01-05T02:19:14.275290', 0, 390),
                        ('with%20space', '2011-01-05T02:19:14.275290', 0, 390))

        objects = [{
            'name': item[0],
            'last_modified': str(item[1]),
            'hash': str(item[2]),
            'bytes': str(item[3])
        } for item in self.objects]
        object_list = json.dumps(objects)

        self.prefixes = ['rose', 'viola', 'lily']
        object_list_subdir = [{"subdir": p} for p in self.prefixes]

        self.swift.register('DELETE', '/v1/AUTH_test/bucket+segments',
                            swob.HTTPNoContent, {}, json.dumps([]))
        for name, _, _, _ in self.objects:
            self.swift.register(
                'DELETE',
                '/v1/AUTH_test/bucket+segments/' + name.encode('utf-8'),
                swob.HTTPNoContent, {}, json.dumps([]))
        self.swift.register(
            'GET',
            '/v1/AUTH_test/bucket+segments?format=json&marker=with%2520space',
            swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
            json.dumps([]))
        self.swift.register(
            'GET', '/v1/AUTH_test/bucket+segments?format=json&marker=',
            swob.HTTPOk, {'Content-Type': 'application/json'}, object_list)
        self.swift.register('HEAD', '/v1/AUTH_test/junk', swob.HTTPNoContent,
                            {}, None)
        self.swift.register('HEAD', '/v1/AUTH_test/nojunk', swob.HTTPNotFound,
                            {}, None)
        self.swift.register('GET', '/v1/AUTH_test/junk', swob.HTTPOk,
                            {'Content-Type': 'application/json'}, object_list)
        self.swift.register(
            'GET',
            '/v1/AUTH_test/junk?delimiter=a&format=json&limit=3&marker=viola',
            swob.HTTPOk, {'Content-Type': 'application/json; charset=utf-8'},
            json.dumps(objects[2:]))
        self.swift.register(
            'GET', '/v1/AUTH_test/junk-subdir', swob.HTTPOk,
            {'Content-Type': 'application/json; charset=utf-8'},
            json.dumps(object_list_subdir))
        self.swift.register(
            'GET', '/v1/AUTH_test/subdirs?delimiter=/&format=json&limit=3',
            swob.HTTPOk, {},
            json.dumps([
                {
                    'subdir': 'nothing/'
                },
                {
                    'subdir': 'but/'
                },
                {
                    'subdir': 'subdirs/'
                },
            ]))
Пример #32
0
     data = []
     for (name, created_at, size, content_type, etag) in container_list:
         if content_type is None:
             data.append({"subdir": name})
         else:
             created_at = datetime.utcfromtimestamp(
                 float(created_at)).isoformat()
             # python isoformat() doesn't include msecs when zero
             if len(created_at) < len("1970-01-01T00:00:00.000000"):
                 created_at += ".000000"
             content_type, size = self.derive_content_type_metadata(
                 content_type, size)
             data.append({'last_modified': created_at, 'bytes': size,
                         'content_type': content_type, 'hash': etag,
                         'name': name})
     container_list = json.dumps(data)
 elif out_content_type.endswith('/xml'):
     xml_output = []
     for (name, created_at, size, content_type, etag) in container_list:
         # escape name and format date here
         name = saxutils.escape(name)
         created_at = datetime.utcfromtimestamp(
             float(created_at)).isoformat()
         # python isoformat() doesn't include msecs when zero
         if len(created_at) < len("1970-01-01T00:00:00.000000"):
             created_at += ".000000"
         if content_type is None:
             xml_output.append('<subdir name="%s"><name>%s</name>'
                               '</subdir>' % (name, name))
         else:
             content_type, size = self.derive_content_type_metadata(
Пример #33
0
 def audit_all_objects(self, mode='once'):
     self.logger.info(
         _('Begin object audit "%s" mode (%s)') % (mode, self.auditor_type))
     begin = reported = time.time()
     self.total_bytes_processed = 0
     self.total_files_processed = 0
     total_quarantines = 0
     total_errors = 0
     time_auditing = 0
     all_locs = audit_location_generator(self.devices,
                                         object_server.DATADIR,
                                         '.data',
                                         mount_check=self.mount_check,
                                         logger=self.logger)
     for path, device, partition in all_locs:
         loop_time = time.time()
         self.failsafe_object_audit(path, device, partition)
         self.logger.timing_since('timing', loop_time)
         self.files_running_time = ratelimit_sleep(
             self.files_running_time, self.max_files_per_second)
         self.total_files_processed += 1
         now = time.time()
         if now - reported >= self.log_time:
             self.logger.info(
                 _('Object audit (%(type)s). '
                   'Since %(start_time)s: Locally: %(passes)d passed, '
                   '%(quars)d quarantined, %(errors)d errors '
                   'files/sec: %(frate).2f , bytes/sec: %(brate).2f, '
                   'Total time: %(total).2f, Auditing time: %(audit).2f, '
                   'Rate: %(audit_rate).2f') % {
                       'type': self.auditor_type,
                       'start_time': time.ctime(reported),
                       'passes': self.passes,
                       'quars': self.quarantines,
                       'errors': self.errors,
                       'frate': self.passes / (now - reported),
                       'brate': self.bytes_processed / (now - reported),
                       'total': (now - begin),
                       'audit': time_auditing,
                       'audit_rate': time_auditing / (now - begin)
                   })
             dump_recon_cache(
                 {
                     'object_auditor_stats_%s' % self.auditor_type: {
                         'errors': self.errors,
                         'passes': self.passes,
                         'quarantined': self.quarantines,
                         'bytes_processed': self.bytes_processed,
                         'start_time': reported,
                         'audit_time': time_auditing
                     }
                 }, self.rcache, self.logger)
             reported = now
             total_quarantines += self.quarantines
             total_errors += self.errors
             self.passes = 0
             self.quarantines = 0
             self.errors = 0
             self.bytes_processed = 0
         time_auditing += (now - loop_time)
     # Avoid divide by zero during very short runs
     elapsed = (time.time() - begin) or 0.000001
     self.logger.info(
         _('Object audit (%(type)s) "%(mode)s" mode '
           'completed: %(elapsed).02fs. Total quarantined: %(quars)d, '
           'Total errors: %(errors)d, Total files/sec: %(frate).2f , '
           'Total bytes/sec: %(brate).2f, Auditing time: %(audit).2f, '
           'Rate: %(audit_rate).2f') % {
               'type': self.auditor_type,
               'mode': mode,
               'elapsed': elapsed,
               'quars': total_quarantines,
               'errors': total_errors,
               'frate': self.total_files_processed / elapsed,
               'brate': self.total_bytes_processed / elapsed,
               'audit': time_auditing,
               'audit_rate': time_auditing / elapsed
           })
     if self.stats_sizes:
         self.logger.info(
             _('Object audit stats: %s') % json.dumps(self.stats_buckets))
Пример #34
0
    def test_bucket_GET_with_versions(self):
        versioned_objects = [
            {'name': '004rose/2', 'hash': '0', 'bytes': '0',
             'last_modified': '2010-03-01T17:09:51.510928',
             'content_type': DELETE_MARKER_CONTENT_TYPE},
            {'name': '004rose/1', 'hash': '1234', 'bytes': '6',
             'last_modified': '2010-03-01T17:09:50.510928'},
        ]
        self.swift.register('GET', '/v1/AUTH_test/junk+versioning',
                            swob.HTTPOk, {}, json.dumps(versioned_objects))
        for obj in self.objects:
            if obj[0] == 'rose':
                headers = {'X-Object-Sysmeta-Version-Id': '3'}
            else:
                headers = {}
            self.swift.register('HEAD', '/v1/AUTH_test/junk/%s' % obj[0],
                                swob.HTTPOk, headers, None)

        req = Request.blank('/junk?versions',
                            environ={'REQUEST_METHOD': 'GET'},
                            headers={'Authorization': 'AWS test:tester:hmac',
                                     'Date': self.get_date_header()})
        status, headers, body = self.call_swift3(req)

        self.assertEqual(status.split()[0], '200')
        elem = fromstring(body, 'ListVersionsResult')
        self.assertEqual(elem.find('./Name').text, 'junk')
        delete_markers = elem.findall('./DeleteMarker')
        self.assertEqual(len(delete_markers), 1)
        self.assertEqual(delete_markers[0].find('./IsLatest').text, 'false')
        self.assertEqual(delete_markers[0].find('./VersionId').text, '2')
        self.assertEqual(delete_markers[0].find('./Key').text, 'rose')
        versions = elem.findall('./Version')
        self.assertEqual(len(versions), len(self.objects) + 1)
        self.assertEqual(versions[2].find('./IsLatest').text, 'false')
        self.assertEqual(versions[2].find('./VersionId').text, '1')
        # Test that version id is retrieved from sysmeta
        self.assertEqual(versions[1].find('./VersionId').text, '3')
        self.assertEqual(versions[2].find('./Key').text, 'rose')

        # with max keys
        req = Request.blank('/junk?versions&max-keys=3',
                            environ={'REQUEST_METHOD': 'GET'},
                            headers={'Authorization': 'AWS test:tester:hmac',
                                     'Date': self.get_date_header()})
        status, headers, body = self.call_swift3(req)

        self.assertEqual(status.split()[0], '200')
        elem = fromstring(body, 'ListVersionsResult')
        self.assertEqual(elem.find('./MaxKeys').text, '3')
        self.assertEqual(elem.find('./IsTruncated').text, 'true')
        delete_markers = elem.findall('./DeleteMarker')
        self.assertEqual(len(delete_markers), 1)
        versions = elem.findall('./Version')
        self.assertEqual(len(versions), 2)
        self.assertEqual([v.find('./Key').text for v in versions],
                         ['lily', 'rose'])
        self.assertEqual([v.find('./VersionId').text for v in versions],
                         ['null', '3'])
        self.assertEqual([v.find('./IsLatest').text for v in versions],
                         ['true', 'true'])

        # with key-marker
        req = Request.blank('/junk?versions&max-keys=2&key-marker=rose',
                            environ={'REQUEST_METHOD': 'GET'},
                            headers={'Authorization': 'AWS test:tester:hmac',
                                     'Date': self.get_date_header()})
        status, headers, body = self.call_swift3(req)

        self.assertEqual(status.split()[0], '200')
        elem = fromstring(body, 'ListVersionsResult')
        self.assertEqual(elem.find('./MaxKeys').text, '2')
        self.assertEqual(elem.find('./IsTruncated').text, 'true')
        delete_markers = elem.findall('./DeleteMarker')
        self.assertEqual(len(delete_markers), 1)
        self.assertEqual(delete_markers[0].find('./Key').text, 'rose')
        self.assertEqual(delete_markers[0].find('./VersionId').text, '2')
        versions = elem.findall('./Version')
        self.assertEqual(len(versions), 1)
        self.assertEqual(versions[0].find('./Key').text, 'rose')
        self.assertEqual(versions[0].find('./VersionId').text, '3')
        self.assertEqual(versions[0].find('./IsLatest').text, 'true')

        # with key-marker and version-id-marker
        req = Request.blank(
            '/junk?versions&key-marker=rose&version-id-marker=2',
            environ={'REQUEST_METHOD': 'GET'},
            headers={'Authorization': 'AWS test:tester:hmac',
                     'Date': self.get_date_header()})
        status, headers, body = self.call_swift3(req)

        self.assertEqual(status.split()[0], '200')
        elem = fromstring(body, 'ListVersionsResult')
        self.assertEqual(elem.find('./IsTruncated').text, 'false')
        delete_markers = elem.findall('./DeleteMarker')
        self.assertEqual(len(delete_markers), 0)
        versions = elem.findall('./Version')
        self.assertEqual(len(versions), len(self.objects) - 1)
        self.assertEqual(versions[0].find('./Key').text, 'rose')
        self.assertEqual(versions[0].find('./VersionId').text, '1')
        self.assertEqual(versions[0].find('./IsLatest').text, 'false')

        # with key-marker and non-existent version-id-marker
        req = Request.blank(
            '/junk?versions&key-marker=rose&version-id-marker=x',
            environ={'REQUEST_METHOD': 'GET'},
            headers={'Authorization': 'AWS test:tester:hmac',
                     'Date': self.get_date_header()})
        status, headers, body = self.call_swift3(req)

        self.assertEqual(status.split()[0], '200')
        elem = fromstring(body, 'ListVersionsResult')
        self.assertEqual(elem.find('./Name').text, 'junk')
        delete_markers = elem.findall('./DeleteMarker')
        self.assertEqual(len(delete_markers), 0)
        versions = elem.findall('./Version')
        self.assertEqual(len(versions), len(self.objects) - 2)
        self.assertEqual(versions[0].find('./Key').text, 'viola')
        self.assertEqual(versions[0].find('./IsLatest').text, 'true')
        self.assertEqual(versions[0].find('./VersionId').text, 'null')
Пример #35
0
def account_listing_response(account,
                             req,
                             response_content_type,
                             info=None,
                             listing=None,
                             s3_buckets_only=False):
    now = time.time()
    if info is None:
        info = {
            'containers': 0,
            'objects': 0,
            'bytes': 0,
            'metadata': {},
            'ctime': Timestamp(now).internal
        }
    if listing is None:
        listing = []
    elif listing and len(listing[0]) < 5:
        # oio-sds < 4.2 does not return mtime
        listing = [x + [now] for x in listing]

    resp_headers = get_response_headers(info)

    if response_content_type == 'application/json':
        data = []
        for (name, object_count, bytes_used, is_subdir, mtime) in listing:
            if is_subdir:
                if not s3_buckets_only:
                    data.append({'subdir': name})
            else:
                data.append({
                    'name': name,
                    'count': object_count,
                    'bytes': bytes_used,
                    'last_modified': Timestamp(mtime).isoformat
                })
        account_list = json.dumps(data)
    elif response_content_type.endswith('/xml'):
        output_list = [
            '<?xml version="1.0" encoding="UTF-8"?>',
            '<account name=%s>' % saxutils.quoteattr(account)
        ]
        for (name, object_count, bytes_used, is_subdir, mtime) in listing:
            if is_subdir:
                if not s3_buckets_only:
                    output_list.append('<subdir name=%s />' %
                                       saxutils.quoteattr(name))
            else:
                item = '<container><name>%s</name><count>%s</count>' \
                       '<bytes>%s</bytes><last_modified>%s</last_modified>' \
                       '</container>' % \
                       (saxutils.escape(name), object_count, bytes_used,
                        Timestamp(mtime).isoformat)
                output_list.append(item)
        output_list.append('</account>')
        account_list = '\n'.join(output_list)
    else:
        if not listing:
            resp = HTTPNoContent(request=req, headers=resp_headers)
            resp.content_type = response_content_type
            resp.charset = 'utf-8'
            return resp
        account_list = '\n'.join(r[0] for r in listing) + '\n'
    ret = HTTPOk(body=account_list, request=req, headers=resp_headers)
    ret.content_type = response_content_type
    ret.charset = 'utf-8'
    return ret
Пример #36
0
    def setUp(self):
        super(TestSloDeleteManifest, self).setUp()

        _submanifest_data = json.dumps([{
            'name': '/deltest/b_2',
            'hash': 'a',
            'bytes': '1'
        }, {
            'name': '/deltest/c_3',
            'hash': 'b',
            'bytes': '2'
        }])

        self.app.register('GET', '/v1/AUTH_test/deltest/man_404',
                          swob.HTTPNotFound, {}, None)
        self.app.register(
            'GET', '/v1/AUTH_test/deltest/man', swob.HTTPOk, {
                'Content-Type': 'application/json',
                'X-Static-Large-Object': 'true'
            },
            json.dumps([{
                'name': '/deltest/gone',
                'hash': 'a',
                'bytes': '1'
            }, {
                'name': '/deltest/b_2',
                'hash': 'b',
                'bytes': '2'
            }]))
        self.app.register('DELETE', '/v1/AUTH_test/deltest/man',
                          swob.HTTPNoContent, {}, None)
        self.app.register(
            'GET', '/v1/AUTH_test/deltest/man-all-there', swob.HTTPOk, {
                'Content-Type': 'application/json',
                'X-Static-Large-Object': 'true'
            },
            json.dumps([{
                'name': '/deltest/b_2',
                'hash': 'a',
                'bytes': '1'
            }, {
                'name': '/deltest/c_3',
                'hash': 'b',
                'bytes': '2'
            }]))
        self.app.register('DELETE', '/v1/AUTH_test/deltest/gone',
                          swob.HTTPNotFound, {}, None)
        self.app.register('GET', '/v1/AUTH_test/deltest/a_1', swob.HTTPOk,
                          {'Content-Length': '1'}, 'a')
        self.app.register('DELETE', '/v1/AUTH_test/deltest/a_1',
                          swob.HTTPNoContent, {}, None)
        self.app.register('DELETE', '/v1/AUTH_test/deltest/b_2',
                          swob.HTTPNoContent, {}, None)
        self.app.register('DELETE', '/v1/AUTH_test/deltest/c_3',
                          swob.HTTPNoContent, {}, None)
        self.app.register('DELETE', '/v1/AUTH_test/deltest/d_3',
                          swob.HTTPNoContent, {}, None)

        self.app.register(
            'GET', '/v1/AUTH_test/deltest/manifest-with-submanifest',
            swob.HTTPOk, {
                'Content-Type': 'application/json',
                'X-Static-Large-Object': 'true'
            },
            json.dumps([{
                'name': '/deltest/a_1',
                'hash': 'a',
                'bytes': '1'
            }, {
                'name': '/deltest/submanifest',
                'sub_slo': True,
                'hash': 'submanifest-etag',
                'bytes': len(_submanifest_data)
            }, {
                'name': '/deltest/d_3',
                'hash': 'd',
                'bytes': '3'
            }]))
        self.app.register('DELETE',
                          '/v1/AUTH_test/deltest/manifest-with-submanifest',
                          swob.HTTPNoContent, {}, None)

        self.app.register('GET', '/v1/AUTH_test/deltest/submanifest',
                          swob.HTTPOk, {
                              'Content-Type': 'application/json',
                              'X-Static-Large-Object': 'true'
                          }, _submanifest_data)
        self.app.register('DELETE', '/v1/AUTH_test/deltest/submanifest',
                          swob.HTTPNoContent, {}, None)

        self.app.register(
            'GET', '/v1/AUTH_test/deltest/manifest-missing-submanifest',
            swob.HTTPOk, {
                'Content-Type': 'application/json',
                'X-Static-Large-Object': 'true'
            },
            json.dumps([{
                'name': '/deltest/a_1',
                'hash': 'a',
                'bytes': '1'
            }, {
                'name': '/deltest/missing-submanifest',
                'hash': 'a',
                'bytes': '2',
                'sub_slo': True
            }, {
                'name': '/deltest/d_3',
                'hash': 'd',
                'bytes': '3'
            }]))
        self.app.register(
            'DELETE', '/v1/AUTH_test/deltest/manifest-missing-submanifest',
            swob.HTTPNoContent, {}, None)

        self.app.register('GET', '/v1/AUTH_test/deltest/missing-submanifest',
                          swob.HTTPNotFound, {}, None)

        self.app.register('GET', '/v1/AUTH_test/deltest/manifest-badjson',
                          swob.HTTPOk, {
                              'Content-Type': 'application/json',
                              'X-Static-Large-Object': 'true'
                          }, "[not {json (at ++++all")

        self.app.register(
            'GET', '/v1/AUTH_test/deltest/manifest-with-unauth-segment',
            swob.HTTPOk, {
                'Content-Type': 'application/json',
                'X-Static-Large-Object': 'true'
            },
            json.dumps([{
                'name': '/deltest/a_1',
                'hash': 'a',
                'bytes': '1'
            }, {
                'name': '/deltest-unauth/q_17',
                'hash': '11',
                'bytes': '17'
            }]))
        self.app.register(
            'DELETE', '/v1/AUTH_test/deltest/manifest-with-unauth-segment',
            swob.HTTPNoContent, {}, None)

        self.app.register('DELETE', '/v1/AUTH_test/deltest-unauth/q_17',
                          swob.HTTPUnauthorized, {}, None)
Пример #37
0
                            headers={'X-Static-Large-Object': 'True'},
                            body=good_data)(env, start_response)


test_xml_data = '''<?xml version="1.0" encoding="UTF-8"?>
<static_large_object>
<object_segment>
<path>/cont/object</path>
<etag>etagoftheobjectsegment</etag>
<size_bytes>100</size_bytes>
</object_segment>
</static_large_object>
'''
test_json_data = json.dumps([{
    'path': '/cont/object',
    'etag': 'etagoftheobjectsegment',
    'size_bytes': 100
}])


def fake_start_response(*args, **kwargs):
    pass


class TestStaticLargeObject(unittest.TestCase):
    def setUp(self):
        self.app = FakeApp()
        self.slo = slo.filter_factory({})(self.app)
        self.slo.min_segment_size = 1

    def tearDown(self):
Пример #38
0
    def handle_container_listing(self, env, start_response):
        # This code may be clearer by using Request(env).get_response()
        # instead of self._app_call(env)
        api_vers, account, container_name = split_path(
            env['PATH_INFO'], 3, 3, True)
        sub_env = env.copy()
        orig_container = get_unversioned_container(container_name)
        if orig_container != container_name:
            # Check that container_name is actually the versioning
            # container for orig_container
            sub_env['PATH_INFO'] = '/%s/%s/%s' % (api_vers, account,
                                                  orig_container)
            info = get_container_info(sub_env, self.app,
                                      swift_source='VW')
            vers_loc = info.get('sysmeta', {}).get('versions-location')
            # Sometimes we receive versioned listing requests whereas
            # versioning is not enabled (vers_loc is None or empty).
            if vers_loc and vers_loc != container_name:
                # The container specified in the request ends with the
                # versioning suffix, but user has asked the versions to
                # be saved elsewhere, thus we will consider this as a
                # regular listing request.
                orig_container = container_name

        if orig_container != container_name:
            qs = parse_qs(sub_env.get('QUERY_STRING', ''))
            if 'marker' in qs:
                marker, _ = swift3_split_object_name_version(qs['marker'][0])
                qs['marker'] = [marker]
            if 'prefix' in qs:
                prefix, _ = swift3_split_object_name_version(qs['prefix'][0])
                qs['prefix'] = prefix
            qs['format'] = 'json'
            sub_env['QUERY_STRING'] = urlencode(qs, True)
            sub_env['oio.query'] = {'versions': True}

        resp = super(OioVersionedWritesContext, self).handle_container_request(
            sub_env, lambda x, y, z: None)

        if orig_container != container_name and \
                self._response_status == '200 OK':
            with closing_if_possible(resp):
                versioned_objects = json.loads("".join(resp))

            # Discard the latest version of each object, because it is
            # not supposed to appear in the versioning container.

            # Also keep object prefixes as some of them may be shadowed
            # from the "main" container.
            latest = dict()
            subdirs = []
            for obj in versioned_objects:
                if 'subdir' in obj:
                    subdirs.append(obj)
                    continue
                ver = int(obj.get('version', '0'))
                # An integer is always strictly greater than None
                if ver > latest.get(obj['name']):
                    latest[obj['name']] = ver
            versioned_objects = [
                obj for obj in versioned_objects
                if 'subdir' not in obj
                and (int(obj.get('version', '0')) != latest[obj['name']]
                     or is_deleted(obj))
            ]

            for obj in versioned_objects:
                obj['name'] = swift3_versioned_object_name(
                    obj['name'], obj.get('version', ''))

            versioned_objects += subdirs
            resp = json.dumps(versioned_objects)
            self._response_headers = [x for x in self._response_headers
                                      if x[0] != 'Content-Length']
            self._response_headers.append(('Content-Length', str(len(resp))))

        start_response(self._response_status, self._response_headers,
                       self._response_exc_info)
        return resp
Пример #39
0
    def test_handle_multipart_put_bad_data(self):
        bad_data = json.dumps([{
            'path': '/cont/object',
            'etag': 'etagoftheobj',
            'size_bytes': 'lala'
        }])
        req = Request.blank(
            '/test_good/AUTH_test/c/man?multipart-manifest=put',
            environ={'REQUEST_METHOD': 'PUT'},
            body=bad_data)
        self.assertRaises(HTTPException, self.slo.handle_multipart_put, req,
                          fake_start_response)

        for bad_data in [
                json.dumps([{
                    'path': '/cont',
                    'etag': 'etagoftheobj',
                    'size_bytes': 100
                }]),
                json.dumps('asdf'),
                json.dumps(None),
                json.dumps(5), 'not json', '1234', None, '',
                json.dumps({'path': None}),
                json.dumps([{
                    'path': '/c/o',
                    'etag': None,
                    'size_bytes': 12
                }]),
                json.dumps([{
                    'path': '/c/o',
                    'etag': 'asdf',
                    'size_bytes': 'sd'
                }]),
                json.dumps([{
                    'path': 12,
                    'etag': 'etagoftheobj',
                    'size_bytes': 100
                }]),
                json.dumps([{
                    'path': u'/cont/object\u2661',
                    'etag': 'etagoftheobj',
                    'size_bytes': 100
                }]),
                json.dumps([{
                    'path': 12,
                    'size_bytes': 100
                }]),
                json.dumps([{
                    'path': 12,
                    'size_bytes': 100
                }]),
                json.dumps([{
                    'path': None,
                    'etag': 'etagoftheobj',
                    'size_bytes': 100
                }])
        ]:
            req = Request.blank(
                '/test_good/AUTH_test/c/man?multipart-manifest=put',
                environ={'REQUEST_METHOD': 'PUT'},
                body=bad_data)
            self.assertRaises(HTTPException, self.slo.handle_multipart_put,
                              req, fake_start_response)
Пример #40
0
    def __call__(self, env, start_response):
        self.calls += 1
        if env['PATH_INFO'] == '/':
            return Response(status=200, body='passed')(env, start_response)
        if env['PATH_INFO'].startswith('/test_good/'):
            j, v, a, cont, obj = env['PATH_INFO'].split('/')
            if obj == 'a_2':
                return Response(status=400)(env, start_response)
            cont_len = 100
            if obj == 'small_object':
                cont_len = 10
            headers = {
                'etag': 'etagoftheobjectsegment',
                'Content-Length': cont_len
            }
            if obj == 'slob':
                headers['X-Static-Large-Object'] = 'true'
            return Response(status=200, headers=headers)(env, start_response)
        if env['PATH_INFO'].startswith('/test_good_check/'):
            j, v, a, cont, obj = env['PATH_INFO'].split('/')
            etag, size = obj.split('_')
            last_mod = 'Fri, 01 Feb 2012 20:38:36 GMT'
            if obj == 'a_1':
                last_mod = ''
            return Response(status=200,
                            headers={
                                'etag': etag,
                                'Last-Modified': last_mod,
                                'Content-Length': size
                            })(env, start_response)
        if env['PATH_INFO'].startswith('/test_get/'):
            good_data = json.dumps([{
                'name': '/c/a_1',
                'hash': 'a',
                'bytes': '1'
            }, {
                'name': '/d/b_2',
                'hash': 'b',
                'bytes': '2'
            }])
            return Response(status=200,
                            headers={
                                'X-Static-Large-Object': 'True',
                                'Content-Type': 'html;swift_bytes=55'
                            },
                            body=good_data)(env, start_response)

        if env['PATH_INFO'].startswith('/test_get_broke_json/'):
            good_data = json.dumps([{
                'name': '/c/a_1',
                'hash': 'a',
                'bytes': '1'
            }, {
                'name': '/d/b_2',
                'hash': 'b',
                'bytes': '2'
            }])
            return Response(status=200,
                            headers={'X-Static-Large-Object': 'True'},
                            body=good_data[:-5])(env, start_response)

        if env['PATH_INFO'].startswith('/test_get_bad_json/'):
            bad_data = json.dumps([{
                'name': '/c/a_1',
                'something': 'a',
                'bytes': '1'
            }, {
                'name': '/d/b_2',
                'bytes': '2'
            }])
            return Response(status=200,
                            headers={'X-Static-Large-Object': 'True'},
                            body=bad_data)(env, start_response)

        if env['PATH_INFO'].startswith('/test_get_not_slo/'):
            return Response(status=200, body='lalala')(env, start_response)

        if env['PATH_INFO'].startswith('/test_delete_404/'):
            good_data = json.dumps([{
                'name': '/c/a_1',
                'hash': 'a',
                'bytes': '1'
            }, {
                'name': '/d/b_2',
                'hash': 'b',
                'bytes': '2'
            }])
            self.req_method_paths.append(
                (env['REQUEST_METHOD'], env['PATH_INFO']))
            if env['PATH_INFO'].endswith('/c/man_404'):
                return Response(status=404)(env, start_response)
            if env['PATH_INFO'].endswith('/c/a_1'):
                return Response(status=404)(env, start_response)
            return Response(status=200,
                            headers={'X-Static-Large-Object': 'True'},
                            body=good_data)(env, start_response)

        if env['PATH_INFO'].startswith('/test_delete/'):
            good_data = json.dumps([{
                'name': '/c/a_1',
                'hash': 'a',
                'bytes': '1'
            }, {
                'name': '/d/b_2',
                'hash': 'b',
                'bytes': '2'
            }])
            self.req_method_paths.append(
                (env['REQUEST_METHOD'], env['PATH_INFO']))
            return Response(status=200,
                            headers={'X-Static-Large-Object': 'True'},
                            body=good_data)(env, start_response)

        if env['PATH_INFO'].startswith('/test_delete_nested/'):
            nested_data = json.dumps([{
                'name': '/b/b_2',
                'hash': 'a',
                'bytes': '1'
            }, {
                'name': '/c/c_3',
                'hash': 'b',
                'bytes': '2'
            }])
            good_data = json.dumps([{
                'name': '/a/a_1',
                'hash': 'a',
                'bytes': '1'
            }, {
                'name': '/a/sub_nest',
                'hash': 'a',
                'sub_slo': True,
                'bytes': len(nested_data)
            }, {
                'name': '/d/d_3',
                'hash': 'b',
                'bytes': '2'
            }])
            self.req_method_paths.append(
                (env['REQUEST_METHOD'], env['PATH_INFO']))
            if 'sub_nest' in env['PATH_INFO']:
                return Response(status=200,
                                headers={'X-Static-Large-Object': 'True'},
                                body=nested_data)(env, start_response)
            else:
                return Response(status=200,
                                headers={'X-Static-Large-Object': 'True'},
                                body=good_data)(env, start_response)

        if env['PATH_INFO'].startswith('/test_delete_nested_404/'):
            good_data = json.dumps([{
                'name': '/a/a_1',
                'hash': 'a',
                'bytes': '1'
            }, {
                'name': '/a/sub_nest',
                'hash': 'a',
                'bytes': '2',
                'sub_slo': True
            }, {
                'name': '/d/d_3',
                'hash': 'b',
                'bytes': '3'
            }])
            self.req_method_paths.append(
                (env['REQUEST_METHOD'], env['PATH_INFO']))
            if 'sub_nest' in env['PATH_INFO']:
                return Response(status=404)(env, start_response)
            else:
                return Response(status=200,
                                headers={'X-Static-Large-Object': 'True'},
                                body=good_data)(env, start_response)

        if env['PATH_INFO'].startswith('/test_delete_bad_json/'):
            self.req_method_paths.append(
                (env['REQUEST_METHOD'], env['PATH_INFO']))
            return Response(status=200,
                            headers={'X-Static-Large-Object': 'True'},
                            body='bad json')(env, start_response)

        if env['PATH_INFO'].startswith('/test_delete_bad_man/'):
            self.req_method_paths.append(
                (env['REQUEST_METHOD'], env['PATH_INFO']))
            return Response(status=200, body='')(env, start_response)

        if env['PATH_INFO'].startswith('/test_delete_401/'):
            good_data = json.dumps([{
                'name': '/c/a_1',
                'hash': 'a',
                'bytes': '1'
            }, {
                'name': '/d/b_2',
                'hash': 'b',
                'bytes': '2'
            }])
            self.req_method_paths.append(
                (env['REQUEST_METHOD'], env['PATH_INFO']))
            if env['PATH_INFO'].endswith('/d/b_2'):
                return Response(status=401)(env, start_response)
            return Response(status=200,
                            headers={'X-Static-Large-Object': 'True'},
                            body=good_data)(env, start_response)
Пример #41
0
    def SERVICES(self, req):
        body = {}
        attr_list = []

        body['min_base_api_version'] = "v1"
        body['max_base_api_version'] = "v1"
        body['search_provider'] = "HP(UCSC)"
        body['search_enabled'] = "true"
        body['min_search_api_version'] = "v1"
        body['max_search_api_version'] = self.version
        body['freshness_complete'] = "false"
        body['freshness_partial'] = "false"
        body['complex_boolean_expr'] = "true"
        body['attr_list'] = attr_list

        #Account Attributes
        attr_list.append({
            "attr_name": "account_uri",
            "data_type": "string",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "account_name",
            "data_type": "string",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "account_tenant_id",
            "data_type": "string",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "account_first_use_time",
            "data_type": "date",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "account_last_modified_time",
            "data_type": "date",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "account_last_changed_time",
            "data_type": "date",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "account_delete_time",
            "data_type": "date",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "account_last_activity_time",
            "data_type": "date",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "account_container_count",
            "data_type": "numeric",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "account_object_count",
            "data_type": "numeric",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "account_bytes_used",
            "data_type": "numeric",
            "sortable": "true"
        })

        #Container Attrobutes
        attr_list.append({
            "attr_name": "container_uri",
            "data_type": "string",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "container_name",
            "data_type": "string",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "container_account_name",
            "data_type": "string",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "container_create_time",
            "data_type": "date",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "container_last_modified_time",
            "data_type": "date",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "container_last_changed_time",
            "data_type": "date",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "container_delete_time",
            "data_type": "date",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "container_last_activity_time",
            "data_type": "date",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "container_read_permissions",
            "data_type": "string",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "container_write_permissions",
            "data_type": "string",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "container_sync_to",
            "data_type": "string",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "container_sync_key",
            "data_type": "string",
            "sortable": "true"
        })
        #        attr_list.append({
        #                "attr_name" : "container_versions_location",
        #                "data_type" : "string",
        #                "sortable"  : "true"})
        attr_list.append({
            "attr_name": "container_object_count",
            "data_type": "numeric",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "container_bytes_used",
            "data_type": "numeric",
            "sortable": "true"
        })

        #Object Attributes
        attr_list.append({
            "attr_name": "object_uri",
            "data_type": "string",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "object_name",
            "data_type": "string",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "object_account_name",
            "data_type": "string",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "object_container_name",
            "data_type": "string",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "object_location",
            "data_type": "string",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "object_uri_create_time",
            "data_type": "date",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "object_last_modified_time",
            "data_type": "date",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "object_last_changed_time",
            "data_type": "date",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "object_delete_time",
            "data_type": "date",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "object_last_activity_time",
            "data_type": "date",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "object_etag_hash",
            "data_type": "string",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "object_content_type",
            "data_type": "string",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "object_content_length",
            "data_type": "numeric",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "object_content_encoding",
            "data_type": "string",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "object_content_disposition",
            "data_type": "string",
            "sortable": "true"
        })
        attr_list.append({
            "attr_name": "object_content_language",
            "data_type": "string",
            "sortable": "true"
        })
        #        attr_list.append({
        #                "attr_name" : "object_cache_control",
        #                "data_type" : "string",
        #                "sortable"  : "true"})
        attr_list.append({
            "attr_name": "object_delete_at",
            "data_type": "date",
            "sortable": "true"
        })
        #        attr_list.append({
        #                "attr_name" : "object_manifest_type",
        #                "data_type" : "numeric",
        #                "sortable"  : "true"})
        #        attr_list.append({
        #                "attr_name" : "object_manifest",
        #                "data_type" : "string",
        #                "sortable"  : "true"})

        #Object CORS Attributes
        #        attr_list.append({
        #                "attr_name" : "object_access_control_allow_origin",
        #                "data_type" : "string",
        #                "sortable"  : "true"})
        #        attr_list.append({
        #                "attr_name" : "object_access_control_allow_credentials",
        #                "data_type" : "string",
        #                "sortable"  : "true"})
        #        attr_list.append({
        #                "attr_name" : "object_access_control_expose_headers",
        #                "data_type" : "string",
        #                "sortable"  : "true"})
        #        attr_list.append({
        #                "attr_name" : "object_access_control_max_age",
        #                "data_type" : "string",
        #                "sortable"  : "true"})
        #        attr_list.append({
        #                "attr_name" : "object_access_control_allow_methods",
        #                "data_type" : "string",
        #                "sortable"  : "true"})
        #        attr_list.append({
        #                "attr_name" : "object_access_control_allow_headers",
        #                "data_type" : "string",
        #                "sortable"  : "true"})
        #        attr_list.append({
        #                "attr_name" : "object_origin",
        #                "data_type" : "string",
        #                "sortable"  : "true"})
        #        attr_list.append({
        #                "attr_name" : "object_access_control_request_method",
        #                "data_type" : "string",
        #                "sortable"  : "true"})
        #        attr_list.append({
        #                "attr_name" : "object_access_control_request_headers",
        #                "data_type" : "string",
        #                "sortable"  : "true"})

        body = json.dumps(body, indent=4, separators=(',', ' : '))
        return Response(request=req, body=body, content_type="json")
Пример #42
0
    def handle_multipart_put(self, req):
        """
        Will handle the PUT of a SLO manifest.
        Heads every object in manifest to check if is valid and if so will
        save a manifest generated from the user input.

        :params req: a swob.Request with an obj in path
        :raises: HttpException on errors
        """
        try:
            vrs, account, container, obj = req.split_path(1, 4, True)
        except ValueError:
            return self.app
        if req.content_length > self.max_manifest_size:
            raise HTTPRequestEntityTooLarge(
                "Manifest File > %d bytes" % self.max_manifest_size)
        if req.headers.get('X-Copy-From'):
            raise HTTPMethodNotAllowed(
                'Multipart Manifest PUTs cannot be Copy requests')
        if req.content_length is None and \
                req.headers.get('transfer-encoding', '').lower() != 'chunked':
            raise HTTPLengthRequired(request=req)
        parsed_data = parse_input(req.body_file.read(self.max_manifest_size))
        problem_segments = []

        if len(parsed_data) > self.max_manifest_segments:
            raise HTTPRequestEntityTooLarge(
                'Number segments must be <= %d' % self.max_manifest_segments)
        total_size = 0
        out_content_type = req.accept.best_match(ACCEPTABLE_FORMATS)
        if not out_content_type:
            out_content_type = 'text/plain'
        data_for_storage = []
        for index, seg_dict in enumerate(parsed_data):
            obj_path = '/'.join(
                ['', vrs, account, seg_dict['path'].lstrip('/')])
            try:
                seg_size = int(seg_dict['size_bytes'])
            except (ValueError, TypeError):
                raise HTTPBadRequest('Invalid Manifest File')
            if seg_size < self.min_segment_size and \
                    (index == 0 or index < len(parsed_data) - 1):
                raise HTTPBadRequest(
                    'Each segment, except the last, must be larger than '
                    '%d bytes.' % self.min_segment_size)

            new_env = req.environ.copy()
            if isinstance(obj_path, unicode):
                obj_path = obj_path.encode('utf-8')
            new_env['PATH_INFO'] = obj_path
            new_env['REQUEST_METHOD'] = 'HEAD'
            new_env['swift.source'] = 'SLO'
            del(new_env['wsgi.input'])
            del(new_env['QUERY_STRING'])
            new_env['CONTENT_LENGTH'] = 0
            new_env['HTTP_USER_AGENT'] = \
                '%s MultipartPUT' % req.environ.get('HTTP_USER_AGENT')
            head_seg_resp = \
                Request.blank(obj_path, new_env).get_response(self.app)
            if head_seg_resp.is_success:
                total_size += seg_size
                if seg_size != head_seg_resp.content_length:
                    problem_segments.append([quote(obj_path), 'Size Mismatch'])
                if seg_dict['etag'] != head_seg_resp.etag:
                    problem_segments.append([quote(obj_path), 'Etag Mismatch'])
                if head_seg_resp.last_modified:
                    last_modified = head_seg_resp.last_modified
                else:
                    # shouldn't happen
                    last_modified = datetime.now()

                last_modified_formatted = \
                    last_modified.strftime('%Y-%m-%dT%H:%M:%S.%f')
                seg_data = {'name': '/' + seg_dict['path'].lstrip('/'),
                            'bytes': seg_size,
                            'hash': seg_dict['etag'],
                            'content_type': head_seg_resp.content_type,
                            'last_modified': last_modified_formatted}
                if config_true_value(
                        head_seg_resp.headers.get('X-Static-Large-Object')):
                    seg_data['sub_slo'] = True
                data_for_storage.append(seg_data)

            else:
                problem_segments.append([quote(obj_path),
                                         head_seg_resp.status])
        if problem_segments:
            resp_body = get_response_body(
                out_content_type, {}, problem_segments)
            raise HTTPBadRequest(resp_body, content_type=out_content_type)
        env = req.environ

        if not env.get('CONTENT_TYPE'):
            guessed_type, _junk = mimetypes.guess_type(req.path_info)
            env['CONTENT_TYPE'] = guessed_type or 'application/octet-stream'
        env['swift.content_type_overriden'] = True
        env['CONTENT_TYPE'] += ";swift_bytes=%d" % total_size
        env['HTTP_X_STATIC_LARGE_OBJECT'] = 'True'
        json_data = json.dumps(data_for_storage)
        env['CONTENT_LENGTH'] = str(len(json_data))
        env['wsgi.input'] = StringIO(json_data)
        return self.app
Пример #43
0
def output_json(metaList):
    """
    Converts the list of dicts into a JSON format 
    """
    return json.dumps(metaList, indent=4, separators=(',', ' : '))
Пример #44
0
        def response_iter():
            # NB: XML requires that the XML declaration, if present, be at the
            # very start of the document. Clients *will* call us out on not
            # being valid XML if we pass through whitespace before it.
            # Track whether we've sent anything yet so we can yield out that
            # declaration *first*
            yielded_anything = False

            try:
                try:
                    # TODO: add support for versioning
                    put_resp = req.get_response(
                        self.app, 'PUT', body=json.dumps(manifest),
                        query={'multipart-manifest': 'put',
                               'heartbeat': 'on'},
                        headers=headers)
                    if put_resp.status_int == 202:
                        body = []
                        put_resp.fix_conditional_response()
                        for chunk in put_resp.response_iter:
                            if not chunk.strip():
                                if time.time() - start_time < 10:
                                    # Include some grace period to keep
                                    # ceph-s3tests happy
                                    continue
                                if not yielded_anything:
                                    yield (b'<?xml version="1.0" '
                                           b'encoding="UTF-8"?>\n')
                                yielded_anything = True
                                yield chunk
                                continue
                            body.append(chunk)
                        body = json.loads(b''.join(body))
                        if body['Response Status'] != '201 Created':
                            for seg, err in body['Errors']:
                                if err == too_small_message:
                                    raise EntityTooSmall()
                                elif err in ('Etag Mismatch', '404 Not Found'):
                                    raise InvalidPart(upload_id=upload_id)
                            raise InvalidRequest(
                                status=body['Response Status'],
                                msg='\n'.join(': '.join(err)
                                              for err in body['Errors']))
                except BadSwiftRequest as e:
                    msg = str(e)
                    if too_small_message in msg:
                        raise EntityTooSmall(msg)
                    elif ', Etag Mismatch' in msg:
                        raise InvalidPart(upload_id=upload_id)
                    elif ', 404 Not Found' in msg:
                        raise InvalidPart(upload_id=upload_id)
                    else:
                        raise

                # clean up the multipart-upload record
                obj = '%s/%s' % (req.object_name, upload_id)
                try:
                    req.get_response(self.app, 'DELETE', container, obj)
                except NoSuchKey:
                    # We know that this existed long enough for us to HEAD
                    pass

                result_elem = Element('CompleteMultipartUploadResult')

                # NOTE: boto with sig v4 appends port to HTTP_HOST value at
                # the request header when the port is non default value and it
                # makes req.host_url like as http://localhost:8080:8080/path
                # that obviously invalid. Probably it should be resolved at
                # swift.common.swob though, tentatively we are parsing and
                # reconstructing the correct host_url info here.
                # in detail, https://github.com/boto/boto/pull/3513
                parsed_url = urlparse(req.host_url)
                host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname)
                # Why are we doing our own port parsing? Because py3 decided
                # to start raising ValueErrors on access after parsing such
                # an invalid port
                netloc = parsed_url.netloc.split('@')[-1].split(']')[-1]
                if ':' in netloc:
                    port = netloc.split(':', 2)[1]
                    host_url += ':%s' % port

                SubElement(result_elem, 'Location').text = host_url + req.path
                SubElement(result_elem, 'Bucket').text = req.container_name
                SubElement(result_elem, 'Key').text = req.object_name
                SubElement(result_elem, 'ETag').text = '"%s"' % s3_etag
                resp.headers.pop('ETag', None)
                if yielded_anything:
                    yield b'\n'
                yield tostring(result_elem,
                               xml_declaration=not yielded_anything)
            except ErrorResponse as err_resp:
                if yielded_anything:
                    err_resp.xml_declaration = False
                    yield b'\n'
                else:
                    # Oh good, we can still change HTTP status code, too!
                    resp.status = err_resp.status
                for chunk in err_resp({}, lambda *a: None):
                    yield chunk
Пример #45
0
    def POST(self, req):
        """
        Handles Complete Multipart Upload.
        """
        upload_id = req.params['uploadId']
        resp = _get_upload_info(req, self.app, upload_id)
        headers = {}
        for key, val in resp.headers.iteritems():
            _key = key.lower()
            if _key.startswith('x-amz-meta-'):
                headers['x-object-meta-' + _key[11:]] = val
            elif _key == 'content-type':
                headers['Content-Type'] = val

        # Query for the objects in the segments area to make sure it completed
        query = {
            'format': 'json',
            'prefix': '%s/%s/' % (req.object_name, upload_id),
            'delimiter': '/'
        }

        container = req.container_name + MULTIUPLOAD_SUFFIX
        resp = req.get_response(self.app, 'GET', container, '', query=query)
        objinfo = json.loads(resp.body)
        objtable = dict((o['name'],
                         {'path': '/'.join(['', container, o['name']]),
                          'etag': o['hash'],
                          'size_bytes': o['bytes']}) for o in objinfo)

        manifest = []
        previous_number = 0
        try:
            xml = req.xml(MAX_COMPLETE_UPLOAD_BODY_SIZE)
            complete_elem = fromstring(xml, 'CompleteMultipartUpload')
            for part_elem in complete_elem.iterchildren('Part'):
                part_number = int(part_elem.find('./PartNumber').text)

                if part_number <= previous_number:
                    raise InvalidPartOrder(upload_id=upload_id)
                previous_number = part_number

                etag = part_elem.find('./ETag').text
                if len(etag) >= 2 and etag[0] == '"' and etag[-1] == '"':
                    # strip double quotes
                    etag = etag[1:-1]

                info = objtable.get("%s/%s/%s" % (req.object_name, upload_id,
                                                  part_number))
                if info is None or info['etag'] != etag:
                    raise InvalidPart(upload_id=upload_id,
                                      part_number=part_number)

                manifest.append(info)
        except (XMLSyntaxError, DocumentInvalid):
            raise MalformedXML()
        except ErrorResponse:
            raise
        except Exception as e:
            exc_type, exc_value, exc_traceback = sys.exc_info()
            LOGGER.error(e)
            raise exc_type, exc_value, exc_traceback

        # Following swift commit 7f636a5, zero-byte segments aren't allowed,
        # even as the final segment
        if int(info['size_bytes']) == 0:
            manifest.pop()

            # Ordinarily, we just let SLO check segment sizes. However, we
            # just popped off a zero-byte segment; if there was a second
            # zero-byte segment and it was at the end, it would succeed on
            # Swift < 2.6.0 and fail on newer Swift. It seems reasonable that
            # it should always fail.
            if manifest and int(manifest[-1]['size_bytes']) == 0:
                raise EntityTooSmall()

        try:
            # TODO: add support for versioning
            if manifest:
                resp = req.get_response(self.app, 'PUT',
                                        body=json.dumps(manifest),
                                        query={'multipart-manifest': 'put'},
                                        headers=headers)
            else:
                # the upload must have consisted of a single zero-length part
                # just write it directly
                resp = req.get_response(self.app, 'PUT', body='',
                                        headers=headers)
        except BadSwiftRequest as e:
            msg = str(e)
            msg_pre_260 = 'Each segment, except the last, must be at least '
            # see https://github.com/openstack/swift/commit/c0866ce
            msg_260 = ('too small; each segment, except the last, must be '
                       'at least ')
            # see https://github.com/openstack/swift/commit/7f636a5
            msg_post_260 = 'too small; each segment must be at least 1 byte'
            if msg.startswith(msg_pre_260) or \
                    msg_260 in msg or msg_post_260 in msg:
                # FIXME: AWS S3 allows a smaller object than 5 MB if there is
                # only one part.  Use a COPY request to copy the part object
                # from the segments container instead.
                raise EntityTooSmall(msg)
            else:
                raise

        if int(info['size_bytes']) == 0:
            # clean up the zero-byte segment
            empty_seg_cont, empty_seg_name = info['path'].split('/', 2)[1:]
            req.get_response(self.app, 'DELETE',
                             container=empty_seg_cont, obj=empty_seg_name)

        # clean up the multipart-upload record
        obj = '%s/%s' % (req.object_name, upload_id)
        req.get_response(self.app, 'DELETE', container, obj)

        result_elem = Element('CompleteMultipartUploadResult')

        # NOTE: boto with sig v4 appends port to HTTP_HOST value at the
        # request header when the port is non default value and it makes
        # req.host_url like as http://localhost:8080:8080/path
        # that obviously invalid. Probably it should be resolved at
        # swift.common.swob though, tentatively we are parsing and
        # reconstructing the correct host_url info here.
        # in detail, https://github.com/boto/boto/pull/3513
        parsed_url = urlparse(req.host_url)
        host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname)
        if parsed_url.port:
            host_url += ':%s' % parsed_url.port

        SubElement(result_elem, 'Location').text = host_url + req.path
        SubElement(result_elem, 'Bucket').text = req.container_name
        SubElement(result_elem, 'Key').text = req.object_name
        SubElement(result_elem, 'ETag').text = resp.etag

        resp.body = tostring(result_elem)
        resp.status = 200
        resp.content_type = "application/xml"

        return resp
Пример #46
0
    def test_admin(self):
        if tf.skip3:
            raise SkipTest

        def get_listing(url, token, parsed, conn):
            conn.request('GET', '%s/%s' % (parsed.path, self.container), '',
                         {'X-Auth-Token': token})
            return check_response(conn)

        def post_account(url, token, parsed, conn, headers):
            new_headers = dict({'X-Auth-Token': token}, **headers)
            conn.request('POST', parsed.path, '', new_headers)
            return check_response(conn)

        def get(url, token, parsed, conn, name):
            conn.request('GET',
                         '%s/%s/%s' % (parsed.path, self.container, name), '',
                         {'X-Auth-Token': token})
            return check_response(conn)

        def put(url, token, parsed, conn, name):
            conn.request('PUT',
                         '%s/%s/%s' % (parsed.path, self.container, name),
                         'test', {'X-Auth-Token': token})
            return check_response(conn)

        def delete(url, token, parsed, conn, name):
            conn.request('DELETE',
                         '%s/%s/%s' % (parsed.path, self.container, name), '',
                         {'X-Auth-Token': token})
            return check_response(conn)

        # cannot list objects
        resp = retry(get_listing, use_account=3)
        resp.read()
        self.assertEquals(resp.status, 403)

        # cannot get object
        resp = retry(get, self.obj, use_account=3)
        resp.read()
        self.assertEquals(resp.status, 403)

        # grant admin access
        acl_user = tf.swift_test_user[2]
        acl = {'admin': [acl_user]}
        headers = {'x-account-access-control': json.dumps(acl)}
        resp = retry(post_account, headers=headers, use_account=1)
        resp.read()
        self.assertEqual(resp.status, 204)

        # can list objects
        resp = retry(get_listing, use_account=3)
        listing = resp.read()
        self.assertEquals(resp.status, 200)
        self.assert_(self.obj in listing)

        # can get object
        resp = retry(get, self.obj, use_account=3)
        body = resp.read()
        self.assertEquals(resp.status, 200)
        self.assertEquals(body, 'test')

        # can put an object
        obj_name = str(uuid4())
        resp = retry(put, obj_name, use_account=3)
        body = resp.read()
        self.assertEquals(resp.status, 201)

        # can delete an object
        resp = retry(delete, self.obj, use_account=3)
        body = resp.read()
        self.assertEquals(resp.status, 204)

        # sanity with account1
        resp = retry(get_listing, use_account=3)
        listing = resp.read()
        self.assertEquals(resp.status, 200)
        self.assert_(obj_name in listing)
        self.assert_(self.obj not in listing)
Пример #47
0
        def response_iter():
            # NB: XML requires that the XML declaration, if present, be at the
            # very start of the document. Clients *will* call us out on not
            # being valid XML if we pass through whitespace before it.
            # Track whether we've sent anything yet so we can yield out that
            # declaration *first*
            yielded_anything = False

            try:
                try:
                    # TODO: add support for versioning
                    put_resp = req.get_response(self.app,
                                                'PUT',
                                                body=json.dumps(manifest),
                                                query={
                                                    'multipart-manifest':
                                                    'put',
                                                    'heartbeat': 'on'
                                                },
                                                headers=headers)
                    if put_resp.status_int == 202:
                        body = []
                        put_resp.fix_conditional_response()
                        for chunk in put_resp.response_iter:
                            if not chunk.strip():
                                if time.time() - start_time < 10:
                                    # Include some grace period to keep
                                    # ceph-s3tests happy
                                    continue
                                if not yielded_anything:
                                    yield (b'<?xml version="1.0" '
                                           b'encoding="UTF-8"?>\n')
                                yielded_anything = True
                                yield chunk
                                continue
                            body.append(chunk)
                        body = json.loads(b''.join(body))
                        if body['Response Status'] != '201 Created':
                            for seg, err in body['Errors']:
                                if err == too_small_message:
                                    raise EntityTooSmall()
                                elif err in ('Etag Mismatch', '404 Not Found'):
                                    raise InvalidPart(upload_id=upload_id)
                            raise InvalidRequest(
                                status=body['Response Status'],
                                msg='\n'.join(': '.join(err)
                                              for err in body['Errors']))
                except BadSwiftRequest as e:
                    msg = str(e)
                    if too_small_message in msg:
                        raise EntityTooSmall(msg)
                    elif ', Etag Mismatch' in msg:
                        raise InvalidPart(upload_id=upload_id)
                    elif ', 404 Not Found' in msg:
                        raise InvalidPart(upload_id=upload_id)
                    else:
                        raise

                # clean up the multipart-upload record
                obj = '%s/%s' % (req.object_name, upload_id)
                try:
                    req.get_response(self.app, 'DELETE', container, obj)
                except NoSuchKey:
                    # The important thing is that we wrote out a tombstone to
                    # make sure the marker got cleaned up. If it's already
                    # gone (e.g., because of concurrent completes or a retried
                    # complete), so much the better.
                    pass

                yield _make_complete_body(req, s3_etag, yielded_anything)
            except ErrorResponse as err_resp:
                if yielded_anything:
                    err_resp.xml_declaration = False
                    yield b'\n'
                else:
                    # Oh good, we can still change HTTP status code, too!
                    resp.status = err_resp.status
                for chunk in err_resp({}, lambda *a: None):
                    yield chunk
Пример #48
0
 def v1_format_response(self, req, endpoints, **kwargs):
     return Response(json.dumps(endpoints), content_type='application/json')
Пример #49
0
 def test_recon_get_version(self):
     req = Request.blank('/recon/version',
                         environ={'REQUEST_METHOD': 'GET'})
     resp = self.app(req.environ, start_response)
     self.assertEquals(resp, [json.dumps({'version': swiftver})])
Пример #50
0
 resp_headers = {
     'X-Container-Object-Count': info['object_count'],
     'X-Container-Bytes-Used': info['bytes_used'],
     'X-Timestamp': info['created_at'],
     'X-PUT-Timestamp': info['put_timestamp'],
 }
 for key, (value, timestamp) in broker.metadata.iteritems():
     if value and (key.lower() in self.save_headers or
                   key.lower().startswith('x-container-meta-')):
         resp_headers[key] = value
 ret = Response(request=req, headers=resp_headers,
                content_type=out_content_type, charset='utf-8')
 container_list = broker.list_objects_iter(limit, marker, end_marker,
                                           prefix, delimiter, path)
 if out_content_type == 'application/json':
     ret.body = json.dumps([self.update_data_record(record)
                            for record in container_list])
 elif out_content_type.endswith('/xml'):
     doc = Element('container', name=container.decode('utf-8'))
     for obj in container_list:
         record = self.update_data_record(obj)
         if 'subdir' in record:
             name = record['subdir'].decode('utf-8')
             sub = SubElement(doc, 'subdir', name=name)
             SubElement(sub, 'name').text = name
         else:
             obj_element = SubElement(doc, 'object')
             for field in ["name", "hash", "bytes", "content_type",
                           "last_modified"]:
                 SubElement(obj_element, field).text = str(
                     record.pop(field)).decode('utf-8')
             for field in sorted(record.keys()):
Пример #51
0
 def GET(self, req):
     """Handle HTTP GET request."""
     drive, part, account, container, obj = split_and_validate_path(
         req, 4, 5, True)
     path = get_param(req, 'path')
     prefix = get_param(req, 'prefix')
     delimiter = get_param(req, 'delimiter')
     if delimiter and (len(delimiter) > 1 or ord(delimiter) > 254):
         # delimiters can be made more flexible later
         return HTTPPreconditionFailed(body='Bad delimiter')
     marker = get_param(req, 'marker', '')
     end_marker = get_param(req, 'end_marker')
     limit = CONTAINER_LISTING_LIMIT
     given_limit = get_param(req, 'limit')
     if given_limit and given_limit.isdigit():
         limit = int(given_limit)
         if limit > CONTAINER_LISTING_LIMIT:
             return HTTPPreconditionFailed(
                 request=req,
                 body='Maximum limit is %d' % CONTAINER_LISTING_LIMIT)
     out_content_type = get_listing_content_type(req)
     if self.mount_check and not check_mount(self.root, drive):
         return HTTPInsufficientStorage(drive=drive, request=req)
     broker = self._get_container_broker(drive, part, account, container,
                                         pending_timeout=0.1,
                                         stale_reads_ok=True)
     if broker.is_deleted():
         return HTTPNotFound(request=req)
     info = broker.get_info()
     resp_headers = {
         'X-Container-Object-Count': info['object_count'],
         'X-Container-Bytes-Used': info['bytes_used'],
         'X-Timestamp': info['created_at'],
         'X-PUT-Timestamp': info['put_timestamp'],
     }
     for key, (value, timestamp) in broker.metadata.iteritems():
         if value and (key.lower() in self.save_headers or
                       key.lower().startswith('x-container-meta-')):
             resp_headers[key] = value
     ret = Response(request=req, headers=resp_headers,
                    content_type=out_content_type, charset='utf-8')
     container_list = broker.list_objects_iter(limit, marker, end_marker,
                                               prefix, delimiter, path)
     if out_content_type == 'application/json':
         ret.body = json.dumps([self.update_data_record(record)
                                for record in container_list])
     elif out_content_type.endswith('/xml'):
         doc = Element('container', name=container.decode('utf-8'))
         for obj in container_list:
             record = self.update_data_record(obj)
             if 'subdir' in record:
                 name = record['subdir'].decode('utf-8')
                 sub = SubElement(doc, 'subdir', name=name)
                 SubElement(sub, 'name').text = name
             else:
                 obj_element = SubElement(doc, 'object')
                 for field in ["name", "hash", "bytes", "content_type",
                               "last_modified"]:
                     SubElement(obj_element, field).text = str(
                         record.pop(field)).decode('utf-8')
                 for field in sorted(record):
                     SubElement(obj_element, field).text = str(
                         record[field]).decode('utf-8')
         ret.body = tostring(doc, encoding='UTF-8').replace(
             "<?xml version='1.0' encoding='UTF-8'?>",
             '<?xml version="1.0" encoding="UTF-8"?>', 1)
     else:
         if not container_list:
             return HTTPNoContent(request=req, headers=resp_headers)
         ret.body = '\n'.join(rec[0] for rec in container_list) + '\n'
     return ret
Пример #52
0
    def setup_objects(self):
        self.objects = (('rose', '2011-01-05T02:19:14.275290', 0,
                         303), ('viola', '2011-01-05T02:19:14.275290', '0',
                                3909), ('lily', '2011-01-05T02:19:14.275290',
                                        '0', '3909'),
                        ('with space', '2011-01-05T02:19:14.275290', 0, 390),
                        ('with%20space', '2011-01-05T02:19:14.275290', 0, 390))

        objects = map(
            lambda item: {
                'name': str(item[0]),
                'last_modified': str(item[1]),
                'hash': str(item[2]),
                'bytes': str(item[3])
            }, list(self.objects))
        object_list = json.dumps(objects)

        self.prefixes = ['rose', 'viola', 'lily']
        object_list_subdir = []
        for p in self.prefixes:
            object_list_subdir.append({"subdir": p})

        self.swift.register('DELETE', '/v1/AUTH_test/bucket+segments',
                            swob.HTTPNoContent, {}, json.dumps([]))
        self.swift.register('DELETE', '/v1/AUTH_test/bucket+segments/rose',
                            swob.HTTPNoContent, {}, json.dumps([]))
        self.swift.register('DELETE', '/v1/AUTH_test/bucket+segments/viola',
                            swob.HTTPNoContent, {}, json.dumps([]))
        self.swift.register('DELETE', '/v1/AUTH_test/bucket+segments/lily',
                            swob.HTTPNoContent, {}, json.dumps([]))
        self.swift.register('DELETE', '/v1/AUTH_test/bucket+segments/with'
                            ' space', swob.HTTPNoContent, {}, json.dumps([]))
        self.swift.register('DELETE', '/v1/AUTH_test/bucket+segments/with%20'
                            'space', swob.HTTPNoContent, {}, json.dumps([]))
        self.swift.register(
            'GET', '/v1/AUTH_test/bucket+segments?format=json'
            '&marker=with%2520space', swob.HTTPOk, {}, json.dumps([]))
        self.swift.register(
            'GET', '/v1/AUTH_test/bucket+segments?format=json'
            '&marker=', swob.HTTPOk, {}, object_list)
        self.swift.register('HEAD', '/v1/AUTH_test/junk', swob.HTTPNoContent,
                            {}, None)
        self.swift.register('HEAD', '/v1/AUTH_test/nojunk', swob.HTTPNotFound,
                            {}, None)
        self.swift.register('GET', '/v1/AUTH_test/junk', swob.HTTPOk, {},
                            object_list)
        self.swift.register(
            'GET',
            '/v1/AUTH_test/junk?delimiter=a&format=json&limit=3&marker=viola',
            swob.HTTPOk, {}, json.dumps(objects[2:]))
        self.swift.register('GET', '/v1/AUTH_test/junk-subdir', swob.HTTPOk,
                            {}, json.dumps(object_list_subdir))
        self.swift.register(
            'GET', '/v1/AUTH_test/subdirs?delimiter=/&format=json&limit=3',
            swob.HTTPOk, {},
            json.dumps([
                {
                    'subdir': 'nothing/'
                },
                {
                    'subdir': 'but/'
                },
                {
                    'subdir': 'subdirs/'
                },
            ]))
Пример #53
0
def account_listing_response(account,
                             req,
                             response_content_type,
                             broker=None,
                             limit='',
                             marker='',
                             end_marker='',
                             prefix='',
                             delimiter='',
                             reverse=False):
    """
    This is an exact copy of swift.account.utis.account_listing_response()
    except for one difference i.e this method passes response_content_type
    to broker.list_containers_iter() method.
    """
    if broker is None:
        broker = FakeAccountBroker()

    resp_headers = get_response_headers(broker)

    account_list = broker.list_containers_iter(limit, marker, end_marker,
                                               prefix, delimiter,
                                               response_content_type, reverse)
    if response_content_type == 'application/json':
        data = []
        for (name, object_count, bytes_used, put_tstamp,
             is_subdir) in account_list:
            if is_subdir:
                data.append({'subdir': name})
            else:
                data.append({
                    'name': name,
                    'count': object_count,
                    'bytes': bytes_used,
                    'last_modified': Timestamp(put_tstamp).isoformat
                })
        account_list = json.dumps(data)
    elif response_content_type.endswith('/xml'):
        output_list = [
            '<?xml version="1.0" encoding="UTF-8"?>',
            '<account name=%s>' % saxutils.quoteattr(account)
        ]
        for (name, object_count, bytes_used, put_tstamp,
             is_subdir) in account_list:
            if is_subdir:
                output_list.append('<subdir name=%s />' %
                                   saxutils.quoteattr(name))
            else:
                item = '<container><name>%s</name><count>%s</count>' \
                       '<bytes>%s</bytes><last_modified>%s</last_modified> \
                        </container>'                                      % \
                       (saxutils.escape(name), object_count, bytes_used,
                        Timestamp(put_tstamp).isoformat)
                output_list.append(item)
        output_list.append('</account>')
        account_list = '\n'.join(output_list)
    else:
        if not account_list:
            resp = HTTPNoContent(request=req, headers=resp_headers)
            resp.content_type = response_content_type
            resp.charset = 'utf-8'
            return resp
        account_list = '\n'.join(r[0] for r in account_list) + '\n'
    ret = HTTPOk(body=account_list, request=req, headers=resp_headers)
    ret.content_type = response_content_type
    ret.charset = 'utf-8'
    return ret
Пример #54
0
 def audit_all_objects(self, mode='once', device_dirs=None):
     description = ''
     if device_dirs:
         device_dir_str = ','.join(sorted(device_dirs))
         description = _(' - %s') % device_dir_str
     self.logger.info(
         _('Begin object audit "%s" mode (%s%s)') %
         (mode, self.auditor_type, description))
     begin = reported = time.time()
     self.total_bytes_processed = 0
     self.total_files_processed = 0
     total_quarantines = 0
     total_errors = 0
     time_auditing = 0
     all_locs = self.diskfile_mgr.object_audit_location_generator(
         device_dirs=device_dirs)
     for location in all_locs:
         loop_time = time.time()
         self.failsafe_object_audit(location)
         self.logger.timing_since('timing', loop_time)
         self.files_running_time = ratelimit_sleep(
             self.files_running_time, self.max_files_per_second)
         self.total_files_processed += 1
         now = time.time()
         if now - reported >= self.log_time:
             self.logger.info(
                 _('Object audit (%(type)s). '
                   'Since %(start_time)s: Locally: %(passes)d passed, '
                   '%(quars)d quarantined, %(errors)d errors '
                   'files/sec: %(frate).2f , bytes/sec: %(brate).2f, '
                   'Total time: %(total).2f, Auditing time: %(audit).2f, '
                   'Rate: %(audit_rate).2f') % {
                       'type': '%s%s' % (self.auditor_type, description),
                       'start_time': time.ctime(reported),
                       'passes': self.passes,
                       'quars': self.quarantines,
                       'errors': self.errors,
                       'frate': self.passes / (now - reported),
                       'brate': self.bytes_processed / (now - reported),
                       'total': (now - begin),
                       'audit': time_auditing,
                       'audit_rate': time_auditing / (now - begin)
                   })
             cache_entry = self.create_recon_nested_dict(
                 'object_auditor_stats_%s' % (self.auditor_type),
                 device_dirs, {
                     'errors': self.errors,
                     'passes': self.passes,
                     'quarantined': self.quarantines,
                     'bytes_processed': self.bytes_processed,
                     'start_time': reported,
                     'audit_time': time_auditing
                 })
             dump_recon_cache(cache_entry, self.rcache, self.logger)
             reported = now
             total_quarantines += self.quarantines
             total_errors += self.errors
             self.passes = 0
             self.quarantines = 0
             self.errors = 0
             self.bytes_processed = 0
         time_auditing += (now - loop_time)
     # Avoid divide by zero during very short runs
     elapsed = (time.time() - begin) or 0.000001
     self.logger.info(
         _('Object audit (%(type)s) "%(mode)s" mode '
           'completed: %(elapsed).02fs. Total quarantined: %(quars)d, '
           'Total errors: %(errors)d, Total files/sec: %(frate).2f, '
           'Total bytes/sec: %(brate).2f, Auditing time: %(audit).2f, '
           'Rate: %(audit_rate).2f') % {
               'type': '%s%s' % (self.auditor_type, description),
               'mode': mode,
               'elapsed': elapsed,
               'quars': total_quarantines + self.quarantines,
               'errors': total_errors + self.errors,
               'frate': self.total_files_processed / elapsed,
               'brate': self.total_bytes_processed / elapsed,
               'audit': time_auditing,
               'audit_rate': time_auditing / elapsed
           })
     # Clear recon cache entry if device_dirs is set
     if device_dirs:
         cache_entry = self.create_recon_nested_dict(
             'object_auditor_stats_%s' % (self.auditor_type), device_dirs,
             {})
         dump_recon_cache(cache_entry, self.rcache, self.logger)
     if self.stats_sizes:
         self.logger.info(
             _('Object audit stats: %s') % json.dumps(self.stats_buckets))
Пример #55
0
    def setUp(self):
        super(TestSwift3MultiUpload, self).setUp()

        segment_bucket = '/v1/AUTH_test/bucket+segments'
        self.etag = '7dfa07a8e59ddbcd1dc84d4c4f82aea1'
        last_modified = 'Fri, 01 Apr 2014 12:00:00 GMT'
        put_headers = {'etag': self.etag, 'last-modified': last_modified}

        objects = map(
            lambda item: {
                'name': item[0],
                'last_modified': item[1],
                'hash': item[2],
                'bytes': item[3]
            }, objects_template)
        object_list = json.dumps(objects)

        self.swift.register('PUT', '/v1/AUTH_test/bucket+segments',
                            swob.HTTPAccepted, {}, None)
        self.swift.register('GET', segment_bucket, swob.HTTPOk, {},
                            object_list)
        self.swift.register('HEAD', segment_bucket + '/object/X', swob.HTTPOk,
                            {'x-object-meta-foo': 'bar'}, None)
        self.swift.register('PUT', segment_bucket + '/object/X',
                            swob.HTTPCreated, {}, None)
        self.swift.register('DELETE', segment_bucket + '/object/X',
                            swob.HTTPNoContent, {}, None)
        self.swift.register('GET', segment_bucket + '/object/invalid',
                            swob.HTTPNotFound, {}, None)
        self.swift.register('PUT', segment_bucket + '/object/X/1',
                            swob.HTTPCreated, put_headers, None)
        self.swift.register('DELETE', segment_bucket + '/object/X/1',
                            swob.HTTPNoContent, {}, None)
        self.swift.register('DELETE', segment_bucket + '/object/X/2',
                            swob.HTTPNoContent, {}, None)

        self.swift.register('HEAD', segment_bucket + '/object/Y', swob.HTTPOk,
                            {}, None)
        self.swift.register('PUT', segment_bucket + '/object/Y',
                            swob.HTTPCreated, {}, None)
        self.swift.register('DELETE', segment_bucket + '/object/Y',
                            swob.HTTPNoContent, {}, None)
        self.swift.register('PUT', segment_bucket + '/object/Y/1',
                            swob.HTTPCreated, {}, None)
        self.swift.register('DELETE', segment_bucket + '/object/Y/1',
                            swob.HTTPNoContent, {}, None)
        self.swift.register('DELETE', segment_bucket + '/object/Y/2',
                            swob.HTTPNoContent, {}, None)

        self.swift.register('HEAD', segment_bucket + '/object2/Z', swob.HTTPOk,
                            {}, None)
        self.swift.register('PUT', segment_bucket + '/object2/Z',
                            swob.HTTPCreated, {}, None)
        self.swift.register('DELETE', segment_bucket + '/object2/Z',
                            swob.HTTPNoContent, {}, None)
        self.swift.register('PUT', segment_bucket + '/object2/Z/1',
                            swob.HTTPCreated, {}, None)
        self.swift.register('DELETE', segment_bucket + '/object2/Z/1',
                            swob.HTTPNoContent, {}, None)
        self.swift.register('DELETE', segment_bucket + '/object2/Z/2',
                            swob.HTTPNoContent, {}, None)
Пример #56
0
 if not out_content_type:
     return HTTPNotAcceptable(request=req)
 account_list = broker.list_containers_iter(limit, marker, end_marker,
                                            prefix, delimiter)
 if out_content_type == 'application/json':
     data = []
     for (name, object_count, bytes_used, is_subdir) in account_list:
         if is_subdir:
             data.append({'subdir': name})
         else:
             data.append({
                 'name': name,
                 'count': object_count,
                 'bytes': bytes_used
             })
     account_list = json.dumps(data)
 elif out_content_type.endswith('/xml'):
     output_list = [
         '<?xml version="1.0" encoding="UTF-8"?>',
         '<account name="%s">' % account
     ]
     for (name, object_count, bytes_used, is_subdir) in account_list:
         name = saxutils.escape(name)
         if is_subdir:
             output_list.append('<subdir name="%s" />' % name)
         else:
             item = '<container><name>%s</name><count>%s</count>' \
                    '<bytes>%s</bytes></container>' % \
                    (name, object_count, bytes_used)
             output_list.append(item)
     output_list.append('</account>')
Пример #57
0
    def POST(self, req):
        """
        Handles Complete Multipart Upload.
        """
        upload_id = req.params['uploadId']
        resp = _get_upload_info(req, self.app, upload_id)
        headers = {}
        for key, val in resp.headers.items():
            _key = key.lower()
            if _key.startswith('x-amz-meta-'):
                headers['x-object-meta-' + _key[11:]] = val

        hct_header = sysmeta_header('object', 'has-content-type')
        if resp.sysmeta_headers.get(hct_header) == 'yes':
            content_type = resp.sysmeta_headers.get(
                sysmeta_header('object', 'content-type'))
        elif hct_header in resp.sysmeta_headers:
            # has-content-type is present but false, so no content type was
            # set on initial upload. In that case, we won't set one on our
            # PUT request. Swift will end up guessing one based on the
            # object name.
            content_type = None
        else:
            content_type = resp.headers.get('Content-Type')

        if content_type:
            headers['Content-Type'] = content_type

        # Query for the objects in the segments area to make sure it completed
        query = {
            'format': 'json',
            'prefix': '%s/%s/' % (req.object_name, upload_id),
            'delimiter': '/'
        }

        container = req.container_name + MULTIUPLOAD_SUFFIX
        resp = req.get_response(self.app, 'GET', container, '', query=query)
        objinfo = json.loads(resp.body)
        objtable = dict((o['name'], {
            'path': '/'.join(['', container, o['name']]),
            'etag': o['hash'],
            'size_bytes': o['bytes']
        }) for o in objinfo)

        s3_etag_hasher = md5()
        manifest = []
        previous_number = 0
        try:
            xml = req.xml(MAX_COMPLETE_UPLOAD_BODY_SIZE)
            if not xml:
                raise InvalidRequest(msg='You must specify at least one part')

            complete_elem = fromstring(xml, 'CompleteMultipartUpload',
                                       self.logger)
            for part_elem in complete_elem.iterchildren('Part'):
                part_number = int(part_elem.find('./PartNumber').text)

                if part_number <= previous_number:
                    raise InvalidPartOrder(upload_id=upload_id)
                previous_number = part_number

                etag = part_elem.find('./ETag').text
                if len(etag) >= 2 and etag[0] == '"' and etag[-1] == '"':
                    # strip double quotes
                    etag = etag[1:-1]

                info = objtable.get("%s/%s/%s" %
                                    (req.object_name, upload_id, part_number))
                if info is None or info['etag'] != etag:
                    raise InvalidPart(upload_id=upload_id,
                                      part_number=part_number)

                s3_etag_hasher.update(etag.decode('hex'))
                info['size_bytes'] = int(info['size_bytes'])
                manifest.append(info)
        except (XMLSyntaxError, DocumentInvalid):
            raise MalformedXML()
        except ErrorResponse:
            raise
        except Exception as e:
            self.logger.error(e)
            raise

        s3_etag = '%s-%d' % (s3_etag_hasher.hexdigest(), len(manifest))
        headers[sysmeta_header('object', 'etag')] = s3_etag
        # Leave base header value blank; SLO will populate
        c_etag = '; s3_etag=%s' % s3_etag
        headers['X-Object-Sysmeta-Container-Update-Override-Etag'] = c_etag

        # Check the size of each segment except the last and make sure they are
        # all more than the minimum upload chunk size
        for info in manifest[:-1]:
            if info['size_bytes'] < self.conf.min_segment_size:
                raise EntityTooSmall()

        try:
            # TODO: add support for versioning
            if manifest:
                resp = req.get_response(self.app,
                                        'PUT',
                                        body=json.dumps(manifest),
                                        query={'multipart-manifest': 'put'},
                                        headers=headers)
            else:
                # the upload must have consisted of a single zero-length part
                # just write it directly
                resp = req.get_response(self.app,
                                        'PUT',
                                        body='',
                                        headers=headers)
        except BadSwiftRequest as e:
            msg = str(e)
            expected_msg = 'too small; each segment must be at least 1 byte'
            if expected_msg in msg:
                # FIXME: AWS S3 allows a smaller object than 5 MB if there is
                # only one part.  Use a COPY request to copy the part object
                # from the segments container instead.
                raise EntityTooSmall(msg)
            else:
                raise

        # clean up the multipart-upload record
        obj = '%s/%s' % (req.object_name, upload_id)
        try:
            req.get_response(self.app, 'DELETE', container, obj)
        except NoSuchKey:
            pass  # We know that this existed long enough for us to HEAD

        result_elem = Element('CompleteMultipartUploadResult')

        # NOTE: boto with sig v4 appends port to HTTP_HOST value at the
        # request header when the port is non default value and it makes
        # req.host_url like as http://localhost:8080:8080/path
        # that obviously invalid. Probably it should be resolved at
        # swift.common.swob though, tentatively we are parsing and
        # reconstructing the correct host_url info here.
        # in detail, https://github.com/boto/boto/pull/3513
        parsed_url = urlparse(req.host_url)
        host_url = '%s://%s' % (parsed_url.scheme, parsed_url.hostname)
        if parsed_url.port:
            host_url += ':%s' % parsed_url.port

        SubElement(result_elem, 'Location').text = host_url + req.path
        SubElement(result_elem, 'Bucket').text = req.container_name
        SubElement(result_elem, 'Key').text = req.object_name
        SubElement(result_elem, 'ETag').text = '"%s"' % s3_etag
        del resp.headers['ETag']

        resp.body = tostring(result_elem)
        resp.status = 200
        resp.content_type = "application/xml"

        return resp