Exemple #1
0
    def test_GET_DLO_outside_container(self):
        seg1 = self.env.container.file(
            "get-dlo-outside-seg1" + Utils.create_name())
        seg2 = self.env.container.file(
            "get-dlo-outside-seg2" + Utils.create_name())
        seg1.write("one fish two fish ")
        seg2.write("red fish blue fish")

        container2 = self.env.account.container(Utils.create_name())
        container2.create()

        manifest = container2.file("manifest" + Utils.create_name())
        manifest.write(
            '',
            hdrs={"X-Object-Manifest": "%s/get-dlo-outside-seg" %
                  (self.env.container.name,)})

        expires = int(time()) + 86400
        parms = self.tempurl_parms(
            'GET', expires, self.env.conn.make_path(manifest.path),
            self.env.tempurl_key)

        # cross container tempurl works fine for account tempurl key
        contents = manifest.read(parms=parms, cfg={'no_auth_token': True})
        self.assertEqual(contents, "one fish two fish red fish blue fish")
        self.assert_status([200])
Exemple #2
0
    def test_GET_DLO_outside_container(self):
        container2 = self.env.account.container(Utils.create_name())
        container2.create()
        seg1 = container2.file("get-dlo-outside-seg1" + Utils.create_name())
        seg2 = container2.file("get-dlo-outside-seg2" + Utils.create_name())
        seg1.write("one fish two fish ")
        seg2.write("red fish blue fish")

        manifest = self.env.container.file("manifest" + Utils.create_name())
        manifest.write('',
                       hdrs={
                           "X-Object-Manifest":
                           "%s/get-dlo-outside-seg" % (container2.name, )
                       })

        expires = int(time()) + 86400
        sig = self.tempurl_sig('GET', expires,
                               self.env.conn.make_path(manifest.path),
                               self.env.tempurl_key)
        parms = {'temp_url_sig': sig, 'temp_url_expires': str(expires)}

        # cross container tempurl does not work for container tempurl key
        try:
            manifest.read(parms=parms, cfg={'no_auth_token': True})
        except ResponseError as e:
            self.assertEqual(e.status, 401)
        else:
            self.fail('request did not error')
        try:
            manifest.info(parms=parms, cfg={'no_auth_token': True})
        except ResponseError as e:
            self.assertEqual(e.status, 401)
        else:
            self.fail('request did not error')
Exemple #3
0
    def setUp(cls):
        cls.conn = Connection(tf.config)
        cls.conn.authenticate()

        if cls.tempurl_enabled is None:
            cls.tempurl_enabled = 'tempurl' in cluster_info
            if not cls.tempurl_enabled:
                return

        cls.tempurl_key = Utils.create_name()
        cls.tempurl_key2 = Utils.create_name()

        cls.account = Account(
            cls.conn, tf.config.get('account', tf.config['username']))
        cls.account.delete_containers()
        cls.account.update_metadata({
            'temp-url-key': cls.tempurl_key,
            'temp-url-key-2': cls.tempurl_key2
        })

        cls.container = cls.account.container(Utils.create_name())
        if not cls.container.create():
            raise ResponseError(cls.conn.response)

        cls.obj = cls.container.file(Utils.create_name())
        cls.obj.write("obj contents")
        cls.other_obj = cls.container.file(Utils.create_name())
        cls.other_obj.write("other obj contents")
Exemple #4
0
    def setUp(cls):
        cls.conn = Connection(tf.config)
        cls.conn.authenticate()

        if cls.tempurl_enabled is None:
            cls.tempurl_enabled = 'tempurl' in cluster_info
            if not cls.tempurl_enabled:
                return

        cls.tempurl_key = Utils.create_name()
        cls.tempurl_key2 = Utils.create_name()

        cls.account = Account(
            cls.conn, tf.config.get('account', tf.config['username']))
        cls.account.delete_containers()
        cls.account.update_metadata({
            'temp-url-key': cls.tempurl_key,
            'temp-url-key-2': cls.tempurl_key2
        })

        cls.container = cls.account.container(Utils.create_name())
        if not cls.container.create():
            raise ResponseError(cls.conn.response)

        cls.obj = cls.container.file(Utils.create_name())
        cls.obj.write("obj contents")
        cls.other_obj = cls.container.file(Utils.create_name())
        cls.other_obj.write("other obj contents")
Exemple #5
0
    def setUp(cls):
        if cls.tempurl_enabled is None:
            cls.tempurl_enabled = 'tempurl' in cluster_info
            if not cls.tempurl_enabled:
                return

        super(TestContainerTempurlEnv, cls).setUp()

        cls.tempurl_key = Utils.create_name()
        cls.tempurl_key2 = Utils.create_name()

        # creating another account and connection
        # for ACL tests
        config2 = deepcopy(tf.config)
        config2['account'] = tf.config['account2']
        config2['username'] = tf.config['username2']
        config2['password'] = tf.config['password2']
        cls.conn2 = Connection(config2)
        cls.conn2.authenticate()
        cls.account2 = Account(
            cls.conn2, config2.get('account', config2['username']))
        cls.account2 = cls.conn2.get_account()

        cls.container = cls.account.container(Utils.create_name())
        if not cls.container.create({
                'x-container-meta-temp-url-key': cls.tempurl_key,
                'x-container-meta-temp-url-key-2': cls.tempurl_key2,
                'x-container-read': cls.account2.name}):
            raise ResponseError(cls.conn.response)

        cls.obj = cls.container.file(Utils.create_name())
        cls.obj.write("obj contents")
        cls.other_obj = cls.container.file(Utils.create_name())
        cls.other_obj.write("other obj contents")
Exemple #6
0
    def test_GET_DLO_outside_container(self):
        seg1 = self.env.container.file("get-dlo-outside-seg1" +
                                       Utils.create_name())
        seg2 = self.env.container.file("get-dlo-outside-seg2" +
                                       Utils.create_name())
        seg1.write("one fish two fish ")
        seg2.write("red fish blue fish")

        container2 = self.env.account.container(Utils.create_name())
        container2.create()

        manifest = container2.file("manifest" + Utils.create_name())
        manifest.write('',
                       hdrs={
                           "X-Object-Manifest":
                           "%s/get-dlo-outside-seg" %
                           (self.env.container.name, )
                       })

        expires = int(time()) + 86400
        parms = self.tempurl_parms('GET', expires,
                                   self.env.conn.make_path(manifest.path),
                                   self.env.tempurl_key)

        # cross container tempurl works fine for account tempurl key
        contents = manifest.read(parms=parms, cfg={'no_auth_token': True})
        self.assertEqual(contents, "one fish two fish red fish blue fish")
        self.assert_status([200])
Exemple #7
0
    def setUp(self):
        if 'slo' not in cluster_info:
            raise SkipTest("SLO not enabled")

        self.conn = Connection(tf.config)
        self.conn.authenticate()
        self.account = Account(self.conn,
                               tf.config.get('account', tf.config['username']))
        self.account.delete_containers()

        # create a container with versioning
        self.versions_container = self.account.container(Utils.create_name())
        self.container = self.account.container(Utils.create_name())
        self.segments_container = self.account.container(Utils.create_name())
        if not self.container.create(
                hdrs={'X-Versions-Location': self.versions_container.name}):
            raise ResponseError(self.conn.response)
        if 'versions' not in self.container.info():
            raise SkipTest("Object versioning not enabled")

        for cont in (self.versions_container, self.segments_container):
            if not cont.create():
                raise ResponseError(self.conn.response)

        # create some segments
        self.seg_info = {}
        for letter, size in (('a', 1024 * 1024), ('b', 1024 * 1024)):
            seg_name = letter
            file_item = self.segments_container.file(seg_name)
            file_item.write(letter * size)
            self.seg_info[seg_name] = {
                'size_bytes': size,
                'etag': file_item.md5,
                'path': '/%s/%s' % (self.segments_container.name, seg_name)
            }
Exemple #8
0
    def setUp(cls):
        if cls.tempurl_enabled is None:
            cls.tempurl_enabled = 'tempurl' in cluster_info
            if not cls.tempurl_enabled:
                return

        super(TestContainerTempurlEnv, cls).setUp()

        cls.tempurl_key = Utils.create_name()
        cls.tempurl_key2 = Utils.create_name()

        # creating another account and connection
        # for ACL tests
        config2 = deepcopy(tf.config)
        config2['account'] = tf.config['account2']
        config2['username'] = tf.config['username2']
        config2['password'] = tf.config['password2']
        cls.conn2 = Connection(config2)
        cls.conn2.authenticate()
        cls.account2 = Account(cls.conn2,
                               config2.get('account', config2['username']))
        cls.account2 = cls.conn2.get_account()

        cls.container = cls.account.container(Utils.create_name())
        if not cls.container.create({
                'x-container-meta-temp-url-key': cls.tempurl_key,
                'x-container-meta-temp-url-key-2': cls.tempurl_key2,
                'x-container-read': cls.account2.name
        }):
            raise ResponseError(cls.conn.response)

        cls.obj = cls.container.file(Utils.create_name())
        cls.obj.write("obj contents")
        cls.other_obj = cls.container.file(Utils.create_name())
        cls.other_obj.write("other obj contents")
    def setUp(self):
        if 'slo' not in cluster_info:
            raise SkipTest("SLO not enabled")

        self.conn = Connection(tf.config)
        self.conn.authenticate()
        self.account = Account(
            self.conn, tf.config.get('account', tf.config['username']))
        self.account.delete_containers()

        # create a container with versioning
        self.versions_container = self.account.container(Utils.create_name())
        self.container = self.account.container(Utils.create_name())
        self.segments_container = self.account.container(Utils.create_name())
        if not self.container.create(
                hdrs={'X-Versions-Location': self.versions_container.name}):
            raise ResponseError(self.conn.response)
        if 'versions' not in self.container.info():
            raise SkipTest("Object versioning not enabled")

        for cont in (self.versions_container, self.segments_container):
            if not cont.create():
                raise ResponseError(self.conn.response)

        # create some segments
        self.seg_info = {}
        for letter, size in (('a', 1024 * 1024),
                             ('b', 1024 * 1024)):
            seg_name = letter
            file_item = self.segments_container.file(seg_name)
            file_item.write(letter * size)
            self.seg_info[seg_name] = {
                'size_bytes': size,
                'etag': file_item.md5,
                'path': '/%s/%s' % (self.segments_container.name, seg_name)}
Exemple #10
0
    def setUp(cls):
        super(TestDloEnv, cls).setUp()
        config2 = tf.config.copy()
        config2['username'] = tf.config['username3']
        config2['password'] = tf.config['password3']
        cls.conn2 = Connection(config2)
        cls.conn2.authenticate()

        cls.container = cls.account.container(Utils.create_name())
        cls.container2 = cls.account.container(Utils.create_name())

        for cont in (cls.container, cls.container2):
            if not cont.create():
                raise ResponseError(cls.conn.response)

        # avoid getting a prefix that stops halfway through an encoded
        # character
        prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8")
        cls.segment_prefix = prefix

        for letter in ('a', 'b', 'c', 'd', 'e'):
            file_item = cls.container.file("%s/seg_lower%s" % (prefix, letter))
            file_item.write(letter * 10)

            file_item = cls.container.file("%s/seg_upper%s" % (prefix, letter))
            file_item.write(letter.upper() * 10)

        for letter in ('f', 'g', 'h', 'i', 'j'):
            file_item = cls.container2.file("%s/seg_lower%s" %
                                            (prefix, letter))
            file_item.write(letter * 10)

        man1 = cls.container.file("man1")
        man1.write('man1-contents',
                   hdrs={
                       "X-Object-Manifest":
                       "%s/%s/seg_lower" % (cls.container.name, prefix)
                   })

        man2 = cls.container.file("man2")
        man2.write('man2-contents',
                   hdrs={
                       "X-Object-Manifest":
                       "%s/%s/seg_upper" % (cls.container.name, prefix)
                   })

        manall = cls.container.file("manall")
        manall.write('manall-contents',
                     hdrs={
                         "X-Object-Manifest":
                         "%s/%s/seg" % (cls.container.name, prefix)
                     })

        mancont2 = cls.container.file("mancont2")
        mancont2.write('mancont2-contents',
                       hdrs={
                           "X-Object-Manifest":
                           "%s/%s/seg_lower" % (cls.container2.name, prefix)
                       })
Exemple #11
0
    def setUp(cls):
        cls.conn = Connection(tf.config)
        cls.conn.authenticate()

        config2 = tf.config.copy()
        config2['username'] = tf.config['username3']
        config2['password'] = tf.config['password3']
        cls.conn2 = Connection(config2)
        cls.conn2.authenticate()

        cls.account = Account(cls.conn, tf.config.get('account',
                                                      tf.config['username']))
        cls.account.delete_containers()

        cls.container = cls.account.container(Utils.create_name())
        cls.container2 = cls.account.container(Utils.create_name())

        for cont in (cls.container, cls.container2):
            if not cont.create():
                raise ResponseError(cls.conn.response)

        # avoid getting a prefix that stops halfway through an encoded
        # character
        prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8")
        cls.segment_prefix = prefix

        for letter in ('a', 'b', 'c', 'd', 'e'):
            file_item = cls.container.file("%s/seg_lower%s" % (prefix, letter))
            file_item.write(letter * 10)

            file_item = cls.container.file("%s/seg_upper%s" % (prefix, letter))
            file_item.write(letter.upper() * 10)

        for letter in ('f', 'g', 'h', 'i', 'j'):
            file_item = cls.container2.file("%s/seg_lower%s" %
                                            (prefix, letter))
            file_item.write(letter * 10)

        man1 = cls.container.file("man1")
        man1.write('man1-contents',
                   hdrs={"X-Object-Manifest": "%s/%s/seg_lower" %
                         (cls.container.name, prefix)})

        man2 = cls.container.file("man2")
        man2.write('man2-contents',
                   hdrs={"X-Object-Manifest": "%s/%s/seg_upper" %
                         (cls.container.name, prefix)})

        manall = cls.container.file("manall")
        manall.write('manall-contents',
                     hdrs={"X-Object-Manifest": "%s/%s/seg" %
                           (cls.container.name, prefix)})

        mancont2 = cls.container.file("mancont2")
        mancont2.write(
            'mancont2-contents',
            hdrs={"X-Object-Manifest": "%s/%s/seg_lower" %
                                       (cls.container2.name, prefix)})
    def setUp(cls):
        super(TestObjectVersioningEnv, cls).setUp()
        if not tf.skip2:
            # Second connection for ACL tests
            config2 = deepcopy(tf.config)
            config2['account'] = tf.config['account2']
            config2['username'] = tf.config['username2']
            config2['password'] = tf.config['password2']
            cls.conn2 = Connection(config2)
            cls.conn2.authenticate()

        if six.PY2:
            # avoid getting a prefix that stops halfway through an encoded
            # character
            prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8")
        else:
            prefix = Utils.create_name()[:10]

        cls.versions_container = cls.account.container(prefix + "-versions")
        if not cls.versions_container.create():
            raise ResponseError(cls.conn.response)

        cls.container = cls.account.container(prefix + "-objs")
        container_headers = {
            cls.location_header_key: quote(cls.versions_container.name)
        }
        if not cls.container.create(hdrs=container_headers):
            if cls.conn.response.status == 412:
                cls.versioning_enabled = False
                return
            raise ResponseError(cls.conn.response)

        container_info = cls.container.info()
        # if versioning is off, then cls.location_header_key won't persist
        cls.versioning_enabled = 'versions' in container_info

        if not tf.skip2:
            # setup another account to test ACLs
            config2 = deepcopy(tf.config)
            config2['account'] = tf.config['account2']
            config2['username'] = tf.config['username2']
            config2['password'] = tf.config['password2']
            cls.conn2 = Connection(config2)
            cls.storage_url2, cls.storage_token2 = cls.conn2.authenticate()
            cls.account2 = cls.conn2.get_account()
            cls.account2.delete_containers()

        if not tf.skip3:
            # setup another account with no access to anything to test ACLs
            config3 = deepcopy(tf.config)
            config3['account'] = tf.config['account']
            config3['username'] = tf.config['username3']
            config3['password'] = tf.config['password3']
            cls.conn3 = Connection(config3)
            cls.storage_url3, cls.storage_token3 = cls.conn3.authenticate()
            cls.account3 = cls.conn3.get_account()
Exemple #13
0
    def setUp(cls):
        super(TestDloEnv, cls).setUp()

        cls.container = cls.account.container(Utils.create_name())
        cls.container2 = cls.account.container(Utils.create_name())

        for cont in (cls.container, cls.container2):
            if not cont.create():
                raise ResponseError(cls.conn.response)

        prefix = Utils.create_name(10)
        cls.segment_prefix = prefix

        for letter in ('a', 'b', 'c', 'd', 'e'):
            file_item = cls.container.file("%s/seg_lower%s" % (prefix, letter))
            file_item.write(letter.encode('ascii') * 10)

            file_item = cls.container.file("%s/seg_upper_%%ff%s" %
                                           (prefix, letter))
            file_item.write(letter.upper().encode('ascii') * 10)

        for letter in ('f', 'g', 'h', 'i', 'j'):
            file_item = cls.container2.file("%s/seg_lower%s" %
                                            (prefix, letter))
            file_item.write(letter.encode('ascii') * 10)

        man1 = cls.container.file("man1")
        man1.write(b'man1-contents',
                   hdrs={
                       "X-Object-Manifest":
                       "%s/%s/seg_lower" % (cls.container.name, prefix)
                   })

        man2 = cls.container.file("man2")
        man2.write(b'man2-contents',
                   hdrs={
                       "X-Object-Manifest":
                       "%s/%s/seg_upper_%%25ff" % (cls.container.name, prefix)
                   })

        manall = cls.container.file("manall")
        manall.write(b'manall-contents',
                     hdrs={
                         "X-Object-Manifest":
                         "%s/%s/seg" % (cls.container.name, prefix)
                     })

        mancont2 = cls.container.file("mancont2")
        mancont2.write(b'mancont2-contents',
                       hdrs={
                           "X-Object-Manifest":
                           "%s/%s/seg_lower" % (cls.container2.name, prefix)
                       })
Exemple #14
0
    def test_PUT_manifest_access(self):
        new_obj = self.env.container.file(Utils.create_name())

        # give out a signature which allows a PUT to new_obj
        expires = int(time.time()) + 86400
        sig = self.tempurl_sig(
            'PUT', expires, self.env.conn.make_path(new_obj.path),
            self.env.tempurl_key)
        put_parms = {'temp_url_sig': sig,
                     'temp_url_expires': str(expires)}

        # try to create manifest pointing to some random container
        try:
            new_obj.write('', {
                'x-object-manifest': '%s/foo' % 'some_random_container'
            }, parms=put_parms, cfg={'no_auth_token': True})
        except ResponseError as e:
            self.assertEqual(e.status, 400)
        else:
            self.fail('request did not error')

        # create some other container
        other_container = self.env.account.container(Utils.create_name())
        if not other_container.create():
            raise ResponseError(self.conn.response)

        # try to create manifest pointing to new container
        try:
            new_obj.write('', {
                'x-object-manifest': '%s/foo' % other_container
            }, parms=put_parms, cfg={'no_auth_token': True})
        except ResponseError as e:
            self.assertEqual(e.status, 400)
        else:
            self.fail('request did not error')

        # try again using a tempurl POST to an already created object
        new_obj.write('', {}, parms=put_parms, cfg={'no_auth_token': True})
        expires = int(time.time()) + 86400
        sig = self.tempurl_sig(
            'POST', expires, self.env.conn.make_path(new_obj.path),
            self.env.tempurl_key)
        post_parms = {'temp_url_sig': sig,
                      'temp_url_expires': str(expires)}
        try:
            new_obj.post({'x-object-manifest': '%s/foo' % other_container},
                         parms=post_parms, cfg={'no_auth_token': True})
        except ResponseError as e:
            self.assertEqual(e.status, 400)
        else:
            self.fail('request did not error')
Exemple #15
0
    def setUp(cls):
        super(TestDloEnv, cls).setUp()

        cls.container = cls.account.container(Utils.create_name())
        cls.container2 = cls.account.container(Utils.create_name())

        for cont in (cls.container, cls.container2):
            if not cont.create():
                raise ResponseError(cls.conn.response)

        # avoid getting a prefix that stops halfway through an encoded
        # character
        prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8")
        cls.segment_prefix = prefix

        for letter in ('a', 'b', 'c', 'd', 'e'):
            file_item = cls.container.file("%s/seg_lower%s" % (prefix, letter))
            file_item.write(letter * 10)

            file_item = cls.container.file(
                "%s/seg_upper_%%ff%s" % (prefix, letter))
            file_item.write(letter.upper() * 10)

        for letter in ('f', 'g', 'h', 'i', 'j'):
            file_item = cls.container2.file("%s/seg_lower%s" %
                                            (prefix, letter))
            file_item.write(letter * 10)

        man1 = cls.container.file("man1")
        man1.write('man1-contents',
                   hdrs={"X-Object-Manifest": "%s/%s/seg_lower" %
                         (cls.container.name, prefix)})

        man2 = cls.container.file("man2")
        man2.write('man2-contents',
                   hdrs={"X-Object-Manifest": "%s/%s/seg_upper_%%25ff" %
                         (cls.container.name, prefix)})

        manall = cls.container.file("manall")
        manall.write('manall-contents',
                     hdrs={"X-Object-Manifest": "%s/%s/seg" %
                           (cls.container.name, prefix)})

        mancont2 = cls.container.file("mancont2")
        mancont2.write(
            'mancont2-contents',
            hdrs={"X-Object-Manifest": "%s/%s/seg_lower" %
                                       (cls.container2.name, prefix)})
Exemple #16
0
    def test_versioning_dlo(self):
        container = self.env.container
        versions_container = self.env.versions_container
        obj_name = Utils.create_name()

        for i in ('1', '2', '3'):
            time.sleep(.01)  # guarantee that the timestamp changes
            obj_name_seg = obj_name + '/' + i
            versioned_obj = container.file(obj_name_seg)
            versioned_obj.write(i)
            versioned_obj.write(i + i)

        self.assertEqual(3, versions_container.info()['object_count'])

        man_file = container.file(obj_name)
        man_file.write('',
                       hdrs={
                           "X-Object-Manifest":
                           "%s/%s/" % (self.env.container.name, obj_name)
                       })

        # guarantee that the timestamp changes
        time.sleep(.01)

        # write manifest file again
        man_file.write('',
                       hdrs={
                           "X-Object-Manifest":
                           "%s/%s/" % (self.env.container.name, obj_name)
                       })

        self.assertEqual(3, versions_container.info()['object_count'])
        self.assertEqual("112233", man_file.read())
    def testObjectMetadataWhenFileModified(self):
        data = "I'm whatever Gotham needs me to be "
        data_hash = hashlib.md5(data).hexdigest()
        # Create an object through object interface
        object_name = Utils.create_name()
        object_item = self.env.container.file(object_name)
        object_item.write(data)
        # Make sure GET works
        self.assertEqual(data, object_item.read())
        self.assert_status(200)
        # Check Etag is right
        self.assertEqual(data_hash, object_item.info()['etag'])
        self.assert_status(200)

        # Extend/append more data to file from filesystem interface
        file_path = os.path.join(self.env.root_dir,
                                 'AUTH_' + self.env.account.name,
                                 self.env.container.name,
                                 object_name)
        more_data = "- Batman"
        with open(file_path, 'a') as f:
            f.write(more_data)
        total_data = data + more_data
        total_data_hash = hashlib.md5(total_data).hexdigest()
        # Make sure GET works
        self.assertEqual(total_data, object_item.read())
        self.assert_status(200)
        # Check Etag and content-length is right
        metadata = object_item.info()
        self.assert_status(200)
        self.assertEqual(total_data_hash, metadata['etag'])
        self.assertEqual(len(total_data), int(metadata['content_length']))

        # Re-write the file to be shorter
        new_data = "I am Batman"
        new_data_hash = hashlib.md5(new_data).hexdigest()
        with open(file_path, 'w') as f:
            f.write(new_data)
        # Make sure GET works
        self.assertEqual(new_data, object_item.read())
        self.assert_status(200)
        # Check Etag and content-length is right
        metadata = object_item.info()
        self.assert_status(200)
        self.assertEqual(new_data_hash, metadata['etag'])
        self.assertEqual(len(new_data), int(metadata['content_length']))

        # Modify the file but let the length remain same
        new_data = "I am Antman"
        new_data_hash = hashlib.md5(new_data).hexdigest()
        with open(file_path, 'w') as f:
            f.write(new_data)
        # Make sure GET works
        self.assertEqual(new_data, object_item.read())
        self.assert_status(200)
        # Check Etag and content-length is right
        metadata = object_item.info()
        self.assert_status(200)
        self.assertEqual(new_data_hash, metadata['etag'])
        self.assertEqual(len(new_data), int(metadata['content_length']))
 def test_PUT_large_object(self):
     file_item = self.env.container.file(Utils.create_name())
     data = File.random_data(1024 * 1024 * 2)
     self.assertTrue(file_item.write(data))
     self.assert_status(201)
     self.assertTrue(data == file_item.read())
     self.assert_status(200)
Exemple #19
0
    def test_versioning_dlo(self):
        container = self.env.container
        versions_container = self.env.versions_container
        obj_name = Utils.create_name()

        for i in ('1', '2', '3'):
            time.sleep(.01)  # guarantee that the timestamp changes
            obj_name_seg = obj_name + '/' + i
            versioned_obj = container.file(obj_name_seg)
            versioned_obj.write(i)
            versioned_obj.write(i + i)

        self.assertEqual(3, versions_container.info()['object_count'])

        man_file = container.file(obj_name)
        man_file.write('', hdrs={"X-Object-Manifest": "%s/%s/" %
                       (self.env.container.name, obj_name)})

        # guarantee that the timestamp changes
        time.sleep(.01)

        # write manifest file again
        man_file.write('', hdrs={"X-Object-Manifest": "%s/%s/" %
                       (self.env.container.name, obj_name)})

        self.assertEqual(3, versions_container.info()['object_count'])
        self.assertEqual("112233", man_file.read())
Exemple #20
0
    def test_dlo_post_with_manifest_regular_object(self):
        # verify that performing a POST to a regular object
        # with a manifest header will create a DLO.

        # Put a regular object
        file_item = self.env.container.file(Utils.create_name())
        file_item.write('file contents', hdrs={})

        # sanity checks
        file_contents = file_item.read(parms={})
        self.assertEqual('file contents', file_contents)

        # get the path associated with man1
        x_o_m = self.env.container.file('man1').info()['x_object_manifest']

        # POST a x-object-manifest value to the regular object
        file_item.post({'x-object-manifest': x_o_m})

        # verify that the file is now a manifest
        manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
        self.assertEqual('file contents', manifest_contents)
        expected_contents = ''.join([(c * 10) for c in 'abcde'])
        contents = file_item.read(parms={})
        self.assertEqual(expected_contents, contents)
        file_item.info()
        resp_headers = file_item.conn.response.getheaders()
        self.assertIn(('x-object-manifest', x_o_m), resp_headers)
Exemple #21
0
    def test_dlo_post_without_manifest_header(self):
        # verify that a POST to a DLO manifest object with no
        # x-object-manifest header will cause the existing x-object-manifest
        # header to be lost

        # create a new manifest for this test to avoid test coupling.
        x_o_m = self.env.container.file('man1').info()['x_object_manifest']
        file_item = self.env.container.file(Utils.create_name())
        file_item.write('manifest-contents', hdrs={"X-Object-Manifest": x_o_m})

        # sanity checks
        manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
        self.assertEqual('manifest-contents', manifest_contents)
        expected_contents = ''.join([(c * 10) for c in 'abcde'])
        contents = file_item.read(parms={})
        self.assertEqual(expected_contents, contents)

        # POST with no x-object-manifest header
        file_item.post({})

        # verify that existing x-object-manifest was removed
        info = file_item.info()
        self.assertNotIn('x_object_manifest', info)

        # verify that object content was not changed
        manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
        self.assertEqual('manifest-contents', manifest_contents)

        # verify that object is no longer a manifest
        contents = file_item.read(parms={})
        self.assertEqual('manifest-contents', contents)
Exemple #22
0
    def test_versioning_check_acl(self):
        container = self.env.container
        versions_container = self.env.versions_container
        versions_container.create(hdrs={'X-Container-Read': '.r:*,.rlistings'})

        obj_name = Utils.create_name()
        versioned_obj = container.file(obj_name)
        versioned_obj.write("aaaaa")
        self.assertEqual("aaaaa", versioned_obj.read())

        versioned_obj.write("bbbbb")
        self.assertEqual("bbbbb", versioned_obj.read())

        # Use token from second account and try to delete the object
        org_token = self.env.account.conn.storage_token
        self.env.account.conn.storage_token = self.env.conn2.storage_token
        try:
            with self.assertRaises(ResponseError) as cm:
                versioned_obj.delete()
            self.assertEqual(403, cm.exception.status)
        finally:
            self.env.account.conn.storage_token = org_token

        # Verify with token from first account
        self.assertEqual("bbbbb", versioned_obj.read())

        versioned_obj.delete()
        self.assertEqual("aaaaa", versioned_obj.read())
    def _test_versioning_check_acl_setup(self):
        container = self.env.container
        versions_container = self.env.versions_container
        versions_container.create(hdrs={'X-Container-Read': '.r:*,.rlistings'})

        obj_name = Utils.create_name()
        versioned_obj = container.file(obj_name)
        versioned_obj.write("aaaaa")
        self.assertEqual("aaaaa", versioned_obj.read())

        versioned_obj.write("bbbbb")
        self.assertEqual("bbbbb", versioned_obj.read())

        # Use token from second account and try to delete the object
        org_token = self.env.account.conn.storage_token
        self.env.account.conn.storage_token = self.env.conn2.storage_token
        try:
            with self.assertRaises(ResponseError) as cm:
                versioned_obj.delete()
            self.assertEqual(403, cm.exception.status)
        finally:
            self.env.account.conn.storage_token = org_token

        # Verify with token from first account
        self.assertEqual("bbbbb", versioned_obj.read())
        return versioned_obj
    def test_object_expiry_X_Delete_At_PUT(self):
        obj = self.env.container.file(Utils.create_name())
        x_delete_at = str(int(time.time()) + 2)
        obj.write_random(self.env.file_size,
                         hdrs={'X-Delete-At': x_delete_at})

        # Object is not expired. Should still be accessible.
        obj.read()
        self.assert_status(200)

        # Ensure X-Delete-At is saved as object metadata.
        self.assertEqual(x_delete_at, str(obj.info()['x_delete_at']))

        # Wait for object to be expired.
        time.sleep(3)

        # Object has expired. Should no longer be accessible.
        self.assertRaises(ResponseError, obj.read)
        self.assert_status(404)

        # Object should still be present on filesystem.
        self.assertTrue(os.path.isfile(os.path.join(self.env.root_dir,
                                                    self.env.container.name,
                                                    obj.name)))

        # But, GET on container should list the expired object.
        result = self.env.container.files()
        self.assertTrue(obj.name in self.env.container.files())

        # Check existence of corresponding tracker object in gsexpiring
        # account.
        enteredLoop = False
        for c in self.env.client.iter_containers("gsexpiring"):
            for o in self.env.client.iter_objects("gsexpiring", c['name']):
                enteredLoop = True
                l = o['name'].split('/')
                self.assertTrue(l[0].endswith('AUTH_' + self.env.account.name))
                self.assertEqual(l[1], self.env.container.name)
                self.assertEqual(l[2], obj.name)
        if not enteredLoop:
            self.fail("Tracker object not found.")

        # Run expirer daemon once.
        self.env.expirer.once()

        # Ensure object is physically deleted from filesystem.
        self.assertFalse(os.path.exists(os.path.join(self.env.root_dir,
                                                     self.env.container.name,
                                                     obj.name)))

        # Ensure tracker object is consumed.
        try:
            self.env.client.iter_containers("gsexpiring").next()
        except StopIteration:
            pass
        else:
            self.fail("Tracker object persists!")

        # GET on container should no longer list the object.
        self.assertFalse(obj.name in self.env.container.files())
Exemple #25
0
    def test_dlo_post_with_manifest_regular_object(self):
        # verify that performing a POST to a regular object
        # with a manifest header will create a DLO.

        # Put a regular object
        file_item = self.env.container.file(Utils.create_name())
        file_item.write(b'file contents', hdrs={})

        # sanity checks
        file_contents = file_item.read(parms={})
        self.assertEqual(b'file contents', file_contents)

        # get the path associated with man1
        x_o_m = self.env.container.file('man1').info()['x_object_manifest']

        # POST a x-object-manifest value to the regular object
        file_item.post({'x-object-manifest': x_o_m})

        # verify that the file is now a manifest
        manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
        self.assertEqual(b'file contents', manifest_contents)
        expected_contents = ''.join([(c * 10) for c in 'abcde']).encode()
        contents = file_item.read(parms={})
        self.assertEqual(expected_contents, contents)
        file_item.info()
        resp_headers = [(h.lower(), v)
                        for h, v in file_item.conn.response.getheaders()]
        self.assertIn(('x-object-manifest', str_to_wsgi(x_o_m)), resp_headers)
Exemple #26
0
    def test_dlo_post_without_manifest_header(self):
        # verify that a POST to a DLO manifest object with no
        # x-object-manifest header will cause the existing x-object-manifest
        # header to be lost

        # create a new manifest for this test to avoid test coupling.
        x_o_m = self.env.container.file('man1').info()['x_object_manifest']
        file_item = self.env.container.file(Utils.create_name())
        file_item.write(b'manifest-contents',
                        hdrs={"X-Object-Manifest": x_o_m})

        # sanity checks
        manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
        self.assertEqual(b'manifest-contents', manifest_contents)
        expected_contents = ''.join((c * 10) for c in 'abcde').encode('ascii')
        contents = file_item.read(parms={})
        self.assertEqual(expected_contents, contents)

        # POST with no x-object-manifest header
        file_item.post({})

        # verify that existing x-object-manifest was removed
        info = file_item.info()
        self.assertNotIn('x_object_manifest', info)

        # verify that object content was not changed
        manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
        self.assertEqual(b'manifest-contents', manifest_contents)

        # verify that object is no longer a manifest
        contents = file_item.read(parms={})
        self.assertEqual(b'manifest-contents', contents)
Exemple #27
0
    def test_dlo_post_with_manifest_header(self):
        # verify that performing a POST to a DLO manifest
        # preserves the fact that it is a manifest file.
        # verify that the x-object-manifest header may be updated.

        # create a new manifest for this test to avoid test coupling.
        x_o_m = self.env.container.file('man1').info()['x_object_manifest']
        file_item = self.env.container.file(Utils.create_name())
        file_item.write(b'manifest-contents',
                        hdrs={"X-Object-Manifest": x_o_m})

        # sanity checks
        manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
        self.assertEqual(b'manifest-contents', manifest_contents)
        expected_contents = ''.join((c * 10) for c in 'abcde').encode('ascii')
        contents = file_item.read(parms={})
        self.assertEqual(expected_contents, contents)

        # POST a modified x-object-manifest value
        new_x_o_m = x_o_m.rstrip('lower') + 'upper'
        file_item.post({
            'x-object-meta-foo': 'bar',
            'x-object-manifest': new_x_o_m
        })

        # verify that x-object-manifest was updated
        file_item.info()
        resp_headers = [(h.lower(), v)
                        for h, v in file_item.conn.response.getheaders()]
        self.assertIn(('x-object-manifest', str_to_wsgi(new_x_o_m)),
                      resp_headers)
        self.assertIn(('x-object-meta-foo', 'bar'), resp_headers)

        # verify that manifest content was not changed
        manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
        self.assertEqual(b'manifest-contents', manifest_contents)

        # verify that updated manifest points to new content
        expected_contents = ''.join((c * 10) for c in 'ABCDE').encode('ascii')
        contents = file_item.read(parms={})
        self.assertEqual(expected_contents, contents)

        # Now revert the manifest to point to original segments, including a
        # multipart-manifest=get param just to check that has no effect
        file_item.post({'x-object-manifest': x_o_m},
                       parms={'multipart-manifest': 'get'})

        # verify that x-object-manifest was reverted
        info = file_item.info()
        self.assertIn('x_object_manifest', info)
        self.assertEqual(x_o_m, info['x_object_manifest'])

        # verify that manifest content was not changed
        manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
        self.assertEqual(b'manifest-contents', manifest_contents)

        # verify that updated manifest points new content
        expected_contents = ''.join((c * 10) for c in 'abcde').encode('ascii')
        contents = file_item.read(parms={})
        self.assertEqual(expected_contents, contents)
Exemple #28
0
    def test_overwriting_with_url_encoded_object_name(self):
        versions_container = self.env.versions_container
        obj_name = Utils.create_name() + '%25ff'
        versioned_obj, expected_headers, expected_content_types = \
            self._test_overwriting_setup(obj_name)

        # test delete
        # at first, delete will succeed with 204
        versioned_obj.delete()
        expected_content_types.append(
            'application/x-deleted;swift_versions_deleted=1')
        # after that, any time the delete doesn't restore the old version
        # and we will get 404 NotFound
        for x in range(3):
            with self.assertRaises(ResponseError) as cm:
                versioned_obj.delete()
            self.assertEqual(404, cm.exception.status)
            expected_content_types.append(
                'application/x-deleted;swift_versions_deleted=1')
        # finally, we have 4 versioned items and 4 delete markers total in
        # the versions container
        self.assertEqual(8, versions_container.info()['object_count'])
        self.assertEqual(expected_content_types, [
            o['content_type']
            for o in versions_container.files(parms={'format': 'json'})
        ])

        # update versioned_obj
        versioned_obj.write("eeee",
                            hdrs={
                                'Content-Type': 'text/thanksgiving',
                                'X-Object-Meta-Bar': 'foo'
                            })
        # verify the PUT object is kept successfully
        obj_info = versioned_obj.info()
        self.assertEqual('text/thanksgiving', obj_info['content_type'])

        # we still have delete-marker there
        self.assertEqual(8, versions_container.info()['object_count'])

        # update versioned_obj
        versioned_obj.write("ffff",
                            hdrs={
                                'Content-Type': 'text/teriyaki',
                                'X-Object-Meta-Food': 'chickin'
                            })
        # verify the PUT object is kept successfully
        obj_info = versioned_obj.info()
        self.assertEqual('text/teriyaki', obj_info['content_type'])

        # new obj will be inserted after delete-marker there
        self.assertEqual(9, versions_container.info()['object_count'])

        versioned_obj.delete()
        with self.assertRaises(ResponseError) as cm:
            versioned_obj.read()
        self.assertEqual(404, cm.exception.status)
Exemple #29
0
    def setUp(cls):
        cls.conn = Connection(tf.config)
        cls.conn.authenticate()

        if cls.static_web_enabled is None:
            cls.static_web_enabled = 'staticweb' in cluster_info
            if not cls.static_web_enabled:
                return

        cls.account = Account(
            cls.conn, tf.config.get('account', tf.config['username']))
        cls.account.delete_containers()

        cls.container = cls.account.container(Utils.create_name())
        if not cls.container.create(
                hdrs={'X-Container-Read': '.r:*,.rlistings'}):
            raise ResponseError(cls.conn.response)

        objects = ['index',
                   'error',
                   'listings_css',
                   'dir/',
                   'dir/obj',
                   'dir/subdir/',
                   'dir/subdir/obj']

        cls.objects = {}
        for item in sorted(objects):
            parent = None
            if '/' in item.rstrip('/'):
                parent, _ = item.rstrip('/').rsplit('/', 1)
                path = '%s/%s' % (cls.objects[parent + '/'].name,
                                  Utils.create_name())
            else:
                path = Utils.create_name()

            if item[-1] == '/':
                cls.objects[item] = cls.container.file(path)
                cls.objects[item].write(hdrs={
                    'Content-Type': 'application/directory'})
            else:
                cls.objects[item] = cls.container.file(path)
                cls.objects[item].write('%s contents' % item)
    def setUp(cls):
        cls.conn = Connection(config)
        cls.conn.authenticate()
        cls.account = Account(cls.conn,
                              config.get('account', config['username']))
        cls.root_dir = os.path.join(
            '/mnt/gluster-object',
            cls.account.conn.storage_url.split('/')[2].split('_')[1])
        cls.account.delete_containers()

        cls.file_size = 8
        cls.container = cls.account.container(Utils.create_name())
        if not cls.container.create():
            raise ResponseError(cls.conn.response)

        cls.dirs = [
            'dir1',
            'dir2',
            'dir1/subdir1',
            'dir1/subdir2',
            'dir1/subdir1/subsubdir1',
            'dir1/subdir1/subsubdir2',
            'dir1/subdir with spaces',
            'dir1/subdir+with{whatever',
        ]

        cls.files = [
            'file1',
            'file A',
            'dir1/file2',
            'dir1/subdir1/file2',
            'dir1/subdir1/file3',
            'dir1/subdir1/file4',
            'dir1/subdir1/subsubdir1/file5',
            'dir1/subdir1/subsubdir1/file6',
            'dir1/subdir1/subsubdir1/file7',
            'dir1/subdir1/subsubdir1/file8',
            'dir1/subdir1/subsubdir2/file9',
            'dir1/subdir1/subsubdir2/file0',
            'dir1/subdir with spaces/file B',
            'dir1/subdir+with{whatever/file D',
        ]

        stored_files = set()
        for d in cls.dirs:
            file = cls.container.file(d)
            file.write(hdrs={'Content-Type': 'application/directory'})
        for f in cls.files:
            file = cls.container.file(f)
            file.write_random(
                cls.file_size,
                hdrs={'Content-Type': 'application/octet-stream'})
            stored_files.add(f)
        cls.stored_files = sorted(stored_files)
        cls.sorted_objects = sorted(set(cls.dirs + cls.files))
Exemple #31
0
    def test_dlo_post_with_manifest_header(self):
        # verify that performing a POST to a DLO manifest
        # preserves the fact that it is a manifest file.
        # verify that the x-object-manifest header may be updated.

        # create a new manifest for this test to avoid test coupling.
        x_o_m = self.env.container.file('man1').info()['x_object_manifest']
        file_item = self.env.container.file(Utils.create_name())
        file_item.write('manifest-contents', hdrs={"X-Object-Manifest": x_o_m})

        # sanity checks
        manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
        self.assertEqual('manifest-contents', manifest_contents)
        expected_contents = ''.join([(c * 10) for c in 'abcde'])
        contents = file_item.read(parms={})
        self.assertEqual(expected_contents, contents)

        # POST a modified x-object-manifest value
        new_x_o_m = x_o_m.rstrip('lower') + 'upper'
        file_item.post({'x-object-meta-foo': 'bar',
                        'x-object-manifest': new_x_o_m})

        # verify that x-object-manifest was updated
        file_item.info()
        resp_headers = file_item.conn.response.getheaders()
        self.assertIn(('x-object-manifest', new_x_o_m), resp_headers)
        self.assertIn(('x-object-meta-foo', 'bar'), resp_headers)

        # verify that manifest content was not changed
        manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
        self.assertEqual('manifest-contents', manifest_contents)

        # verify that updated manifest points to new content
        expected_contents = ''.join([(c * 10) for c in 'ABCDE'])
        contents = file_item.read(parms={})
        self.assertEqual(expected_contents, contents)

        # Now revert the manifest to point to original segments, including a
        # multipart-manifest=get param just to check that has no effect
        file_item.post({'x-object-manifest': x_o_m},
                       parms={'multipart-manifest': 'get'})

        # verify that x-object-manifest was reverted
        info = file_item.info()
        self.assertIn('x_object_manifest', info)
        self.assertEqual(x_o_m, info['x_object_manifest'])

        # verify that manifest content was not changed
        manifest_contents = file_item.read(parms={'multipart-manifest': 'get'})
        self.assertEqual('manifest-contents', manifest_contents)

        # verify that updated manifest points new content
        expected_contents = ''.join([(c * 10) for c in 'abcde'])
        contents = file_item.read(parms={})
        self.assertEqual(expected_contents, contents)
Exemple #32
0
    def setUp(cls):
        cls.conn = Connection(tf.config)
        cls.conn.authenticate()

        if cls.static_web_enabled is None:
            cls.static_web_enabled = 'staticweb' in tf.cluster_info
            if not cls.static_web_enabled:
                return

        cls.account = Account(
            cls.conn, tf.config.get('account', tf.config['username']))
        cls.account.delete_containers()

        cls.container = cls.account.container(Utils.create_name())
        if not cls.container.create(
                hdrs={'X-Container-Read': '.r:*,.rlistings'}):
            raise ResponseError(cls.conn.response)

        objects = ['index',
                   'error',
                   'listings_css',
                   'dir/',
                   'dir/obj',
                   'dir/some sub%dir/',
                   'dir/some sub%dir/obj']

        cls.objects = {}
        for item in sorted(objects):
            if '/' in item.rstrip('/'):
                parent, _ = item.rstrip('/').rsplit('/', 1)
                path = '%s/%s' % (cls.objects[parent + '/'].name,
                                  Utils.create_name())
            else:
                path = Utils.create_name()

            if item[-1] == '/':
                cls.objects[item] = cls.container.file(path)
                cls.objects[item].write(hdrs={
                    'Content-Type': 'application/directory'})
            else:
                cls.objects[item] = cls.container.file(path)
                cls.objects[item].write(('%s contents' % item).encode('utf8'))
 def testObjectContentFromMountPoint(self):
     file_name = Utils.create_name()
     file_item = self.env.container.file(file_name)
     data = file_item.write_random()
     self.assert_status(201)
     file_info = file_item.info()
     fhOnMountPoint = open(os.path.join(self.env.root_dir,self.env.container.name,file_name),'r')
     data_read_from_mountP = fhOnMountPoint.read()
     md5_returned = hashlib.md5(data_read_from_mountP).hexdigest()
     self.assertEquals(md5_returned,file_info['etag'])
     fhOnMountPoint.close()
    def setUp(cls):
        cls.conn = Connection(tf.config)
        cls.storage_url, cls.storage_token = cls.conn.authenticate()

        cls.account = Account(cls.conn, tf.config.get('account',
                                                      tf.config['username']))

        # Second connection for ACL tests
        config2 = deepcopy(tf.config)
        config2['account'] = tf.config['account2']
        config2['username'] = tf.config['username2']
        config2['password'] = tf.config['password2']
        cls.conn2 = Connection(config2)
        cls.conn2.authenticate()

        # avoid getting a prefix that stops halfway through an encoded
        # character
        prefix = Utils.create_name().decode("utf-8")[:10].encode("utf-8")

        cls.versions_container = cls.account.container(prefix + "-versions")
        if not cls.versions_container.create():
            raise ResponseError(cls.conn.response)

        cls.container = cls.account.container(prefix + "-objs")
        container_headers = {
            cls.location_header_key: cls.versions_container.name}
        if not cls.container.create(hdrs=container_headers):
            if cls.conn.response.status == 412:
                cls.versioning_enabled = False
                return
            raise ResponseError(cls.conn.response)

        container_info = cls.container.info()
        # if versioning is off, then cls.location_header_key won't persist
        cls.versioning_enabled = 'versions' in container_info

        # setup another account to test ACLs
        config2 = deepcopy(tf.config)
        config2['account'] = tf.config['account2']
        config2['username'] = tf.config['username2']
        config2['password'] = tf.config['password2']
        cls.conn2 = Connection(config2)
        cls.storage_url2, cls.storage_token2 = cls.conn2.authenticate()
        cls.account2 = cls.conn2.get_account()
        cls.account2.delete_containers()

        # setup another account with no access to anything to test ACLs
        config3 = deepcopy(tf.config)
        config3['account'] = tf.config['account']
        config3['username'] = tf.config['username3']
        config3['password'] = tf.config['password3']
        cls.conn3 = Connection(config3)
        cls.storage_url3, cls.storage_token3 = cls.conn3.authenticate()
        cls.account3 = cls.conn3.get_account()
    def setUp(cls):
        cls.conn = Connection(tf.config)
        cls.conn.authenticate()
        cls.account = Account(cls.conn, tf.config.get('account',
                                                      tf.config['username']))

        #cls.root_dir = os.path.join('/mnt/swiftonhpss/test')
        cls.root_dir = os.path.join('/srv/hpss')
        cls.account.delete_containers()

        cls.file_size = 8
        cls.container = cls.account.container(Utils.create_name())
        if not cls.container.create(hdrs={'X-Storage-Policy': 'swiftonhpss'}):
            raise ResponseError(cls.conn.response)

        cls.dirs = [
            'dir1',
            'dir2',
            'dir1/subdir1',
            'dir1/subdir2',
            'dir1/subdir1/subsubdir1',
            'dir1/subdir1/subsubdir2',
            'dir1/subdir with spaces',
            'dir1/subdir+with{whatever',
        ]

        cls.files = [
            'file1',
            'file A',
            'dir1/file2',
            'dir1/subdir1/file2',
            'dir1/subdir1/file3',
            'dir1/subdir1/file4',
            'dir1/subdir1/subsubdir1/file5',
            'dir1/subdir1/subsubdir1/file6',
            'dir1/subdir1/subsubdir1/file7',
            'dir1/subdir1/subsubdir1/file8',
            'dir1/subdir1/subsubdir2/file9',
            'dir1/subdir1/subsubdir2/file0',
            'dir1/subdir with spaces/file B',
            'dir1/subdir+with{whatever/file D',
        ]

        stored_files = set()
        for d in cls.dirs:
            file = cls.container.file(d)
            file.write(hdrs={'Content-Type': 'application/directory'})
        for f in cls.files:
            file = cls.container.file(f)
            file.write_random(cls.file_size, hdrs={'Content-Type':
                              'application/octet-stream'})
            stored_files.add(f)
        cls.stored_files = sorted(stored_files)
        cls.sorted_objects = sorted(set(cls.dirs + cls.files))
    def setUp(cls):
        cls.conn = Connection(tf.config)
        cls.conn.authenticate()
        cls.account = Account(cls.conn, tf.config.get("account", tf.config["username"]))

        # PGB - change hardcoded SoF mountpoint
        # cls.root_dir = os.path.join('/mnt/swiftonhpss/test')
        cls.root_dir = os.path.join("/srv/swift/hpss")
        cls.account.delete_containers()

        cls.file_size = 8
        cls.container = cls.account.container(Utils.create_name())
        if not cls.container.create(None, None):
            raise ResponseError(cls.conn.response)

        cls.dirs = [
            "dir1",
            "dir2",
            "dir1/subdir1",
            "dir1/subdir2",
            "dir1/subdir1/subsubdir1",
            "dir1/subdir1/subsubdir2",
            "dir1/subdir with spaces",
            "dir1/subdir+with{whatever",
        ]

        cls.files = [
            "file1",
            "file A",
            "dir1/file2",
            "dir1/subdir1/file2",
            "dir1/subdir1/file3",
            "dir1/subdir1/file4",
            "dir1/subdir1/subsubdir1/file5",
            "dir1/subdir1/subsubdir1/file6",
            "dir1/subdir1/subsubdir1/file7",
            "dir1/subdir1/subsubdir1/file8",
            "dir1/subdir1/subsubdir2/file9",
            "dir1/subdir1/subsubdir2/file0",
            "dir1/subdir with spaces/file B",
            "dir1/subdir+with{whatever/file D",
        ]

        stored_files = set()
        for d in cls.dirs:
            file = cls.container.file(d)
            file.write(hdrs={"Content-Type": "application/directory"})
        for f in cls.files:
            file = cls.container.file(f)
            file.write_random(cls.file_size, hdrs={"Content-Type": "application/octet-stream"})
            stored_files.add(f)
        cls.stored_files = sorted(stored_files)
        cls.sorted_objects = sorted(set(cls.dirs + cls.files))
 def testInvalidHeadersPUT(self):
     file = self.env.container.file(Utils.create_name())
     self.assertRaises(ResponseError,
                       file.write_random,
                       self.env.file_size,
                       hdrs={'X-Delete-At': '9876545321'})
     self.assert_status(400)
     self.assertRaises(ResponseError,
                       file.write_random,
                       self.env.file_size,
                       hdrs={'X-Delete-After': '60'})
     self.assert_status(400)
Exemple #38
0
    def test_PUT_remapped_container(self):
        for domain in (self.cont_domain_dash, self.cont_domain_underscore):
            new_obj_name = Utils.create_name()
            path = '/'.join(['', new_obj_name])
            self.env.account.conn.make_request('PUT', path,
                                               data='new obj contents',
                                               hdrs={'Host': domain},
                                               cfg={'absolute_path': True})
            self.assert_status(201)

            new_obj = self.env.container.file(new_obj_name)
            self.assertEqual(new_obj.read(), 'new obj contents')
Exemple #39
0
    def test_GET_DLO_inside_container(self):
        seg1 = self.env.container.file(
            "get-dlo-inside-seg1" + Utils.create_name())
        seg2 = self.env.container.file(
            "get-dlo-inside-seg2" + Utils.create_name())
        seg1.write(b"one fish two fish ")
        seg2.write(b"red fish blue fish")

        manifest = self.env.container.file("manifest" + Utils.create_name())
        manifest.write(
            b'',
            hdrs={"X-Object-Manifest": "%s/get-dlo-inside-seg" %
                  (self.env.container.name,)})

        expires = int(time()) + 86400
        parms = self.tempurl_parms(
            'GET', expires, self.env.conn.make_path(manifest.path),
            self.env.tempurl_key)

        contents = manifest.read(parms=parms, cfg={'no_auth_token': True})
        self.assertEqual(contents, b"one fish two fish red fish blue fish")
Exemple #40
0
    def test_GET_DLO_inside_container(self):
        seg1 = self.env.container.file(
            "get-dlo-inside-seg1" + Utils.create_name())
        seg2 = self.env.container.file(
            "get-dlo-inside-seg2" + Utils.create_name())
        seg1.write("one fish two fish ")
        seg2.write("red fish blue fish")

        manifest = self.env.container.file("manifest" + Utils.create_name())
        manifest.write(
            '',
            hdrs={"X-Object-Manifest": "%s/get-dlo-inside-seg" %
                  (self.env.container.name,)})

        expires = int(time.time()) + 86400
        parms = self.tempurl_parms(
            'GET', expires, self.env.conn.make_path(manifest.path),
            self.env.tempurl_key)

        contents = manifest.read(parms=parms, cfg={'no_auth_token': True})
        self.assertEqual(contents, "one fish two fish red fish blue fish")
Exemple #41
0
    def setUp(cls):
        super(TestSloTempurlEnv, cls).setUp()
        if cls.enabled is None:
            cls.enabled = 'tempurl' in cluster_info and 'slo' in cluster_info

        cls.tempurl_key = Utils.create_name()

        cls.account.update_metadata({'temp-url-key': cls.tempurl_key})

        cls.manifest_container = cls.account.container(Utils.create_name())
        cls.segments_container = cls.account.container(Utils.create_name())
        if not cls.manifest_container.create():
            raise ResponseError(cls.conn.response)
        if not cls.segments_container.create():
            raise ResponseError(cls.conn.response)

        seg1 = cls.segments_container.file(Utils.create_name())
        seg1.write(b'1' * 1024 * 1024)

        seg2 = cls.segments_container.file(Utils.create_name())
        seg2.write(b'2' * 1024 * 1024)

        cls.manifest_data = [{'size_bytes': 1024 * 1024,
                              'etag': seg1.md5,
                              'path': '/%s/%s' % (cls.segments_container.name,
                                                  seg1.name)},
                             {'size_bytes': 1024 * 1024,
                              'etag': seg2.md5,
                              'path': '/%s/%s' % (cls.segments_container.name,
                                                  seg2.name)}]

        cls.manifest = cls.manifest_container.file(Utils.create_name())
        cls.manifest.write(
            json.dumps(cls.manifest_data).encode('ascii'),
            parms={'multipart-manifest': 'put'})
Exemple #42
0
    def setUp(cls):
        super(TestSloTempurlEnv, cls).setUp()
        if cls.enabled is None:
            cls.enabled = 'tempurl' in cluster_info and 'slo' in cluster_info

        cls.tempurl_key = Utils.create_name()

        cls.account.update_metadata({'temp-url-key': cls.tempurl_key})

        cls.manifest_container = cls.account.container(Utils.create_name())
        cls.segments_container = cls.account.container(Utils.create_name())
        if not cls.manifest_container.create():
            raise ResponseError(cls.conn.response)
        if not cls.segments_container.create():
            raise ResponseError(cls.conn.response)

        seg1 = cls.segments_container.file(Utils.create_name())
        seg1.write('1' * 1024 * 1024)

        seg2 = cls.segments_container.file(Utils.create_name())
        seg2.write('2' * 1024 * 1024)

        cls.manifest_data = [{'size_bytes': 1024 * 1024,
                              'etag': seg1.md5,
                              'path': '/%s/%s' % (cls.segments_container.name,
                                                  seg1.name)},
                             {'size_bytes': 1024 * 1024,
                              'etag': seg2.md5,
                              'path': '/%s/%s' % (cls.segments_container.name,
                                                  seg2.name)}]

        cls.manifest = cls.manifest_container.file(Utils.create_name())
        cls.manifest.write(
            json.dumps(cls.manifest_data),
            parms={'multipart-manifest': 'put'})
Exemple #43
0
    def setUp(cls):
        cls.conn = Connection(tf.config)
        cls.conn.authenticate()

        if cls.domain_remap_enabled is None:
            cls.domain_remap_enabled = 'domain_remap' in cluster_info
            if not cls.domain_remap_enabled:
                return

        cls.account = Account(
            cls.conn, tf.config.get('account', tf.config['username']))
        cls.account.delete_containers()

        cls.container = cls.account.container(Utils.create_name())
        if not cls.container.create():
            raise ResponseError(cls.conn.response)

        cls.obj = cls.container.file(Utils.create_name())
        cls.obj.write('obj contents')

        cls.obj_slash = cls.container.file('/v1')
        cls.obj_slash.write('obj contents')
Exemple #44
0
    def setUp(cls):
        cls.conn = Connection(tf.config)
        cls.conn.authenticate()

        if cls.domain_remap_enabled is None:
            cls.domain_remap_enabled = 'domain_remap' in cluster_info
            if not cls.domain_remap_enabled:
                return

        cls.account = Account(cls.conn,
                              tf.config.get('account', tf.config['username']))
        cls.account.delete_containers()

        cls.container = cls.account.container(Utils.create_name())
        if not cls.container.create():
            raise ResponseError(cls.conn.response)

        cls.obj = cls.container.file(Utils.create_name())
        cls.obj.write('obj contents')

        cls.obj_slash = cls.container.file('/v1')
        cls.obj_slash.write('obj contents')
Exemple #45
0
    def test_PUT_remapped_container(self):
        for domain in (self.cont_domain_dash, self.cont_domain_underscore):
            new_obj_name = Utils.create_name()
            path = '/'.join(['', new_obj_name])
            self.env.account.conn.make_request('PUT',
                                               path,
                                               data='new obj contents',
                                               hdrs={'Host': domain},
                                               cfg={'absolute_path': True})
            self.assert_status(201)

            new_obj = self.env.container.file(new_obj_name)
            self.assertEqual(new_obj.read(), 'new obj contents')
    def test_overwriting_with_url_encoded_object_name(self):
        versions_container = self.env.versions_container
        obj_name = Utils.create_name() + '%25ff'
        versioned_obj, expected_headers, expected_content_types = \
            self._test_overwriting_setup(obj_name)

        # test delete
        # at first, delete will succeed with 204
        versioned_obj.delete()
        expected_content_types.append(
            'application/x-deleted;swift_versions_deleted=1')
        # after that, any time the delete doesn't restore the old version
        # and we will get 404 NotFound
        for x in range(3):
            with self.assertRaises(ResponseError) as cm:
                versioned_obj.delete()
            self.assertEqual(404, cm.exception.status)
            expected_content_types.append(
                'application/x-deleted;swift_versions_deleted=1')
        # finally, we have 4 versioned items and 4 delete markers total in
        # the versions container
        self.assertEqual(8, versions_container.info()['object_count'])
        self.assertEqual(expected_content_types, [
            o['content_type'] for o in versions_container.files(
                parms={'format': 'json'})])

        # update versioned_obj
        versioned_obj.write("eeee", hdrs={'Content-Type': 'text/thanksgiving',
                            'X-Object-Meta-Bar': 'foo'})
        # verify the PUT object is kept successfully
        obj_info = versioned_obj.info()
        self.assertEqual('text/thanksgiving', obj_info['content_type'])

        # we still have delete-marker there
        self.assertEqual(8, versions_container.info()['object_count'])

        # update versioned_obj
        versioned_obj.write("ffff", hdrs={'Content-Type': 'text/teriyaki',
                            'X-Object-Meta-Food': 'chickin'})
        # verify the PUT object is kept successfully
        obj_info = versioned_obj.info()
        self.assertEqual('text/teriyaki', obj_info['content_type'])

        # new obj will be inserted after delete-marker there
        self.assertEqual(9, versions_container.info()['object_count'])

        versioned_obj.delete()
        with self.assertRaises(ResponseError) as cm:
            versioned_obj.read()
        self.assertEqual(404, cm.exception.status)
Exemple #47
0
    def setUp(cls):
        if cls.tempurl_enabled is None:
            cls.tempurl_enabled = 'tempurl' in cluster_info
            if not cls.tempurl_enabled:
                return

        super(TestTempurlEnv, cls).setUp()

        cls.tempurl_key = Utils.create_name()
        cls.tempurl_key2 = Utils.create_name()

        cls.account.update_metadata({
            'temp-url-key': cls.tempurl_key,
            'temp-url-key-2': cls.tempurl_key2
        })

        cls.container = cls.account.container(Utils.create_name())
        if not cls.container.create():
            raise ResponseError(cls.conn.response)

        cls.obj = cls.container.file(Utils.create_name())
        cls.obj.write(b"obj contents")
        cls.other_obj = cls.container.file(Utils.create_name())
        cls.other_obj.write(b"other obj contents")
Exemple #48
0
    def test_PUT(self):
        new_obj = self.env.container.file(Utils.create_name())

        expires = int(time.time()) + 86400
        put_parms = self.tempurl_parms(
            'PUT', expires, self.env.conn.make_path(new_obj.path),
            self.env.tempurl_key)

        new_obj.write('new obj contents',
                      parms=put_parms, cfg={'no_auth_token': True})
        self.assertEqual(new_obj.read(), "new obj contents")

        # PUT tempurls also allow HEAD requests
        self.assertTrue(new_obj.info(parms=put_parms,
                                     cfg={'no_auth_token': True}))
Exemple #49
0
    def test_PUT(self):
        new_obj = self.env.container.file(Utils.create_name())

        expires = int(time.time()) + 86400
        put_parms = self.tempurl_parms(
            'PUT', expires, self.env.conn.make_path(new_obj.path),
            self.env.tempurl_key)

        new_obj.write('new obj contents',
                      parms=put_parms, cfg={'no_auth_token': True})
        self.assertEqual(new_obj.read(), "new obj contents")

        # PUT tempurls also allow HEAD requests
        self.assertTrue(new_obj.info(parms=put_parms,
                                     cfg={'no_auth_token': True}))
Exemple #50
0
    def test_GET_DLO_outside_container(self):
        container2 = self.env.account.container(Utils.create_name())
        container2.create()
        seg1 = container2.file(
            "get-dlo-outside-seg1" + Utils.create_name())
        seg2 = container2.file(
            "get-dlo-outside-seg2" + Utils.create_name())
        seg1.write("one fish two fish ")
        seg2.write("red fish blue fish")

        manifest = self.env.container.file("manifest" + Utils.create_name())
        manifest.write(
            '',
            hdrs={"X-Object-Manifest": "%s/get-dlo-outside-seg" %
                  (container2.name,)})

        expires = int(time()) + 86400
        sig = self.tempurl_sig(
            'GET', expires, self.env.conn.make_path(manifest.path),
            self.env.tempurl_key)
        parms = {'temp_url_sig': sig,
                 'temp_url_expires': str(expires)}

        # cross container tempurl does not work for container tempurl key
        try:
            manifest.read(parms=parms, cfg={'no_auth_token': True})
        except ResponseError as e:
            self.assertEqual(e.status, 401)
        else:
            self.fail('request did not error')
        try:
            manifest.info(parms=parms, cfg={'no_auth_token': True})
        except ResponseError as e:
            self.assertEqual(e.status, 401)
        else:
            self.fail('request did not error')
Exemple #51
0
    def setUp(cls):
        if cls.tempurl_enabled is None:
            cls.tempurl_enabled = 'tempurl' in cluster_info
            if not cls.tempurl_enabled:
                return

        super(TestTempurlEnv, cls).setUp()

        cls.tempurl_key = Utils.create_name()
        cls.tempurl_key2 = Utils.create_name()

        cls.account.update_metadata({
            'temp-url-key': cls.tempurl_key,
            'temp-url-key-2': cls.tempurl_key2
        })

        cls.container = cls.account.container(Utils.create_name())
        if not cls.container.create():
            raise ResponseError(cls.conn.response)

        cls.obj = cls.container.file(Utils.create_name())
        cls.obj.write("obj contents")
        cls.other_obj = cls.container.file(Utils.create_name())
        cls.other_obj.write("other obj contents")
Exemple #52
0
    def _check_overwriting_symlink(self):
        # assertions common to x-versions-location and x-history-location modes
        container = self.env.container
        versions_container = self.env.versions_container

        tgt_a_name = Utils.create_name()
        tgt_b_name = Utils.create_name()

        tgt_a = container.file(tgt_a_name)
        tgt_a.write("aaaaa")

        tgt_b = container.file(tgt_b_name)
        tgt_b.write("bbbbb")

        symlink_name = Utils.create_name()
        sym_tgt_header = '%s/%s' % (container.name, tgt_a_name)
        sym_headers_a = {'X-Symlink-Target': sym_tgt_header}
        symlink = container.file(symlink_name)
        symlink.write("", hdrs=sym_headers_a)
        self.assertEqual("aaaaa", symlink.read())

        sym_headers_b = {
            'X-Symlink-Target': '%s/%s' % (container.name, tgt_b_name)
        }
        symlink.write("", hdrs=sym_headers_b)
        self.assertEqual("bbbbb", symlink.read())

        # the old version got saved off
        self.assertEqual(1, versions_container.info()['object_count'])
        versioned_obj_name = versions_container.files()[0]
        prev_version = versions_container.file(versioned_obj_name)
        prev_version_info = prev_version.info(parms={'symlink': 'get'})
        self.assertEqual("aaaaa", prev_version.read())
        self.assertEqual(MD5_OF_EMPTY_STRING, prev_version_info['etag'])
        self.assertEqual(sym_tgt_header, prev_version_info['x_symlink_target'])
        return symlink, tgt_a
    def _check_overwriting_symlink(self):
        # assertions common to x-versions-location and x-history-location modes
        container = self.env.container
        versions_container = self.env.versions_container

        tgt_a_name = Utils.create_name()
        tgt_b_name = Utils.create_name()

        tgt_a = container.file(tgt_a_name)
        tgt_a.write("aaaaa")

        tgt_b = container.file(tgt_b_name)
        tgt_b.write("bbbbb")

        symlink_name = Utils.create_name()
        sym_tgt_header = '%s/%s' % (container.name, tgt_a_name)
        sym_headers_a = {'X-Symlink-Target': sym_tgt_header}
        symlink = container.file(symlink_name)
        symlink.write("", hdrs=sym_headers_a)
        self.assertEqual("aaaaa", symlink.read())

        sym_headers_b = {'X-Symlink-Target': '%s/%s' % (container.name,
                                                        tgt_b_name)}
        symlink.write("", hdrs=sym_headers_b)
        self.assertEqual("bbbbb", symlink.read())

        # the old version got saved off
        self.assertEqual(1, versions_container.info()['object_count'])
        versioned_obj_name = versions_container.files()[0]
        prev_version = versions_container.file(versioned_obj_name)
        prev_version_info = prev_version.info(parms={'symlink': 'get'})
        self.assertEqual("aaaaa", prev_version.read())
        self.assertEqual(MD5_OF_EMPTY_STRING, prev_version_info['etag'])
        self.assertEqual(sym_tgt_header,
                         prev_version_info['x_symlink_target'])
        return symlink, tgt_a
 def testInvalidHeadersPUT(self):
     #TODO: Although we now support x-delete-at and x-delete-after,
     #retained this test case as we may add some other header to
     #unsupported list in future
     raise SkipTest()
     file = self.env.container.file(Utils.create_name())
     self.assertRaises(ResponseError,
                       file.write_random,
                       self.env.file_size,
                       hdrs={'X-Delete-At': '9876545321'})
     self.assert_status(400)
     self.assertRaises(ResponseError,
                       file.write_random,
                       self.env.file_size,
                       hdrs={'X-Delete-After': '60'})
     self.assert_status(400)
    def _test_versioning_dlo_setup(self):
        if tf.in_process:
            tf.skip_if_no_xattrs()

        container = self.env.container
        versions_container = self.env.versions_container
        obj_name = Utils.create_name()

        for i in ('1', '2', '3'):
            time.sleep(.01)  # guarantee that the timestamp changes
            obj_name_seg = obj_name + '/' + i
            versioned_obj = container.file(obj_name_seg)
            versioned_obj.write(i.encode('ascii'))
            # immediately overwrite
            versioned_obj.write((i + i).encode('ascii'))

        self.assertEqual(3, versions_container.info()['object_count'])

        man_file = container.file(obj_name)

        # write a normal file first
        man_file.write(b'old content')

        # guarantee that the timestamp changes
        time.sleep(.01)

        # overwrite with a dlo manifest
        man_file.write(b'',
                       hdrs={
                           "X-Object-Manifest":
                           "%s/%s/" % (self.env.container.name, obj_name)
                       })

        self.assertEqual(4, versions_container.info()['object_count'])
        self.assertEqual(b"112233", man_file.read())
        self.assert_most_recent_version(obj_name, b'old content')

        # overwrite the manifest with a normal file
        man_file.write(b'new content')
        self.assertEqual(5, versions_container.info()['object_count'])

        # new most-recent archive is the dlo
        self.assert_most_recent_version(obj_name,
                                        b'112233',
                                        should_be_dlo=True)

        return obj_name, man_file