Beispiel #1
0
    def test_is_sup(self):
        backend = AccountBackend({}, self.conn)
        compare = (backend.lua_is_sup + """
            if (is_sup(KEYS[1], KEYS[2])) then
              return redis.status_reply('IS SUP');
            else
              return redis.error_reply('IS NOT SUP');
            end;
                   """)

        compare_script = backend.register_script(compare)
        self.assertRaises(redis.exceptions.ResponseError,
                          compare_script,
                          keys=["12457.00245", "12457.00245"],
                          client=self.conn)
        self.assertEqual(
            compare_script(keys=["42457.00245", "12457.00245"],
                           client=self.conn), "IS SUP")
        self.assertRaises(redis.exceptions.ResponseError,
                          compare_script,
                          keys=["12457.00245", "12457.002450"],
                          client=self.conn)
        self.assertEqual(
            compare_script(keys=["42457.00245", "12457.002450"],
                           client=self.conn), "IS SUP")
        self.assertRaises(redis.exceptions.ResponseError,
                          compare_script,
                          keys=["12457.00245", "12457.002456"],
                          client=self.conn)
        self.assertEqual(
            compare_script(keys=["42457.00245", "12457.002456"],
                           client=self.conn), "IS SUP")
        self.assertEqual(
            compare_script(keys=["42457.00246", "42457.002458"],
                           client=self.conn), "IS SUP")
Beispiel #2
0
    def test_delete_container(self):
        backend = AccountBackend({}, self.conn)
        account_id = "test"
        self.assertEqual(backend.create_account(account_id), account_id)
        name = "c"
        old_mtime = Timestamp(time() - 1).normal
        mtime = Timestamp(time()).normal

        # initial container
        backend.update_container(account_id, name, mtime, 0, 0, 0)
        res = self.conn.zrangebylex("containers:%s" % account_id, "-", "+")
        self.assertEqual(res[0], name)

        # delete event
        sleep(0.00001)
        dtime = Timestamp(time()).normal
        backend.update_container(account_id, name, mtime, dtime, 0, 0)
        res = self.conn.zrangebylex("containers:%s" % account_id, "-", "+")
        self.assertEqual(len(res), 0)
        self.assertTrue(self.conn.ttl("container:%s:%s" % (account_id, name)) >= 1)

        # same event
        backend.update_container(account_id, name, mtime, dtime, 0, 0)
        res = self.conn.zrangebylex("containers:%s" % account_id, "-", "+")
        self.assertEqual(len(res), 0)
        self.assertTrue(self.conn.ttl("container:%s:%s" % (account_id, name)) >= 1)

        # old event
        backend.update_container(account_id, name, old_mtime, 0, 0, 0)
        res = self.conn.zrangebylex("containers:%s" % account_id, "-", "+")
        self.assertEqual(len(res), 0)
        self.assertTrue(self.conn.ttl("container:%s:%s" % (account_id, name)) >= 1)
Beispiel #3
0
    def test_utf8_container(self):
        backend = AccountBackend({}, self.conn)
        account_id = 'test'
        self.assertEqual(backend.create_account(account_id), account_id)
        name = u'La fête à la maison'
        mtime = Timestamp(time()).normal

        # create container
        backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(unicode(res[0], 'utf8'), name)

        # ensure it appears in listing
        listing = backend.list_containers(account_id,
                                          marker='',
                                          delimiter='',
                                          limit=100)
        self.assertIn(name, [entry[0] for entry in listing])

        # delete container
        sleep(.00001)
        dtime = Timestamp(time()).normal
        backend.update_container(account_id, name, 0, dtime, 0, 0, 0, 0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(len(res), 0)
        self.assertTrue(
            self.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)

        # ensure it has been removed
        with ExpectedException(Conflict):
            backend.update_container(account_id, name, 0, dtime, 0, 0, 0, 0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(len(res), 0)
        self.assertTrue(
            self.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)
Beispiel #4
0
 def setUp(self):
     super(TestAccountBackend, self).setUp()
     _, _, self.redis_host, self.redis_port = self.get_service('redis')
     self.conf_account = {
         'redis_host': self.redis_host,
         'redis_port': self.redis_port
     }
     self.backend = AccountBackend(self.conf_account)
     self.backend.conn.flushdb()
    def test_delete_container(self):
        backend = AccountBackend({}, self.conn)
        account_id = 'test'
        self.assertEqual(backend.create_account(account_id), account_id)
        name = 'c'
        mtime = Timestamp(time()).normal

        # initial container
        backend.update_container(account_id, name, mtime, 0, 0, 0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)

        # delete event
        sleep(.00001)
        dtime = Timestamp(time()).normal
        backend.update_container(account_id, name, 0, dtime, 0, 0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(len(res), 0)
        self.assertFalse(
            self.conn.exists('container:%s:%s' % (account_id, name)))

        # same event
        backend.update_container(account_id, name, 0, dtime, 0, 0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(len(res), 0)
        self.assertFalse(
            self.conn.exists('container:%s:%s' % (account_id, name)))
Beispiel #6
0
    def test_utf8_container(self):
        backend = AccountBackend({}, self.conn)
        account_id = 'test'
        self.assertEqual(backend.create_account(account_id), account_id)
        name = u'La fête à la maison'
        mtime = Timestamp(time()).normal

        # create container
        backend.update_container(account_id, name, mtime, 0, 0, 0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(unicode(res[0], 'utf8'), name)

        # ensure it appears in listing
        listing = backend.list_containers(account_id, marker='',
                                          delimiter='', limit=100)
        self.assertIn(name, [entry[0] for entry in listing])

        # delete container
        sleep(.00001)
        dtime = Timestamp(time()).normal
        backend.update_container(account_id, name, 0, dtime, 0, 0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(len(res), 0)
        self.assertFalse(self.conn.exists('container:%s:%s' % (account_id,
                                                               name)))

        # ensure it ha been removed
        backend.update_container(account_id, name, 0, dtime, 0, 0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(len(res), 0)
        self.assertFalse(self.conn.exists('container:%s:%s' % (account_id,
                                                               name)))
Beispiel #7
0
    def test_utf8_container(self):
        backend = AccountBackend({}, self.conn)
        account_id = "test"
        self.assertEqual(backend.create_account(account_id), account_id)
        name = u"La fête à la maison"
        mtime = Timestamp(time()).normal

        # create container
        backend.update_container(account_id, name, mtime, 0, 0, 0)
        res = self.conn.zrangebylex("containers:%s" % account_id, "-", "+")
        self.assertEqual(unicode(res[0], "utf8"), name)

        # ensure it appears in listing
        listing = backend.list_containers(account_id, marker="", delimiter="", limit=100)
        self.assertIn(name, [entry[0] for entry in listing])

        # delete container
        sleep(0.00001)
        dtime = Timestamp(time()).normal
        backend.update_container(account_id, name, 0, dtime, 0, 0)
        res = self.conn.zrangebylex("containers:%s" % account_id, "-", "+")
        self.assertEqual(len(res), 0)
        self.assertTrue(self.conn.ttl("container:%s:%s" % (account_id, name)) >= 1)

        # ensure it has been removed
        backend.update_container(account_id, name, 0, dtime, 0, 0)
        res = self.conn.zrangebylex("containers:%s" % account_id, "-", "+")
        self.assertEqual(len(res), 0)
        self.assertTrue(self.conn.ttl("container:%s:%s" % (account_id, name)) >= 1)
Beispiel #8
0
    def test_list_account(self):
        backend = AccountBackend({}, self.conn)

        # Create and check if in list
        account_id = 'test_list'
        backend.create_account(account_id)
        account_list = backend.list_account()
        self.assertTrue(account_id in account_list)

        # Check the result of a nonexistent account
        self.assertFalse("Should_not_exist" in account_list)
Beispiel #9
0
    def test_delete_container(self):
        backend = AccountBackend({}, self.conn)
        account_id = 'test'
        self.assertEqual(backend.create_account(account_id), account_id)
        name = 'c'
        mtime = Timestamp(time()).normal

        # initial container
        backend.update_container(account_id, name, mtime, 0, 0, 0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)

        # delete event
        sleep(.00001)
        dtime = Timestamp(time()).normal
        backend.update_container(account_id, name, 0, dtime, 0, 0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(len(res), 0)
        self.assertFalse(self.conn.exists('container:%s:%s' % (account_id,
                                                               name)))

        # same event
        backend.update_container(account_id, name, 0, dtime, 0, 0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(len(res), 0)
        self.assertFalse(self.conn.exists('container:%s:%s' % (account_id,
                                                               name)))
Beispiel #10
0
def create_app(conf, **kwargs):
    app = flask.Flask(__name__)
    app.backend = AccountBackend(conf)
    app.register_blueprint(account_api)
    # we want exceptions to be logged
    if conf.get('log_level') == 'DEBUG':
        app.config['PROPAGATE_EXCEPTIONS'] = True
    return app
Beispiel #11
0
    def test_flush_account(self):
        backend = AccountBackend({}, self.conn)
        account_id = random_str(16)
        account_key = 'account:%s' % account_id

        self.assertEqual(backend.create_account(account_id), account_id)

        total_bytes = 0
        total_objects = 0

        # 10 containers with bytes and objects
        for i in range(10):
            name = "container%d" % i
            mtime = Timestamp(time()).normal
            nb_bytes = random.randrange(100)
            total_bytes += nb_bytes
            nb_objets = random.randrange(100)
            total_objects += nb_objets
            backend.update_container(account_id, name, mtime, 0, nb_objets,
                                     nb_bytes)

        self.assertEqual(self.conn.hget(account_key, 'bytes'),
                         str(total_bytes))
        self.assertEqual(self.conn.hget(account_key, 'objects'),
                         str(total_objects))

        backend.flush_account(account_id)
        self.assertEqual(self.conn.hget(account_key, 'bytes'), '0')
        self.assertEqual(self.conn.hget(account_key, 'objects'), '0')
        self.assertEqual(self.conn.zcard("containers:%s" % account_id), 0)
        self.assertEqual(self.conn.exists("container:test:*"), 0)
Beispiel #12
0
    def test_refresh_account(self):
        backend = AccountBackend({}, self.conn)
        account_id = random_str(16)
        account_key = 'account:%s' % account_id

        self.assertEqual(backend.create_account(account_id), account_id)

        total_bytes = 0
        total_objects = 0

        # 10 containers with bytes and objects
        for i in range(10):
            name = "container%d" % i
            mtime = Timestamp(time()).normal
            nb_bytes = random.randrange(100)
            total_bytes += nb_bytes
            nb_objets = random.randrange(100)
            total_objects += nb_objets
            backend.update_container(account_id, name, mtime, 0, nb_objets,
                                     nb_bytes)

        # change values
        self.conn.hset(account_key, 'bytes', 1)
        self.conn.hset(account_key, 'objects', 2)
        self.assertEqual(self.conn.hget(account_key, 'bytes'), '1')
        self.assertEqual(self.conn.hget(account_key, 'objects'), '2')

        backend.refresh_account(account_id)
        self.assertEqual(self.conn.hget(account_key, 'bytes'),
                         str(total_bytes))
        self.assertEqual(self.conn.hget(account_key, 'objects'),
                         str(total_objects))
Beispiel #13
0
 def test_update_container_missing_damaged_object(self):
     backend = AccountBackend({}, self.conn)
     account_id = random_str(16)
     self.assertEqual(backend.create_account(account_id), account_id)
     # initial container
     name = '"{<container \'&\' name>}"'
     backend.update_container(account_id, name, 0, 0, 0, 0, 0, 0)
     self.conn.hdel('container:%s:%s' % (account_id, name),
                    'damaged_objects')
     backend.refresh_account(account_id)
     self.assertEqual(
         self.conn.hget('account:%s' % (account_id), 'damaged_objects'),
         '0')
Beispiel #14
0
def create_app(conf, **kwargs):
    logger = get_logger(conf)
    iam_conn = conf.get('iam.connection')
    if not iam_conn:
        ns_conf = load_namespace_conf(conf['namespace'], failsafe=True)
        iam_conn = ns_conf.get('iam.connection', DEFAULT_IAM_CONNECTION)
    if iam_conn == DEFAULT_IAM_CONNECTION:
        logger.warning(
            'Using the default connection (%s) is probably '
            'not what you want to do.', DEFAULT_IAM_CONNECTION)
    scheme, netloc, iam_kwargs = parse_conn_str(iam_conn)
    if scheme == 'redis+sentinel':
        iam_kwargs['sentinel_hosts'] = netloc
    else:
        iam_kwargs['host'] = netloc
    backend = AccountBackend(conf)
    iam_db = RedisIamDb(logger=logger, **iam_kwargs)
    app = Account(conf, backend, iam_db, logger=logger)
    return app
Beispiel #15
0
def run(args):
    pool = get_pool_manager()

    v = vars(args)

    dirclient = DirectoryClient(v)
    backend = AccountBackend(v)

    for entry, _, _, partial in full_list(backend, prefix=args.prefix):
        if partial:
            if args.verbose:
                print(":%s: partial, skip" % entry)
            continue
        try:
            dirclient.show(account=ACCOUNT, reference=entry)
            if args.verbose:
                print("%s: OK" % entry)
            continue
        except NotFound:
            pass
        except Exception as exc:
            print("Exception not managed for %s: %s" % (entry, str(exc)))
            continue
        print("%s: meta2 not found" % entry)
        if args.dry_run:
            continue

        data = {"dtime": time(), "name": entry}
        # post event to Account service
        res = pool.request('POST',
                           HOST +
                           '/v1.0/account/container/update?id=%s' % ACCOUNT,
                           headers={'Content-Type': 'application/json'},
                           body=json.dumps(data))
        if res.status_int / 100 != 2:
            print(res.status)
Beispiel #16
0
 def test_create_account(self):
     backend = AccountBackend({}, self.conn)
     account_id = 'a'
     self.assertEqual(backend.create_account(account_id), account_id)
     self.assertEqual(backend.create_account(account_id), None)
Beispiel #17
0
    def test_update_container(self):
        backend = AccountBackend({}, self.conn)
        account_id = 'test'
        self.assertEqual(backend.create_account(account_id), account_id)

        # initial container
        name = '"{<container \'&\' name>}"'
        mtime = Timestamp(time()).normal
        backend.update_container(account_id, name, mtime, 0, 0, 0)

        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)

        self.assertEqual(self.conn.hget('container:%s:%s' %
                                        (account_id, name), 'mtime'), mtime)

        # same event
        backend.update_container(account_id, name, mtime, 0, 0, 0)

        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)

        self.assertEqual(self.conn.hget('container:%s:%s' %
                                        (account_id, name), 'mtime'), mtime)

        # New event
        sleep(.00001)
        mtime = Timestamp(time()).normal
        backend.update_container(account_id, name, mtime, 0, 0, 0)

        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)

        self.assertEqual(self.conn.hget('container:%s:%s' %
                                        (account_id, name), 'mtime'), mtime)

        # Old event
        old_mtime = Timestamp(time() - 1).normal
        backend.update_container(account_id, name, old_mtime, 0, 0, 0)

        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)

        self.assertEqual(self.conn.hget('container:%s:%s' %
                                        (account_id, name), 'mtime'), mtime)

        # Old delete event
        dtime = Timestamp(time() - 1).normal
        backend.update_container(account_id, name, 0, dtime, 0, 0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)
        self.assertEqual(self.conn.hget('container:%s:%s' %
                                        (account_id, name), 'mtime'), mtime)

        # New delete event
        sleep(.00001)
        mtime = Timestamp(time()).normal
        backend.update_container(account_id, name, 0, mtime, 0, 0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(len(res), 0)
        self.assertFalse(self.conn.exists('container:%s:%s' % (account_id,
                                                               name)))

        # New event
        sleep(.00001)
        mtime = Timestamp(time()).normal
        backend.update_container(account_id, name, mtime, 0, 0, 0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)
        self.assertEqual(self.conn.hget('container:%s:%s' %
                                        (account_id, name), 'mtime'), mtime)
Beispiel #18
0
    def test_list_containers(self):
        backend = AccountBackend({}, self.conn)
        account_id = "test"

        backend.create_account(account_id)
        for cont1 in xrange(4):
            for cont2 in xrange(125):
                name = "%d-%04d" % (cont1, cont2)
                backend.update_container(account_id, name, Timestamp(time()).normal, 0, 0, 0)

        for cont in xrange(125):
            name = "2-0051-%04d" % cont
            backend.update_container(account_id, name, Timestamp(time()).normal, 0, 0, 0)

        for cont in xrange(125):
            name = "3-%04d-0049" % cont
            backend.update_container(account_id, name, Timestamp(time()).normal, 0, 0, 0)

        listing = backend.list_containers(account_id, marker="", delimiter="", limit=100)
        self.assertEqual(len(listing), 100)
        self.assertEqual(listing[0][0], "0-0000")
        self.assertEqual(listing[-1][0], "0-0099")

        listing = backend.list_containers(account_id, marker="", end_marker="0-0050", delimiter="", limit=100)
        self.assertEqual(len(listing), 50)
        self.assertEqual(listing[0][0], "0-0000")
        self.assertEqual(listing[-1][0], "0-0049")

        listing = backend.list_containers(account_id, marker="0-0099", delimiter="", limit=100)
        self.assertEqual(len(listing), 100)
        self.assertEqual(listing[0][0], "0-0100")
        self.assertEqual(listing[-1][0], "1-0074")

        listing = backend.list_containers(account_id, marker="1-0074", delimiter="", limit=55)
        self.assertEqual(len(listing), 55)
        self.assertEqual(listing[0][0], "1-0075")
        self.assertEqual(listing[-1][0], "2-0004")

        listing = backend.list_containers(account_id, marker="", prefix="0-01", delimiter="", limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual(listing[0][0], "0-0100")
        self.assertEqual(listing[-1][0], "0-0109")

        listing = backend.list_containers(account_id, marker="", prefix="0-01", delimiter="-", limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual(listing[0][0], "0-0100")
        self.assertEqual(listing[-1][0], "0-0109")

        listing = backend.list_containers(account_id, marker="", prefix="0-", delimiter="-", limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual(listing[0][0], "0-0000")
        self.assertEqual(listing[-1][0], "0-0009")

        listing = backend.list_containers(account_id, marker="", prefix="", delimiter="-", limit=10)
        self.assertEqual(len(listing), 4)
        self.assertEqual([c[0] for c in listing], ["0-", "1-", "2-", "3-"])

        listing = backend.list_containers(account_id, marker="2-", delimiter="-", limit=10)
        self.assertEqual(len(listing), 1)
        self.assertEqual([c[0] for c in listing], ["3-"])

        listing = backend.list_containers(account_id, marker="", prefix="2", delimiter="-", limit=10)
        self.assertEqual(len(listing), 1)
        self.assertEqual([c[0] for c in listing], ["2-"])

        listing = backend.list_containers(account_id, marker="2-0050", prefix="2-", delimiter="-", limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual(listing[0][0], "2-0051")
        self.assertEqual(listing[1][0], "2-0051-")
        self.assertEqual(listing[2][0], "2-0052")
        self.assertEqual(listing[-1][0], "2-0059")

        listing = backend.list_containers(account_id, marker="3-0045", prefix="3-", delimiter="-", limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual(
            [c[0] for c in listing],
            ["3-0045-", "3-0046", "3-0046-", "3-0047", "3-0047-", "3-0048", "3-0048-", "3-0049", "3-0049-", "3-0050"],
        )

        name = "3-0049-"
        backend.update_container(account_id, name, Timestamp(time()).normal, 0, 0, 0)
        listing = backend.list_containers(account_id, marker="3-0048", limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual(
            [c[0] for c in listing],
            [
                "3-0048-0049",
                "3-0049",
                "3-0049-",
                "3-0049-0049",
                "3-0050",
                "3-0050-0049",
                "3-0051",
                "3-0051-0049",
                "3-0052",
                "3-0052-0049",
            ],
        )

        listing = backend.list_containers(account_id, marker="3-0048", prefix="3-", delimiter="-", limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual(
            [c[0] for c in listing],
            ["3-0048-", "3-0049", "3-0049-", "3-0050", "3-0050-", "3-0051", "3-0051-", "3-0052", "3-0052-", "3-0053"],
        )

        listing = backend.list_containers(account_id, prefix="3-0049-", delimiter="-", limit=10)
        self.assertEqual(len(listing), 2)
        self.assertEqual([c[0] for c in listing], ["3-0049-", "3-0049-0049"])
Beispiel #19
0
    def test_update_account_metadata(self):
        backend = AccountBackend({}, self.conn)
        account_id = "test"
        self.assertEqual(backend.create_account(account_id), account_id)

        # first meta
        backend.update_account_metadata(account_id, {"a": "1"})
        metadata = backend.get_account_metadata(account_id)
        self.assert_("a" in metadata)
        self.assertEqual(metadata["a"], "1")

        # second meta
        backend.update_account_metadata(account_id, {"b": "2"})
        metadata = backend.get_account_metadata(account_id)
        self.assert_("a" in metadata)
        self.assertEqual(metadata["a"], "1")
        self.assert_("b" in metadata)
        self.assertEqual(metadata["b"], "2")

        # update first meta
        backend.update_account_metadata(account_id, {"a": "1b"})
        metadata = backend.get_account_metadata(account_id)
        self.assert_("a" in metadata)
        self.assertEqual(metadata["a"], "1b")
        self.assert_("b" in metadata)
        self.assertEqual(metadata["b"], "2")

        # delete second meta
        backend.update_account_metadata(account_id, None, ["b"])
        metadata = backend.get_account_metadata(account_id)
        self.assert_("a" in metadata)
        self.assertEqual(metadata["a"], "1b")
        self.assert_("b" not in metadata)
Beispiel #20
0
    def test_update_container(self):
        backend = AccountBackend({}, self.conn)
        account_id = 'test'
        self.assertEqual(backend.create_account(account_id), account_id)

        # initial container
        name = '"{<container \'&\' name>}"'
        mtime = Timestamp(time()).normal
        backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)

        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)

        self.assertEqual(
            self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
            mtime)

        # same event
        with ExpectedException(Conflict):
            backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)

        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)

        self.assertEqual(
            self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
            mtime)

        # New event
        sleep(.00001)
        mtime = Timestamp(time()).normal
        backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)

        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)

        self.assertEqual(
            self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
            mtime)

        # Old event
        old_mtime = Timestamp(time() - 1).normal
        with ExpectedException(Conflict):
            backend.update_container(account_id, name, old_mtime, 0, 0, 0, 0,
                                     0)

        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)

        self.assertEqual(
            self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
            mtime)

        # Old delete event
        dtime = Timestamp(time() - 1).normal
        with ExpectedException(Conflict):
            backend.update_container(account_id, name, 0, dtime, 0, 0, 0, 0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)
        self.assertEqual(
            self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
            mtime)

        # New delete event
        sleep(.00001)
        mtime = Timestamp(time()).normal
        backend.update_container(account_id, name, 0, mtime, 0, 0, 0, 0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(len(res), 0)
        self.assertTrue(
            self.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)

        # New event
        sleep(.00001)
        mtime = Timestamp(time()).normal
        backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)
        self.assertEqual(
            self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
            mtime)
        # ensure ttl has been removed
        self.assertEqual(self.conn.ttl('container:%s:%s' % (account_id, name)),
                         -1)
Beispiel #21
0
class TestAccountBackend(BaseTestCase):
    def setUp(self):
        super(TestAccountBackend, self).setUp()
        _, _, self.redis_host, self.redis_port = self.get_service('redis')
        self.conf_account = {
            'redis_host': self.redis_host,
            'redis_port': self.redis_port
        }
        self.backend = AccountBackend(self.conf_account)
        self.backend.conn.flushdb()

    def tearDown(self):
        super(TestAccountBackend, self).tearDown()
        self.backend.conn.flushdb()

    def test_create_account(self):
        account_id = 'a'
        self.assertEqual(self.backend.create_account(account_id), account_id)
        self.assertEqual(self.backend.create_account(account_id), None)

    def test_update_account_metadata(self):
        account_id = 'test'
        self.assertEqual(self.backend.create_account(account_id), account_id)

        # first meta
        self.backend.update_account_metadata(account_id, {'a': '1'})
        metadata = self.backend.get_account_metadata(account_id)
        self.assert_('a' in metadata)
        self.assertEqual(metadata['a'], '1')

        # second meta
        self.backend.update_account_metadata(account_id, {'b': '2'})
        metadata = self.backend.get_account_metadata(account_id)
        self.assert_('a' in metadata)
        self.assertEqual(metadata['a'], '1')
        self.assert_('b' in metadata)
        self.assertEqual(metadata['b'], '2')

        # update first meta
        self.backend.update_account_metadata(account_id, {'a': '1b'})
        metadata = self.backend.get_account_metadata(account_id)
        self.assert_('a' in metadata)
        self.assertEqual(metadata['a'], '1b')
        self.assert_('b' in metadata)
        self.assertEqual(metadata['b'], '2')

        # delete second meta
        self.backend.update_account_metadata(account_id, None, ['b'])
        metadata = self.backend.get_account_metadata(account_id)
        self.assert_('a' in metadata)
        self.assertEqual(metadata['a'], '1b')
        self.assert_('b' not in metadata)

    def test_list_account(self):

        # Create and check if in list
        account_id = 'test_list'
        self.backend.create_account(account_id)
        account_list = self.backend.list_account()
        self.assertTrue(account_id in account_list)

        # Check the result of a nonexistent account
        self.assertFalse("Should_not_exist" in account_list)

    def test_info_account(self):
        account_id = 'test'
        self.assertEqual(self.backend.create_account(account_id), account_id)
        info = self.backend.info_account(account_id)
        self.assertEqual(info['id'], account_id)
        self.assertEqual(info['objects'], 0)
        self.assertEqual(info['bytes'], 0)
        self.assertEqual(info['damaged_objects'], 0)
        self.assertEqual(info['missing_chunks'], 0)
        self.assertEqual(info['containers'], 0)
        self.assertTrue(info['ctime'])

        # first container
        self.backend.update_container(account_id, 'c1',
                                      Timestamp(time()).normal, 0, 1, 1, 1, 1)
        info = self.backend.info_account(account_id)
        self.assertEqual(info['containers'], 1)
        self.assertEqual(info['objects'], 1)
        self.assertEqual(info['bytes'], 1)
        self.assertEqual(info['damaged_objects'], 1)
        self.assertEqual(info['missing_chunks'], 1)

        # second container
        sleep(.00001)
        self.backend.update_container(account_id, 'c2',
                                      Timestamp(time()).normal, 0, 0, 0, 0, 0)
        info = self.backend.info_account(account_id)
        self.assertEqual(info['containers'], 2)
        self.assertEqual(info['objects'], 1)
        self.assertEqual(info['bytes'], 1)
        self.assertEqual(info['damaged_objects'], 1)
        self.assertEqual(info['missing_chunks'], 1)

        # update second container
        sleep(.00001)
        self.backend.update_container(account_id, 'c2',
                                      Timestamp(time()).normal, 0, 1, 1, 1, 2)
        info = self.backend.info_account(account_id)
        self.assertEqual(info['containers'], 2)
        self.assertEqual(info['objects'], 2)
        self.assertEqual(info['bytes'], 2)
        self.assertEqual(info['damaged_objects'], 2)
        self.assertEqual(info['missing_chunks'], 3)

        # delete first container
        sleep(.00001)
        self.backend.update_container(account_id, 'c1', 0,
                                      Timestamp(time()).normal, 0, 0, 0, 0)
        info = self.backend.info_account(account_id)
        self.assertEqual(info['containers'], 1)
        self.assertEqual(info['objects'], 1)
        self.assertEqual(info['bytes'], 1)
        self.assertEqual(info['damaged_objects'], 1)
        self.assertEqual(info['missing_chunks'], 2)

        # delete second container
        sleep(.00001)
        self.backend.update_container(account_id, 'c2', 0,
                                      Timestamp(time()).normal, 0, 0, 0, 0)
        info = self.backend.info_account(account_id)
        self.assertEqual(info['containers'], 0)
        self.assertEqual(info['objects'], 0)
        self.assertEqual(info['bytes'], 0)
        self.assertEqual(info['damaged_objects'], 0)
        self.assertEqual(info['missing_chunks'], 0)

    def test_update_after_container_deletion(self):
        account_id = 'test-%06x' % int(time())
        self.assertEqual(self.backend.create_account(account_id), account_id)

        # Container create event, sent immediately after creation
        self.backend.update_container(account_id, 'c1',
                                      Timestamp(time()).normal, None, None,
                                      None, None, None)

        # Container update event
        self.backend.update_container(account_id, 'c1',
                                      Timestamp(time()).normal, None, 3, 30, 7,
                                      5)
        info = self.backend.info_account(account_id)
        self.assertEqual(info['containers'], 1)
        self.assertEqual(info['objects'], 3)
        self.assertEqual(info['bytes'], 30)
        self.assertEqual(info['damaged_objects'], 7)
        self.assertEqual(info['missing_chunks'], 5)

        # Container is flushed, but the event is deferred
        flush_timestamp = Timestamp(time()).normal

        sleep(.00001)
        # Container delete event, sent immediately after deletion
        self.backend.update_container(account_id, 'c1', None,
                                      Timestamp(time()).normal, None, None,
                                      None, None)

        # Deferred container update event (with lower timestamp)
        self.backend.update_container(account_id, 'c1', flush_timestamp, None,
                                      0, 0, 0, 0)
        info = self.backend.info_account(account_id)
        self.assertEqual(info['containers'], 0)
        self.assertEqual(info['objects'], 0)
        self.assertEqual(info['bytes'], 0)
        self.assertEqual(info['damaged_objects'], 0)
        self.assertEqual(info['missing_chunks'], 0)

    def test_delete_container(self):
        account_id = 'test'
        self.assertEqual(self.backend.create_account(account_id), account_id)
        name = 'c'
        old_mtime = Timestamp(time() - 1).normal
        mtime = Timestamp(time()).normal

        # initial container
        self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)
        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(res[0], name)

        # delete event
        sleep(.00001)
        dtime = Timestamp(time()).normal
        self.backend.update_container(account_id, name, mtime, dtime, 0, 0, 0,
                                      0)
        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(len(res), 0)
        self.assertTrue(
            self.backend.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)

        # same event
        with ExpectedException(Conflict):
            self.backend.update_container(account_id, name, mtime, dtime, 0, 0,
                                          0, 0)
        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(len(res), 0)
        self.assertTrue(
            self.backend.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)

        # old event
        with ExpectedException(Conflict):
            self.backend.update_container(account_id, name, old_mtime, 0, 0, 0,
                                          0, 0)
        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(len(res), 0)
        self.assertTrue(
            self.backend.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)

    def test_utf8_container(self):
        account_id = 'test'
        self.assertEqual(self.backend.create_account(account_id), account_id)
        name = u'La fête à la maison'
        mtime = Timestamp(time()).normal

        # create container
        self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)
        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(unicode(res[0], 'utf8'), name)

        # ensure it appears in listing
        listing = self.backend.list_containers(account_id,
                                               marker='',
                                               delimiter='',
                                               limit=100)
        self.assertIn(name, [entry[0] for entry in listing])

        # delete container
        sleep(.00001)
        dtime = Timestamp(time()).normal
        self.backend.update_container(account_id, name, 0, dtime, 0, 0, 0, 0)
        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(len(res), 0)
        self.assertTrue(
            self.backend.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)

        # ensure it has been removed
        with ExpectedException(Conflict):
            self.backend.update_container(account_id, name, 0, dtime, 0, 0, 0,
                                          0)
        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(len(res), 0)
        self.assertTrue(
            self.backend.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)

    def test_update_container(self):
        account_id = 'test'
        self.assertEqual(self.backend.create_account(account_id), account_id)

        # initial container
        name = '"{<container \'&\' name>}"'
        mtime = Timestamp(time()).normal
        self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)

        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(res[0], name)

        self.assertEqual(
            self.backend.conn.hget('container:%s:%s' % (account_id, name),
                                   'mtime'), mtime)

        # same event
        with ExpectedException(Conflict):
            self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0,
                                          0)

        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(res[0], name)

        self.assertEqual(
            self.backend.conn.hget('container:%s:%s' % (account_id, name),
                                   'mtime'), mtime)

        # New event
        sleep(.00001)
        mtime = Timestamp(time()).normal
        self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)

        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(res[0], name)

        self.assertEqual(
            self.backend.conn.hget('container:%s:%s' % (account_id, name),
                                   'mtime'), mtime)

        # Old event
        old_mtime = Timestamp(time() - 1).normal
        with ExpectedException(Conflict):
            self.backend.update_container(account_id, name, old_mtime, 0, 0, 0,
                                          0, 0)

        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(res[0], name)

        self.assertEqual(
            self.backend.conn.hget('container:%s:%s' % (account_id, name),
                                   'mtime'), mtime)

        # Old delete event
        dtime = Timestamp(time() - 1).normal
        with ExpectedException(Conflict):
            self.backend.update_container(account_id, name, 0, dtime, 0, 0, 0,
                                          0)
        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(res[0], name)
        self.assertEqual(
            self.backend.conn.hget('container:%s:%s' % (account_id, name),
                                   'mtime'), mtime)

        # New delete event
        sleep(.00001)
        mtime = Timestamp(time()).normal
        self.backend.update_container(account_id, name, 0, mtime, 0, 0, 0, 0)
        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(len(res), 0)
        self.assertTrue(
            self.backend.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)

        # New event
        sleep(.00001)
        mtime = Timestamp(time()).normal
        self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)
        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(res[0], name)
        self.assertEqual(
            self.backend.conn.hget('container:%s:%s' % (account_id, name),
                                   'mtime'), mtime)
        # ensure ttl has been removed
        self.assertEqual(
            self.backend.conn.ttl('container:%s:%s' % (account_id, name)), -1)

    def test_list_containers(self):
        account_id = 'test'

        self.backend.create_account(account_id)
        for cont1 in xrange(4):
            for cont2 in xrange(125):
                name = '%d-%04d' % (cont1, cont2)
                self.backend.update_container(account_id, name,
                                              Timestamp(time()).normal, 0, 0,
                                              0, 0, 0)

        for cont in xrange(125):
            name = '2-0051-%04d' % cont
            self.backend.update_container(account_id, name,
                                          Timestamp(time()).normal, 0, 0, 0, 0,
                                          0)

        for cont in xrange(125):
            name = '3-%04d-0049' % cont
            self.backend.update_container(account_id, name,
                                          Timestamp(time()).normal, 0, 0, 0, 0,
                                          0)

        listing = self.backend.list_containers(account_id,
                                               marker='',
                                               delimiter='',
                                               limit=100)
        self.assertEqual(len(listing), 100)
        self.assertEqual(listing[0][0], '0-0000')
        self.assertEqual(listing[-1][0], '0-0099')

        listing = self.backend.list_containers(account_id,
                                               marker='',
                                               end_marker='0-0050',
                                               delimiter='',
                                               limit=100)
        self.assertEqual(len(listing), 50)
        self.assertEqual(listing[0][0], '0-0000')
        self.assertEqual(listing[-1][0], '0-0049')

        listing = self.backend.list_containers(account_id,
                                               marker='0-0099',
                                               delimiter='',
                                               limit=100)
        self.assertEqual(len(listing), 100)
        self.assertEqual(listing[0][0], '0-0100')
        self.assertEqual(listing[-1][0], '1-0074')

        listing = self.backend.list_containers(account_id,
                                               marker='1-0074',
                                               delimiter='',
                                               limit=55)
        self.assertEqual(len(listing), 55)
        self.assertEqual(listing[0][0], '1-0075')
        self.assertEqual(listing[-1][0], '2-0004')

        listing = self.backend.list_containers(account_id,
                                               marker='',
                                               prefix='0-01',
                                               delimiter='',
                                               limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual(listing[0][0], '0-0100')
        self.assertEqual(listing[-1][0], '0-0109')

        listing = self.backend.list_containers(account_id,
                                               marker='',
                                               prefix='0-01',
                                               delimiter='-',
                                               limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual(listing[0][0], '0-0100')
        self.assertEqual(listing[-1][0], '0-0109')

        listing = self.backend.list_containers(account_id,
                                               marker='',
                                               prefix='0-',
                                               delimiter='-',
                                               limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual(listing[0][0], '0-0000')
        self.assertEqual(listing[-1][0], '0-0009')

        listing = self.backend.list_containers(account_id,
                                               marker='',
                                               prefix='',
                                               delimiter='-',
                                               limit=10)
        self.assertEqual(len(listing), 4)
        self.assertEqual([c[0] for c in listing], ['0-', '1-', '2-', '3-'])

        listing = self.backend.list_containers(account_id,
                                               marker='2-',
                                               delimiter='-',
                                               limit=10)
        self.assertEqual(len(listing), 1)
        self.assertEqual([c[0] for c in listing], ['3-'])

        listing = self.backend.list_containers(account_id,
                                               marker='',
                                               prefix='2',
                                               delimiter='-',
                                               limit=10)
        self.assertEqual(len(listing), 1)
        self.assertEqual([c[0] for c in listing], ['2-'])

        listing = self.backend.list_containers(account_id,
                                               marker='2-0050',
                                               prefix='2-',
                                               delimiter='-',
                                               limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual(listing[0][0], '2-0051')
        self.assertEqual(listing[1][0], '2-0051-')
        self.assertEqual(listing[2][0], '2-0052')
        self.assertEqual(listing[-1][0], '2-0059')

        listing = self.backend.list_containers(account_id,
                                               marker='3-0045',
                                               prefix='3-',
                                               delimiter='-',
                                               limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual([c[0] for c in listing], [
            '3-0045-', '3-0046', '3-0046-', '3-0047', '3-0047-', '3-0048',
            '3-0048-', '3-0049', '3-0049-', '3-0050'
        ])

        name = '3-0049-'
        self.backend.update_container(account_id, name,
                                      Timestamp(time()).normal, 0, 0, 0, 0, 0)
        listing = self.backend.list_containers(account_id,
                                               marker='3-0048',
                                               limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual([c[0] for c in listing], [
            '3-0048-0049', '3-0049', '3-0049-', '3-0049-0049', '3-0050',
            '3-0050-0049', '3-0051', '3-0051-0049', '3-0052', '3-0052-0049'
        ])

        listing = self.backend.list_containers(account_id,
                                               marker='3-0048',
                                               prefix='3-',
                                               delimiter='-',
                                               limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual([c[0] for c in listing], [
            '3-0048-', '3-0049', '3-0049-', '3-0050', '3-0050-', '3-0051',
            '3-0051-', '3-0052', '3-0052-', '3-0053'
        ])

        listing = self.backend.list_containers(account_id,
                                               prefix='3-0049-',
                                               delimiter='-',
                                               limit=10)
        self.assertEqual(len(listing), 2)
        self.assertEqual([c[0] for c in listing], ['3-0049-', '3-0049-0049'])

    def test_refresh_account(self):
        account_id = random_str(16)
        account_key = 'account:%s' % account_id

        self.assertEqual(self.backend.create_account(account_id), account_id)

        total_bytes = 0
        total_objects = 0
        total_damaged_objects = 0
        total_missing_chunks = 0

        # 10 containers with bytes and objects
        for i in range(10):
            name = "container%d" % i
            mtime = Timestamp(time()).normal
            nb_bytes = random.randrange(100)
            total_bytes += nb_bytes
            nb_objets = random.randrange(100)
            total_objects += nb_objets
            damaged_objects = random.randrange(100)
            total_damaged_objects += damaged_objects
            missing_chunks = random.randrange(100)
            total_missing_chunks += missing_chunks
            self.backend.update_container(account_id, name, mtime, 0,
                                          nb_objets, nb_bytes, damaged_objects,
                                          missing_chunks)

        # change values
        self.backend.conn.hset(account_key, 'bytes', 1)
        self.backend.conn.hset(account_key, 'objects', 2)
        self.backend.conn.hset(account_key, 'damaged_objects', 3)
        self.backend.conn.hset(account_key, 'missing_chunks', 4)
        self.assertEqual(self.backend.conn.hget(account_key, 'bytes'), '1')
        self.assertEqual(self.backend.conn.hget(account_key, 'objects'), '2')
        self.assertEqual(
            self.backend.conn.hget(account_key, 'damaged_objects'), '3')
        self.assertEqual(self.backend.conn.hget(account_key, 'missing_chunks'),
                         '4')

        self.backend.refresh_account(account_id)
        self.assertEqual(self.backend.conn.hget(account_key, 'bytes'),
                         str(total_bytes))
        self.assertEqual(self.backend.conn.hget(account_key, 'objects'),
                         str(total_objects))
        self.assertEqual(
            self.backend.conn.hget(account_key, 'damaged_objects'),
            str(total_damaged_objects))
        self.assertEqual(self.backend.conn.hget(account_key, 'missing_chunks'),
                         str(total_missing_chunks))

    def test_update_container_wrong_timestamp_format(self):
        account_id = 'test'
        self.assertEqual(self.backend.create_account(account_id), account_id)

        # initial container
        name = '"{<container \'&\' name>}"'
        mtime = "12456.0000076"
        self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)

        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(res[0], name)

        # same event
        with ExpectedException(Conflict):
            self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0,
                                          0)

        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(res[0], name)

        mtime = "0000012456.00005"
        self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)

        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(res[0], name)

        self.assertEqual(
            self.backend.conn.hget('container:%s:%s' % (account_id, name),
                                   'mtime'), mtime)

        mtime = "0000012456.00035"
        self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)

        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(res[0], name)

        self.assertEqual(
            self.backend.conn.hget('container:%s:%s' % (account_id, name),
                                   'mtime'), mtime)

    def test_update_container_missing_damaged_object(self):
        account_id = random_str(16)
        self.assertEqual(self.backend.create_account(account_id), account_id)
        # initial container
        name = '"{<container \'&\' name>}"'
        self.backend.update_container(account_id, name, 0, 0, 0, 0, 0, 0)
        self.backend.conn.hdel('container:%s:%s' % (account_id, name),
                               'damaged_objects')
        self.backend.refresh_account(account_id)
        self.assertEqual(
            self.backend.conn.hget('account:%s' % (account_id),
                                   'damaged_objects'), '0')

    def test_is_sup(self):
        compare = (self.backend.lua_is_sup + """
            if (is_sup(KEYS[1], KEYS[2])) then
              return redis.status_reply('IS SUP');
            else
              return redis.error_reply('IS NOT SUP');
            end;
                   """)

        compare_script = self.backend.register_script(compare)
        self.assertRaises(redis.exceptions.ResponseError,
                          compare_script,
                          keys=["12457.00245", "12457.00245"],
                          client=self.backend.conn)
        self.assertEqual(
            compare_script(keys=["42457.00245", "12457.00245"],
                           client=self.backend.conn), "IS SUP")
        self.assertRaises(redis.exceptions.ResponseError,
                          compare_script,
                          keys=["12457.00245", "12457.002450"],
                          client=self.backend.conn)
        self.assertEqual(
            compare_script(keys=["42457.00245", "12457.002450"],
                           client=self.backend.conn), "IS SUP")
        self.assertRaises(redis.exceptions.ResponseError,
                          compare_script,
                          keys=["12457.00245", "12457.002456"],
                          client=self.backend.conn)
        self.assertEqual(
            compare_script(keys=["42457.00245", "12457.002456"],
                           client=self.backend.conn), "IS SUP")
        self.assertEqual(
            compare_script(keys=["42457.00246", "42457.002458"],
                           client=self.backend.conn), "IS SUP")

    def test_flush_account(self):
        account_id = random_str(16)
        account_key = 'account:%s' % account_id

        self.assertEqual(self.backend.create_account(account_id), account_id)

        total_bytes = 0
        total_objects = 0
        total_damaged_objects = 0
        total_missing_chunks = 0

        # 10 containers with bytes and objects
        for i in range(10):
            name = "container%d" % i
            mtime = Timestamp(time()).normal
            nb_bytes = random.randrange(100)
            total_bytes += nb_bytes
            nb_objets = random.randrange(100)
            total_objects += nb_objets
            damaged_objects = random.randrange(100)
            total_damaged_objects += damaged_objects
            missing_chunks = random.randrange(100)
            total_missing_chunks += missing_chunks
            self.backend.update_container(account_id, name, mtime, 0,
                                          nb_objets, nb_bytes, damaged_objects,
                                          missing_chunks)

        self.assertEqual(self.backend.conn.hget(account_key, 'bytes'),
                         str(total_bytes))
        self.assertEqual(self.backend.conn.hget(account_key, 'objects'),
                         str(total_objects))
        self.assertEqual(
            self.backend.conn.hget(account_key, 'damaged_objects'),
            str(total_damaged_objects))
        self.assertEqual(self.backend.conn.hget(account_key, 'missing_chunks'),
                         str(total_missing_chunks))

        self.backend.flush_account(account_id)
        self.assertEqual(self.backend.conn.hget(account_key, 'bytes'), '0')
        self.assertEqual(self.backend.conn.hget(account_key, 'objects'), '0')
        self.assertEqual(
            self.backend.conn.hget(account_key, 'damaged_objects'), '0')
        self.assertEqual(self.backend.conn.hget(account_key, 'missing_chunks'),
                         '0')
        self.assertEqual(self.backend.conn.zcard("containers:%s" % account_id),
                         0)
        self.assertEqual(self.backend.conn.exists("container:test:*"), 0)
Beispiel #22
0
    def test_info_account(self):
        backend = AccountBackend({}, self.conn)
        account_id = 'test'
        self.assertEqual(backend.create_account(account_id), account_id)
        info = backend.info_account(account_id)
        self.assertEqual(info['id'], account_id)
        self.assertEqual(info['bytes'], 0)
        self.assertEqual(info['objects'], 0)
        self.assertEqual(info['containers'], 0)
        self.assertTrue(info['ctime'])

        # first container
        backend.update_container(account_id, 'c1', Timestamp(time()).normal, 0,
                                 1, 1)
        info = backend.info_account(account_id)
        self.assertEqual(info['containers'], 1)
        self.assertEqual(info['objects'], 1)
        self.assertEqual(info['bytes'], 1)

        # second container
        sleep(.00001)
        backend.update_container(account_id, 'c2', Timestamp(time()).normal, 0,
                                 0, 0)
        info = backend.info_account(account_id)
        self.assertEqual(info['containers'], 2)
        self.assertEqual(info['objects'], 1)
        self.assertEqual(info['bytes'], 1)

        # update second container
        sleep(.00001)
        backend.update_container(account_id, 'c2', Timestamp(time()).normal, 0,
                                 1, 1)
        info = backend.info_account(account_id)
        self.assertEqual(info['containers'], 2)
        self.assertEqual(info['objects'], 2)
        self.assertEqual(info['bytes'], 2)

        # delete first container
        sleep(.00001)
        backend.update_container(account_id, 'c1', 0, Timestamp(time()).normal,
                                 0, 0)
        info = backend.info_account(account_id)
        self.assertEqual(info['containers'], 1)
        self.assertEqual(info['objects'], 1)
        self.assertEqual(info['bytes'], 1)

        # delete second container
        sleep(.00001)
        backend.update_container(account_id, 'c2', 0, Timestamp(time()).normal,
                                 0, 0)
        info = backend.info_account(account_id)
        self.assertEqual(info['containers'], 0)
        self.assertEqual(info['objects'], 0)
        self.assertEqual(info['bytes'], 0)
Beispiel #23
0
    def test_info_account(self):
        backend = AccountBackend({}, self.conn)
        account_id = "test"
        self.assertEqual(backend.create_account(account_id), account_id)
        info = backend.info_account(account_id)
        self.assertEqual(info["id"], account_id)
        self.assertEqual(info["bytes"], 0)
        self.assertEqual(info["objects"], 0)
        self.assertEqual(info["containers"], 0)
        self.assertTrue(info["ctime"])

        # first container
        backend.update_container(account_id, "c1", Timestamp(time()).normal, 0, 1, 1)
        info = backend.info_account(account_id)
        self.assertEqual(info["containers"], 1)
        self.assertEqual(info["objects"], 1)
        self.assertEqual(info["bytes"], 1)

        # second container
        sleep(0.00001)
        backend.update_container(account_id, "c2", Timestamp(time()).normal, 0, 0, 0)
        info = backend.info_account(account_id)
        self.assertEqual(info["containers"], 2)
        self.assertEqual(info["objects"], 1)
        self.assertEqual(info["bytes"], 1)

        # update second container
        sleep(0.00001)
        backend.update_container(account_id, "c2", Timestamp(time()).normal, 0, 1, 1)
        info = backend.info_account(account_id)
        self.assertEqual(info["containers"], 2)
        self.assertEqual(info["objects"], 2)
        self.assertEqual(info["bytes"], 2)

        # delete first container
        sleep(0.00001)
        backend.update_container(account_id, "c1", 0, Timestamp(time()).normal, 0, 0)
        info = backend.info_account(account_id)
        self.assertEqual(info["containers"], 1)
        self.assertEqual(info["objects"], 1)
        self.assertEqual(info["bytes"], 1)

        # delete second container
        sleep(0.00001)
        backend.update_container(account_id, "c2", 0, Timestamp(time()).normal, 0, 0)
        info = backend.info_account(account_id)
        self.assertEqual(info["containers"], 0)
        self.assertEqual(info["objects"], 0)
        self.assertEqual(info["bytes"], 0)
    def test_update_container(self):
        backend = AccountBackend({}, self.conn)
        account_id = 'test'
        self.assertEqual(backend.create_account(account_id), account_id)

        # initial container
        name = '"{<container \'&\' name>}"'
        mtime = Timestamp(time()).normal
        backend.update_container(account_id, name, mtime, 0, 0, 0)

        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)

        self.assertEqual(
            self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
            mtime)

        # same event
        backend.update_container(account_id, name, mtime, 0, 0, 0)

        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)

        self.assertEqual(
            self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
            mtime)

        # New event
        sleep(.00001)
        mtime = Timestamp(time()).normal
        backend.update_container(account_id, name, mtime, 0, 0, 0)

        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)

        self.assertEqual(
            self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
            mtime)

        # Old event
        old_mtime = Timestamp(time() - 1).normal
        backend.update_container(account_id, name, old_mtime, 0, 0, 0)

        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)

        self.assertEqual(
            self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
            mtime)

        # Old delete event
        dtime = Timestamp(time() - 1).normal
        backend.update_container(account_id, name, 0, dtime, 0, 0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)
        self.assertEqual(
            self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
            mtime)

        # New delete event
        sleep(.00001)
        mtime = Timestamp(time()).normal
        backend.update_container(account_id, name, 0, mtime, 0, 0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(len(res), 0)
        self.assertFalse(
            self.conn.exists('container:%s:%s' % (account_id, name)))

        # New event
        sleep(.00001)
        mtime = Timestamp(time()).normal
        backend.update_container(account_id, name, mtime, 0, 0, 0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)
        self.assertEqual(
            self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
            mtime)
Beispiel #25
0
    def test_update_after_container_deletion(self):
        backend = AccountBackend({}, self.conn)
        account_id = 'test-%06x' % int(time())
        self.assertEqual(backend.create_account(account_id), account_id)

        # Container create event, sent immediately after creation
        backend.update_container(account_id, 'c1',
                                 Timestamp(time()).normal, None, None, None,
                                 None, None)

        # Container update event
        backend.update_container(account_id, 'c1',
                                 Timestamp(time()).normal, None, 3, 30, 7, 5)
        info = backend.info_account(account_id)
        self.assertEqual(info['containers'], 1)
        self.assertEqual(info['objects'], 3)
        self.assertEqual(info['bytes'], 30)
        self.assertEqual(info['damaged_objects'], 7)
        self.assertEqual(info['missing_chunks'], 5)

        # Container is flushed, but the event is deferred
        flush_timestamp = Timestamp(time()).normal

        sleep(.00001)
        # Container delete event, sent immediately after deletion
        backend.update_container(account_id, 'c1', None,
                                 Timestamp(time()).normal, None, None, None,
                                 None)

        # Deferred container update event (with lower timestamp)
        backend.update_container(account_id, 'c1', flush_timestamp, None, 0, 0,
                                 0, 0)
        info = backend.info_account(account_id)
        self.assertEqual(info['containers'], 0)
        self.assertEqual(info['objects'], 0)
        self.assertEqual(info['bytes'], 0)
        self.assertEqual(info['damaged_objects'], 0)
        self.assertEqual(info['missing_chunks'], 0)
Beispiel #26
0
    def test_update_container_wrong_timestamp_format(self):
        backend = AccountBackend({}, self.conn)
        account_id = 'test'
        self.assertEqual(backend.create_account(account_id), account_id)

        # initial container
        name = '"{<container \'&\' name>}"'
        mtime = "12456.0000076"
        backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)

        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)

        # same event
        with ExpectedException(Conflict):
            backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)

        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)

        mtime = "0000012456.00005"
        backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)

        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)

        self.assertEqual(
            self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
            mtime)

        mtime = "0000012456.00035"
        backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)

        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)

        self.assertEqual(
            self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
            mtime)
Beispiel #27
0
 def test_create_account(self):
     backend = AccountBackend({}, self.conn)
     account_id = 'a'
     self.assertEqual(backend.create_account(account_id), account_id)
     self.assertEqual(backend.create_account(account_id), None)
Beispiel #28
0
    def test_list_containers(self):
        backend = AccountBackend({}, self.conn)
        account_id = 'test'

        backend.create_account(account_id)
        for cont1 in xrange(4):
            for cont2 in xrange(125):
                name = '%d-%04d' % (cont1, cont2)
                backend.update_container(account_id, name,
                                         Timestamp(time()).normal, 0, 0, 0, 0,
                                         0)

        for cont in xrange(125):
            name = '2-0051-%04d' % cont
            backend.update_container(account_id, name,
                                     Timestamp(time()).normal, 0, 0, 0, 0, 0)

        for cont in xrange(125):
            name = '3-%04d-0049' % cont
            backend.update_container(account_id, name,
                                     Timestamp(time()).normal, 0, 0, 0, 0, 0)

        listing = backend.list_containers(account_id,
                                          marker='',
                                          delimiter='',
                                          limit=100)
        self.assertEqual(len(listing), 100)
        self.assertEqual(listing[0][0], '0-0000')
        self.assertEqual(listing[-1][0], '0-0099')

        listing = backend.list_containers(account_id,
                                          marker='',
                                          end_marker='0-0050',
                                          delimiter='',
                                          limit=100)
        self.assertEqual(len(listing), 50)
        self.assertEqual(listing[0][0], '0-0000')
        self.assertEqual(listing[-1][0], '0-0049')

        listing = backend.list_containers(account_id,
                                          marker='0-0099',
                                          delimiter='',
                                          limit=100)
        self.assertEqual(len(listing), 100)
        self.assertEqual(listing[0][0], '0-0100')
        self.assertEqual(listing[-1][0], '1-0074')

        listing = backend.list_containers(account_id,
                                          marker='1-0074',
                                          delimiter='',
                                          limit=55)
        self.assertEqual(len(listing), 55)
        self.assertEqual(listing[0][0], '1-0075')
        self.assertEqual(listing[-1][0], '2-0004')

        listing = backend.list_containers(account_id,
                                          marker='',
                                          prefix='0-01',
                                          delimiter='',
                                          limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual(listing[0][0], '0-0100')
        self.assertEqual(listing[-1][0], '0-0109')

        listing = backend.list_containers(account_id,
                                          marker='',
                                          prefix='0-01',
                                          delimiter='-',
                                          limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual(listing[0][0], '0-0100')
        self.assertEqual(listing[-1][0], '0-0109')

        listing = backend.list_containers(account_id,
                                          marker='',
                                          prefix='0-',
                                          delimiter='-',
                                          limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual(listing[0][0], '0-0000')
        self.assertEqual(listing[-1][0], '0-0009')

        listing = backend.list_containers(account_id,
                                          marker='',
                                          prefix='',
                                          delimiter='-',
                                          limit=10)
        self.assertEqual(len(listing), 4)
        self.assertEqual([c[0] for c in listing], ['0-', '1-', '2-', '3-'])

        listing = backend.list_containers(account_id,
                                          marker='2-',
                                          delimiter='-',
                                          limit=10)
        self.assertEqual(len(listing), 1)
        self.assertEqual([c[0] for c in listing], ['3-'])

        listing = backend.list_containers(account_id,
                                          marker='',
                                          prefix='2',
                                          delimiter='-',
                                          limit=10)
        self.assertEqual(len(listing), 1)
        self.assertEqual([c[0] for c in listing], ['2-'])

        listing = backend.list_containers(account_id,
                                          marker='2-0050',
                                          prefix='2-',
                                          delimiter='-',
                                          limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual(listing[0][0], '2-0051')
        self.assertEqual(listing[1][0], '2-0051-')
        self.assertEqual(listing[2][0], '2-0052')
        self.assertEqual(listing[-1][0], '2-0059')

        listing = backend.list_containers(account_id,
                                          marker='3-0045',
                                          prefix='3-',
                                          delimiter='-',
                                          limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual([c[0] for c in listing], [
            '3-0045-', '3-0046', '3-0046-', '3-0047', '3-0047-', '3-0048',
            '3-0048-', '3-0049', '3-0049-', '3-0050'
        ])

        name = '3-0049-'
        backend.update_container(account_id, name,
                                 Timestamp(time()).normal, 0, 0, 0, 0, 0)
        listing = backend.list_containers(account_id,
                                          marker='3-0048',
                                          limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual([c[0] for c in listing], [
            '3-0048-0049', '3-0049', '3-0049-', '3-0049-0049', '3-0050',
            '3-0050-0049', '3-0051', '3-0051-0049', '3-0052', '3-0052-0049'
        ])

        listing = backend.list_containers(account_id,
                                          marker='3-0048',
                                          prefix='3-',
                                          delimiter='-',
                                          limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual([c[0] for c in listing], [
            '3-0048-', '3-0049', '3-0049-', '3-0050', '3-0050-', '3-0051',
            '3-0051-', '3-0052', '3-0052-', '3-0053'
        ])

        listing = backend.list_containers(account_id,
                                          prefix='3-0049-',
                                          delimiter='-',
                                          limit=10)
        self.assertEqual(len(listing), 2)
        self.assertEqual([c[0] for c in listing], ['3-0049-', '3-0049-0049'])
Beispiel #29
0
    def test_list_containers(self):
        backend = AccountBackend({}, self.conn)
        account_id = 'test'

        backend.create_account(account_id)
        for cont1 in xrange(4):
            for cont2 in xrange(125):
                name = '%d-%04d' % (cont1, cont2)
                backend.update_container(account_id, name,
                                         Timestamp(time()).normal, 0, 0, 0)

        for cont in xrange(125):
            name = '2-0051-%04d' % cont
            backend.update_container(
                account_id, name, Timestamp(time()).normal, 0, 0, 0)

        for cont in xrange(125):
            name = '3-%04d-0049' % cont
            backend.update_container(
                account_id, name, Timestamp(time()).normal, 0, 0, 0)

        listing = backend.list_containers(account_id, marker='',
                                          delimiter='', limit=100)
        self.assertEqual(len(listing), 100)
        self.assertEqual(listing[0][0], '0-0000')
        self.assertEqual(listing[-1][0], '0-0099')

        listing = backend.list_containers(account_id, marker='',
                                          end_marker='0-0050',
                                          delimiter='', limit=100)
        self.assertEqual(len(listing), 50)
        self.assertEqual(listing[0][0], '0-0000')
        self.assertEqual(listing[-1][0], '0-0049')

        listing = backend.list_containers(account_id, marker='0-0099',
                                          delimiter='', limit=100)
        self.assertEqual(len(listing), 100)
        self.assertEqual(listing[0][0], '0-0100')
        self.assertEqual(listing[-1][0], '1-0074')

        listing = backend.list_containers(account_id, marker='1-0074',
                                          delimiter='', limit=55)
        self.assertEqual(len(listing), 55)
        self.assertEqual(listing[0][0], '1-0075')
        self.assertEqual(listing[-1][0], '2-0004')

        listing = backend.list_containers(account_id, marker='', prefix='0-01',
                                          delimiter='', limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual(listing[0][0], '0-0100')
        self.assertEqual(listing[-1][0], '0-0109')

        listing = backend.list_containers(account_id, marker='',
                                          prefix='0-01', delimiter='-',
                                          limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual(listing[0][0], '0-0100')
        self.assertEqual(listing[-1][0], '0-0109')

        listing = backend.list_containers(account_id, marker='',
                                          prefix='0-', delimiter='-',
                                          limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual(listing[0][0], '0-0000')
        self.assertEqual(listing[-1][0], '0-0009')

        listing = backend.list_containers(account_id, marker='',
                                          prefix='', delimiter='-',
                                          limit=10)
        self.assertEqual(len(listing), 4)
        self.assertEqual([c[0] for c in listing],
                         ['0-', '1-', '2-', '3-'])

        listing = backend.list_containers(account_id, marker='2-',
                                          delimiter='-', limit=10)
        self.assertEqual(len(listing), 1)
        self.assertEqual([c[0] for c in listing], ['3-'])

        listing = backend.list_containers(account_id, marker='', prefix='2',
                                          delimiter='-', limit=10)
        self.assertEqual(len(listing), 1)
        self.assertEqual([c[0] for c in listing], ['2-'])

        listing = backend.list_containers(account_id, marker='2-0050',
                                          prefix='2-',
                                          delimiter='-', limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual(listing[0][0], '2-0051')
        self.assertEqual(listing[1][0], '2-0051-')
        self.assertEqual(listing[2][0], '2-0052')
        self.assertEqual(listing[-1][0], '2-0059')

        listing = backend.list_containers(account_id, marker='3-0045',
                                          prefix='3-', delimiter='-', limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual([c[0] for c in listing],
                         ['3-0045-', '3-0046', '3-0046-', '3-0047',
                          '3-0047-', '3-0048', '3-0048-', '3-0049',
                          '3-0049-', '3-0050'])

        name = '3-0049-'
        backend.update_container(account_id, name, Timestamp(time()).normal, 0,
                                 0, 0)
        listing = backend.list_containers(
            account_id, marker='3-0048', limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual([c[0] for c in listing],
                         ['3-0048-0049', '3-0049', '3-0049-', '3-0049-0049',
                          '3-0050', '3-0050-0049', '3-0051', '3-0051-0049',
                          '3-0052', '3-0052-0049'])

        listing = backend.list_containers(account_id, marker='3-0048',
                                          prefix='3-',
                                          delimiter='-', limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual([c[0] for c in listing],
                         ['3-0048-', '3-0049', '3-0049-', '3-0050',
                          '3-0050-', '3-0051', '3-0051-', '3-0052',
                          '3-0052-', '3-0053'])

        listing = backend.list_containers(account_id,
                                          prefix='3-0049-',
                                          delimiter='-', limit=10)
        self.assertEqual(len(listing), 2)
        self.assertEqual([c[0] for c in listing],
                         ['3-0049-', '3-0049-0049'])
Beispiel #30
0
    def test_info_account(self):
        backend = AccountBackend({}, self.conn)
        account_id = 'test'
        self.assertEqual(backend.create_account(account_id), account_id)
        info = backend.info_account(account_id)
        self.assertEqual(info['id'], account_id)
        self.assertEqual(info['objects'], 0)
        self.assertEqual(info['bytes'], 0)
        self.assertEqual(info['damaged_objects'], 0)
        self.assertEqual(info['missing_chunks'], 0)
        self.assertEqual(info['containers'], 0)
        self.assertTrue(info['ctime'])

        # first container
        backend.update_container(account_id, 'c1',
                                 Timestamp(time()).normal, 0, 1, 1, 1, 1)
        info = backend.info_account(account_id)
        self.assertEqual(info['containers'], 1)
        self.assertEqual(info['objects'], 1)
        self.assertEqual(info['bytes'], 1)
        self.assertEqual(info['damaged_objects'], 1)
        self.assertEqual(info['missing_chunks'], 1)

        # second container
        sleep(.00001)
        backend.update_container(account_id, 'c2',
                                 Timestamp(time()).normal, 0, 0, 0, 0, 0)
        info = backend.info_account(account_id)
        self.assertEqual(info['containers'], 2)
        self.assertEqual(info['objects'], 1)
        self.assertEqual(info['bytes'], 1)
        self.assertEqual(info['damaged_objects'], 1)
        self.assertEqual(info['missing_chunks'], 1)

        # update second container
        sleep(.00001)
        backend.update_container(account_id, 'c2',
                                 Timestamp(time()).normal, 0, 1, 1, 1, 2)
        info = backend.info_account(account_id)
        self.assertEqual(info['containers'], 2)
        self.assertEqual(info['objects'], 2)
        self.assertEqual(info['bytes'], 2)
        self.assertEqual(info['damaged_objects'], 2)
        self.assertEqual(info['missing_chunks'], 3)

        # delete first container
        sleep(.00001)
        backend.update_container(account_id, 'c1', 0,
                                 Timestamp(time()).normal, 0, 0, 0, 0)
        info = backend.info_account(account_id)
        self.assertEqual(info['containers'], 1)
        self.assertEqual(info['objects'], 1)
        self.assertEqual(info['bytes'], 1)
        self.assertEqual(info['damaged_objects'], 1)
        self.assertEqual(info['missing_chunks'], 2)

        # delete second container
        sleep(.00001)
        backend.update_container(account_id, 'c2', 0,
                                 Timestamp(time()).normal, 0, 0, 0, 0)
        info = backend.info_account(account_id)
        self.assertEqual(info['containers'], 0)
        self.assertEqual(info['objects'], 0)
        self.assertEqual(info['bytes'], 0)
        self.assertEqual(info['damaged_objects'], 0)
        self.assertEqual(info['missing_chunks'], 0)
Beispiel #31
0
    def test_update_account_metadata(self):
        backend = AccountBackend({}, self.conn)
        account_id = 'test'
        self.assertEqual(backend.create_account(account_id), account_id)

        # first meta
        backend.update_account_metadata(account_id, {'a': '1'})
        metadata = backend.get_account_metadata(account_id)
        self.assert_('a' in metadata)
        self.assertEqual(metadata['a'], '1')

        # second meta
        backend.update_account_metadata(account_id, {'b': '2'})
        metadata = backend.get_account_metadata(account_id)
        self.assert_('a' in metadata)
        self.assertEqual(metadata['a'], '1')
        self.assert_('b' in metadata)
        self.assertEqual(metadata['b'], '2')

        # update first meta
        backend.update_account_metadata(account_id, {'a': '1b'})
        metadata = backend.get_account_metadata(account_id)
        self.assert_('a' in metadata)
        self.assertEqual(metadata['a'], '1b')
        self.assert_('b' in metadata)
        self.assertEqual(metadata['b'], '2')

        # delete second meta
        backend.update_account_metadata(account_id, None, ['b'])
        metadata = backend.get_account_metadata(account_id)
        self.assert_('a' in metadata)
        self.assertEqual(metadata['a'], '1b')
        self.assert_('b' not in metadata)
Beispiel #32
0
def create_app(conf, **kwargs):
    backend = AccountBackend(conf)
    app = Account(conf, backend)
    return app
Beispiel #33
0
    def test_update_container(self):
        backend = AccountBackend({}, self.conn)
        account_id = "test"
        self.assertEqual(backend.create_account(account_id), account_id)

        # initial container
        name = "\"{<container '&' name>}\""
        mtime = Timestamp(time()).normal
        backend.update_container(account_id, name, mtime, 0, 0, 0)

        res = self.conn.zrangebylex("containers:%s" % account_id, "-", "+")
        self.assertEqual(res[0], name)

        self.assertEqual(self.conn.hget("container:%s:%s" % (account_id, name), "mtime"), mtime)

        # same event
        backend.update_container(account_id, name, mtime, 0, 0, 0)

        res = self.conn.zrangebylex("containers:%s" % account_id, "-", "+")
        self.assertEqual(res[0], name)

        self.assertEqual(self.conn.hget("container:%s:%s" % (account_id, name), "mtime"), mtime)

        # New event
        sleep(0.00001)
        mtime = Timestamp(time()).normal
        backend.update_container(account_id, name, mtime, 0, 0, 0)

        res = self.conn.zrangebylex("containers:%s" % account_id, "-", "+")
        self.assertEqual(res[0], name)

        self.assertEqual(self.conn.hget("container:%s:%s" % (account_id, name), "mtime"), mtime)

        # Old event
        old_mtime = Timestamp(time() - 1).normal
        backend.update_container(account_id, name, old_mtime, 0, 0, 0)

        res = self.conn.zrangebylex("containers:%s" % account_id, "-", "+")
        self.assertEqual(res[0], name)

        self.assertEqual(self.conn.hget("container:%s:%s" % (account_id, name), "mtime"), mtime)

        # Old delete event
        dtime = Timestamp(time() - 1).normal
        backend.update_container(account_id, name, 0, dtime, 0, 0)
        res = self.conn.zrangebylex("containers:%s" % account_id, "-", "+")
        self.assertEqual(res[0], name)
        self.assertEqual(self.conn.hget("container:%s:%s" % (account_id, name), "mtime"), mtime)

        # New delete event
        sleep(0.00001)
        mtime = Timestamp(time()).normal
        backend.update_container(account_id, name, 0, mtime, 0, 0)
        res = self.conn.zrangebylex("containers:%s" % account_id, "-", "+")
        self.assertEqual(len(res), 0)
        self.assertTrue(self.conn.ttl("container:%s:%s" % (account_id, name)) >= 1)

        # New event
        sleep(0.00001)
        mtime = Timestamp(time()).normal
        backend.update_container(account_id, name, mtime, 0, 0, 0)
        res = self.conn.zrangebylex("containers:%s" % account_id, "-", "+")
        self.assertEqual(res[0], name)
        self.assertEqual(self.conn.hget("container:%s:%s" % (account_id, name), "mtime"), mtime)
        # ensure ttl has been removed
        self.assertEqual(self.conn.ttl("container:%s:%s" % (account_id, name)), -1)
Beispiel #34
0
class TestAccountBackend(BaseTestCase):
    def setUp(self):
        super(TestAccountBackend, self).setUp()
        _, _, self.redis_host, self.redis_port = self.get_service('redis')
        self.account_conf = {
            'redis_host': self.redis_host,
            'redis_port': self.redis_port
        }
        self.backend = AccountBackend(self.account_conf)
        self.backend.conn.flushdb()

    def tearDown(self):
        super(TestAccountBackend, self).tearDown()
        self.backend.conn.flushdb()

    def test_create_account(self):
        account_id = 'a'
        self.assertEqual(self.backend.create_account(account_id), account_id)
        self.assertEqual(self.backend.create_account(account_id), None)

    def test_update_account_metadata(self):
        account_id = 'test'
        self.assertEqual(self.backend.create_account(account_id), account_id)

        # first meta
        self.backend.update_account_metadata(account_id, {'a': '1'})
        metadata = self.backend.get_account_metadata(account_id)
        self.assertIn('a', metadata)
        self.assertEqual(metadata['a'], '1')

        # second meta
        self.backend.update_account_metadata(account_id, {'b': '2'})
        metadata = self.backend.get_account_metadata(account_id)
        self.assertIn('a', metadata)
        self.assertEqual(metadata['a'], '1')
        self.assertIn('b', metadata)
        self.assertEqual(metadata['b'], '2')

        # update first meta
        self.backend.update_account_metadata(account_id, {'a': '1b'})
        metadata = self.backend.get_account_metadata(account_id)
        self.assertIn('a', metadata)
        self.assertEqual(metadata['a'], '1b')
        self.assertIn('b', metadata)
        self.assertEqual(metadata['b'], '2')

        # delete second meta
        self.backend.update_account_metadata(account_id, None, ['b'])
        metadata = self.backend.get_account_metadata(account_id)
        self.assertIn('a', metadata)
        self.assertEqual(metadata['a'], '1b')
        self.assertNotIn('b', metadata)

    def test_list_account(self):

        # Create and check if in list
        account_id = 'test_list'
        self.backend.create_account(account_id)
        account_list = self.backend.list_accounts()
        self.assertIn(account_id, account_list)

        # Check the result of a nonexistent account
        self.assertFalse("Should_not_exist" in account_list)

    def test_info_account(self):
        account_id = 'test'
        self.assertEqual(self.backend.create_account(account_id), account_id)
        info = self.backend.info_account(account_id)
        self.assertEqual(info['id'], account_id)
        self.assertEqual(info['objects'], 0)
        self.assertEqual(info['bytes'], 0)
        self.assertEqual(info['containers'], 0)
        self.assertTrue(info['ctime'])

        # first container
        self.backend.update_container(account_id, 'c1',
                                      Timestamp().normal, 0, 1, 1)
        info = self.backend.info_account(account_id)
        self.assertEqual(info['containers'], 1)
        self.assertEqual(info['objects'], 1)
        self.assertEqual(info['bytes'], 1)

        # second container
        sleep(.00001)
        self.backend.update_container(account_id, 'c2',
                                      Timestamp().normal, 0, 0, 0)
        info = self.backend.info_account(account_id)
        self.assertEqual(info['containers'], 2)
        self.assertEqual(info['objects'], 1)
        self.assertEqual(info['bytes'], 1)

        # update second container
        sleep(.00001)
        self.backend.update_container(account_id, 'c2',
                                      Timestamp().normal, 0, 1, 1)
        info = self.backend.info_account(account_id)
        self.assertEqual(info['containers'], 2)
        self.assertEqual(info['objects'], 2)
        self.assertEqual(info['bytes'], 2)

        # delete first container
        sleep(.00001)
        self.backend.update_container(account_id, 'c1', 0,
                                      Timestamp().normal, 0, 0)
        info = self.backend.info_account(account_id)
        self.assertEqual(info['containers'], 1)
        self.assertEqual(info['objects'], 1)
        self.assertEqual(info['bytes'], 1)

        # delete second container
        sleep(.00001)
        self.backend.update_container(account_id, 'c2', 0,
                                      Timestamp().normal, 0, 0)
        info = self.backend.info_account(account_id)
        self.assertEqual(info['containers'], 0)
        self.assertEqual(info['objects'], 0)
        self.assertEqual(info['bytes'], 0)

    def test_update_after_container_deletion(self):
        account_id = 'test-%06x' % int(time())
        self.assertEqual(self.backend.create_account(account_id), account_id)

        # Container create event, sent immediately after creation
        self.backend.update_container(account_id, 'c1',
                                      Timestamp().normal, None, None, None)

        # Container update event
        self.backend.update_container(account_id, 'c1',
                                      Timestamp().normal, None, 3, 30)
        info = self.backend.info_account(account_id)
        self.assertEqual(info['containers'], 1)
        self.assertEqual(info['objects'], 3)
        self.assertEqual(info['bytes'], 30)

        # Container is flushed, but the event is deferred
        flush_timestamp = Timestamp().normal

        sleep(.00001)
        # Container delete event, sent immediately after deletion
        self.backend.update_container(account_id, 'c1', None,
                                      Timestamp().normal, None, None)

        # Deferred container update event (with lower timestamp)
        self.backend.update_container(account_id, 'c1', flush_timestamp, None,
                                      0, 0)
        info = self.backend.info_account(account_id)
        self.assertEqual(info['containers'], 0)
        self.assertEqual(info['objects'], 0)
        self.assertEqual(info['bytes'], 0)

    def test_delete_container(self):
        account_id = 'test'
        self.assertEqual(self.backend.create_account(account_id), account_id)
        name = 'c'
        old_mtime = Timestamp(time() - 1).normal
        mtime = Timestamp().normal

        # initial container
        self.backend.update_container(account_id, name, mtime, 0, 0, 0)
        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(res[0].decode('utf-8'), name)

        # delete event
        sleep(.00001)
        dtime = Timestamp().normal
        self.backend.update_container(account_id, name, mtime, dtime, 0, 0)
        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(len(res), 0)
        self.assertGreaterEqual(
            self.backend.conn.ttl('container:%s:%s' % (account_id, name)), 1)

        # same event
        with ExpectedException(Conflict):
            self.backend.update_container(account_id, name, mtime, dtime, 0, 0)
        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(len(res), 0)
        self.assertGreaterEqual(
            self.backend.conn.ttl('container:%s:%s' % (account_id, name)), 1)

        # old event
        with ExpectedException(Conflict):
            self.backend.update_container(account_id, name, old_mtime, 0, 0, 0)
        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(len(res), 0)
        self.assertGreaterEqual(
            self.backend.conn.ttl('container:%s:%s' % (account_id, name)), 1)

    def test_utf8_container(self):
        account_id = 'test'
        self.assertEqual(self.backend.create_account(account_id), account_id)
        name = u'La fête à la maison'
        mtime = Timestamp().normal

        # create container
        self.backend.update_container(account_id, name, mtime, 0, 0, 0)
        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(text_type(res[0], 'utf8'), name)

        # ensure it appears in listing
        listing = self.backend.list_containers(account_id,
                                               marker='',
                                               delimiter='',
                                               limit=100)
        self.assertIn(name, [entry[0] for entry in listing])

        # delete container
        sleep(.00001)
        dtime = Timestamp().normal
        self.backend.update_container(account_id, name, 0, dtime, 0, 0)
        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(len(res), 0)
        self.assertGreaterEqual(
            self.backend.conn.ttl('container:%s:%s' % (account_id, name)), 1)

        # ensure it has been removed
        with ExpectedException(Conflict):
            self.backend.update_container(account_id, name, 0, dtime, 0, 0)
        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(len(res), 0)
        self.assertGreaterEqual(
            self.backend.conn.ttl('container:%s:%s' % (account_id, name)), 1)

    def test_update_container(self):
        account_id = 'test'
        self.assertEqual(self.backend.create_account(account_id), account_id)

        # initial container
        name = '"{<container \'&\' name>}"'
        mtime = Timestamp().normal
        self.backend.update_container(account_id, name, mtime, 0, 0, 0)

        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(res[0].decode('utf-8'), name)

        tmtime = self.backend.conn.hget('container:%s:%s' % (account_id, name),
                                        'mtime')
        self.assertEqual(tmtime.decode('utf-8'), mtime)

        # same event
        with ExpectedException(Conflict):
            self.backend.update_container(account_id, name, mtime, 0, 0, 0)

        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(res[0].decode('utf-8'), name)

        tmtime = self.backend.conn.hget('container:%s:%s' % (account_id, name),
                                        'mtime')
        self.assertEqual(tmtime.decode('utf-8'), mtime)

        # New event
        sleep(.00001)
        mtime = Timestamp().normal
        self.backend.update_container(account_id, name, mtime, 0, 0, 0)

        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(res[0].decode('utf-8'), name)

        tmtime = self.backend.conn.hget('container:%s:%s' % (account_id, name),
                                        'mtime')
        self.assertEqual(tmtime.decode('utf-8'), mtime)

        # Old event
        old_mtime = Timestamp(time() - 1).normal
        with ExpectedException(Conflict):
            self.backend.update_container(account_id, name, old_mtime, 0, 0, 0)

        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(res[0].decode('utf-8'), name)

        tmtime = self.backend.conn.hget('container:%s:%s' % (account_id, name),
                                        'mtime')
        self.assertEqual(tmtime.decode('utf-8'), mtime)

        # Old delete event
        dtime = Timestamp(time() - 1).normal
        with ExpectedException(Conflict):
            self.backend.update_container(account_id, name, 0, dtime, 0, 0)
        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(res[0].decode('utf-8'), name)
        tmtime = self.backend.conn.hget('container:%s:%s' % (account_id, name),
                                        'mtime')
        self.assertEqual(tmtime.decode('utf-8'), mtime)

        # New delete event
        sleep(.00001)
        mtime = Timestamp().normal
        self.backend.update_container(account_id, name, 0, mtime, 0, 0)
        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(len(res), 0)
        self.assertGreaterEqual(
            self.backend.conn.ttl('container:%s:%s' % (account_id, name)), 1)

        # New event
        sleep(.00001)
        mtime = Timestamp().normal
        self.backend.update_container(account_id, name, mtime, 0, 0, 0)
        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(res[0].decode('utf-8'), name)
        tmtime = self.backend.conn.hget('container:%s:%s' % (account_id, name),
                                        'mtime')
        self.assertEqual(tmtime.decode('utf-8'), mtime)
        # ensure ttl has been removed
        self.assertEqual(
            self.backend.conn.ttl('container:%s:%s' % (account_id, name)), -1)

    def test_list_containers(self):
        account_id = 'test'

        self.backend.create_account(account_id)
        for cont1 in xrange(4):
            for cont2 in xrange(125):
                name = '%d-%04d' % (cont1, cont2)
                self.backend.update_container(account_id, name,
                                              Timestamp().normal, 0, 0, 0)

        for cont in xrange(125):
            name = '2-0051-%04d' % cont
            self.backend.update_container(account_id, name,
                                          Timestamp().normal, 0, 0, 0)

        for cont in xrange(125):
            name = '3-%04d-0049' % cont
            self.backend.update_container(account_id, name,
                                          Timestamp().normal, 0, 0, 0)

        listing = self.backend.list_containers(account_id,
                                               marker='',
                                               delimiter='',
                                               limit=100)
        self.assertEqual(len(listing), 100)
        self.assertEqual(listing[0][0], '0-0000')
        self.assertEqual(listing[-1][0], '0-0099')

        listing = self.backend.list_containers(account_id,
                                               marker='',
                                               end_marker='0-0050',
                                               delimiter='',
                                               limit=100)
        self.assertEqual(len(listing), 50)
        self.assertEqual(listing[0][0], '0-0000')
        self.assertEqual(listing[-1][0], '0-0049')

        listing = self.backend.list_containers(account_id,
                                               marker='0-0099',
                                               delimiter='',
                                               limit=100)
        self.assertEqual(len(listing), 100)
        self.assertEqual(listing[0][0], '0-0100')
        self.assertEqual(listing[-1][0], '1-0074')

        listing = self.backend.list_containers(account_id,
                                               marker='1-0074',
                                               delimiter='',
                                               limit=55)
        self.assertEqual(len(listing), 55)
        self.assertEqual(listing[0][0], '1-0075')
        self.assertEqual(listing[-1][0], '2-0004')

        listing = self.backend.list_containers(account_id,
                                               marker='',
                                               prefix='0-01',
                                               delimiter='',
                                               limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual(listing[0][0], '0-0100')
        self.assertEqual(listing[-1][0], '0-0109')

        listing = self.backend.list_containers(account_id,
                                               marker='',
                                               prefix='0-01',
                                               delimiter='-',
                                               limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual(listing[0][0], '0-0100')
        self.assertEqual(listing[-1][0], '0-0109')

        listing = self.backend.list_containers(account_id,
                                               marker='',
                                               prefix='0-',
                                               delimiter='-',
                                               limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual(listing[0][0], '0-0000')
        self.assertEqual(listing[-1][0], '0-0009')

        listing = self.backend.list_containers(account_id,
                                               marker='',
                                               prefix='',
                                               delimiter='-',
                                               limit=10)
        self.assertEqual(len(listing), 4)
        self.assertEqual([c[0] for c in listing], ['0-', '1-', '2-', '3-'])

        listing = self.backend.list_containers(account_id,
                                               marker='2-',
                                               delimiter='-',
                                               limit=10)
        self.assertEqual(len(listing), 1)
        self.assertEqual([c[0] for c in listing], ['3-'])

        listing = self.backend.list_containers(account_id,
                                               marker='',
                                               prefix='2',
                                               delimiter='-',
                                               limit=10)
        self.assertEqual(len(listing), 1)
        self.assertEqual([c[0] for c in listing], ['2-'])

        listing = self.backend.list_containers(account_id,
                                               marker='2-0050',
                                               prefix='2-',
                                               delimiter='-',
                                               limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual(listing[0][0], '2-0051')
        self.assertEqual(listing[1][0], '2-0051-')
        self.assertEqual(listing[2][0], '2-0052')
        self.assertEqual(listing[-1][0], '2-0059')

        listing = self.backend.list_containers(account_id,
                                               marker='3-0045',
                                               prefix='3-',
                                               delimiter='-',
                                               limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual([c[0] for c in listing], [
            '3-0045-', '3-0046', '3-0046-', '3-0047', '3-0047-', '3-0048',
            '3-0048-', '3-0049', '3-0049-', '3-0050'
        ])

        name = '3-0049-'
        self.backend.update_container(account_id, name,
                                      Timestamp().normal, 0, 0, 0)
        listing = self.backend.list_containers(account_id,
                                               marker='3-0048',
                                               limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual([c[0] for c in listing], [
            '3-0048-0049', '3-0049', '3-0049-', '3-0049-0049', '3-0050',
            '3-0050-0049', '3-0051', '3-0051-0049', '3-0052', '3-0052-0049'
        ])

        listing = self.backend.list_containers(account_id,
                                               marker='3-0048',
                                               prefix='3-',
                                               delimiter='-',
                                               limit=10)
        self.assertEqual(len(listing), 10)
        self.assertEqual([c[0] for c in listing], [
            '3-0048-', '3-0049', '3-0049-', '3-0050', '3-0050-', '3-0051',
            '3-0051-', '3-0052', '3-0052-', '3-0053'
        ])

        listing = self.backend.list_containers(account_id,
                                               prefix='3-0049-',
                                               delimiter='-',
                                               limit=10)
        self.assertEqual(len(listing), 2)
        self.assertEqual([c[0] for c in listing], ['3-0049-', '3-0049-0049'])

    def test_refresh_account(self):
        account_id = random_str(16)
        account_key = 'account:%s' % account_id

        self.assertEqual(self.backend.create_account(account_id), account_id)

        total_bytes = 0
        total_objects = 0

        # 10 containers with bytes and objects
        for i in range(10):
            name = "container%d" % i
            mtime = Timestamp().normal
            nb_bytes = random.randrange(100)
            total_bytes += nb_bytes
            nb_objets = random.randrange(100)
            total_objects += nb_objets
            self.backend.update_container(account_id, name, mtime, 0,
                                          nb_objets, nb_bytes)

        # change values
        self.backend.conn.hset(account_key, 'bytes', 1)
        self.backend.conn.hset(account_key, 'objects', 2)
        self.assertEqual(self.backend.conn.hget(account_key, 'bytes'), b'1')
        self.assertEqual(self.backend.conn.hget(account_key, 'objects'), b'2')

        self.backend.refresh_account(account_id)
        self.assertEqual(self.backend.conn.hget(account_key, 'bytes'),
                         str(total_bytes).encode('utf-8'))
        self.assertEqual(self.backend.conn.hget(account_key, 'objects'),
                         str(total_objects).encode('utf-8'))

    def test_update_container_wrong_timestamp_format(self):
        account_id = 'test'
        self.assertEqual(self.backend.create_account(account_id), account_id)

        # initial container
        name = '"{<container \'&\' name>}"'
        mtime = "12456.0000076"
        self.backend.update_container(account_id, name, mtime, 0, 0, 0)

        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(res[0].decode('utf-8'), name)

        # same event
        with ExpectedException(Conflict):
            self.backend.update_container(account_id, name, mtime, 0, 0, 0)

        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(res[0].decode('utf-8'), name)

        mtime = "0000012456.00005"
        self.backend.update_container(account_id, name, mtime, 0, 0, 0)

        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(res[0].decode('utf-8'), name)

        tmtime = self.backend.conn.hget('container:%s:%s' % (account_id, name),
                                        'mtime')
        self.assertEqual(tmtime.decode('utf-8'), mtime)

        mtime = "0000012456.00035"
        self.backend.update_container(account_id, name, mtime, 0, 0, 0)

        res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
                                            '+')
        self.assertEqual(res[0].decode('utf-8'), name)

        tmtime = self.backend.conn.hget('container:%s:%s' % (account_id, name),
                                        'mtime')
        self.assertEqual(tmtime.decode('utf-8'), mtime)

    def test_is_sup(self):
        # HACK: replace %%d by %d
        compare = (self.backend.lua_is_sup % (()) + """
            if (is_sup(KEYS[1], KEYS[2])) then
              return redis.status_reply('IS SUP');
            else
              return redis.error_reply('IS NOT SUP');
            end;
                   """)

        compare_script = self.backend.register_script(compare)
        self.assertRaises(redis.exceptions.ResponseError,
                          compare_script,
                          keys=["12457.00245", "12457.00245"],
                          client=self.backend.conn)
        self.assertEqual(
            compare_script(keys=["42457.00245", "12457.00245"],
                           client=self.backend.conn), b"IS SUP")
        self.assertRaises(redis.exceptions.ResponseError,
                          compare_script,
                          keys=["12457.00245", "12457.002450"],
                          client=self.backend.conn)
        self.assertEqual(
            compare_script(keys=["42457.00245", "12457.002450"],
                           client=self.backend.conn), b"IS SUP")
        self.assertRaises(redis.exceptions.ResponseError,
                          compare_script,
                          keys=["12457.00245", "12457.002456"],
                          client=self.backend.conn)
        self.assertEqual(
            compare_script(keys=["42457.00245", "12457.002456"],
                           client=self.backend.conn), b"IS SUP")
        self.assertEqual(
            compare_script(keys=["42457.00246", "42457.002458"],
                           client=self.backend.conn), b"IS SUP")

    def test_flush_account(self):
        account_id = random_str(16)
        account_key = 'account:%s' % account_id

        self.assertEqual(self.backend.create_account(account_id), account_id)

        total_bytes = 0
        total_objects = 0

        # 10 containers with bytes and objects
        for i in range(10):
            name = "container%d" % i
            mtime = Timestamp().normal
            nb_bytes = random.randrange(100)
            total_bytes += nb_bytes
            nb_objets = random.randrange(100)
            total_objects += nb_objets
            self.backend.update_container(account_id, name, mtime, 0,
                                          nb_objets, nb_bytes)

        self.assertEqual(self.backend.conn.hget(account_key, 'bytes'),
                         str(total_bytes).encode('utf-8'))
        self.assertEqual(self.backend.conn.hget(account_key, 'objects'),
                         str(total_objects).encode('utf-8'))

        self.backend.flush_account(account_id)
        self.assertEqual(self.backend.conn.hget(account_key, 'bytes'), b'0')
        self.assertEqual(self.backend.conn.hget(account_key, 'objects'), b'0')
        self.assertEqual(self.backend.conn.zcard("containers:%s" % account_id),
                         0)
        self.assertEqual(self.backend.conn.exists("container:test:*"), 0)

    def test_refresh_bucket(self):
        account_id = random_str(16)
        account_key = 'account:%s' % account_id
        bucket = random_str(16)

        self.assertEqual(self.backend.create_account(account_id), account_id)

        total_bytes = 0
        total_objects = 0

        # 10 containers with bytes and objects
        for i in range(10):
            name = "container%d" % i
            mtime = Timestamp().normal
            nb_bytes = random.randrange(100)
            total_bytes += nb_bytes
            nb_objets = random.randrange(100)
            total_objects += nb_objets
            self.backend.update_container(account_id,
                                          name,
                                          mtime,
                                          0,
                                          nb_objets,
                                          nb_bytes,
                                          bucket_name=bucket)

        bkey = self.backend.bkey(bucket)
        # change values
        self.backend.conn.hset(bkey, 'bytes', 1)
        self.backend.conn.hset(bkey, 'objects', 2)
        self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'1')
        self.assertEqual(self.backend.conn.hget(bkey, 'objects'), b'2')

        # force pagination
        self.backend.refresh_bucket(bucket, batch_size=6)
        conn = self.backend.conn
        self.assertEqual(
            conn.hget(account_key, 'bytes').decode('utf-8'), str(total_bytes))
        self.assertEqual(
            conn.hget(account_key, 'objects').decode('utf-8'),
            str(total_objects))

    def test_refresh_bucket_with_lock(self):
        account_id = random_str(16)
        bucket = random_str(16)

        self.assertEqual(self.backend.create_account(account_id), account_id)

        mtime = Timestamp().normal
        self.backend.update_container(account_id,
                                      "cnt-1",
                                      mtime,
                                      0,
                                      100,
                                      100,
                                      bucket_name=bucket)
        self.backend.update_container(account_id,
                                      "cnt-2",
                                      mtime,
                                      0,
                                      500,
                                      500,
                                      bucket_name=bucket)
        self.backend.update_container(account_id,
                                      "cnt-3",
                                      mtime,
                                      0,
                                      999,
                                      999,
                                      bucket_name=bucket)

        bkey = self.backend.bkey(bucket)

        # create lock
        block = self.backend.blockkey(bucket)
        self.backend.conn.hset(block,
                               mapping={
                                   "marker": "cnt-2",
                                   "mtime": int(time()),
                                   "ctime": int(time())
                               })

        with ExpectedException(Conflict):
            self.backend.refresh_bucket(bucket)

        # change value of first container
        self.backend.conn.hset(bkey, 'bytes', 100)
        self.backend.conn.hset(bkey, 'objects', 100)

        # update a container before marker, update is ok
        mtime = Timestamp().normal
        self.backend.update_container(account_id,
                                      "cnt-1",
                                      mtime,
                                      0,
                                      200,
                                      200,
                                      bucket_name=bucket)
        self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'200')

        # update a container equal to marker, update is ok
        mtime = Timestamp().normal
        self.backend.update_container(account_id,
                                      "cnt-2",
                                      mtime,
                                      0,
                                      600,
                                      600,
                                      bucket_name=bucket)
        self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'300')

        # update container after marker, update is not done
        mtime = Timestamp().normal
        self.backend.update_container(account_id,
                                      "cnt-3",
                                      mtime,
                                      0,
                                      998,
                                      998,
                                      bucket_name=bucket)
        self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'300')

        # update lock with very old timestamp
        self.backend.conn.hset(block, "mtime", 1000)
        self.backend.refresh_bucket(bucket)
        self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'1798')
Beispiel #35
0
    def test_update_account_metadata(self):
        backend = AccountBackend({}, self.conn)
        account_id = 'test'
        self.assertEqual(backend.create_account(account_id), account_id)

        # first meta
        backend.update_account_metadata(account_id, {'a': '1'})
        metadata = backend.get_account_metadata(account_id)
        self.assert_('a' in metadata)
        self.assertEqual(metadata['a'], '1')

        # second meta
        backend.update_account_metadata(account_id, {'b': '2'})
        metadata = backend.get_account_metadata(account_id)
        self.assert_('a' in metadata)
        self.assertEqual(metadata['a'], '1')
        self.assert_('b' in metadata)
        self.assertEqual(metadata['b'], '2')

        # update first meta
        backend.update_account_metadata(account_id, {'a': '1b'})
        metadata = backend.get_account_metadata(account_id)
        self.assert_('a' in metadata)
        self.assertEqual(metadata['a'], '1b')
        self.assert_('b' in metadata)
        self.assertEqual(metadata['b'], '2')

        # delete second meta
        backend.update_account_metadata(account_id, None, ['b'])
        metadata = backend.get_account_metadata(account_id)
        self.assert_('a' in metadata)
        self.assertEqual(metadata['a'], '1b')
        self.assert_('b' not in metadata)
Beispiel #36
0
    def test_delete_container(self):
        backend = AccountBackend({}, self.conn)
        account_id = 'test'
        self.assertEqual(backend.create_account(account_id), account_id)
        name = 'c'
        old_mtime = Timestamp(time() - 1).normal
        mtime = Timestamp(time()).normal

        # initial container
        backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(res[0], name)

        # delete event
        sleep(.00001)
        dtime = Timestamp(time()).normal
        backend.update_container(account_id, name, mtime, dtime, 0, 0, 0, 0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(len(res), 0)
        self.assertTrue(
            self.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)

        # same event
        with ExpectedException(Conflict):
            backend.update_container(account_id, name, mtime, dtime, 0, 0, 0,
                                     0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(len(res), 0)
        self.assertTrue(
            self.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)

        # old event
        with ExpectedException(Conflict):
            backend.update_container(account_id, name, old_mtime, 0, 0, 0, 0,
                                     0)
        res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
        self.assertEqual(len(res), 0)
        self.assertTrue(
            self.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)