def test_update_account_metadata(self): backend = AccountBackend({}, self.conn) account_id = 'test' self.assertEqual(backend.create_account(account_id), account_id) # first meta backend.update_account_metadata(account_id, {'a': '1'}) metadata = backend.get_account_metadata(account_id) self.assert_('a' in metadata) self.assertEqual(metadata['a'], '1') # second meta backend.update_account_metadata(account_id, {'b': '2'}) metadata = backend.get_account_metadata(account_id) self.assert_('a' in metadata) self.assertEqual(metadata['a'], '1') self.assert_('b' in metadata) self.assertEqual(metadata['b'], '2') # update first meta backend.update_account_metadata(account_id, {'a': '1b'}) metadata = backend.get_account_metadata(account_id) self.assert_('a' in metadata) self.assertEqual(metadata['a'], '1b') self.assert_('b' in metadata) self.assertEqual(metadata['b'], '2') # delete second meta backend.update_account_metadata(account_id, None, ['b']) metadata = backend.get_account_metadata(account_id) self.assert_('a' in metadata) self.assertEqual(metadata['a'], '1b') self.assert_('b' not in metadata)
def test_update_account_metadata(self): backend = AccountBackend({}, self.conn) account_id = 'test' self.assertEqual(backend.create_account(account_id), account_id) # first meta backend.update_account_metadata(account_id, {'a': '1'}) metadata = backend.get_account_metadata(account_id) self.assert_('a' in metadata) self.assertEqual(metadata['a'], '1') # second meta backend.update_account_metadata(account_id, {'b': '2'}) metadata = backend.get_account_metadata(account_id) self.assert_('a' in metadata) self.assertEqual(metadata['a'], '1') self.assert_('b' in metadata) self.assertEqual(metadata['b'], '2') # update first meta backend.update_account_metadata(account_id, {'a': '1b'}) metadata = backend.get_account_metadata(account_id) self.assert_('a' in metadata) self.assertEqual(metadata['a'], '1b') self.assert_('b' in metadata) self.assertEqual(metadata['b'], '2') # delete second meta backend.update_account_metadata(account_id, None, ['b']) metadata = backend.get_account_metadata(account_id) self.assert_('a' in metadata) self.assertEqual(metadata['a'], '1b') self.assert_('b' not in metadata)
def test_update_account_metadata(self): backend = AccountBackend({}, self.conn) account_id = "test" self.assertEqual(backend.create_account(account_id), account_id) # first meta backend.update_account_metadata(account_id, {"a": "1"}) metadata = backend.get_account_metadata(account_id) self.assert_("a" in metadata) self.assertEqual(metadata["a"], "1") # second meta backend.update_account_metadata(account_id, {"b": "2"}) metadata = backend.get_account_metadata(account_id) self.assert_("a" in metadata) self.assertEqual(metadata["a"], "1") self.assert_("b" in metadata) self.assertEqual(metadata["b"], "2") # update first meta backend.update_account_metadata(account_id, {"a": "1b"}) metadata = backend.get_account_metadata(account_id) self.assert_("a" in metadata) self.assertEqual(metadata["a"], "1b") self.assert_("b" in metadata) self.assertEqual(metadata["b"], "2") # delete second meta backend.update_account_metadata(account_id, None, ["b"]) metadata = backend.get_account_metadata(account_id) self.assert_("a" in metadata) self.assertEqual(metadata["a"], "1b") self.assert_("b" not in metadata)
class TestAccountBackend(BaseTestCase): def setUp(self): super(TestAccountBackend, self).setUp() _, _, self.redis_host, self.redis_port = self.get_service('redis') self.conf_account = { 'redis_host': self.redis_host, 'redis_port': self.redis_port } self.backend = AccountBackend(self.conf_account) self.backend.conn.flushdb() def tearDown(self): super(TestAccountBackend, self).tearDown() self.backend.conn.flushdb() def test_create_account(self): account_id = 'a' self.assertEqual(self.backend.create_account(account_id), account_id) self.assertEqual(self.backend.create_account(account_id), None) def test_update_account_metadata(self): account_id = 'test' self.assertEqual(self.backend.create_account(account_id), account_id) # first meta self.backend.update_account_metadata(account_id, {'a': '1'}) metadata = self.backend.get_account_metadata(account_id) self.assert_('a' in metadata) self.assertEqual(metadata['a'], '1') # second meta self.backend.update_account_metadata(account_id, {'b': '2'}) metadata = self.backend.get_account_metadata(account_id) self.assert_('a' in metadata) self.assertEqual(metadata['a'], '1') self.assert_('b' in metadata) self.assertEqual(metadata['b'], '2') # update first meta self.backend.update_account_metadata(account_id, {'a': '1b'}) metadata = self.backend.get_account_metadata(account_id) self.assert_('a' in metadata) self.assertEqual(metadata['a'], '1b') self.assert_('b' in metadata) self.assertEqual(metadata['b'], '2') # delete second meta self.backend.update_account_metadata(account_id, None, ['b']) metadata = self.backend.get_account_metadata(account_id) self.assert_('a' in metadata) self.assertEqual(metadata['a'], '1b') self.assert_('b' not in metadata) def test_list_account(self): # Create and check if in list account_id = 'test_list' self.backend.create_account(account_id) account_list = self.backend.list_account() self.assertTrue(account_id in account_list) # Check the result of a nonexistent account self.assertFalse("Should_not_exist" in account_list) def test_info_account(self): account_id = 'test' self.assertEqual(self.backend.create_account(account_id), account_id) info = self.backend.info_account(account_id) self.assertEqual(info['id'], account_id) self.assertEqual(info['objects'], 0) self.assertEqual(info['bytes'], 0) self.assertEqual(info['damaged_objects'], 0) self.assertEqual(info['missing_chunks'], 0) self.assertEqual(info['containers'], 0) self.assertTrue(info['ctime']) # first container self.backend.update_container(account_id, 'c1', Timestamp(time()).normal, 0, 1, 1, 1, 1) info = self.backend.info_account(account_id) self.assertEqual(info['containers'], 1) self.assertEqual(info['objects'], 1) self.assertEqual(info['bytes'], 1) self.assertEqual(info['damaged_objects'], 1) self.assertEqual(info['missing_chunks'], 1) # second container sleep(.00001) self.backend.update_container(account_id, 'c2', Timestamp(time()).normal, 0, 0, 0, 0, 0) info = self.backend.info_account(account_id) self.assertEqual(info['containers'], 2) self.assertEqual(info['objects'], 1) self.assertEqual(info['bytes'], 1) self.assertEqual(info['damaged_objects'], 1) self.assertEqual(info['missing_chunks'], 1) # update second container sleep(.00001) self.backend.update_container(account_id, 'c2', Timestamp(time()).normal, 0, 1, 1, 1, 2) info = self.backend.info_account(account_id) self.assertEqual(info['containers'], 2) self.assertEqual(info['objects'], 2) self.assertEqual(info['bytes'], 2) self.assertEqual(info['damaged_objects'], 2) self.assertEqual(info['missing_chunks'], 3) # delete first container sleep(.00001) self.backend.update_container(account_id, 'c1', 0, Timestamp(time()).normal, 0, 0, 0, 0) info = self.backend.info_account(account_id) self.assertEqual(info['containers'], 1) self.assertEqual(info['objects'], 1) self.assertEqual(info['bytes'], 1) self.assertEqual(info['damaged_objects'], 1) self.assertEqual(info['missing_chunks'], 2) # delete second container sleep(.00001) self.backend.update_container(account_id, 'c2', 0, Timestamp(time()).normal, 0, 0, 0, 0) info = self.backend.info_account(account_id) self.assertEqual(info['containers'], 0) self.assertEqual(info['objects'], 0) self.assertEqual(info['bytes'], 0) self.assertEqual(info['damaged_objects'], 0) self.assertEqual(info['missing_chunks'], 0) def test_update_after_container_deletion(self): account_id = 'test-%06x' % int(time()) self.assertEqual(self.backend.create_account(account_id), account_id) # Container create event, sent immediately after creation self.backend.update_container(account_id, 'c1', Timestamp(time()).normal, None, None, None, None, None) # Container update event self.backend.update_container(account_id, 'c1', Timestamp(time()).normal, None, 3, 30, 7, 5) info = self.backend.info_account(account_id) self.assertEqual(info['containers'], 1) self.assertEqual(info['objects'], 3) self.assertEqual(info['bytes'], 30) self.assertEqual(info['damaged_objects'], 7) self.assertEqual(info['missing_chunks'], 5) # Container is flushed, but the event is deferred flush_timestamp = Timestamp(time()).normal sleep(.00001) # Container delete event, sent immediately after deletion self.backend.update_container(account_id, 'c1', None, Timestamp(time()).normal, None, None, None, None) # Deferred container update event (with lower timestamp) self.backend.update_container(account_id, 'c1', flush_timestamp, None, 0, 0, 0, 0) info = self.backend.info_account(account_id) self.assertEqual(info['containers'], 0) self.assertEqual(info['objects'], 0) self.assertEqual(info['bytes'], 0) self.assertEqual(info['damaged_objects'], 0) self.assertEqual(info['missing_chunks'], 0) def test_delete_container(self): account_id = 'test' self.assertEqual(self.backend.create_account(account_id), account_id) name = 'c' old_mtime = Timestamp(time() - 1).normal mtime = Timestamp(time()).normal # initial container self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0], name) # delete event sleep(.00001) dtime = Timestamp(time()).normal self.backend.update_container(account_id, name, mtime, dtime, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(len(res), 0) self.assertTrue( self.backend.conn.ttl('container:%s:%s' % (account_id, name)) >= 1) # same event with ExpectedException(Conflict): self.backend.update_container(account_id, name, mtime, dtime, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(len(res), 0) self.assertTrue( self.backend.conn.ttl('container:%s:%s' % (account_id, name)) >= 1) # old event with ExpectedException(Conflict): self.backend.update_container(account_id, name, old_mtime, 0, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(len(res), 0) self.assertTrue( self.backend.conn.ttl('container:%s:%s' % (account_id, name)) >= 1) def test_utf8_container(self): account_id = 'test' self.assertEqual(self.backend.create_account(account_id), account_id) name = u'La fête à la maison' mtime = Timestamp(time()).normal # create container self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(unicode(res[0], 'utf8'), name) # ensure it appears in listing listing = self.backend.list_containers(account_id, marker='', delimiter='', limit=100) self.assertIn(name, [entry[0] for entry in listing]) # delete container sleep(.00001) dtime = Timestamp(time()).normal self.backend.update_container(account_id, name, 0, dtime, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(len(res), 0) self.assertTrue( self.backend.conn.ttl('container:%s:%s' % (account_id, name)) >= 1) # ensure it has been removed with ExpectedException(Conflict): self.backend.update_container(account_id, name, 0, dtime, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(len(res), 0) self.assertTrue( self.backend.conn.ttl('container:%s:%s' % (account_id, name)) >= 1) def test_update_container(self): account_id = 'test' self.assertEqual(self.backend.create_account(account_id), account_id) # initial container name = '"{<container \'&\' name>}"' mtime = Timestamp(time()).normal self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0], name) self.assertEqual( self.backend.conn.hget('container:%s:%s' % (account_id, name), 'mtime'), mtime) # same event with ExpectedException(Conflict): self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0], name) self.assertEqual( self.backend.conn.hget('container:%s:%s' % (account_id, name), 'mtime'), mtime) # New event sleep(.00001) mtime = Timestamp(time()).normal self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0], name) self.assertEqual( self.backend.conn.hget('container:%s:%s' % (account_id, name), 'mtime'), mtime) # Old event old_mtime = Timestamp(time() - 1).normal with ExpectedException(Conflict): self.backend.update_container(account_id, name, old_mtime, 0, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0], name) self.assertEqual( self.backend.conn.hget('container:%s:%s' % (account_id, name), 'mtime'), mtime) # Old delete event dtime = Timestamp(time() - 1).normal with ExpectedException(Conflict): self.backend.update_container(account_id, name, 0, dtime, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0], name) self.assertEqual( self.backend.conn.hget('container:%s:%s' % (account_id, name), 'mtime'), mtime) # New delete event sleep(.00001) mtime = Timestamp(time()).normal self.backend.update_container(account_id, name, 0, mtime, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(len(res), 0) self.assertTrue( self.backend.conn.ttl('container:%s:%s' % (account_id, name)) >= 1) # New event sleep(.00001) mtime = Timestamp(time()).normal self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0], name) self.assertEqual( self.backend.conn.hget('container:%s:%s' % (account_id, name), 'mtime'), mtime) # ensure ttl has been removed self.assertEqual( self.backend.conn.ttl('container:%s:%s' % (account_id, name)), -1) def test_list_containers(self): account_id = 'test' self.backend.create_account(account_id) for cont1 in xrange(4): for cont2 in xrange(125): name = '%d-%04d' % (cont1, cont2) self.backend.update_container(account_id, name, Timestamp(time()).normal, 0, 0, 0, 0, 0) for cont in xrange(125): name = '2-0051-%04d' % cont self.backend.update_container(account_id, name, Timestamp(time()).normal, 0, 0, 0, 0, 0) for cont in xrange(125): name = '3-%04d-0049' % cont self.backend.update_container(account_id, name, Timestamp(time()).normal, 0, 0, 0, 0, 0) listing = self.backend.list_containers(account_id, marker='', delimiter='', limit=100) self.assertEqual(len(listing), 100) self.assertEqual(listing[0][0], '0-0000') self.assertEqual(listing[-1][0], '0-0099') listing = self.backend.list_containers(account_id, marker='', end_marker='0-0050', delimiter='', limit=100) self.assertEqual(len(listing), 50) self.assertEqual(listing[0][0], '0-0000') self.assertEqual(listing[-1][0], '0-0049') listing = self.backend.list_containers(account_id, marker='0-0099', delimiter='', limit=100) self.assertEqual(len(listing), 100) self.assertEqual(listing[0][0], '0-0100') self.assertEqual(listing[-1][0], '1-0074') listing = self.backend.list_containers(account_id, marker='1-0074', delimiter='', limit=55) self.assertEqual(len(listing), 55) self.assertEqual(listing[0][0], '1-0075') self.assertEqual(listing[-1][0], '2-0004') listing = self.backend.list_containers(account_id, marker='', prefix='0-01', delimiter='', limit=10) self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '0-0100') self.assertEqual(listing[-1][0], '0-0109') listing = self.backend.list_containers(account_id, marker='', prefix='0-01', delimiter='-', limit=10) self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '0-0100') self.assertEqual(listing[-1][0], '0-0109') listing = self.backend.list_containers(account_id, marker='', prefix='0-', delimiter='-', limit=10) self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '0-0000') self.assertEqual(listing[-1][0], '0-0009') listing = self.backend.list_containers(account_id, marker='', prefix='', delimiter='-', limit=10) self.assertEqual(len(listing), 4) self.assertEqual([c[0] for c in listing], ['0-', '1-', '2-', '3-']) listing = self.backend.list_containers(account_id, marker='2-', delimiter='-', limit=10) self.assertEqual(len(listing), 1) self.assertEqual([c[0] for c in listing], ['3-']) listing = self.backend.list_containers(account_id, marker='', prefix='2', delimiter='-', limit=10) self.assertEqual(len(listing), 1) self.assertEqual([c[0] for c in listing], ['2-']) listing = self.backend.list_containers(account_id, marker='2-0050', prefix='2-', delimiter='-', limit=10) self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '2-0051') self.assertEqual(listing[1][0], '2-0051-') self.assertEqual(listing[2][0], '2-0052') self.assertEqual(listing[-1][0], '2-0059') listing = self.backend.list_containers(account_id, marker='3-0045', prefix='3-', delimiter='-', limit=10) self.assertEqual(len(listing), 10) self.assertEqual([c[0] for c in listing], [ '3-0045-', '3-0046', '3-0046-', '3-0047', '3-0047-', '3-0048', '3-0048-', '3-0049', '3-0049-', '3-0050' ]) name = '3-0049-' self.backend.update_container(account_id, name, Timestamp(time()).normal, 0, 0, 0, 0, 0) listing = self.backend.list_containers(account_id, marker='3-0048', limit=10) self.assertEqual(len(listing), 10) self.assertEqual([c[0] for c in listing], [ '3-0048-0049', '3-0049', '3-0049-', '3-0049-0049', '3-0050', '3-0050-0049', '3-0051', '3-0051-0049', '3-0052', '3-0052-0049' ]) listing = self.backend.list_containers(account_id, marker='3-0048', prefix='3-', delimiter='-', limit=10) self.assertEqual(len(listing), 10) self.assertEqual([c[0] for c in listing], [ '3-0048-', '3-0049', '3-0049-', '3-0050', '3-0050-', '3-0051', '3-0051-', '3-0052', '3-0052-', '3-0053' ]) listing = self.backend.list_containers(account_id, prefix='3-0049-', delimiter='-', limit=10) self.assertEqual(len(listing), 2) self.assertEqual([c[0] for c in listing], ['3-0049-', '3-0049-0049']) def test_refresh_account(self): account_id = random_str(16) account_key = 'account:%s' % account_id self.assertEqual(self.backend.create_account(account_id), account_id) total_bytes = 0 total_objects = 0 total_damaged_objects = 0 total_missing_chunks = 0 # 10 containers with bytes and objects for i in range(10): name = "container%d" % i mtime = Timestamp(time()).normal nb_bytes = random.randrange(100) total_bytes += nb_bytes nb_objets = random.randrange(100) total_objects += nb_objets damaged_objects = random.randrange(100) total_damaged_objects += damaged_objects missing_chunks = random.randrange(100) total_missing_chunks += missing_chunks self.backend.update_container(account_id, name, mtime, 0, nb_objets, nb_bytes, damaged_objects, missing_chunks) # change values self.backend.conn.hset(account_key, 'bytes', 1) self.backend.conn.hset(account_key, 'objects', 2) self.backend.conn.hset(account_key, 'damaged_objects', 3) self.backend.conn.hset(account_key, 'missing_chunks', 4) self.assertEqual(self.backend.conn.hget(account_key, 'bytes'), '1') self.assertEqual(self.backend.conn.hget(account_key, 'objects'), '2') self.assertEqual( self.backend.conn.hget(account_key, 'damaged_objects'), '3') self.assertEqual(self.backend.conn.hget(account_key, 'missing_chunks'), '4') self.backend.refresh_account(account_id) self.assertEqual(self.backend.conn.hget(account_key, 'bytes'), str(total_bytes)) self.assertEqual(self.backend.conn.hget(account_key, 'objects'), str(total_objects)) self.assertEqual( self.backend.conn.hget(account_key, 'damaged_objects'), str(total_damaged_objects)) self.assertEqual(self.backend.conn.hget(account_key, 'missing_chunks'), str(total_missing_chunks)) def test_update_container_wrong_timestamp_format(self): account_id = 'test' self.assertEqual(self.backend.create_account(account_id), account_id) # initial container name = '"{<container \'&\' name>}"' mtime = "12456.0000076" self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0], name) # same event with ExpectedException(Conflict): self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0], name) mtime = "0000012456.00005" self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0], name) self.assertEqual( self.backend.conn.hget('container:%s:%s' % (account_id, name), 'mtime'), mtime) mtime = "0000012456.00035" self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0], name) self.assertEqual( self.backend.conn.hget('container:%s:%s' % (account_id, name), 'mtime'), mtime) def test_update_container_missing_damaged_object(self): account_id = random_str(16) self.assertEqual(self.backend.create_account(account_id), account_id) # initial container name = '"{<container \'&\' name>}"' self.backend.update_container(account_id, name, 0, 0, 0, 0, 0, 0) self.backend.conn.hdel('container:%s:%s' % (account_id, name), 'damaged_objects') self.backend.refresh_account(account_id) self.assertEqual( self.backend.conn.hget('account:%s' % (account_id), 'damaged_objects'), '0') def test_is_sup(self): compare = (self.backend.lua_is_sup + """ if (is_sup(KEYS[1], KEYS[2])) then return redis.status_reply('IS SUP'); else return redis.error_reply('IS NOT SUP'); end; """) compare_script = self.backend.register_script(compare) self.assertRaises(redis.exceptions.ResponseError, compare_script, keys=["12457.00245", "12457.00245"], client=self.backend.conn) self.assertEqual( compare_script(keys=["42457.00245", "12457.00245"], client=self.backend.conn), "IS SUP") self.assertRaises(redis.exceptions.ResponseError, compare_script, keys=["12457.00245", "12457.002450"], client=self.backend.conn) self.assertEqual( compare_script(keys=["42457.00245", "12457.002450"], client=self.backend.conn), "IS SUP") self.assertRaises(redis.exceptions.ResponseError, compare_script, keys=["12457.00245", "12457.002456"], client=self.backend.conn) self.assertEqual( compare_script(keys=["42457.00245", "12457.002456"], client=self.backend.conn), "IS SUP") self.assertEqual( compare_script(keys=["42457.00246", "42457.002458"], client=self.backend.conn), "IS SUP") def test_flush_account(self): account_id = random_str(16) account_key = 'account:%s' % account_id self.assertEqual(self.backend.create_account(account_id), account_id) total_bytes = 0 total_objects = 0 total_damaged_objects = 0 total_missing_chunks = 0 # 10 containers with bytes and objects for i in range(10): name = "container%d" % i mtime = Timestamp(time()).normal nb_bytes = random.randrange(100) total_bytes += nb_bytes nb_objets = random.randrange(100) total_objects += nb_objets damaged_objects = random.randrange(100) total_damaged_objects += damaged_objects missing_chunks = random.randrange(100) total_missing_chunks += missing_chunks self.backend.update_container(account_id, name, mtime, 0, nb_objets, nb_bytes, damaged_objects, missing_chunks) self.assertEqual(self.backend.conn.hget(account_key, 'bytes'), str(total_bytes)) self.assertEqual(self.backend.conn.hget(account_key, 'objects'), str(total_objects)) self.assertEqual( self.backend.conn.hget(account_key, 'damaged_objects'), str(total_damaged_objects)) self.assertEqual(self.backend.conn.hget(account_key, 'missing_chunks'), str(total_missing_chunks)) self.backend.flush_account(account_id) self.assertEqual(self.backend.conn.hget(account_key, 'bytes'), '0') self.assertEqual(self.backend.conn.hget(account_key, 'objects'), '0') self.assertEqual( self.backend.conn.hget(account_key, 'damaged_objects'), '0') self.assertEqual(self.backend.conn.hget(account_key, 'missing_chunks'), '0') self.assertEqual(self.backend.conn.zcard("containers:%s" % account_id), 0) self.assertEqual(self.backend.conn.exists("container:test:*"), 0)
class TestAccountBackend(BaseTestCase): def setUp(self): super(TestAccountBackend, self).setUp() _, _, self.redis_host, self.redis_port = self.get_service('redis') self.account_conf = { 'redis_host': self.redis_host, 'redis_port': self.redis_port } self.backend = AccountBackend(self.account_conf) self.backend.conn.flushdb() def tearDown(self): super(TestAccountBackend, self).tearDown() self.backend.conn.flushdb() def test_create_account(self): account_id = 'a' self.assertEqual(self.backend.create_account(account_id), account_id) self.assertEqual(self.backend.create_account(account_id), None) def test_update_account_metadata(self): account_id = 'test' self.assertEqual(self.backend.create_account(account_id), account_id) # first meta self.backend.update_account_metadata(account_id, {'a': '1'}) metadata = self.backend.get_account_metadata(account_id) self.assertIn('a', metadata) self.assertEqual(metadata['a'], '1') # second meta self.backend.update_account_metadata(account_id, {'b': '2'}) metadata = self.backend.get_account_metadata(account_id) self.assertIn('a', metadata) self.assertEqual(metadata['a'], '1') self.assertIn('b', metadata) self.assertEqual(metadata['b'], '2') # update first meta self.backend.update_account_metadata(account_id, {'a': '1b'}) metadata = self.backend.get_account_metadata(account_id) self.assertIn('a', metadata) self.assertEqual(metadata['a'], '1b') self.assertIn('b', metadata) self.assertEqual(metadata['b'], '2') # delete second meta self.backend.update_account_metadata(account_id, None, ['b']) metadata = self.backend.get_account_metadata(account_id) self.assertIn('a', metadata) self.assertEqual(metadata['a'], '1b') self.assertNotIn('b', metadata) def test_list_account(self): # Create and check if in list account_id = 'test_list' self.backend.create_account(account_id) account_list = self.backend.list_accounts() self.assertIn(account_id, account_list) # Check the result of a nonexistent account self.assertFalse("Should_not_exist" in account_list) def test_info_account(self): account_id = 'test' self.assertEqual(self.backend.create_account(account_id), account_id) info = self.backend.info_account(account_id) self.assertEqual(info['id'], account_id) self.assertEqual(info['objects'], 0) self.assertEqual(info['bytes'], 0) self.assertEqual(info['containers'], 0) self.assertTrue(info['ctime']) # first container self.backend.update_container(account_id, 'c1', Timestamp().normal, 0, 1, 1) info = self.backend.info_account(account_id) self.assertEqual(info['containers'], 1) self.assertEqual(info['objects'], 1) self.assertEqual(info['bytes'], 1) # second container sleep(.00001) self.backend.update_container(account_id, 'c2', Timestamp().normal, 0, 0, 0) info = self.backend.info_account(account_id) self.assertEqual(info['containers'], 2) self.assertEqual(info['objects'], 1) self.assertEqual(info['bytes'], 1) # update second container sleep(.00001) self.backend.update_container(account_id, 'c2', Timestamp().normal, 0, 1, 1) info = self.backend.info_account(account_id) self.assertEqual(info['containers'], 2) self.assertEqual(info['objects'], 2) self.assertEqual(info['bytes'], 2) # delete first container sleep(.00001) self.backend.update_container(account_id, 'c1', 0, Timestamp().normal, 0, 0) info = self.backend.info_account(account_id) self.assertEqual(info['containers'], 1) self.assertEqual(info['objects'], 1) self.assertEqual(info['bytes'], 1) # delete second container sleep(.00001) self.backend.update_container(account_id, 'c2', 0, Timestamp().normal, 0, 0) info = self.backend.info_account(account_id) self.assertEqual(info['containers'], 0) self.assertEqual(info['objects'], 0) self.assertEqual(info['bytes'], 0) def test_update_after_container_deletion(self): account_id = 'test-%06x' % int(time()) self.assertEqual(self.backend.create_account(account_id), account_id) # Container create event, sent immediately after creation self.backend.update_container(account_id, 'c1', Timestamp().normal, None, None, None) # Container update event self.backend.update_container(account_id, 'c1', Timestamp().normal, None, 3, 30) info = self.backend.info_account(account_id) self.assertEqual(info['containers'], 1) self.assertEqual(info['objects'], 3) self.assertEqual(info['bytes'], 30) # Container is flushed, but the event is deferred flush_timestamp = Timestamp().normal sleep(.00001) # Container delete event, sent immediately after deletion self.backend.update_container(account_id, 'c1', None, Timestamp().normal, None, None) # Deferred container update event (with lower timestamp) self.backend.update_container(account_id, 'c1', flush_timestamp, None, 0, 0) info = self.backend.info_account(account_id) self.assertEqual(info['containers'], 0) self.assertEqual(info['objects'], 0) self.assertEqual(info['bytes'], 0) def test_delete_container(self): account_id = 'test' self.assertEqual(self.backend.create_account(account_id), account_id) name = 'c' old_mtime = Timestamp(time() - 1).normal mtime = Timestamp().normal # initial container self.backend.update_container(account_id, name, mtime, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0].decode('utf-8'), name) # delete event sleep(.00001) dtime = Timestamp().normal self.backend.update_container(account_id, name, mtime, dtime, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(len(res), 0) self.assertGreaterEqual( self.backend.conn.ttl('container:%s:%s' % (account_id, name)), 1) # same event with ExpectedException(Conflict): self.backend.update_container(account_id, name, mtime, dtime, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(len(res), 0) self.assertGreaterEqual( self.backend.conn.ttl('container:%s:%s' % (account_id, name)), 1) # old event with ExpectedException(Conflict): self.backend.update_container(account_id, name, old_mtime, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(len(res), 0) self.assertGreaterEqual( self.backend.conn.ttl('container:%s:%s' % (account_id, name)), 1) def test_utf8_container(self): account_id = 'test' self.assertEqual(self.backend.create_account(account_id), account_id) name = u'La fête à la maison' mtime = Timestamp().normal # create container self.backend.update_container(account_id, name, mtime, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(text_type(res[0], 'utf8'), name) # ensure it appears in listing listing = self.backend.list_containers(account_id, marker='', delimiter='', limit=100) self.assertIn(name, [entry[0] for entry in listing]) # delete container sleep(.00001) dtime = Timestamp().normal self.backend.update_container(account_id, name, 0, dtime, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(len(res), 0) self.assertGreaterEqual( self.backend.conn.ttl('container:%s:%s' % (account_id, name)), 1) # ensure it has been removed with ExpectedException(Conflict): self.backend.update_container(account_id, name, 0, dtime, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(len(res), 0) self.assertGreaterEqual( self.backend.conn.ttl('container:%s:%s' % (account_id, name)), 1) def test_update_container(self): account_id = 'test' self.assertEqual(self.backend.create_account(account_id), account_id) # initial container name = '"{<container \'&\' name>}"' mtime = Timestamp().normal self.backend.update_container(account_id, name, mtime, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0].decode('utf-8'), name) tmtime = self.backend.conn.hget('container:%s:%s' % (account_id, name), 'mtime') self.assertEqual(tmtime.decode('utf-8'), mtime) # same event with ExpectedException(Conflict): self.backend.update_container(account_id, name, mtime, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0].decode('utf-8'), name) tmtime = self.backend.conn.hget('container:%s:%s' % (account_id, name), 'mtime') self.assertEqual(tmtime.decode('utf-8'), mtime) # New event sleep(.00001) mtime = Timestamp().normal self.backend.update_container(account_id, name, mtime, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0].decode('utf-8'), name) tmtime = self.backend.conn.hget('container:%s:%s' % (account_id, name), 'mtime') self.assertEqual(tmtime.decode('utf-8'), mtime) # Old event old_mtime = Timestamp(time() - 1).normal with ExpectedException(Conflict): self.backend.update_container(account_id, name, old_mtime, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0].decode('utf-8'), name) tmtime = self.backend.conn.hget('container:%s:%s' % (account_id, name), 'mtime') self.assertEqual(tmtime.decode('utf-8'), mtime) # Old delete event dtime = Timestamp(time() - 1).normal with ExpectedException(Conflict): self.backend.update_container(account_id, name, 0, dtime, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0].decode('utf-8'), name) tmtime = self.backend.conn.hget('container:%s:%s' % (account_id, name), 'mtime') self.assertEqual(tmtime.decode('utf-8'), mtime) # New delete event sleep(.00001) mtime = Timestamp().normal self.backend.update_container(account_id, name, 0, mtime, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(len(res), 0) self.assertGreaterEqual( self.backend.conn.ttl('container:%s:%s' % (account_id, name)), 1) # New event sleep(.00001) mtime = Timestamp().normal self.backend.update_container(account_id, name, mtime, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0].decode('utf-8'), name) tmtime = self.backend.conn.hget('container:%s:%s' % (account_id, name), 'mtime') self.assertEqual(tmtime.decode('utf-8'), mtime) # ensure ttl has been removed self.assertEqual( self.backend.conn.ttl('container:%s:%s' % (account_id, name)), -1) def test_list_containers(self): account_id = 'test' self.backend.create_account(account_id) for cont1 in xrange(4): for cont2 in xrange(125): name = '%d-%04d' % (cont1, cont2) self.backend.update_container(account_id, name, Timestamp().normal, 0, 0, 0) for cont in xrange(125): name = '2-0051-%04d' % cont self.backend.update_container(account_id, name, Timestamp().normal, 0, 0, 0) for cont in xrange(125): name = '3-%04d-0049' % cont self.backend.update_container(account_id, name, Timestamp().normal, 0, 0, 0) listing = self.backend.list_containers(account_id, marker='', delimiter='', limit=100) self.assertEqual(len(listing), 100) self.assertEqual(listing[0][0], '0-0000') self.assertEqual(listing[-1][0], '0-0099') listing = self.backend.list_containers(account_id, marker='', end_marker='0-0050', delimiter='', limit=100) self.assertEqual(len(listing), 50) self.assertEqual(listing[0][0], '0-0000') self.assertEqual(listing[-1][0], '0-0049') listing = self.backend.list_containers(account_id, marker='0-0099', delimiter='', limit=100) self.assertEqual(len(listing), 100) self.assertEqual(listing[0][0], '0-0100') self.assertEqual(listing[-1][0], '1-0074') listing = self.backend.list_containers(account_id, marker='1-0074', delimiter='', limit=55) self.assertEqual(len(listing), 55) self.assertEqual(listing[0][0], '1-0075') self.assertEqual(listing[-1][0], '2-0004') listing = self.backend.list_containers(account_id, marker='', prefix='0-01', delimiter='', limit=10) self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '0-0100') self.assertEqual(listing[-1][0], '0-0109') listing = self.backend.list_containers(account_id, marker='', prefix='0-01', delimiter='-', limit=10) self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '0-0100') self.assertEqual(listing[-1][0], '0-0109') listing = self.backend.list_containers(account_id, marker='', prefix='0-', delimiter='-', limit=10) self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '0-0000') self.assertEqual(listing[-1][0], '0-0009') listing = self.backend.list_containers(account_id, marker='', prefix='', delimiter='-', limit=10) self.assertEqual(len(listing), 4) self.assertEqual([c[0] for c in listing], ['0-', '1-', '2-', '3-']) listing = self.backend.list_containers(account_id, marker='2-', delimiter='-', limit=10) self.assertEqual(len(listing), 1) self.assertEqual([c[0] for c in listing], ['3-']) listing = self.backend.list_containers(account_id, marker='', prefix='2', delimiter='-', limit=10) self.assertEqual(len(listing), 1) self.assertEqual([c[0] for c in listing], ['2-']) listing = self.backend.list_containers(account_id, marker='2-0050', prefix='2-', delimiter='-', limit=10) self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '2-0051') self.assertEqual(listing[1][0], '2-0051-') self.assertEqual(listing[2][0], '2-0052') self.assertEqual(listing[-1][0], '2-0059') listing = self.backend.list_containers(account_id, marker='3-0045', prefix='3-', delimiter='-', limit=10) self.assertEqual(len(listing), 10) self.assertEqual([c[0] for c in listing], [ '3-0045-', '3-0046', '3-0046-', '3-0047', '3-0047-', '3-0048', '3-0048-', '3-0049', '3-0049-', '3-0050' ]) name = '3-0049-' self.backend.update_container(account_id, name, Timestamp().normal, 0, 0, 0) listing = self.backend.list_containers(account_id, marker='3-0048', limit=10) self.assertEqual(len(listing), 10) self.assertEqual([c[0] for c in listing], [ '3-0048-0049', '3-0049', '3-0049-', '3-0049-0049', '3-0050', '3-0050-0049', '3-0051', '3-0051-0049', '3-0052', '3-0052-0049' ]) listing = self.backend.list_containers(account_id, marker='3-0048', prefix='3-', delimiter='-', limit=10) self.assertEqual(len(listing), 10) self.assertEqual([c[0] for c in listing], [ '3-0048-', '3-0049', '3-0049-', '3-0050', '3-0050-', '3-0051', '3-0051-', '3-0052', '3-0052-', '3-0053' ]) listing = self.backend.list_containers(account_id, prefix='3-0049-', delimiter='-', limit=10) self.assertEqual(len(listing), 2) self.assertEqual([c[0] for c in listing], ['3-0049-', '3-0049-0049']) def test_refresh_account(self): account_id = random_str(16) account_key = 'account:%s' % account_id self.assertEqual(self.backend.create_account(account_id), account_id) total_bytes = 0 total_objects = 0 # 10 containers with bytes and objects for i in range(10): name = "container%d" % i mtime = Timestamp().normal nb_bytes = random.randrange(100) total_bytes += nb_bytes nb_objets = random.randrange(100) total_objects += nb_objets self.backend.update_container(account_id, name, mtime, 0, nb_objets, nb_bytes) # change values self.backend.conn.hset(account_key, 'bytes', 1) self.backend.conn.hset(account_key, 'objects', 2) self.assertEqual(self.backend.conn.hget(account_key, 'bytes'), b'1') self.assertEqual(self.backend.conn.hget(account_key, 'objects'), b'2') self.backend.refresh_account(account_id) self.assertEqual(self.backend.conn.hget(account_key, 'bytes'), str(total_bytes).encode('utf-8')) self.assertEqual(self.backend.conn.hget(account_key, 'objects'), str(total_objects).encode('utf-8')) def test_update_container_wrong_timestamp_format(self): account_id = 'test' self.assertEqual(self.backend.create_account(account_id), account_id) # initial container name = '"{<container \'&\' name>}"' mtime = "12456.0000076" self.backend.update_container(account_id, name, mtime, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0].decode('utf-8'), name) # same event with ExpectedException(Conflict): self.backend.update_container(account_id, name, mtime, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0].decode('utf-8'), name) mtime = "0000012456.00005" self.backend.update_container(account_id, name, mtime, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0].decode('utf-8'), name) tmtime = self.backend.conn.hget('container:%s:%s' % (account_id, name), 'mtime') self.assertEqual(tmtime.decode('utf-8'), mtime) mtime = "0000012456.00035" self.backend.update_container(account_id, name, mtime, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0].decode('utf-8'), name) tmtime = self.backend.conn.hget('container:%s:%s' % (account_id, name), 'mtime') self.assertEqual(tmtime.decode('utf-8'), mtime) def test_is_sup(self): # HACK: replace %%d by %d compare = (self.backend.lua_is_sup % (()) + """ if (is_sup(KEYS[1], KEYS[2])) then return redis.status_reply('IS SUP'); else return redis.error_reply('IS NOT SUP'); end; """) compare_script = self.backend.register_script(compare) self.assertRaises(redis.exceptions.ResponseError, compare_script, keys=["12457.00245", "12457.00245"], client=self.backend.conn) self.assertEqual( compare_script(keys=["42457.00245", "12457.00245"], client=self.backend.conn), b"IS SUP") self.assertRaises(redis.exceptions.ResponseError, compare_script, keys=["12457.00245", "12457.002450"], client=self.backend.conn) self.assertEqual( compare_script(keys=["42457.00245", "12457.002450"], client=self.backend.conn), b"IS SUP") self.assertRaises(redis.exceptions.ResponseError, compare_script, keys=["12457.00245", "12457.002456"], client=self.backend.conn) self.assertEqual( compare_script(keys=["42457.00245", "12457.002456"], client=self.backend.conn), b"IS SUP") self.assertEqual( compare_script(keys=["42457.00246", "42457.002458"], client=self.backend.conn), b"IS SUP") def test_flush_account(self): account_id = random_str(16) account_key = 'account:%s' % account_id self.assertEqual(self.backend.create_account(account_id), account_id) total_bytes = 0 total_objects = 0 # 10 containers with bytes and objects for i in range(10): name = "container%d" % i mtime = Timestamp().normal nb_bytes = random.randrange(100) total_bytes += nb_bytes nb_objets = random.randrange(100) total_objects += nb_objets self.backend.update_container(account_id, name, mtime, 0, nb_objets, nb_bytes) self.assertEqual(self.backend.conn.hget(account_key, 'bytes'), str(total_bytes).encode('utf-8')) self.assertEqual(self.backend.conn.hget(account_key, 'objects'), str(total_objects).encode('utf-8')) self.backend.flush_account(account_id) self.assertEqual(self.backend.conn.hget(account_key, 'bytes'), b'0') self.assertEqual(self.backend.conn.hget(account_key, 'objects'), b'0') self.assertEqual(self.backend.conn.zcard("containers:%s" % account_id), 0) self.assertEqual(self.backend.conn.exists("container:test:*"), 0) def test_refresh_bucket(self): account_id = random_str(16) account_key = 'account:%s' % account_id bucket = random_str(16) self.assertEqual(self.backend.create_account(account_id), account_id) total_bytes = 0 total_objects = 0 # 10 containers with bytes and objects for i in range(10): name = "container%d" % i mtime = Timestamp().normal nb_bytes = random.randrange(100) total_bytes += nb_bytes nb_objets = random.randrange(100) total_objects += nb_objets self.backend.update_container(account_id, name, mtime, 0, nb_objets, nb_bytes, bucket_name=bucket) bkey = self.backend.bkey(bucket) # change values self.backend.conn.hset(bkey, 'bytes', 1) self.backend.conn.hset(bkey, 'objects', 2) self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'1') self.assertEqual(self.backend.conn.hget(bkey, 'objects'), b'2') # force pagination self.backend.refresh_bucket(bucket, batch_size=6) conn = self.backend.conn self.assertEqual( conn.hget(account_key, 'bytes').decode('utf-8'), str(total_bytes)) self.assertEqual( conn.hget(account_key, 'objects').decode('utf-8'), str(total_objects)) def test_refresh_bucket_with_lock(self): account_id = random_str(16) bucket = random_str(16) self.assertEqual(self.backend.create_account(account_id), account_id) mtime = Timestamp().normal self.backend.update_container(account_id, "cnt-1", mtime, 0, 100, 100, bucket_name=bucket) self.backend.update_container(account_id, "cnt-2", mtime, 0, 500, 500, bucket_name=bucket) self.backend.update_container(account_id, "cnt-3", mtime, 0, 999, 999, bucket_name=bucket) bkey = self.backend.bkey(bucket) # create lock block = self.backend.blockkey(bucket) self.backend.conn.hset(block, mapping={ "marker": "cnt-2", "mtime": int(time()), "ctime": int(time()) }) with ExpectedException(Conflict): self.backend.refresh_bucket(bucket) # change value of first container self.backend.conn.hset(bkey, 'bytes', 100) self.backend.conn.hset(bkey, 'objects', 100) # update a container before marker, update is ok mtime = Timestamp().normal self.backend.update_container(account_id, "cnt-1", mtime, 0, 200, 200, bucket_name=bucket) self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'200') # update a container equal to marker, update is ok mtime = Timestamp().normal self.backend.update_container(account_id, "cnt-2", mtime, 0, 600, 600, bucket_name=bucket) self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'300') # update container after marker, update is not done mtime = Timestamp().normal self.backend.update_container(account_id, "cnt-3", mtime, 0, 998, 998, bucket_name=bucket) self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'300') # update lock with very old timestamp self.backend.conn.hset(block, "mtime", 1000) self.backend.refresh_bucket(bucket) self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'1798')