def refresh_bucket(self, bucket_name, **kwargs):
"""
Refresh the counters of a bucket. Recompute them from the counters
of all shards (containers).
"""
lkey = self.blockkey(bucket_name)
batch_size = kwargs.get("batch_size", 10000)
try:
ctime = Timestamp().normal
self.script_get_lock_bucket(keys=[lkey], args=[ctime])
except redis.exceptions.ResponseError as exc:
if text_type(exc).endswith("bucket_lock"):
raise Conflict("Refresh on bucket already in progress")
raise
account_id = '__account__'
keys = [
AccountBackend.ckey(account_id, ''),
self.clistkey(account_id),
self.bkey(bucket_name), lkey
]
try:
while True:
args = [bucket_name, Timestamp().normal, batch_size]
res = self.script_refresh_bucket(keys=keys, args=args)
if res[0]:
break
except redis.exceptions.ResponseError as exc:
self.conn.delete(lkey)
if text_type(exc).endswith("no_account"):
raise NotFound("Account %s not found" % account_id)
if text_type(exc).endswith("no_bucket"):
raise NotFound("Bucket %s not found" % bucket_name)
raise
def test_utf8_container(self):
backend = AccountBackend({}, self.conn)
account_id = 'test'
self.assertEqual(backend.create_account(account_id), account_id)
name = u'La fête à la maison'
mtime = Timestamp(time()).normal
# create container
backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(unicode(res[0], 'utf8'), name)
# ensure it appears in listing
listing = backend.list_containers(account_id,
marker='',
delimiter='',
limit=100)
self.assertIn(name, [entry[0] for entry in listing])
# delete container
sleep(.00001)
dtime = Timestamp(time()).normal
backend.update_container(account_id, name, 0, dtime, 0, 0, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(len(res), 0)
self.assertTrue(
self.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)
# ensure it has been removed
with ExpectedException(Conflict):
backend.update_container(account_id, name, 0, dtime, 0, 0, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(len(res), 0)
self.assertTrue(
self.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)
def test_update_after_container_deletion(self):
account_id = 'test-%06x' % int(time())
self.assertEqual(self.backend.create_account(account_id), account_id)
# Container create event, sent immediately after creation
self.backend.update_container(account_id, 'c1',
Timestamp().normal, None, None, None)
# Container update event
self.backend.update_container(account_id, 'c1',
Timestamp().normal, None, 3, 30)
info = self.backend.info_account(account_id)
self.assertEqual(info['containers'], 1)
self.assertEqual(info['objects'], 3)
self.assertEqual(info['bytes'], 30)
# Container is flushed, but the event is deferred
flush_timestamp = Timestamp().normal
sleep(.00001)
# Container delete event, sent immediately after deletion
self.backend.update_container(account_id, 'c1', None,
Timestamp().normal, None, None)
# Deferred container update event (with lower timestamp)
self.backend.update_container(account_id, 'c1', flush_timestamp, None,
0, 0)
info = self.backend.info_account(account_id)
self.assertEqual(info['containers'], 0)
self.assertEqual(info['objects'], 0)
self.assertEqual(info['bytes'], 0)
def test_account_container_reset(self):
data = {
'name': 'foo',
'mtime': Timestamp().normal,
'objects': 12,
'bytes': 42
}
dataj = json.dumps(data)
resp = self.app.put('/v1.0/account/container/update',
data=dataj,
query_string={'id': self.account_id})
data = {'name': 'foo', 'mtime': Timestamp().normal}
dataj = json.dumps(data)
resp = self.app.put('/v1.0/account/container/reset',
data=dataj,
query_string={'id': self.account_id})
self.assertEqual(resp.status_code, 204)
resp = self.app.get('/v1.0/account/containers',
query_string={
'id': self.account_id,
'prefix': 'foo'
})
resp = self.json_loads(resp.data)
for container in resp["listing"]:
name, nb_objects, nb_bytes, _, mtime = container
if not name.startswith('foo'):
self.fail("No prefix foo: %s" % name)
if name == 'foo':
self.assertEqual(0, nb_objects)
self.assertEqual(0, nb_bytes)
self.assertEqual(float(data['mtime']), mtime)
return
self.fail("No container foo")
def update_container(self,
account_id,
name,
mtime,
dtime,
object_count,
bytes_used,
autocreate_account=None,
autocreate_container=True):
conn = self.conn
if not account_id or not name:
raise BadRequest("Missing account or container")
if autocreate_account is None:
autocreate_account = self.autocreate
if mtime is None:
mtime = '0'
else:
mtime = Timestamp(float(mtime)).normal
if dtime is None:
dtime = '0'
else:
dtime = Timestamp(float(dtime)).normal
if object_count is None:
object_count = 0
if bytes_used is None:
bytes_used = 0
keys = [
account_id,
AccountBackend.ckey(account_id, name),
("containers:%s" % (account_id)), ("account:%s" % (account_id))
]
args = [
name, mtime, dtime, object_count, bytes_used,
str(autocreate_account),
Timestamp(time()).normal, EXPIRE_TIME,
str(autocreate_container)
]
try:
self.script_update_container(keys=keys, args=args, client=conn)
except redis.exceptions.ResponseError as exc:
if str(exc) == "no_account":
raise NotFound("Account %s not found" % account_id)
if str(exc) == "no_container":
raise NotFound("Container %s not found" % name)
elif str(exc) == "no_update_needed":
raise Conflict("No update needed, "
"event older than last container update")
else:
raise
return name
def test_info_account(self):
backend = AccountBackend({}, self.conn)
account_id = 'test'
self.assertEqual(backend.create_account(account_id), account_id)
info = backend.info_account(account_id)
self.assertEqual(info['id'], account_id)
self.assertEqual(info['bytes'], 0)
self.assertEqual(info['objects'], 0)
self.assertEqual(info['containers'], 0)
self.assertTrue(info['ctime'])
# first container
backend.update_container(account_id, 'c1',
Timestamp(time()).normal, 0, 1, 1)
info = backend.info_account(account_id)
self.assertEqual(info['containers'], 1)
self.assertEqual(info['objects'], 1)
self.assertEqual(info['bytes'], 1)
# second container
sleep(.00001)
backend.update_container(account_id, 'c2',
Timestamp(time()).normal, 0, 0, 0)
info = backend.info_account(account_id)
self.assertEqual(info['containers'], 2)
self.assertEqual(info['objects'], 1)
self.assertEqual(info['bytes'], 1)
# update second container
sleep(.00001)
backend.update_container(account_id, 'c2',
Timestamp(time()).normal, 0, 1, 1)
info = backend.info_account(account_id)
self.assertEqual(info['containers'], 2)
self.assertEqual(info['objects'], 2)
self.assertEqual(info['bytes'], 2)
# delete first container
sleep(.00001)
backend.update_container(account_id, 'c1', 0,
Timestamp(time()).normal, 0, 0)
info = backend.info_account(account_id)
self.assertEqual(info['containers'], 1)
self.assertEqual(info['objects'], 1)
self.assertEqual(info['bytes'], 1)
# delete second container
sleep(.00001)
backend.update_container(account_id, 'c2', 0,
Timestamp(time()).normal, 0, 0)
info = backend.info_account(account_id)
self.assertEqual(info['containers'], 0)
self.assertEqual(info['objects'], 0)
self.assertEqual(info['bytes'], 0)
def test_account_flush(self):
data = {
'name': 'foo',
'mtime': Timestamp().normal,
'objects': 12,
'bytes': 42
}
data = json.dumps(data)
resp = self.app.put('/v1.0/account/container/update',
data=data,
query_string={'id': self.account_id})
resp = self.app.post('/v1.0/account/flush',
query_string={'id': self.account_id})
self.assertEqual(resp.status_code, 204)
resp = self.app.get('/v1.0/account/show',
query_string={'id': self.account_id})
resp = self.json_loads(resp.data)
self.assertEqual(resp["bytes"], 0)
self.assertEqual(resp["objects"], 0)
resp = self.app.get('/v1.0/account/containers',
query_string={'id': self.account_id})
resp = self.json_loads(resp.data)
self.assertEqual(len(resp["listing"]), 0)
def take_action(self, parsed_args):
self.log.debug('take_action(%s)', parsed_args)
self.take_action_container(parsed_args)
cid = parsed_args.cid
if cid is None:
account = self.app.client_manager.account
container = parsed_args.container
else:
account, container = \
self.app.client_manager.storage.resolve_cid(cid)
deadline = timeout_to_deadline(parsed_args.timeout)
dst_account = parsed_args.dst_account or account
dst_container = (parsed_args.dst_container
or (container + "-" + Timestamp().normal))
batch_size = parsed_args.chunk_batch_size
self.app.client_manager.storage.container_snapshot(
account,
container,
dst_account,
dst_container,
batch_size=batch_size,
deadline=deadline)
lines = [(dst_account, dst_container, "OK")]
return ('Account', 'Container', 'Status'), lines
def test_account_container_update(self):
data = {'name': 'foo', 'mtime': Timestamp(time()).normal,
'objects': 0, 'bytes': 0}
data = json.dumps(data)
resp = self.app.put('/v1.0/account/container/update',
data=data, query_string={'id': self.account_id})
self.assertEqual(resp.status_code, 200)
def create_account(self, account_id):
conn = self.conn
if not account_id:
return None
if conn.hget('accounts:', account_id):
return None
lock = self.acquire_lock_with_timeout('account:%s' % account_id, 1)
if not lock:
return None
pipeline = conn.pipeline(True)
pipeline.hset('accounts:', account_id, 1)
pipeline.hmset(
'account:%s' % account_id, {
'id': account_id,
'objects': 0,
'bytes': 0,
'damaged_objects': 0,
'missing_chunks': 0,
'ctime': Timestamp(time()).normal
})
pipeline.execute()
self.release_lock('account:%s' % account_id, lock)
return account_id
def test_refresh_account(self):
account_id = random_str(16)
account_key = 'account:%s' % account_id
self.assertEqual(self.backend.create_account(account_id), account_id)
total_bytes = 0
total_objects = 0
# 10 containers with bytes and objects
for i in range(10):
name = "container%d" % i
mtime = Timestamp().normal
nb_bytes = random.randrange(100)
total_bytes += nb_bytes
nb_objets = random.randrange(100)
total_objects += nb_objets
self.backend.update_container(account_id, name, mtime, 0,
nb_objets, nb_bytes)
# change values
self.backend.conn.hset(account_key, 'bytes', 1)
self.backend.conn.hset(account_key, 'objects', 2)
self.assertEqual(self.backend.conn.hget(account_key, 'bytes'), b'1')
self.assertEqual(self.backend.conn.hget(account_key, 'objects'), b'2')
self.backend.refresh_account(account_id)
self.assertEqual(self.backend.conn.hget(account_key, 'bytes'),
str(total_bytes).encode('utf-8'))
self.assertEqual(self.backend.conn.hget(account_key, 'objects'),
str(total_objects).encode('utf-8'))
def _gen_results(objects):
for obj in objects:
result = (obj['name'], obj['size'], obj['hash'],
obj['version'], obj['deleted'], obj['mime_type'],
Timestamp(obj['ctime']).isoformat, obj['policy'],
_format_props(obj.get('properties', {})))
yield result
def test_flush_account(self):
account_id = random_str(16)
account_key = 'account:%s' % account_id
self.assertEqual(self.backend.create_account(account_id), account_id)
total_bytes = 0
total_objects = 0
# 10 containers with bytes and objects
for i in range(10):
name = "container%d" % i
mtime = Timestamp().normal
nb_bytes = random.randrange(100)
total_bytes += nb_bytes
nb_objets = random.randrange(100)
total_objects += nb_objets
self.backend.update_container(account_id, name, mtime, 0,
nb_objets, nb_bytes)
self.assertEqual(self.backend.conn.hget(account_key, 'bytes'),
str(total_bytes).encode('utf-8'))
self.assertEqual(self.backend.conn.hget(account_key, 'objects'),
str(total_objects).encode('utf-8'))
self.backend.flush_account(account_id)
self.assertEqual(self.backend.conn.hget(account_key, 'bytes'), b'0')
self.assertEqual(self.backend.conn.hget(account_key, 'objects'), b'0')
self.assertEqual(self.backend.conn.zcard("containers:%s" % account_id),
0)
self.assertEqual(self.backend.conn.exists("container:test:*"), 0)
def test_refresh_bucket(self):
account_id = random_str(16)
account_key = 'account:%s' % account_id
bucket = random_str(16)
self.assertEqual(self.backend.create_account(account_id), account_id)
total_bytes = 0
total_objects = 0
total_damaged_objects = 0
total_missing_chunks = 0
# 10 containers with bytes and objects
for i in range(10):
name = "container%d" % i
mtime = Timestamp().normal
nb_bytes = random.randrange(100)
total_bytes += nb_bytes
nb_objets = random.randrange(100)
total_objects += nb_objets
damaged_objects = random.randrange(100)
total_damaged_objects += damaged_objects
missing_chunks = random.randrange(100)
total_missing_chunks += missing_chunks
self.backend.update_container(account_id,
name,
mtime,
0,
nb_objets,
nb_bytes,
damaged_objects,
missing_chunks,
bucket_name=bucket)
bkey = self.backend.bkey(bucket)
# change values
self.backend.conn.hset(bkey, 'bytes', 1)
self.backend.conn.hset(bkey, 'objects', 2)
self.backend.conn.hset(bkey, 'damaged_objects', 3)
self.backend.conn.hset(bkey, 'missing_chunks', 4)
self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'1')
self.assertEqual(self.backend.conn.hget(bkey, 'objects'), b'2')
self.assertEqual(self.backend.conn.hget(bkey, 'damaged_objects'), b'3')
self.assertEqual(self.backend.conn.hget(bkey, 'missing_chunks'), b'4')
# force pagination
self.backend.refresh_bucket(bucket, batch_size=6)
conn = self.backend.conn
self.assertEqual(
conn.hget(account_key, 'bytes').decode('utf-8'), str(total_bytes))
self.assertEqual(
conn.hget(account_key, 'objects').decode('utf-8'),
str(total_objects))
self.assertEqual(
conn.hget(account_key, 'damaged_objects').decode('utf-8'),
str(total_damaged_objects))
self.assertEqual(
conn.hget(account_key, 'missing_chunks').decode('utf-8'),
str(total_missing_chunks))
def test_delete_container(self):
account_id = 'test'
self.assertEqual(self.backend.create_account(account_id), account_id)
name = 'c'
old_mtime = Timestamp(time() - 1).normal
mtime = Timestamp(time()).normal
# initial container
self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)
res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
'+')
self.assertEqual(res[0], name)
# delete event
sleep(.00001)
dtime = Timestamp(time()).normal
self.backend.update_container(account_id, name, mtime, dtime, 0, 0, 0,
0)
res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
'+')
self.assertEqual(len(res), 0)
self.assertTrue(
self.backend.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)
# same event
with ExpectedException(Conflict):
self.backend.update_container(account_id, name, mtime, dtime, 0, 0,
0, 0)
res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
'+')
self.assertEqual(len(res), 0)
self.assertTrue(
self.backend.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)
# old event
with ExpectedException(Conflict):
self.backend.update_container(account_id, name, old_mtime, 0, 0, 0,
0, 0)
res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
'+')
self.assertEqual(len(res), 0)
self.assertTrue(
self.backend.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)
def take_action(self, parsed_args):
self.log.debug('take_action(%s)', parsed_args)
reqid = request_id(prefix='CLI-BUCKET-')
acct_client = self.app.client_manager.storage.account
data = acct_client.bucket_show(parsed_args.bucket, reqid=reqid)
if parsed_args.formatter == 'table':
from oio.common.easy_value import convert_size
data['bytes'] = convert_size(data['bytes'])
data['mtime'] = Timestamp(data.get('mtime', 0.0)).isoformat
return zip(*sorted(data.items()))
def test_update_after_container_deletion(self):
backend = AccountBackend({}, self.conn)
account_id = 'test-%06x' % int(time())
self.assertEqual(backend.create_account(account_id), account_id)
# Container create event, sent immediately after creation
backend.update_container(account_id, 'c1',
Timestamp(time()).normal, None,
None, None, None, None)
# Container update event
backend.update_container(account_id, 'c1',
Timestamp(time()).normal, None,
3, 30, 7, 5)
info = backend.info_account(account_id)
self.assertEqual(info['containers'], 1)
self.assertEqual(info['objects'], 3)
self.assertEqual(info['bytes'], 30)
self.assertEqual(info['damaged_objects'], 7)
self.assertEqual(info['missing_chunks'], 5)
# Container is flushed, but the event is deferred
flush_timestamp = Timestamp(time()).normal
sleep(.00001)
# Container delete event, sent immediately after deletion
backend.update_container(account_id, 'c1',
None, Timestamp(time()).normal,
None, None, None, None)
# Deferred container update event (with lower timestamp)
backend.update_container(account_id, 'c1',
flush_timestamp, None,
0, 0, 0, 0)
info = backend.info_account(account_id)
self.assertEqual(info['containers'], 0)
self.assertEqual(info['objects'], 0)
self.assertEqual(info['bytes'], 0)
self.assertEqual(info['damaged_objects'], 0)
self.assertEqual(info['missing_chunks'], 0)
def _gen_results(objects):
for obj in objects:
try:
result = (obj['name'], obj['size'], obj['hash'],
obj['version'],
obj['deleted'], obj['mime_type'],
Timestamp(obj['mtime']).isoformat,
obj['policy'],
_format_props(obj.get('properties', {})))
yield result
except KeyError as exc:
self.log.warn("Bad object entry, missing '%s': %s",
exc, obj)
def take_action(self, parsed_args):
self.log.debug('take_action(%s)', parsed_args)
account = self.app.client_manager.account
container = parsed_args.container
dst_account = parsed_args.dst_account or account
dst_container = (parsed_args.dst_container or
(container + "-" + Timestamp(time()).normal))
batch = parsed_args.chunk_batch_size
self.app.client_manager.storage.container_snapshot(
account, container, dst_account, dst_container, batch=batch)
lines = [(dst_account, dst_container, "OK")]
return ('Account', 'Container', 'Status'), lines
def test_refresh_account(self):
account_id = random_str(16)
account_key = 'account:%s' % account_id
self.assertEqual(self.backend.create_account(account_id), account_id)
total_bytes = 0
total_objects = 0
total_damaged_objects = 0
total_missing_chunks = 0
# 10 containers with bytes and objects
for i in range(10):
name = "container%d" % i
mtime = Timestamp(time()).normal
nb_bytes = random.randrange(100)
total_bytes += nb_bytes
nb_objets = random.randrange(100)
total_objects += nb_objets
damaged_objects = random.randrange(100)
total_damaged_objects += damaged_objects
missing_chunks = random.randrange(100)
total_missing_chunks += missing_chunks
self.backend.update_container(account_id, name, mtime, 0,
nb_objets, nb_bytes, damaged_objects,
missing_chunks)
# change values
self.backend.conn.hset(account_key, 'bytes', 1)
self.backend.conn.hset(account_key, 'objects', 2)
self.backend.conn.hset(account_key, 'damaged_objects', 3)
self.backend.conn.hset(account_key, 'missing_chunks', 4)
self.assertEqual(self.backend.conn.hget(account_key, 'bytes'), '1')
self.assertEqual(self.backend.conn.hget(account_key, 'objects'), '2')
self.assertEqual(
self.backend.conn.hget(account_key, 'damaged_objects'), '3')
self.assertEqual(self.backend.conn.hget(account_key, 'missing_chunks'),
'4')
self.backend.refresh_account(account_id)
self.assertEqual(self.backend.conn.hget(account_key, 'bytes'),
str(total_bytes))
self.assertEqual(self.backend.conn.hget(account_key, 'objects'),
str(total_objects))
self.assertEqual(
self.backend.conn.hget(account_key, 'damaged_objects'),
str(total_damaged_objects))
self.assertEqual(self.backend.conn.hget(account_key, 'missing_chunks'),
str(total_missing_chunks))
def test_flush_account(self):
account_id = random_str(16)
account_key = 'account:%s' % account_id
self.assertEqual(self.backend.create_account(account_id), account_id)
total_bytes = 0
total_objects = 0
total_damaged_objects = 0
total_missing_chunks = 0
# 10 containers with bytes and objects
for i in range(10):
name = "container%d" % i
mtime = Timestamp(time()).normal
nb_bytes = random.randrange(100)
total_bytes += nb_bytes
nb_objets = random.randrange(100)
total_objects += nb_objets
damaged_objects = random.randrange(100)
total_damaged_objects += damaged_objects
missing_chunks = random.randrange(100)
total_missing_chunks += missing_chunks
self.backend.update_container(account_id, name, mtime, 0,
nb_objets, nb_bytes, damaged_objects,
missing_chunks)
self.assertEqual(self.backend.conn.hget(account_key, 'bytes'),
str(total_bytes))
self.assertEqual(self.backend.conn.hget(account_key, 'objects'),
str(total_objects))
self.assertEqual(
self.backend.conn.hget(account_key, 'damaged_objects'),
str(total_damaged_objects))
self.assertEqual(self.backend.conn.hget(account_key, 'missing_chunks'),
str(total_missing_chunks))
self.backend.flush_account(account_id)
self.assertEqual(self.backend.conn.hget(account_key, 'bytes'), '0')
self.assertEqual(self.backend.conn.hget(account_key, 'objects'), '0')
self.assertEqual(
self.backend.conn.hget(account_key, 'damaged_objects'), '0')
self.assertEqual(self.backend.conn.hget(account_key, 'missing_chunks'),
'0')
self.assertEqual(self.backend.conn.zcard("containers:%s" % account_id),
0)
self.assertEqual(self.backend.conn.exists("container:test:*"), 0)
def take_action(self, parsed_args):
self.log.debug('take_action(%s)', parsed_args)
reqid = request_id(prefix='CLI-BUCKET-')
acct_client = self.app.client_manager.storage.account
metadata = dict()
if parsed_args.replicate is not None:
metadata[BUCKET_PROP_REPLI_ENABLED] = str(parsed_args.replicate)
data = acct_client.bucket_update(parsed_args.bucket,
metadata=metadata, to_delete=None,
reqid=reqid)
if parsed_args.formatter == 'table':
from oio.common.easy_value import convert_size
data['bytes'] = convert_size(data['bytes'])
data['mtime'] = Timestamp(data.get('mtime', 0.0)).isoformat
return zip(*sorted(data.items()))
def test_account_refresh(self):
data = {
'name': 'foo',
'mtime': Timestamp(time()).normal,
'objects': 12,
'bytes': 42
}
data = json.dumps(data)
resp = self.app.post('/v1.0/account/container/update',
data=data,
query_string={'id': self.account_id})
resp = self.app.post('/v1.0/account/refresh',
query_string={'id': self.account_id})
self.assertEqual(resp.status_code, 204)
resp = self.app.post('/v1.0/account/show',
query_string={'id': self.account_id})
resp = self.json_loads(resp.data)
self.assertEqual(resp["bytes"], 42)
self.assertEqual(resp["objects"], 12)
def create_account(self, account_id, **kwargs):
conn = self.conn
if not account_id:
return None
if conn.hget('accounts:', account_id):
return None
lock = self.acquire_lock_with_timeout(self.akey(account_id), 1)
if not lock:
return None
pipeline = conn.pipeline(True)
pipeline.hset('accounts:', account_id, 1)
pipeline.hmset(
self.akey(account_id), {
'id': account_id,
'objects': 0,
'bytes': 0,
'ctime': Timestamp().normal
})
pipeline.execute()
self.release_lock(self.akey(account_id), lock)
return account_id
def test_info_account(self):
account_id = 'test'
self.assertEqual(self.backend.create_account(account_id), account_id)
info = self.backend.info_account(account_id)
self.assertEqual(info['id'], account_id)
self.assertEqual(info['objects'], 0)
self.assertEqual(info['bytes'], 0)
self.assertEqual(info['damaged_objects'], 0)
self.assertEqual(info['missing_chunks'], 0)
self.assertEqual(info['containers'], 0)
self.assertTrue(info['ctime'])
# first container
self.backend.update_container(account_id, 'c1',
Timestamp(time()).normal, 0, 1, 1, 1, 1)
info = self.backend.info_account(account_id)
self.assertEqual(info['containers'], 1)
self.assertEqual(info['objects'], 1)
self.assertEqual(info['bytes'], 1)
self.assertEqual(info['damaged_objects'], 1)
self.assertEqual(info['missing_chunks'], 1)
# second container
sleep(.00001)
self.backend.update_container(account_id, 'c2',
Timestamp(time()).normal, 0, 0, 0, 0, 0)
info = self.backend.info_account(account_id)
self.assertEqual(info['containers'], 2)
self.assertEqual(info['objects'], 1)
self.assertEqual(info['bytes'], 1)
self.assertEqual(info['damaged_objects'], 1)
self.assertEqual(info['missing_chunks'], 1)
# update second container
sleep(.00001)
self.backend.update_container(account_id, 'c2',
Timestamp(time()).normal, 0, 1, 1, 1, 2)
info = self.backend.info_account(account_id)
self.assertEqual(info['containers'], 2)
self.assertEqual(info['objects'], 2)
self.assertEqual(info['bytes'], 2)
self.assertEqual(info['damaged_objects'], 2)
self.assertEqual(info['missing_chunks'], 3)
# delete first container
sleep(.00001)
self.backend.update_container(account_id, 'c1', 0,
Timestamp(time()).normal, 0, 0, 0, 0)
info = self.backend.info_account(account_id)
self.assertEqual(info['containers'], 1)
self.assertEqual(info['objects'], 1)
self.assertEqual(info['bytes'], 1)
self.assertEqual(info['damaged_objects'], 1)
self.assertEqual(info['missing_chunks'], 2)
# delete second container
sleep(.00001)
self.backend.update_container(account_id, 'c2', 0,
Timestamp(time()).normal, 0, 0, 0, 0)
info = self.backend.info_account(account_id)
self.assertEqual(info['containers'], 0)
self.assertEqual(info['objects'], 0)
self.assertEqual(info['bytes'], 0)
self.assertEqual(info['damaged_objects'], 0)
self.assertEqual(info['missing_chunks'], 0)
def test_list_containers(self):
account_id = 'test'
self.backend.create_account(account_id)
for cont1 in xrange(4):
for cont2 in xrange(125):
name = '%d-%04d' % (cont1, cont2)
self.backend.update_container(account_id, name,
Timestamp(time()).normal, 0, 0,
0, 0, 0)
for cont in xrange(125):
name = '2-0051-%04d' % cont
self.backend.update_container(account_id, name,
Timestamp(time()).normal, 0, 0, 0, 0,
0)
for cont in xrange(125):
name = '3-%04d-0049' % cont
self.backend.update_container(account_id, name,
Timestamp(time()).normal, 0, 0, 0, 0,
0)
listing = self.backend.list_containers(account_id,
marker='',
delimiter='',
limit=100)
self.assertEqual(len(listing), 100)
self.assertEqual(listing[0][0], '0-0000')
self.assertEqual(listing[-1][0], '0-0099')
listing = self.backend.list_containers(account_id,
marker='',
end_marker='0-0050',
delimiter='',
limit=100)
self.assertEqual(len(listing), 50)
self.assertEqual(listing[0][0], '0-0000')
self.assertEqual(listing[-1][0], '0-0049')
listing = self.backend.list_containers(account_id,
marker='0-0099',
delimiter='',
limit=100)
self.assertEqual(len(listing), 100)
self.assertEqual(listing[0][0], '0-0100')
self.assertEqual(listing[-1][0], '1-0074')
listing = self.backend.list_containers(account_id,
marker='1-0074',
delimiter='',
limit=55)
self.assertEqual(len(listing), 55)
self.assertEqual(listing[0][0], '1-0075')
self.assertEqual(listing[-1][0], '2-0004')
listing = self.backend.list_containers(account_id,
marker='',
prefix='0-01',
delimiter='',
limit=10)
self.assertEqual(len(listing), 10)
self.assertEqual(listing[0][0], '0-0100')
self.assertEqual(listing[-1][0], '0-0109')
listing = self.backend.list_containers(account_id,
marker='',
prefix='0-01',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 10)
self.assertEqual(listing[0][0], '0-0100')
self.assertEqual(listing[-1][0], '0-0109')
listing = self.backend.list_containers(account_id,
marker='',
prefix='0-',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 10)
self.assertEqual(listing[0][0], '0-0000')
self.assertEqual(listing[-1][0], '0-0009')
listing = self.backend.list_containers(account_id,
marker='',
prefix='',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 4)
self.assertEqual([c[0] for c in listing], ['0-', '1-', '2-', '3-'])
listing = self.backend.list_containers(account_id,
marker='2-',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 1)
self.assertEqual([c[0] for c in listing], ['3-'])
listing = self.backend.list_containers(account_id,
marker='',
prefix='2',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 1)
self.assertEqual([c[0] for c in listing], ['2-'])
listing = self.backend.list_containers(account_id,
marker='2-0050',
prefix='2-',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 10)
self.assertEqual(listing[0][0], '2-0051')
self.assertEqual(listing[1][0], '2-0051-')
self.assertEqual(listing[2][0], '2-0052')
self.assertEqual(listing[-1][0], '2-0059')
listing = self.backend.list_containers(account_id,
marker='3-0045',
prefix='3-',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 10)
self.assertEqual([c[0] for c in listing], [
'3-0045-', '3-0046', '3-0046-', '3-0047', '3-0047-', '3-0048',
'3-0048-', '3-0049', '3-0049-', '3-0050'
])
name = '3-0049-'
self.backend.update_container(account_id, name,
Timestamp(time()).normal, 0, 0, 0, 0, 0)
listing = self.backend.list_containers(account_id,
marker='3-0048',
limit=10)
self.assertEqual(len(listing), 10)
self.assertEqual([c[0] for c in listing], [
'3-0048-0049', '3-0049', '3-0049-', '3-0049-0049', '3-0050',
'3-0050-0049', '3-0051', '3-0051-0049', '3-0052', '3-0052-0049'
])
listing = self.backend.list_containers(account_id,
marker='3-0048',
prefix='3-',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 10)
self.assertEqual([c[0] for c in listing], [
'3-0048-', '3-0049', '3-0049-', '3-0050', '3-0050-', '3-0051',
'3-0051-', '3-0052', '3-0052-', '3-0053'
])
listing = self.backend.list_containers(account_id,
prefix='3-0049-',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 2)
self.assertEqual([c[0] for c in listing], ['3-0049-', '3-0049-0049'])
def test_update_container(self):
account_id = 'test'
self.assertEqual(self.backend.create_account(account_id), account_id)
# initial container
name = '"{<container \'&\' name>}"'
mtime = Timestamp(time()).normal
self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)
res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
'+')
self.assertEqual(res[0], name)
self.assertEqual(
self.backend.conn.hget('container:%s:%s' % (account_id, name),
'mtime'), mtime)
# same event
with ExpectedException(Conflict):
self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0,
0)
res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
'+')
self.assertEqual(res[0], name)
self.assertEqual(
self.backend.conn.hget('container:%s:%s' % (account_id, name),
'mtime'), mtime)
# New event
sleep(.00001)
mtime = Timestamp(time()).normal
self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)
res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
'+')
self.assertEqual(res[0], name)
self.assertEqual(
self.backend.conn.hget('container:%s:%s' % (account_id, name),
'mtime'), mtime)
# Old event
old_mtime = Timestamp(time() - 1).normal
with ExpectedException(Conflict):
self.backend.update_container(account_id, name, old_mtime, 0, 0, 0,
0, 0)
res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
'+')
self.assertEqual(res[0], name)
self.assertEqual(
self.backend.conn.hget('container:%s:%s' % (account_id, name),
'mtime'), mtime)
# Old delete event
dtime = Timestamp(time() - 1).normal
with ExpectedException(Conflict):
self.backend.update_container(account_id, name, 0, dtime, 0, 0, 0,
0)
res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
'+')
self.assertEqual(res[0], name)
self.assertEqual(
self.backend.conn.hget('container:%s:%s' % (account_id, name),
'mtime'), mtime)
# New delete event
sleep(.00001)
mtime = Timestamp(time()).normal
self.backend.update_container(account_id, name, 0, mtime, 0, 0, 0, 0)
res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
'+')
self.assertEqual(len(res), 0)
self.assertTrue(
self.backend.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)
# New event
sleep(.00001)
mtime = Timestamp(time()).normal
self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)
res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
'+')
self.assertEqual(res[0], name)
self.assertEqual(
self.backend.conn.hget('container:%s:%s' % (account_id, name),
'mtime'), mtime)
# ensure ttl has been removed
self.assertEqual(
self.backend.conn.ttl('container:%s:%s' % (account_id, name)), -1)
def test_refresh_bucket_with_lock(self):
account_id = random_str(16)
bucket = random_str(16)
self.assertEqual(self.backend.create_account(account_id), account_id)
mtime = Timestamp().normal
self.backend.update_container(account_id,
"cnt-1",
mtime,
0,
100,
100,
bucket_name=bucket)
self.backend.update_container(account_id,
"cnt-2",
mtime,
0,
500,
500,
bucket_name=bucket)
self.backend.update_container(account_id,
"cnt-3",
mtime,
0,
999,
999,
bucket_name=bucket)
bkey = self.backend.bkey(bucket)
# create lock
block = self.backend.blockkey(bucket)
self.backend.conn.hset(block,
mapping={
"marker": "cnt-2",
"mtime": int(time()),
"ctime": int(time())
})
with ExpectedException(Conflict):
self.backend.refresh_bucket(bucket)
# change value of first container
self.backend.conn.hset(bkey, 'bytes', 100)
self.backend.conn.hset(bkey, 'objects', 100)
# update a container before marker, update is ok
mtime = Timestamp().normal
self.backend.update_container(account_id,
"cnt-1",
mtime,
0,
200,
200,
bucket_name=bucket)
self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'200')
# update a container equal to marker, update is ok
mtime = Timestamp().normal
self.backend.update_container(account_id,
"cnt-2",
mtime,
0,
600,
600,
bucket_name=bucket)
self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'300')
# update container after marker, update is not done
mtime = Timestamp().normal
self.backend.update_container(account_id,
"cnt-3",
mtime,
0,
998,
998,
bucket_name=bucket)
self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'300')
# update lock with very old timestamp
self.backend.conn.hset(block, "mtime", 1000)
self.backend.refresh_bucket(bucket)
self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'1798')
def update_container(self,
account_id,
name,
mtime,
dtime,
object_count,
bytes_used,
damaged_objects,
missing_chunks,
bucket_name=None,
autocreate_account=None,
autocreate_container=True,
**kwargs):
if not account_id or not name:
raise BadRequest("Missing account or container")
if autocreate_account is None:
autocreate_account = self.autocreate
if mtime is None:
mtime = '0'
else:
mtime = Timestamp(mtime).normal
if dtime is None:
dtime = '0'
else:
dtime = Timestamp(dtime).normal
if object_count is None:
object_count = 0
if bytes_used is None:
bytes_used = 0
if damaged_objects is None:
damaged_objects = 0
if missing_chunks is None:
missing_chunks = 0
# If no bucket name is provided, set it to ''
# (we cannot pass None to the Lua script).
bucket_name = bucket_name or ''
bucket_lock = self.blockkey(bucket_name)
now = Timestamp().normal
ckey = AccountBackend.ckey(account_id, name)
keys = [
self.akey(account_id), ckey,
self.clistkey(account_id), self._bucket_prefix,
self.blistkey(account_id)
]
args = [
account_id, name, bucket_name, bucket_lock, mtime, dtime,
object_count, bytes_used, damaged_objects, missing_chunks,
str(autocreate_account), now, EXPIRE_TIME,
str(autocreate_container)
]
try:
self.script_update_container(keys=keys, args=args)
except redis.exceptions.ResponseError as exc:
if text_type(exc).endswith("no_account"):
raise NotFound("Account %s not found" % account_id)
if text_type(exc).endswith("no_container"):
raise NotFound("Container %s not found" % name)
elif text_type(exc).endswith("no_update_needed"):
raise Conflict("No update needed, "
"event older than last container update")
else:
raise
return name
def _get_timestamp(self):
return Timestamp().normal
