def test_account_refresh(self):
data = {
'name': 'foo',
'mtime': Timestamp(time()).normal,
'objects': 12,
'bytes': 42
}
data = json.dumps(data)
resp = self.app.post('/v1.0/account/container/update',
data=data,
query_string={'id': self.account_id})
resp = self.app.post('/v1.0/account/refresh',
query_string={'id': self.account_id})
self.assertEqual(resp.status_code, 204)
resp = self.app.post('/v1.0/account/show',
query_string={'id': self.account_id})
resp = self.json_loads(resp.data)
self.assertEqual(resp["bytes"], 42)
self.assertEqual(resp["objects"], 12)
def create_account(self, account_id):
conn = self.conn
if not account_id:
return None
if conn.hget('accounts:', account_id):
return None
lock = self.acquire_lock_with_timeout('account:%s' % account_id, 1)
if not lock:
return None
pipeline = conn.pipeline(True)
pipeline.hset('accounts:', account_id, 1)
pipeline.hmset('account:%s' % account_id, {
'id': account_id,
'bytes': 0,
'objects': 0,
'ctime': Timestamp(time()).normal
})
pipeline.execute()
self.release_lock('account:%s' % account_id, lock)
return account_id
def take_action(self, parsed_args):
self.log.debug('take_action(%s)', parsed_args)
self.take_action_container(parsed_args)
cid = parsed_args.cid
if cid is None:
account = self.app.client_manager.account
container = parsed_args.container
else:
account, container = \
self.app.client_manager.storage.resolve_cid(cid)
deadline = timeout_to_deadline(parsed_args.timeout)
dst_account = parsed_args.dst_account or account
dst_container = (parsed_args.dst_container or
(container + "-" + Timestamp(time()).normal))
batch_size = parsed_args.chunk_batch_size
self.app.client_manager.storage.container_snapshot(
account, container, dst_account,
dst_container, batch_size=batch_size,
deadline=deadline)
lines = [(dst_account, dst_container, "OK")]
return ('Account', 'Container', 'Status'), lines
def test_info_account(self):
account_id = 'test'
self.assertEqual(self.backend.create_account(account_id), account_id)
info = self.backend.info_account(account_id)
self.assertEqual(info['id'], account_id)
self.assertEqual(info['objects'], 0)
self.assertEqual(info['bytes'], 0)
self.assertEqual(info['damaged_objects'], 0)
self.assertEqual(info['missing_chunks'], 0)
self.assertEqual(info['containers'], 0)
self.assertTrue(info['ctime'])
# first container
self.backend.update_container(account_id, 'c1',
Timestamp(time()).normal, 0, 1, 1, 1, 1)
info = self.backend.info_account(account_id)
self.assertEqual(info['containers'], 1)
self.assertEqual(info['objects'], 1)
self.assertEqual(info['bytes'], 1)
self.assertEqual(info['damaged_objects'], 1)
self.assertEqual(info['missing_chunks'], 1)
# second container
sleep(.00001)
self.backend.update_container(account_id, 'c2',
Timestamp(time()).normal, 0, 0, 0, 0, 0)
info = self.backend.info_account(account_id)
self.assertEqual(info['containers'], 2)
self.assertEqual(info['objects'], 1)
self.assertEqual(info['bytes'], 1)
self.assertEqual(info['damaged_objects'], 1)
self.assertEqual(info['missing_chunks'], 1)
# update second container
sleep(.00001)
self.backend.update_container(account_id, 'c2',
Timestamp(time()).normal, 0, 1, 1, 1, 2)
info = self.backend.info_account(account_id)
self.assertEqual(info['containers'], 2)
self.assertEqual(info['objects'], 2)
self.assertEqual(info['bytes'], 2)
self.assertEqual(info['damaged_objects'], 2)
self.assertEqual(info['missing_chunks'], 3)
# delete first container
sleep(.00001)
self.backend.update_container(account_id, 'c1', 0,
Timestamp(time()).normal, 0, 0, 0, 0)
info = self.backend.info_account(account_id)
self.assertEqual(info['containers'], 1)
self.assertEqual(info['objects'], 1)
self.assertEqual(info['bytes'], 1)
self.assertEqual(info['damaged_objects'], 1)
self.assertEqual(info['missing_chunks'], 2)
# delete second container
sleep(.00001)
self.backend.update_container(account_id, 'c2', 0,
Timestamp(time()).normal, 0, 0, 0, 0)
info = self.backend.info_account(account_id)
self.assertEqual(info['containers'], 0)
self.assertEqual(info['objects'], 0)
self.assertEqual(info['bytes'], 0)
self.assertEqual(info['damaged_objects'], 0)
self.assertEqual(info['missing_chunks'], 0)
def test_list_containers(self):
account_id = 'test'
self.backend.create_account(account_id)
for cont1 in xrange(4):
for cont2 in xrange(125):
name = '%d-%04d' % (cont1, cont2)
self.backend.update_container(account_id, name,
Timestamp(time()).normal, 0, 0,
0, 0, 0)
for cont in xrange(125):
name = '2-0051-%04d' % cont
self.backend.update_container(account_id, name,
Timestamp(time()).normal, 0, 0, 0, 0,
0)
for cont in xrange(125):
name = '3-%04d-0049' % cont
self.backend.update_container(account_id, name,
Timestamp(time()).normal, 0, 0, 0, 0,
0)
listing = self.backend.list_containers(account_id,
marker='',
delimiter='',
limit=100)
self.assertEqual(len(listing), 100)
self.assertEqual(listing[0][0], '0-0000')
self.assertEqual(listing[-1][0], '0-0099')
listing = self.backend.list_containers(account_id,
marker='',
end_marker='0-0050',
delimiter='',
limit=100)
self.assertEqual(len(listing), 50)
self.assertEqual(listing[0][0], '0-0000')
self.assertEqual(listing[-1][0], '0-0049')
listing = self.backend.list_containers(account_id,
marker='0-0099',
delimiter='',
limit=100)
self.assertEqual(len(listing), 100)
self.assertEqual(listing[0][0], '0-0100')
self.assertEqual(listing[-1][0], '1-0074')
listing = self.backend.list_containers(account_id,
marker='1-0074',
delimiter='',
limit=55)
self.assertEqual(len(listing), 55)
self.assertEqual(listing[0][0], '1-0075')
self.assertEqual(listing[-1][0], '2-0004')
listing = self.backend.list_containers(account_id,
marker='',
prefix='0-01',
delimiter='',
limit=10)
self.assertEqual(len(listing), 10)
self.assertEqual(listing[0][0], '0-0100')
self.assertEqual(listing[-1][0], '0-0109')
listing = self.backend.list_containers(account_id,
marker='',
prefix='0-01',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 10)
self.assertEqual(listing[0][0], '0-0100')
self.assertEqual(listing[-1][0], '0-0109')
listing = self.backend.list_containers(account_id,
marker='',
prefix='0-',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 10)
self.assertEqual(listing[0][0], '0-0000')
self.assertEqual(listing[-1][0], '0-0009')
listing = self.backend.list_containers(account_id,
marker='',
prefix='',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 4)
self.assertEqual([c[0] for c in listing], ['0-', '1-', '2-', '3-'])
listing = self.backend.list_containers(account_id,
marker='2-',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 1)
self.assertEqual([c[0] for c in listing], ['3-'])
listing = self.backend.list_containers(account_id,
marker='',
prefix='2',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 1)
self.assertEqual([c[0] for c in listing], ['2-'])
listing = self.backend.list_containers(account_id,
marker='2-0050',
prefix='2-',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 10)
self.assertEqual(listing[0][0], '2-0051')
self.assertEqual(listing[1][0], '2-0051-')
self.assertEqual(listing[2][0], '2-0052')
self.assertEqual(listing[-1][0], '2-0059')
listing = self.backend.list_containers(account_id,
marker='3-0045',
prefix='3-',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 10)
self.assertEqual([c[0] for c in listing], [
'3-0045-', '3-0046', '3-0046-', '3-0047', '3-0047-', '3-0048',
'3-0048-', '3-0049', '3-0049-', '3-0050'
])
name = '3-0049-'
self.backend.update_container(account_id, name,
Timestamp(time()).normal, 0, 0, 0, 0, 0)
listing = self.backend.list_containers(account_id,
marker='3-0048',
limit=10)
self.assertEqual(len(listing), 10)
self.assertEqual([c[0] for c in listing], [
'3-0048-0049', '3-0049', '3-0049-', '3-0049-0049', '3-0050',
'3-0050-0049', '3-0051', '3-0051-0049', '3-0052', '3-0052-0049'
])
listing = self.backend.list_containers(account_id,
marker='3-0048',
prefix='3-',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 10)
self.assertEqual([c[0] for c in listing], [
'3-0048-', '3-0049', '3-0049-', '3-0050', '3-0050-', '3-0051',
'3-0051-', '3-0052', '3-0052-', '3-0053'
])
listing = self.backend.list_containers(account_id,
prefix='3-0049-',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 2)
self.assertEqual([c[0] for c in listing], ['3-0049-', '3-0049-0049'])
def test_update_container(self):
account_id = 'test'
self.assertEqual(self.backend.create_account(account_id), account_id)
# initial container
name = '"{<container \'&\' name>}"'
mtime = Timestamp(time()).normal
self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)
res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
'+')
self.assertEqual(res[0], name)
self.assertEqual(
self.backend.conn.hget('container:%s:%s' % (account_id, name),
'mtime'), mtime)
# same event
with ExpectedException(Conflict):
self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0,
0)
res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
'+')
self.assertEqual(res[0], name)
self.assertEqual(
self.backend.conn.hget('container:%s:%s' % (account_id, name),
'mtime'), mtime)
# New event
sleep(.00001)
mtime = Timestamp(time()).normal
self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)
res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
'+')
self.assertEqual(res[0], name)
self.assertEqual(
self.backend.conn.hget('container:%s:%s' % (account_id, name),
'mtime'), mtime)
# Old event
old_mtime = Timestamp(time() - 1).normal
with ExpectedException(Conflict):
self.backend.update_container(account_id, name, old_mtime, 0, 0, 0,
0, 0)
res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
'+')
self.assertEqual(res[0], name)
self.assertEqual(
self.backend.conn.hget('container:%s:%s' % (account_id, name),
'mtime'), mtime)
# Old delete event
dtime = Timestamp(time() - 1).normal
with ExpectedException(Conflict):
self.backend.update_container(account_id, name, 0, dtime, 0, 0, 0,
0)
res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
'+')
self.assertEqual(res[0], name)
self.assertEqual(
self.backend.conn.hget('container:%s:%s' % (account_id, name),
'mtime'), mtime)
# New delete event
sleep(.00001)
mtime = Timestamp(time()).normal
self.backend.update_container(account_id, name, 0, mtime, 0, 0, 0, 0)
res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
'+')
self.assertEqual(len(res), 0)
self.assertTrue(
self.backend.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)
# New event
sleep(.00001)
mtime = Timestamp(time()).normal
self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0)
res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-',
'+')
self.assertEqual(res[0], name)
self.assertEqual(
self.backend.conn.hget('container:%s:%s' % (account_id, name),
'mtime'), mtime)
# ensure ttl has been removed
self.assertEqual(
self.backend.conn.ttl('container:%s:%s' % (account_id, name)), -1)
def test_refresh_bucket_with_lock(self):
account_id = random_str(16)
bucket = random_str(16)
self.assertEqual(self.backend.create_account(account_id), account_id)
mtime = Timestamp().normal
self.backend.update_container(account_id,
"cnt-1",
mtime,
0,
100,
100,
bucket_name=bucket)
self.backend.update_container(account_id,
"cnt-2",
mtime,
0,
500,
500,
bucket_name=bucket)
self.backend.update_container(account_id,
"cnt-3",
mtime,
0,
999,
999,
bucket_name=bucket)
bkey = self.backend.bkey(bucket)
# create lock
block = self.backend.blockkey(bucket)
self.backend.conn.hset(block,
mapping={
"marker": "cnt-2",
"mtime": int(time()),
"ctime": int(time())
})
with ExpectedException(Conflict):
self.backend.refresh_bucket(bucket)
# change value of first container
self.backend.conn.hset(bkey, 'bytes', 100)
self.backend.conn.hset(bkey, 'objects', 100)
# update a container before marker, update is ok
mtime = Timestamp().normal
self.backend.update_container(account_id,
"cnt-1",
mtime,
0,
200,
200,
bucket_name=bucket)
self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'200')
# update a container equal to marker, update is ok
mtime = Timestamp().normal
self.backend.update_container(account_id,
"cnt-2",
mtime,
0,
600,
600,
bucket_name=bucket)
self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'300')
# update container after marker, update is not done
mtime = Timestamp().normal
self.backend.update_container(account_id,
"cnt-3",
mtime,
0,
998,
998,
bucket_name=bucket)
self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'300')
# update lock with very old timestamp
self.backend.conn.hset(block, "mtime", 1000)
self.backend.refresh_bucket(bucket)
self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'1798')
def update_container(self,
account_id,
name,
mtime,
dtime,
object_count,
bytes_used,
damaged_objects,
missing_chunks,
autocreate_account=None,
autocreate_container=True):
conn = self.conn
if not account_id or not name:
raise BadRequest("Missing account or container")
if autocreate_account is None:
autocreate_account = self.autocreate
if mtime is None:
mtime = '0'
else:
mtime = Timestamp(float(mtime)).normal
if dtime is None:
dtime = '0'
else:
dtime = Timestamp(float(dtime)).normal
if object_count is None:
object_count = 0
if bytes_used is None:
bytes_used = 0
if damaged_objects is None:
damaged_objects = 0
if missing_chunks is None:
missing_chunks = 0
keys = [
account_id,
AccountBackend.ckey(account_id, name),
("containers:%s" % (account_id)), ("account:%s" % (account_id))
]
args = [
name, mtime, dtime, object_count, bytes_used, damaged_objects,
missing_chunks,
str(autocreate_account),
Timestamp(time()).normal, EXPIRE_TIME,
str(autocreate_container)
]
try:
self.script_update_container(keys=keys, args=args, client=conn)
except redis.exceptions.ResponseError as exc:
if str(exc) == "no_account":
raise NotFound("Account %s not found" % account_id)
if str(exc) == "no_container":
raise NotFound("Container %s not found" % name)
elif str(exc) == "no_update_needed":
raise Conflict("No update needed, "
"event older than last container update")
else:
raise
return name
def _get_timestamp(self):
return Timestamp().normal
def update_container(self,
account_id,
name,
mtime,
dtime,
object_count,
bytes_used,
damaged_objects,
missing_chunks,
bucket_name=None,
autocreate_account=None,
autocreate_container=True,
**kwargs):
if not account_id or not name:
raise BadRequest("Missing account or container")
if autocreate_account is None:
autocreate_account = self.autocreate
if mtime is None:
mtime = '0'
else:
mtime = Timestamp(mtime).normal
if dtime is None:
dtime = '0'
else:
dtime = Timestamp(dtime).normal
if object_count is None:
object_count = 0
if bytes_used is None:
bytes_used = 0
if damaged_objects is None:
damaged_objects = 0
if missing_chunks is None:
missing_chunks = 0
# If no bucket name is provided, set it to ''
# (we cannot pass None to the Lua script).
bucket_name = bucket_name or ''
bucket_lock = self.blockkey(bucket_name)
now = Timestamp().normal
ckey = AccountBackend.ckey(account_id, name)
keys = [
self.akey(account_id), ckey,
self.clistkey(account_id), self._bucket_prefix,
self.blistkey(account_id)
]
args = [
account_id, name, bucket_name, bucket_lock, mtime, dtime,
object_count, bytes_used, damaged_objects, missing_chunks,
str(autocreate_account), now, EXPIRE_TIME,
str(autocreate_container)
]
try:
self.script_update_container(keys=keys, args=args)
except redis.exceptions.ResponseError as exc:
if text_type(exc).endswith("no_account"):
raise NotFound("Account %s not found" % account_id)
if text_type(exc).endswith("no_container"):
raise NotFound("Container %s not found" % name)
elif text_type(exc).endswith("no_update_needed"):
raise Conflict("No update needed, "
"event older than last container update")
else:
raise
return name
def update_container(self,
account_id,
name,
mtime,
dtime,
object_count,
bytes_used,
damaged_objects,
missing_chunks,
bucket_name=None,
autocreate_account=None,
autocreate_container=True):
if not account_id or not name:
raise BadRequest("Missing account or container")
if autocreate_account is None:
autocreate_account = self.autocreate
if mtime is None:
mtime = '0'
else:
mtime = Timestamp(mtime).normal
if dtime is None:
dtime = '0'
else:
dtime = Timestamp(dtime).normal
deleted = float(dtime) > float(mtime)
if object_count is None:
object_count = 0
if bytes_used is None:
bytes_used = 0
if damaged_objects is None:
damaged_objects = 0
if missing_chunks is None:
missing_chunks = 0
# If no bucket name is provided, set it to 'False'
# (we cannot pass None to the Lua script).
bucket_key = self.bkey(bucket_name) if bucket_name else str(False)
now = Timestamp().normal
# With some sharding middlewares, the suffix may be
# in the middle of the container name.
update_bucket_object_count = SEGMENTS_BUCKET_SUFFIX not in name
ckey = AccountBackend.ckey(account_id, name)
keys = [
account_id, ckey,
self.clistkey(account_id),
self.akey(account_id), bucket_key
]
args = [
name, mtime, dtime, object_count, bytes_used, damaged_objects,
missing_chunks,
str(autocreate_account), now, EXPIRE_TIME,
str(autocreate_container),
str(update_bucket_object_count)
]
pipeline = self.conn.pipeline(True)
try:
self.script_update_container(keys=keys, args=args, client=pipeline)
if bucket_name and not deleted:
pipeline.hset(ckey, "bucket", bucket_name)
# Only execute when the main shard is created/deleted.
if bucket_name == name:
self.script_update_bucket_list(
keys=[self.blistkey(account_id)],
args=[bucket_name, str(deleted)],
client=pipeline)
pipeline.execute()
except redis.exceptions.ResponseError as exc:
if text_type(exc).endswith("no_account"):
raise NotFound("Account %s not found" % account_id)
if text_type(exc).endswith("no_container"):
raise NotFound("Container %s not found" % name)
elif text_type(exc).endswith("no_update_needed"):
raise Conflict("No update needed, "
"event older than last container update")
else:
raise
return name