def test_delete_container(self):
backend = AccountBackend({}, self.conn)
account_id = 'test'
self.assertEqual(backend.create_account(account_id), account_id)
name = 'c'
mtime = Timestamp(time()).normal
# initial container
backend.update_container(account_id, name, mtime, 0, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(res[0], name)
# delete event
sleep(.00001)
dtime = Timestamp(time()).normal
backend.update_container(account_id, name, 0, dtime, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(len(res), 0)
self.assertFalse(
self.conn.exists('container:%s:%s' % (account_id, name)))
# same event
backend.update_container(account_id, name, 0, dtime, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(len(res), 0)
self.assertFalse(
self.conn.exists('container:%s:%s' % (account_id, name)))
def test_account_container_reset(self):
data = {
'name': 'foo',
'mtime': Timestamp(time()).normal,
'objects': 12,
'bytes': 42
}
data = json.dumps(data)
resp = self.app.post('/v1.0/account/container/update',
data=data,
query_string={'id': self.account_id})
data = {'name': 'foo', 'mtime': Timestamp(time()).normal}
data = json.dumps(data)
resp = self.app.post('/v1.0/account/container/reset',
data=data,
query_string={'id': self.account_id})
self.assertEqual(resp.status_code, 204)
data = {'prefix': 'foo'}
data = json.dumps(data)
resp = self.app.post('/v1.0/account/containers',
data=data,
query_string={'id': self.account_id})
resp = self.json_loads(resp.data)
for container in resp["listing"]:
name, nb_objects, nb_bytes, _ = container
if name == 'foo':
self.assertEqual(nb_objects, 0)
self.assertEqual(nb_bytes, 0)
return
self.fail("No container foo")
def test_utf8_container(self):
backend = AccountBackend({}, self.conn)
account_id = 'test'
self.assertEqual(backend.create_account(account_id), account_id)
name = u'La fête à la maison'
mtime = Timestamp(time()).normal
# create container
backend.update_container(account_id, name, mtime, 0, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(unicode(res[0], 'utf8'), name)
# ensure it appears in listing
listing = backend.list_containers(account_id,
marker='',
delimiter='',
limit=100)
self.assertIn(name, [entry[0] for entry in listing])
# delete container
sleep(.00001)
dtime = Timestamp(time()).normal
backend.update_container(account_id, name, 0, dtime, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(len(res), 0)
self.assertTrue(
self.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)
# ensure it has been removed
with ExpectedException(Conflict):
backend.update_container(account_id, name, 0, dtime, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(len(res), 0)
self.assertTrue(
self.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)
def test_update_after_container_deletion(self):
backend = AccountBackend({}, self.conn)
account_id = 'test-%06x' % int(time())
self.assertEqual(backend.create_account(account_id), account_id)
# Container create event, sent immediately after creation
backend.update_container(account_id, 'c1',
Timestamp(time()).normal, None, None, None)
# Container update event
backend.update_container(account_id, 'c1',
Timestamp(time()).normal, None, 3, 30)
info = backend.info_account(account_id)
self.assertEqual(info['containers'], 1)
self.assertEqual(info['objects'], 3)
self.assertEqual(info['bytes'], 30)
# Container is flushed, but the event is deferred
flush_timestamp = Timestamp(time()).normal
sleep(.00001)
# Container delete event, sent immediately after deletion
backend.update_container(account_id, 'c1', None,
Timestamp(time()).normal, None, None)
# Deferred container update event (with lower timestamp)
backend.update_container(account_id, 'c1', flush_timestamp, None, 0, 0)
info = backend.info_account(account_id)
self.assertEqual(info['containers'], 0)
self.assertEqual(info['objects'], 0)
self.assertEqual(info['bytes'], 0)
def test_info_account(self):
backend = AccountBackend({}, self.conn)
account_id = 'test'
self.assertEqual(backend.create_account(account_id), account_id)
info = backend.info_account(account_id)
self.assertEqual(info['id'], account_id)
self.assertEqual(info['bytes'], 0)
self.assertEqual(info['objects'], 0)
self.assertEqual(info['containers'], 0)
self.assertTrue(info['ctime'])
# first container
backend.update_container(account_id, 'c1',
Timestamp(time()).normal, 0, 1, 1)
info = backend.info_account(account_id)
self.assertEqual(info['containers'], 1)
self.assertEqual(info['objects'], 1)
self.assertEqual(info['bytes'], 1)
# second container
sleep(.00001)
backend.update_container(account_id, 'c2',
Timestamp(time()).normal, 0, 0, 0)
info = backend.info_account(account_id)
self.assertEqual(info['containers'], 2)
self.assertEqual(info['objects'], 1)
self.assertEqual(info['bytes'], 1)
# update second container
sleep(.00001)
backend.update_container(account_id, 'c2',
Timestamp(time()).normal, 0, 1, 1)
info = backend.info_account(account_id)
self.assertEqual(info['containers'], 2)
self.assertEqual(info['objects'], 2)
self.assertEqual(info['bytes'], 2)
# delete first container
sleep(.00001)
backend.update_container(account_id, 'c1', 0,
Timestamp(time()).normal, 0, 0)
info = backend.info_account(account_id)
self.assertEqual(info['containers'], 1)
self.assertEqual(info['objects'], 1)
self.assertEqual(info['bytes'], 1)
# delete second container
sleep(.00001)
backend.update_container(account_id, 'c2', 0,
Timestamp(time()).normal, 0, 0)
info = backend.info_account(account_id)
self.assertEqual(info['containers'], 0)
self.assertEqual(info['objects'], 0)
self.assertEqual(info['bytes'], 0)
def update_container(self,
account_id,
name,
mtime,
dtime,
object_count,
bytes_used,
autocreate_account=None,
autocreate_container=True):
conn = self.conn
if not account_id or not name:
raise BadRequest("Missing account or container")
if autocreate_account is None:
autocreate_account = self.autocreate
if mtime is None:
mtime = '0'
else:
mtime = Timestamp(float(mtime)).normal
if dtime is None:
dtime = '0'
else:
dtime = Timestamp(float(dtime)).normal
if object_count is None:
object_count = 0
if bytes_used is None:
bytes_used = 0
keys = [
account_id,
AccountBackend.ckey(account_id, name),
("containers:%s" % (account_id)), ("account:%s" % (account_id))
]
args = [
name, mtime, dtime, object_count, bytes_used, autocreate_account,
Timestamp(time()).normal, EXPIRE_TIME, autocreate_container
]
try:
self.script_update_container(keys=keys, args=args, client=conn)
except redis.exceptions.ResponseError as exc:
if str(exc) == "no_account":
raise NotFound("Account %s not found" % account_id)
if str(exc) == "no_container":
raise NotFound("Container %s not found" % name)
elif str(exc) == "no_update_needed":
raise Conflict("No update needed, "
"event older than last container update")
else:
raise
return name
def test_refresh_account(self):
backend = AccountBackend({}, self.conn)
account_id = random_str(16)
account_key = 'account:%s' % account_id
self.assertEqual(backend.create_account(account_id), account_id)
total_bytes = 0
total_objects = 0
# 10 containers with bytes and objects
for i in range(10):
name = "container%d" % i
mtime = Timestamp(time()).normal
nb_bytes = random.randrange(100)
total_bytes += nb_bytes
nb_objets = random.randrange(100)
total_objects += nb_objets
backend.update_container(account_id, name, mtime, 0, nb_objets,
nb_bytes)
# change values
self.conn.hset(account_key, 'bytes', 1)
self.conn.hset(account_key, 'objects', 2)
self.assertEqual(self.conn.hget(account_key, 'bytes'), '1')
self.assertEqual(self.conn.hget(account_key, 'objects'), '2')
backend.refresh_account(account_id)
self.assertEqual(self.conn.hget(account_key, 'bytes'),
str(total_bytes))
self.assertEqual(self.conn.hget(account_key, 'objects'),
str(total_objects))
def update_container(self, account_id, name, mtime, dtime, object_count,
bytes_used):
conn = self.conn
if not account_id or not name:
raise NotFound("Missing account or container")
if mtime is None:
mtime = '0'
if dtime is None:
dtime = '0'
if object_count is None:
object_count = 0
if bytes_used is None:
bytes_used = 0
keys = [
account_id,
AccountBackend.ckey(account_id, name),
("containers:%s" % (account_id)), ("account:%s" % (account_id))
]
args = [
name, mtime, dtime, object_count, bytes_used, self.autocreate,
Timestamp(time()).normal, EXPIRE_TIME
]
try:
self.script_update_container(keys=keys, args=args, client=conn)
except redis.exceptions.ResponseError as e:
if str(e) == "no_account":
raise NotFound(account_id)
elif str(e) == "no_update_needed":
raise Conflict("No updated needed")
else:
raise e
return name
def test_account_flush(self):
data = {
'name': 'foo',
'mtime': Timestamp(time()).normal,
'objects': 12,
'bytes': 42
}
data = json.dumps(data)
resp = self.app.post('/v1.0/account/container/update',
data=data,
query_string={'id': self.account_id})
resp = self.app.post('/v1.0/account/flush',
query_string={'id': self.account_id})
self.assertEqual(resp.status_code, 204)
resp = self.app.post('/v1.0/account/show',
query_string={'id': self.account_id})
resp = self.json_loads(resp.data)
self.assertEqual(resp["bytes"], 0)
self.assertEqual(resp["objects"], 0)
resp = self.app.post('/v1.0/account/containers',
query_string={'id': self.account_id})
resp = self.json_loads(resp.data)
self.assertEqual(len(resp["listing"]), 0)
def test_flush_account(self):
backend = AccountBackend({}, self.conn)
account_id = random_str(16)
account_key = 'account:%s' % account_id
self.assertEqual(backend.create_account(account_id), account_id)
total_bytes = 0
total_objects = 0
# 10 containers with bytes and objects
for i in range(10):
name = "container%d" % i
mtime = Timestamp(time()).normal
nb_bytes = random.randrange(100)
total_bytes += nb_bytes
nb_objets = random.randrange(100)
total_objects += nb_objets
backend.update_container(account_id, name, mtime, 0, nb_objets,
nb_bytes)
self.assertEqual(self.conn.hget(account_key, 'bytes'),
str(total_bytes))
self.assertEqual(self.conn.hget(account_key, 'objects'),
str(total_objects))
backend.flush_account(account_id)
self.assertEqual(self.conn.hget(account_key, 'bytes'), '0')
self.assertEqual(self.conn.hget(account_key, 'objects'), '0')
self.assertEqual(self.conn.zcard("containers:%s" % account_id), 0)
self.assertEqual(self.conn.exists("container:test:*"), 0)
def _gen_results(objects):
for obj in objects:
result = (obj['name'], obj['size'], obj['hash'],
obj['version'], obj['deleted'], obj['mime_type'],
Timestamp(obj['ctime']).isoformat, obj['policy'],
_format_props(obj.get('properties', {})))
yield result
def test_account_container_update(self):
data = {
'name': 'foo',
'mtime': Timestamp(time()).normal,
'objects': 0,
'bytes': 0
}
data = json.dumps(data)
resp = self.app.post('/v1.0/account/container/update',
data=data,
query_string={'id': self.account_id})
self.assertEqual(resp.status_code, 200)
def test_delete_container(self):
backend = AccountBackend({}, self.conn)
account_id = 'test'
self.assertEqual(backend.create_account(account_id), account_id)
name = 'c'
old_mtime = Timestamp(time() - 1).normal
mtime = Timestamp(time()).normal
# initial container
backend.update_container(account_id, name, mtime, 0, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(res[0], name)
# delete event
sleep(.00001)
dtime = Timestamp(time()).normal
backend.update_container(account_id, name, mtime, dtime, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(len(res), 0)
self.assertTrue(
self.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)
# same event
with ExpectedException(Conflict):
backend.update_container(account_id, name, mtime, dtime, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(len(res), 0)
self.assertTrue(
self.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)
# old event
with ExpectedException(Conflict):
backend.update_container(account_id, name, old_mtime, 0, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(len(res), 0)
self.assertTrue(
self.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)
def create_account(self, account_id):
conn = self.conn
if not account_id:
return None
if conn.hget('accounts:', account_id):
return None
lock = acquire_lock_with_timeout(conn, 'account:%s' % account_id, 1)
if not lock:
return None
pipeline = conn.pipeline(True)
pipeline.hset('accounts:', account_id, 1)
pipeline.hmset('account:%s' % account_id, {
'id': account_id,
'bytes': 0,
'objects': 0,
'ctime': Timestamp(time()).normal
})
pipeline.execute()
release_lock(conn, 'account:%s' % account_id, lock)
return account_id
def test_list_containers(self):
backend = AccountBackend({}, self.conn)
account_id = 'test'
backend.create_account(account_id)
for cont1 in xrange(4):
for cont2 in xrange(125):
name = '%d-%04d' % (cont1, cont2)
backend.update_container(account_id, name,
Timestamp(time()).normal, 0, 0, 0)
for cont in xrange(125):
name = '2-0051-%04d' % cont
backend.update_container(account_id, name,
Timestamp(time()).normal, 0, 0, 0)
for cont in xrange(125):
name = '3-%04d-0049' % cont
backend.update_container(account_id, name,
Timestamp(time()).normal, 0, 0, 0)
listing = backend.list_containers(account_id,
marker='',
delimiter='',
limit=100)
self.assertEqual(len(listing), 100)
self.assertEqual(listing[0][0], '0-0000')
self.assertEqual(listing[-1][0], '0-0099')
listing = backend.list_containers(account_id,
marker='',
end_marker='0-0050',
delimiter='',
limit=100)
self.assertEqual(len(listing), 50)
self.assertEqual(listing[0][0], '0-0000')
self.assertEqual(listing[-1][0], '0-0049')
listing = backend.list_containers(account_id,
marker='0-0099',
delimiter='',
limit=100)
self.assertEqual(len(listing), 100)
self.assertEqual(listing[0][0], '0-0100')
self.assertEqual(listing[-1][0], '1-0074')
listing = backend.list_containers(account_id,
marker='1-0074',
delimiter='',
limit=55)
self.assertEqual(len(listing), 55)
self.assertEqual(listing[0][0], '1-0075')
self.assertEqual(listing[-1][0], '2-0004')
listing = backend.list_containers(account_id,
marker='',
prefix='0-01',
delimiter='',
limit=10)
self.assertEqual(len(listing), 10)
self.assertEqual(listing[0][0], '0-0100')
self.assertEqual(listing[-1][0], '0-0109')
listing = backend.list_containers(account_id,
marker='',
prefix='0-01',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 10)
self.assertEqual(listing[0][0], '0-0100')
self.assertEqual(listing[-1][0], '0-0109')
listing = backend.list_containers(account_id,
marker='',
prefix='0-',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 10)
self.assertEqual(listing[0][0], '0-0000')
self.assertEqual(listing[-1][0], '0-0009')
listing = backend.list_containers(account_id,
marker='',
prefix='',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 4)
self.assertEqual([c[0] for c in listing], ['0-', '1-', '2-', '3-'])
listing = backend.list_containers(account_id,
marker='2-',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 1)
self.assertEqual([c[0] for c in listing], ['3-'])
listing = backend.list_containers(account_id,
marker='',
prefix='2',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 1)
self.assertEqual([c[0] for c in listing], ['2-'])
listing = backend.list_containers(account_id,
marker='2-0050',
prefix='2-',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 10)
self.assertEqual(listing[0][0], '2-0051')
self.assertEqual(listing[1][0], '2-0051-')
self.assertEqual(listing[2][0], '2-0052')
self.assertEqual(listing[-1][0], '2-0059')
listing = backend.list_containers(account_id,
marker='3-0045',
prefix='3-',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 10)
self.assertEqual([c[0] for c in listing], [
'3-0045-', '3-0046', '3-0046-', '3-0047', '3-0047-', '3-0048',
'3-0048-', '3-0049', '3-0049-', '3-0050'
])
name = '3-0049-'
backend.update_container(account_id, name,
Timestamp(time()).normal, 0, 0, 0)
listing = backend.list_containers(account_id,
marker='3-0048',
limit=10)
self.assertEqual(len(listing), 10)
self.assertEqual([c[0] for c in listing], [
'3-0048-0049', '3-0049', '3-0049-', '3-0049-0049', '3-0050',
'3-0050-0049', '3-0051', '3-0051-0049', '3-0052', '3-0052-0049'
])
listing = backend.list_containers(account_id,
marker='3-0048',
prefix='3-',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 10)
self.assertEqual([c[0] for c in listing], [
'3-0048-', '3-0049', '3-0049-', '3-0050', '3-0050-', '3-0051',
'3-0051-', '3-0052', '3-0052-', '3-0053'
])
listing = backend.list_containers(account_id,
prefix='3-0049-',
delimiter='-',
limit=10)
self.assertEqual(len(listing), 2)
self.assertEqual([c[0] for c in listing], ['3-0049-', '3-0049-0049'])
def test_update_container(self):
backend = AccountBackend({}, self.conn)
account_id = 'test'
self.assertEqual(backend.create_account(account_id), account_id)
# initial container
name = '"{<container \'&\' name>}"'
mtime = Timestamp(time()).normal
backend.update_container(account_id, name, mtime, 0, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(res[0], name)
self.assertEqual(
self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
mtime)
# same event
backend.update_container(account_id, name, mtime, 0, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(res[0], name)
self.assertEqual(
self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
mtime)
# New event
sleep(.00001)
mtime = Timestamp(time()).normal
backend.update_container(account_id, name, mtime, 0, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(res[0], name)
self.assertEqual(
self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
mtime)
# Old event
old_mtime = Timestamp(time() - 1).normal
backend.update_container(account_id, name, old_mtime, 0, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(res[0], name)
self.assertEqual(
self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
mtime)
# Old delete event
dtime = Timestamp(time() - 1).normal
backend.update_container(account_id, name, 0, dtime, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(res[0], name)
self.assertEqual(
self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
mtime)
# New delete event
sleep(.00001)
mtime = Timestamp(time()).normal
backend.update_container(account_id, name, 0, mtime, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(len(res), 0)
self.assertFalse(
self.conn.exists('container:%s:%s' % (account_id, name)))
# New event
sleep(.00001)
mtime = Timestamp(time()).normal
backend.update_container(account_id, name, mtime, 0, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(res[0], name)
self.assertEqual(
self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
mtime)
def test_update_container(self):
backend = AccountBackend({}, self.conn)
account_id = 'test'
self.assertEqual(backend.create_account(account_id), account_id)
# initial container
name = '"{<container \'&\' name>}"'
mtime = Timestamp(time()).normal
backend.update_container(account_id, name, mtime, 0, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(res[0], name)
self.assertEqual(
self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
mtime)
# same event
with ExpectedException(Conflict):
backend.update_container(account_id, name, mtime, 0, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(res[0], name)
self.assertEqual(
self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
mtime)
# New event
sleep(.00001)
mtime = Timestamp(time()).normal
backend.update_container(account_id, name, mtime, 0, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(res[0], name)
self.assertEqual(
self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
mtime)
# Old event
old_mtime = Timestamp(time() - 1).normal
with ExpectedException(Conflict):
backend.update_container(account_id, name, old_mtime, 0, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(res[0], name)
self.assertEqual(
self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
mtime)
# Old delete event
dtime = Timestamp(time() - 1).normal
with ExpectedException(Conflict):
backend.update_container(account_id, name, 0, dtime, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(res[0], name)
self.assertEqual(
self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
mtime)
# New delete event
sleep(.00001)
mtime = Timestamp(time()).normal
backend.update_container(account_id, name, 0, mtime, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(len(res), 0)
self.assertTrue(
self.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)
# New event
sleep(.00001)
mtime = Timestamp(time()).normal
backend.update_container(account_id, name, mtime, 0, 0, 0)
res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+')
self.assertEqual(res[0], name)
self.assertEqual(
self.conn.hget('container:%s:%s' % (account_id, name), 'mtime'),
mtime)
# ensure ttl has been removed
self.assertEqual(self.conn.ttl('container:%s:%s' % (account_id, name)),
-1)
def update_container(self, account_id, name, mtime, dtime, object_count,
bytes_used):
conn = self.conn
if not account_id or not name:
return None
_account_id = conn.hget('account:%s' % account_id, 'id')
if not _account_id:
if self.autocreate:
self.create_account(account_id)
else:
return None
lock = acquire_lock_with_timeout(
conn, 'container:%s:%s' % (account_id, name), 1)
if not lock:
return None
data = conn.hgetall('container:%s:%s' % (account_id, name))
record = {
'name': name,
'mtime': mtime,
'dtime': dtime,
'objects': object_count,
'bytes': bytes_used
}
deleted = False
if data:
data['mtime'] = Timestamp(data['mtime'])
data['dtime'] = Timestamp(data['dtime'])
for r in ['name', 'mtime', 'dtime', 'objects', 'bytes']:
if record[r] is None and data[r] is not None:
record[r] = data[r]
if data['mtime'] > record['mtime']:
record['mtime'] = data['mtime']
if data['dtime'] > record['dtime']:
record['dtime'] = data['dtime']
if record['dtime'] > record['mtime']:
deleted = True
if not deleted:
incr_bytes_used = int_value(record.get('bytes'), 0) -\
int_value(data.get('bytes'), 0)
incr_object_count = int_value(record.get('objects'), 0) -\
int_value(data.get('objects'), 0)
else:
incr_bytes_used = -int_value(data.get('bytes'), 0)
incr_object_count = -int_value(data.get('objects'), 0)
record.update({'name': name, 'account_uid': account_id})
# replace None values
for r in ['bytes', 'objects', 'mtime', 'dtime']:
if record[r] is None:
record[r] = 0
ct = {str(name): 0}
pipeline = conn.pipeline(True)
if deleted:
pipeline.delete('container:%s:%s' % (account_id, name))
pipeline.zrem('containers:%s' % account_id, str(name))
else:
pipeline.hmset('container:%s:%s' % (account_id, name), record)
pipeline.zadd('containers:%s' % account_id, **ct)
if incr_object_count:
pipeline.hincrby('account:%s' % account_id, 'objects',
incr_object_count)
if incr_bytes_used:
pipeline.hincrby('account:%s' % account_id, 'bytes',
incr_bytes_used)
pipeline.execute()
release_lock(conn, 'container:%s:%s' % (account_id, name), lock)
return name
def update_container(self, account_id, name, mtime, dtime, object_count,
bytes_used):
conn = self.conn
if not account_id or not name:
return None
_account_id = conn.hget('account:%s' % account_id, 'id')
if not _account_id:
if self.autocreate:
self.create_account(account_id)
else:
return None
lock = acquire_lock_with_timeout(conn,
AccountBackend.ckey(account_id, name),
1)
if not lock:
return None
data = conn.hgetall(AccountBackend.ckey(account_id, name))
record = {
'name': name,
'mtime': mtime or dtime,
'dtime': dtime,
'objects': object_count,
'bytes': bytes_used
}
if data:
data['mtime'] = Timestamp(data['mtime'])
data['dtime'] = Timestamp(data['dtime'])
for r in ['name', 'mtime', 'dtime', 'objects', 'bytes']:
if record[r] is None and data[r] is not None:
record[r] = data[r]
if data['mtime'] > record['mtime']:
record['mtime'] = data['mtime']
if data['dtime'] > record['dtime']:
record['dtime'] = data['dtime']
deleted = record['dtime'] >= record['mtime']
if not deleted:
incr_bytes_used = int_value(record.get('bytes'), 0) -\
int_value(data.get('bytes'), 0)
incr_object_count = int_value(record.get('objects'), 0) -\
int_value(data.get('objects'), 0)
elif record.get('mtime') > data.get('mtime'):
incr_bytes_used = -int_value(data.get('bytes'), 0)
incr_object_count = -int_value(data.get('objects'), 0)
else:
# The event has been delayed, the container has already
# been deleted, and the object and bytes statistics have
# already been reported to the account.
incr_bytes_used = 0
incr_object_count = 0
record.update({'name': name, 'account_uid': account_id})
# replace None values
for r in ['bytes', 'objects', 'mtime', 'dtime']:
if record[r] is None:
record[r] = 0
ct = {name: 0}
pipeline = conn.pipeline(True)
pipeline.hmset(AccountBackend.ckey(account_id, name), record)
if deleted:
pipeline.expire(AccountBackend.ckey(account_id, name), EXPIRE_TIME)
pipeline.zrem('containers:%s' % account_id, name)
else:
pipeline.persist(AccountBackend.ckey(account_id, name))
pipeline.zadd('containers:%s' % account_id, **ct)
if incr_object_count:
pipeline.hincrby('account:%s' % account_id, 'objects',
incr_object_count)
if incr_bytes_used:
pipeline.hincrby('account:%s' % account_id, 'bytes',
incr_bytes_used)
pipeline.execute()
release_lock(conn, AccountBackend.ckey(account_id, name), lock)
return name