def refresh_bucket(self, bucket_name, **kwargs): """ Refresh the counters of a bucket. Recompute them from the counters of all shards (containers). """ lkey = self.blockkey(bucket_name) batch_size = kwargs.get("batch_size", 10000) try: ctime = Timestamp().normal self.script_get_lock_bucket(keys=[lkey], args=[ctime]) except redis.exceptions.ResponseError as exc: if text_type(exc).endswith("bucket_lock"): raise Conflict("Refresh on bucket already in progress") raise account_id = '__account__' keys = [ AccountBackend.ckey(account_id, ''), self.clistkey(account_id), self.bkey(bucket_name), lkey ] try: while True: args = [bucket_name, Timestamp().normal, batch_size] res = self.script_refresh_bucket(keys=keys, args=args) if res[0]: break except redis.exceptions.ResponseError as exc: self.conn.delete(lkey) if text_type(exc).endswith("no_account"): raise NotFound("Account %s not found" % account_id) if text_type(exc).endswith("no_bucket"): raise NotFound("Bucket %s not found" % bucket_name) raise
def test_utf8_container(self): backend = AccountBackend({}, self.conn) account_id = 'test' self.assertEqual(backend.create_account(account_id), account_id) name = u'La fête à la maison' mtime = Timestamp(time()).normal # create container backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0) res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(unicode(res[0], 'utf8'), name) # ensure it appears in listing listing = backend.list_containers(account_id, marker='', delimiter='', limit=100) self.assertIn(name, [entry[0] for entry in listing]) # delete container sleep(.00001) dtime = Timestamp(time()).normal backend.update_container(account_id, name, 0, dtime, 0, 0, 0, 0) res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(len(res), 0) self.assertTrue( self.conn.ttl('container:%s:%s' % (account_id, name)) >= 1) # ensure it has been removed with ExpectedException(Conflict): backend.update_container(account_id, name, 0, dtime, 0, 0, 0, 0) res = self.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(len(res), 0) self.assertTrue( self.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)
def test_update_after_container_deletion(self): account_id = 'test-%06x' % int(time()) self.assertEqual(self.backend.create_account(account_id), account_id) # Container create event, sent immediately after creation self.backend.update_container(account_id, 'c1', Timestamp().normal, None, None, None) # Container update event self.backend.update_container(account_id, 'c1', Timestamp().normal, None, 3, 30) info = self.backend.info_account(account_id) self.assertEqual(info['containers'], 1) self.assertEqual(info['objects'], 3) self.assertEqual(info['bytes'], 30) # Container is flushed, but the event is deferred flush_timestamp = Timestamp().normal sleep(.00001) # Container delete event, sent immediately after deletion self.backend.update_container(account_id, 'c1', None, Timestamp().normal, None, None) # Deferred container update event (with lower timestamp) self.backend.update_container(account_id, 'c1', flush_timestamp, None, 0, 0) info = self.backend.info_account(account_id) self.assertEqual(info['containers'], 0) self.assertEqual(info['objects'], 0) self.assertEqual(info['bytes'], 0)
def test_account_container_reset(self): data = { 'name': 'foo', 'mtime': Timestamp().normal, 'objects': 12, 'bytes': 42 } dataj = json.dumps(data) resp = self.app.put('/v1.0/account/container/update', data=dataj, query_string={'id': self.account_id}) data = {'name': 'foo', 'mtime': Timestamp().normal} dataj = json.dumps(data) resp = self.app.put('/v1.0/account/container/reset', data=dataj, query_string={'id': self.account_id}) self.assertEqual(resp.status_code, 204) resp = self.app.get('/v1.0/account/containers', query_string={ 'id': self.account_id, 'prefix': 'foo' }) resp = self.json_loads(resp.data) for container in resp["listing"]: name, nb_objects, nb_bytes, _, mtime = container if not name.startswith('foo'): self.fail("No prefix foo: %s" % name) if name == 'foo': self.assertEqual(0, nb_objects) self.assertEqual(0, nb_bytes) self.assertEqual(float(data['mtime']), mtime) return self.fail("No container foo")
def update_container(self, account_id, name, mtime, dtime, object_count, bytes_used, autocreate_account=None, autocreate_container=True): conn = self.conn if not account_id or not name: raise BadRequest("Missing account or container") if autocreate_account is None: autocreate_account = self.autocreate if mtime is None: mtime = '0' else: mtime = Timestamp(float(mtime)).normal if dtime is None: dtime = '0' else: dtime = Timestamp(float(dtime)).normal if object_count is None: object_count = 0 if bytes_used is None: bytes_used = 0 keys = [ account_id, AccountBackend.ckey(account_id, name), ("containers:%s" % (account_id)), ("account:%s" % (account_id)) ] args = [ name, mtime, dtime, object_count, bytes_used, str(autocreate_account), Timestamp(time()).normal, EXPIRE_TIME, str(autocreate_container) ] try: self.script_update_container(keys=keys, args=args, client=conn) except redis.exceptions.ResponseError as exc: if str(exc) == "no_account": raise NotFound("Account %s not found" % account_id) if str(exc) == "no_container": raise NotFound("Container %s not found" % name) elif str(exc) == "no_update_needed": raise Conflict("No update needed, " "event older than last container update") else: raise return name
def test_info_account(self): backend = AccountBackend({}, self.conn) account_id = 'test' self.assertEqual(backend.create_account(account_id), account_id) info = backend.info_account(account_id) self.assertEqual(info['id'], account_id) self.assertEqual(info['bytes'], 0) self.assertEqual(info['objects'], 0) self.assertEqual(info['containers'], 0) self.assertTrue(info['ctime']) # first container backend.update_container(account_id, 'c1', Timestamp(time()).normal, 0, 1, 1) info = backend.info_account(account_id) self.assertEqual(info['containers'], 1) self.assertEqual(info['objects'], 1) self.assertEqual(info['bytes'], 1) # second container sleep(.00001) backend.update_container(account_id, 'c2', Timestamp(time()).normal, 0, 0, 0) info = backend.info_account(account_id) self.assertEqual(info['containers'], 2) self.assertEqual(info['objects'], 1) self.assertEqual(info['bytes'], 1) # update second container sleep(.00001) backend.update_container(account_id, 'c2', Timestamp(time()).normal, 0, 1, 1) info = backend.info_account(account_id) self.assertEqual(info['containers'], 2) self.assertEqual(info['objects'], 2) self.assertEqual(info['bytes'], 2) # delete first container sleep(.00001) backend.update_container(account_id, 'c1', 0, Timestamp(time()).normal, 0, 0) info = backend.info_account(account_id) self.assertEqual(info['containers'], 1) self.assertEqual(info['objects'], 1) self.assertEqual(info['bytes'], 1) # delete second container sleep(.00001) backend.update_container(account_id, 'c2', 0, Timestamp(time()).normal, 0, 0) info = backend.info_account(account_id) self.assertEqual(info['containers'], 0) self.assertEqual(info['objects'], 0) self.assertEqual(info['bytes'], 0)
def test_account_flush(self): data = { 'name': 'foo', 'mtime': Timestamp().normal, 'objects': 12, 'bytes': 42 } data = json.dumps(data) resp = self.app.put('/v1.0/account/container/update', data=data, query_string={'id': self.account_id}) resp = self.app.post('/v1.0/account/flush', query_string={'id': self.account_id}) self.assertEqual(resp.status_code, 204) resp = self.app.get('/v1.0/account/show', query_string={'id': self.account_id}) resp = self.json_loads(resp.data) self.assertEqual(resp["bytes"], 0) self.assertEqual(resp["objects"], 0) resp = self.app.get('/v1.0/account/containers', query_string={'id': self.account_id}) resp = self.json_loads(resp.data) self.assertEqual(len(resp["listing"]), 0)
def take_action(self, parsed_args): self.log.debug('take_action(%s)', parsed_args) self.take_action_container(parsed_args) cid = parsed_args.cid if cid is None: account = self.app.client_manager.account container = parsed_args.container else: account, container = \ self.app.client_manager.storage.resolve_cid(cid) deadline = timeout_to_deadline(parsed_args.timeout) dst_account = parsed_args.dst_account or account dst_container = (parsed_args.dst_container or (container + "-" + Timestamp().normal)) batch_size = parsed_args.chunk_batch_size self.app.client_manager.storage.container_snapshot( account, container, dst_account, dst_container, batch_size=batch_size, deadline=deadline) lines = [(dst_account, dst_container, "OK")] return ('Account', 'Container', 'Status'), lines
def test_account_container_update(self): data = {'name': 'foo', 'mtime': Timestamp(time()).normal, 'objects': 0, 'bytes': 0} data = json.dumps(data) resp = self.app.put('/v1.0/account/container/update', data=data, query_string={'id': self.account_id}) self.assertEqual(resp.status_code, 200)
def create_account(self, account_id): conn = self.conn if not account_id: return None if conn.hget('accounts:', account_id): return None lock = self.acquire_lock_with_timeout('account:%s' % account_id, 1) if not lock: return None pipeline = conn.pipeline(True) pipeline.hset('accounts:', account_id, 1) pipeline.hmset( 'account:%s' % account_id, { 'id': account_id, 'objects': 0, 'bytes': 0, 'damaged_objects': 0, 'missing_chunks': 0, 'ctime': Timestamp(time()).normal }) pipeline.execute() self.release_lock('account:%s' % account_id, lock) return account_id
def test_refresh_account(self): account_id = random_str(16) account_key = 'account:%s' % account_id self.assertEqual(self.backend.create_account(account_id), account_id) total_bytes = 0 total_objects = 0 # 10 containers with bytes and objects for i in range(10): name = "container%d" % i mtime = Timestamp().normal nb_bytes = random.randrange(100) total_bytes += nb_bytes nb_objets = random.randrange(100) total_objects += nb_objets self.backend.update_container(account_id, name, mtime, 0, nb_objets, nb_bytes) # change values self.backend.conn.hset(account_key, 'bytes', 1) self.backend.conn.hset(account_key, 'objects', 2) self.assertEqual(self.backend.conn.hget(account_key, 'bytes'), b'1') self.assertEqual(self.backend.conn.hget(account_key, 'objects'), b'2') self.backend.refresh_account(account_id) self.assertEqual(self.backend.conn.hget(account_key, 'bytes'), str(total_bytes).encode('utf-8')) self.assertEqual(self.backend.conn.hget(account_key, 'objects'), str(total_objects).encode('utf-8'))
def _gen_results(objects): for obj in objects: result = (obj['name'], obj['size'], obj['hash'], obj['version'], obj['deleted'], obj['mime_type'], Timestamp(obj['ctime']).isoformat, obj['policy'], _format_props(obj.get('properties', {}))) yield result
def test_flush_account(self): account_id = random_str(16) account_key = 'account:%s' % account_id self.assertEqual(self.backend.create_account(account_id), account_id) total_bytes = 0 total_objects = 0 # 10 containers with bytes and objects for i in range(10): name = "container%d" % i mtime = Timestamp().normal nb_bytes = random.randrange(100) total_bytes += nb_bytes nb_objets = random.randrange(100) total_objects += nb_objets self.backend.update_container(account_id, name, mtime, 0, nb_objets, nb_bytes) self.assertEqual(self.backend.conn.hget(account_key, 'bytes'), str(total_bytes).encode('utf-8')) self.assertEqual(self.backend.conn.hget(account_key, 'objects'), str(total_objects).encode('utf-8')) self.backend.flush_account(account_id) self.assertEqual(self.backend.conn.hget(account_key, 'bytes'), b'0') self.assertEqual(self.backend.conn.hget(account_key, 'objects'), b'0') self.assertEqual(self.backend.conn.zcard("containers:%s" % account_id), 0) self.assertEqual(self.backend.conn.exists("container:test:*"), 0)
def test_refresh_bucket(self): account_id = random_str(16) account_key = 'account:%s' % account_id bucket = random_str(16) self.assertEqual(self.backend.create_account(account_id), account_id) total_bytes = 0 total_objects = 0 total_damaged_objects = 0 total_missing_chunks = 0 # 10 containers with bytes and objects for i in range(10): name = "container%d" % i mtime = Timestamp().normal nb_bytes = random.randrange(100) total_bytes += nb_bytes nb_objets = random.randrange(100) total_objects += nb_objets damaged_objects = random.randrange(100) total_damaged_objects += damaged_objects missing_chunks = random.randrange(100) total_missing_chunks += missing_chunks self.backend.update_container(account_id, name, mtime, 0, nb_objets, nb_bytes, damaged_objects, missing_chunks, bucket_name=bucket) bkey = self.backend.bkey(bucket) # change values self.backend.conn.hset(bkey, 'bytes', 1) self.backend.conn.hset(bkey, 'objects', 2) self.backend.conn.hset(bkey, 'damaged_objects', 3) self.backend.conn.hset(bkey, 'missing_chunks', 4) self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'1') self.assertEqual(self.backend.conn.hget(bkey, 'objects'), b'2') self.assertEqual(self.backend.conn.hget(bkey, 'damaged_objects'), b'3') self.assertEqual(self.backend.conn.hget(bkey, 'missing_chunks'), b'4') # force pagination self.backend.refresh_bucket(bucket, batch_size=6) conn = self.backend.conn self.assertEqual( conn.hget(account_key, 'bytes').decode('utf-8'), str(total_bytes)) self.assertEqual( conn.hget(account_key, 'objects').decode('utf-8'), str(total_objects)) self.assertEqual( conn.hget(account_key, 'damaged_objects').decode('utf-8'), str(total_damaged_objects)) self.assertEqual( conn.hget(account_key, 'missing_chunks').decode('utf-8'), str(total_missing_chunks))
def test_delete_container(self): account_id = 'test' self.assertEqual(self.backend.create_account(account_id), account_id) name = 'c' old_mtime = Timestamp(time() - 1).normal mtime = Timestamp(time()).normal # initial container self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0], name) # delete event sleep(.00001) dtime = Timestamp(time()).normal self.backend.update_container(account_id, name, mtime, dtime, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(len(res), 0) self.assertTrue( self.backend.conn.ttl('container:%s:%s' % (account_id, name)) >= 1) # same event with ExpectedException(Conflict): self.backend.update_container(account_id, name, mtime, dtime, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(len(res), 0) self.assertTrue( self.backend.conn.ttl('container:%s:%s' % (account_id, name)) >= 1) # old event with ExpectedException(Conflict): self.backend.update_container(account_id, name, old_mtime, 0, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(len(res), 0) self.assertTrue( self.backend.conn.ttl('container:%s:%s' % (account_id, name)) >= 1)
def take_action(self, parsed_args): self.log.debug('take_action(%s)', parsed_args) reqid = request_id(prefix='CLI-BUCKET-') acct_client = self.app.client_manager.storage.account data = acct_client.bucket_show(parsed_args.bucket, reqid=reqid) if parsed_args.formatter == 'table': from oio.common.easy_value import convert_size data['bytes'] = convert_size(data['bytes']) data['mtime'] = Timestamp(data.get('mtime', 0.0)).isoformat return zip(*sorted(data.items()))
def test_update_after_container_deletion(self): backend = AccountBackend({}, self.conn) account_id = 'test-%06x' % int(time()) self.assertEqual(backend.create_account(account_id), account_id) # Container create event, sent immediately after creation backend.update_container(account_id, 'c1', Timestamp(time()).normal, None, None, None, None, None) # Container update event backend.update_container(account_id, 'c1', Timestamp(time()).normal, None, 3, 30, 7, 5) info = backend.info_account(account_id) self.assertEqual(info['containers'], 1) self.assertEqual(info['objects'], 3) self.assertEqual(info['bytes'], 30) self.assertEqual(info['damaged_objects'], 7) self.assertEqual(info['missing_chunks'], 5) # Container is flushed, but the event is deferred flush_timestamp = Timestamp(time()).normal sleep(.00001) # Container delete event, sent immediately after deletion backend.update_container(account_id, 'c1', None, Timestamp(time()).normal, None, None, None, None) # Deferred container update event (with lower timestamp) backend.update_container(account_id, 'c1', flush_timestamp, None, 0, 0, 0, 0) info = backend.info_account(account_id) self.assertEqual(info['containers'], 0) self.assertEqual(info['objects'], 0) self.assertEqual(info['bytes'], 0) self.assertEqual(info['damaged_objects'], 0) self.assertEqual(info['missing_chunks'], 0)
def _gen_results(objects): for obj in objects: try: result = (obj['name'], obj['size'], obj['hash'], obj['version'], obj['deleted'], obj['mime_type'], Timestamp(obj['mtime']).isoformat, obj['policy'], _format_props(obj.get('properties', {}))) yield result except KeyError as exc: self.log.warn("Bad object entry, missing '%s': %s", exc, obj)
def take_action(self, parsed_args): self.log.debug('take_action(%s)', parsed_args) account = self.app.client_manager.account container = parsed_args.container dst_account = parsed_args.dst_account or account dst_container = (parsed_args.dst_container or (container + "-" + Timestamp(time()).normal)) batch = parsed_args.chunk_batch_size self.app.client_manager.storage.container_snapshot( account, container, dst_account, dst_container, batch=batch) lines = [(dst_account, dst_container, "OK")] return ('Account', 'Container', 'Status'), lines
def test_refresh_account(self): account_id = random_str(16) account_key = 'account:%s' % account_id self.assertEqual(self.backend.create_account(account_id), account_id) total_bytes = 0 total_objects = 0 total_damaged_objects = 0 total_missing_chunks = 0 # 10 containers with bytes and objects for i in range(10): name = "container%d" % i mtime = Timestamp(time()).normal nb_bytes = random.randrange(100) total_bytes += nb_bytes nb_objets = random.randrange(100) total_objects += nb_objets damaged_objects = random.randrange(100) total_damaged_objects += damaged_objects missing_chunks = random.randrange(100) total_missing_chunks += missing_chunks self.backend.update_container(account_id, name, mtime, 0, nb_objets, nb_bytes, damaged_objects, missing_chunks) # change values self.backend.conn.hset(account_key, 'bytes', 1) self.backend.conn.hset(account_key, 'objects', 2) self.backend.conn.hset(account_key, 'damaged_objects', 3) self.backend.conn.hset(account_key, 'missing_chunks', 4) self.assertEqual(self.backend.conn.hget(account_key, 'bytes'), '1') self.assertEqual(self.backend.conn.hget(account_key, 'objects'), '2') self.assertEqual( self.backend.conn.hget(account_key, 'damaged_objects'), '3') self.assertEqual(self.backend.conn.hget(account_key, 'missing_chunks'), '4') self.backend.refresh_account(account_id) self.assertEqual(self.backend.conn.hget(account_key, 'bytes'), str(total_bytes)) self.assertEqual(self.backend.conn.hget(account_key, 'objects'), str(total_objects)) self.assertEqual( self.backend.conn.hget(account_key, 'damaged_objects'), str(total_damaged_objects)) self.assertEqual(self.backend.conn.hget(account_key, 'missing_chunks'), str(total_missing_chunks))
def test_flush_account(self): account_id = random_str(16) account_key = 'account:%s' % account_id self.assertEqual(self.backend.create_account(account_id), account_id) total_bytes = 0 total_objects = 0 total_damaged_objects = 0 total_missing_chunks = 0 # 10 containers with bytes and objects for i in range(10): name = "container%d" % i mtime = Timestamp(time()).normal nb_bytes = random.randrange(100) total_bytes += nb_bytes nb_objets = random.randrange(100) total_objects += nb_objets damaged_objects = random.randrange(100) total_damaged_objects += damaged_objects missing_chunks = random.randrange(100) total_missing_chunks += missing_chunks self.backend.update_container(account_id, name, mtime, 0, nb_objets, nb_bytes, damaged_objects, missing_chunks) self.assertEqual(self.backend.conn.hget(account_key, 'bytes'), str(total_bytes)) self.assertEqual(self.backend.conn.hget(account_key, 'objects'), str(total_objects)) self.assertEqual( self.backend.conn.hget(account_key, 'damaged_objects'), str(total_damaged_objects)) self.assertEqual(self.backend.conn.hget(account_key, 'missing_chunks'), str(total_missing_chunks)) self.backend.flush_account(account_id) self.assertEqual(self.backend.conn.hget(account_key, 'bytes'), '0') self.assertEqual(self.backend.conn.hget(account_key, 'objects'), '0') self.assertEqual( self.backend.conn.hget(account_key, 'damaged_objects'), '0') self.assertEqual(self.backend.conn.hget(account_key, 'missing_chunks'), '0') self.assertEqual(self.backend.conn.zcard("containers:%s" % account_id), 0) self.assertEqual(self.backend.conn.exists("container:test:*"), 0)
def take_action(self, parsed_args): self.log.debug('take_action(%s)', parsed_args) reqid = request_id(prefix='CLI-BUCKET-') acct_client = self.app.client_manager.storage.account metadata = dict() if parsed_args.replicate is not None: metadata[BUCKET_PROP_REPLI_ENABLED] = str(parsed_args.replicate) data = acct_client.bucket_update(parsed_args.bucket, metadata=metadata, to_delete=None, reqid=reqid) if parsed_args.formatter == 'table': from oio.common.easy_value import convert_size data['bytes'] = convert_size(data['bytes']) data['mtime'] = Timestamp(data.get('mtime', 0.0)).isoformat return zip(*sorted(data.items()))
def test_account_refresh(self): data = { 'name': 'foo', 'mtime': Timestamp(time()).normal, 'objects': 12, 'bytes': 42 } data = json.dumps(data) resp = self.app.post('/v1.0/account/container/update', data=data, query_string={'id': self.account_id}) resp = self.app.post('/v1.0/account/refresh', query_string={'id': self.account_id}) self.assertEqual(resp.status_code, 204) resp = self.app.post('/v1.0/account/show', query_string={'id': self.account_id}) resp = self.json_loads(resp.data) self.assertEqual(resp["bytes"], 42) self.assertEqual(resp["objects"], 12)
def create_account(self, account_id, **kwargs): conn = self.conn if not account_id: return None if conn.hget('accounts:', account_id): return None lock = self.acquire_lock_with_timeout(self.akey(account_id), 1) if not lock: return None pipeline = conn.pipeline(True) pipeline.hset('accounts:', account_id, 1) pipeline.hmset( self.akey(account_id), { 'id': account_id, 'objects': 0, 'bytes': 0, 'ctime': Timestamp().normal }) pipeline.execute() self.release_lock(self.akey(account_id), lock) return account_id
def test_info_account(self): account_id = 'test' self.assertEqual(self.backend.create_account(account_id), account_id) info = self.backend.info_account(account_id) self.assertEqual(info['id'], account_id) self.assertEqual(info['objects'], 0) self.assertEqual(info['bytes'], 0) self.assertEqual(info['damaged_objects'], 0) self.assertEqual(info['missing_chunks'], 0) self.assertEqual(info['containers'], 0) self.assertTrue(info['ctime']) # first container self.backend.update_container(account_id, 'c1', Timestamp(time()).normal, 0, 1, 1, 1, 1) info = self.backend.info_account(account_id) self.assertEqual(info['containers'], 1) self.assertEqual(info['objects'], 1) self.assertEqual(info['bytes'], 1) self.assertEqual(info['damaged_objects'], 1) self.assertEqual(info['missing_chunks'], 1) # second container sleep(.00001) self.backend.update_container(account_id, 'c2', Timestamp(time()).normal, 0, 0, 0, 0, 0) info = self.backend.info_account(account_id) self.assertEqual(info['containers'], 2) self.assertEqual(info['objects'], 1) self.assertEqual(info['bytes'], 1) self.assertEqual(info['damaged_objects'], 1) self.assertEqual(info['missing_chunks'], 1) # update second container sleep(.00001) self.backend.update_container(account_id, 'c2', Timestamp(time()).normal, 0, 1, 1, 1, 2) info = self.backend.info_account(account_id) self.assertEqual(info['containers'], 2) self.assertEqual(info['objects'], 2) self.assertEqual(info['bytes'], 2) self.assertEqual(info['damaged_objects'], 2) self.assertEqual(info['missing_chunks'], 3) # delete first container sleep(.00001) self.backend.update_container(account_id, 'c1', 0, Timestamp(time()).normal, 0, 0, 0, 0) info = self.backend.info_account(account_id) self.assertEqual(info['containers'], 1) self.assertEqual(info['objects'], 1) self.assertEqual(info['bytes'], 1) self.assertEqual(info['damaged_objects'], 1) self.assertEqual(info['missing_chunks'], 2) # delete second container sleep(.00001) self.backend.update_container(account_id, 'c2', 0, Timestamp(time()).normal, 0, 0, 0, 0) info = self.backend.info_account(account_id) self.assertEqual(info['containers'], 0) self.assertEqual(info['objects'], 0) self.assertEqual(info['bytes'], 0) self.assertEqual(info['damaged_objects'], 0) self.assertEqual(info['missing_chunks'], 0)
def test_list_containers(self): account_id = 'test' self.backend.create_account(account_id) for cont1 in xrange(4): for cont2 in xrange(125): name = '%d-%04d' % (cont1, cont2) self.backend.update_container(account_id, name, Timestamp(time()).normal, 0, 0, 0, 0, 0) for cont in xrange(125): name = '2-0051-%04d' % cont self.backend.update_container(account_id, name, Timestamp(time()).normal, 0, 0, 0, 0, 0) for cont in xrange(125): name = '3-%04d-0049' % cont self.backend.update_container(account_id, name, Timestamp(time()).normal, 0, 0, 0, 0, 0) listing = self.backend.list_containers(account_id, marker='', delimiter='', limit=100) self.assertEqual(len(listing), 100) self.assertEqual(listing[0][0], '0-0000') self.assertEqual(listing[-1][0], '0-0099') listing = self.backend.list_containers(account_id, marker='', end_marker='0-0050', delimiter='', limit=100) self.assertEqual(len(listing), 50) self.assertEqual(listing[0][0], '0-0000') self.assertEqual(listing[-1][0], '0-0049') listing = self.backend.list_containers(account_id, marker='0-0099', delimiter='', limit=100) self.assertEqual(len(listing), 100) self.assertEqual(listing[0][0], '0-0100') self.assertEqual(listing[-1][0], '1-0074') listing = self.backend.list_containers(account_id, marker='1-0074', delimiter='', limit=55) self.assertEqual(len(listing), 55) self.assertEqual(listing[0][0], '1-0075') self.assertEqual(listing[-1][0], '2-0004') listing = self.backend.list_containers(account_id, marker='', prefix='0-01', delimiter='', limit=10) self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '0-0100') self.assertEqual(listing[-1][0], '0-0109') listing = self.backend.list_containers(account_id, marker='', prefix='0-01', delimiter='-', limit=10) self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '0-0100') self.assertEqual(listing[-1][0], '0-0109') listing = self.backend.list_containers(account_id, marker='', prefix='0-', delimiter='-', limit=10) self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '0-0000') self.assertEqual(listing[-1][0], '0-0009') listing = self.backend.list_containers(account_id, marker='', prefix='', delimiter='-', limit=10) self.assertEqual(len(listing), 4) self.assertEqual([c[0] for c in listing], ['0-', '1-', '2-', '3-']) listing = self.backend.list_containers(account_id, marker='2-', delimiter='-', limit=10) self.assertEqual(len(listing), 1) self.assertEqual([c[0] for c in listing], ['3-']) listing = self.backend.list_containers(account_id, marker='', prefix='2', delimiter='-', limit=10) self.assertEqual(len(listing), 1) self.assertEqual([c[0] for c in listing], ['2-']) listing = self.backend.list_containers(account_id, marker='2-0050', prefix='2-', delimiter='-', limit=10) self.assertEqual(len(listing), 10) self.assertEqual(listing[0][0], '2-0051') self.assertEqual(listing[1][0], '2-0051-') self.assertEqual(listing[2][0], '2-0052') self.assertEqual(listing[-1][0], '2-0059') listing = self.backend.list_containers(account_id, marker='3-0045', prefix='3-', delimiter='-', limit=10) self.assertEqual(len(listing), 10) self.assertEqual([c[0] for c in listing], [ '3-0045-', '3-0046', '3-0046-', '3-0047', '3-0047-', '3-0048', '3-0048-', '3-0049', '3-0049-', '3-0050' ]) name = '3-0049-' self.backend.update_container(account_id, name, Timestamp(time()).normal, 0, 0, 0, 0, 0) listing = self.backend.list_containers(account_id, marker='3-0048', limit=10) self.assertEqual(len(listing), 10) self.assertEqual([c[0] for c in listing], [ '3-0048-0049', '3-0049', '3-0049-', '3-0049-0049', '3-0050', '3-0050-0049', '3-0051', '3-0051-0049', '3-0052', '3-0052-0049' ]) listing = self.backend.list_containers(account_id, marker='3-0048', prefix='3-', delimiter='-', limit=10) self.assertEqual(len(listing), 10) self.assertEqual([c[0] for c in listing], [ '3-0048-', '3-0049', '3-0049-', '3-0050', '3-0050-', '3-0051', '3-0051-', '3-0052', '3-0052-', '3-0053' ]) listing = self.backend.list_containers(account_id, prefix='3-0049-', delimiter='-', limit=10) self.assertEqual(len(listing), 2) self.assertEqual([c[0] for c in listing], ['3-0049-', '3-0049-0049'])
def test_update_container(self): account_id = 'test' self.assertEqual(self.backend.create_account(account_id), account_id) # initial container name = '"{<container \'&\' name>}"' mtime = Timestamp(time()).normal self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0], name) self.assertEqual( self.backend.conn.hget('container:%s:%s' % (account_id, name), 'mtime'), mtime) # same event with ExpectedException(Conflict): self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0], name) self.assertEqual( self.backend.conn.hget('container:%s:%s' % (account_id, name), 'mtime'), mtime) # New event sleep(.00001) mtime = Timestamp(time()).normal self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0], name) self.assertEqual( self.backend.conn.hget('container:%s:%s' % (account_id, name), 'mtime'), mtime) # Old event old_mtime = Timestamp(time() - 1).normal with ExpectedException(Conflict): self.backend.update_container(account_id, name, old_mtime, 0, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0], name) self.assertEqual( self.backend.conn.hget('container:%s:%s' % (account_id, name), 'mtime'), mtime) # Old delete event dtime = Timestamp(time() - 1).normal with ExpectedException(Conflict): self.backend.update_container(account_id, name, 0, dtime, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0], name) self.assertEqual( self.backend.conn.hget('container:%s:%s' % (account_id, name), 'mtime'), mtime) # New delete event sleep(.00001) mtime = Timestamp(time()).normal self.backend.update_container(account_id, name, 0, mtime, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(len(res), 0) self.assertTrue( self.backend.conn.ttl('container:%s:%s' % (account_id, name)) >= 1) # New event sleep(.00001) mtime = Timestamp(time()).normal self.backend.update_container(account_id, name, mtime, 0, 0, 0, 0, 0) res = self.backend.conn.zrangebylex('containers:%s' % account_id, '-', '+') self.assertEqual(res[0], name) self.assertEqual( self.backend.conn.hget('container:%s:%s' % (account_id, name), 'mtime'), mtime) # ensure ttl has been removed self.assertEqual( self.backend.conn.ttl('container:%s:%s' % (account_id, name)), -1)
def test_refresh_bucket_with_lock(self): account_id = random_str(16) bucket = random_str(16) self.assertEqual(self.backend.create_account(account_id), account_id) mtime = Timestamp().normal self.backend.update_container(account_id, "cnt-1", mtime, 0, 100, 100, bucket_name=bucket) self.backend.update_container(account_id, "cnt-2", mtime, 0, 500, 500, bucket_name=bucket) self.backend.update_container(account_id, "cnt-3", mtime, 0, 999, 999, bucket_name=bucket) bkey = self.backend.bkey(bucket) # create lock block = self.backend.blockkey(bucket) self.backend.conn.hset(block, mapping={ "marker": "cnt-2", "mtime": int(time()), "ctime": int(time()) }) with ExpectedException(Conflict): self.backend.refresh_bucket(bucket) # change value of first container self.backend.conn.hset(bkey, 'bytes', 100) self.backend.conn.hset(bkey, 'objects', 100) # update a container before marker, update is ok mtime = Timestamp().normal self.backend.update_container(account_id, "cnt-1", mtime, 0, 200, 200, bucket_name=bucket) self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'200') # update a container equal to marker, update is ok mtime = Timestamp().normal self.backend.update_container(account_id, "cnt-2", mtime, 0, 600, 600, bucket_name=bucket) self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'300') # update container after marker, update is not done mtime = Timestamp().normal self.backend.update_container(account_id, "cnt-3", mtime, 0, 998, 998, bucket_name=bucket) self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'300') # update lock with very old timestamp self.backend.conn.hset(block, "mtime", 1000) self.backend.refresh_bucket(bucket) self.assertEqual(self.backend.conn.hget(bkey, 'bytes'), b'1798')
def update_container(self, account_id, name, mtime, dtime, object_count, bytes_used, damaged_objects, missing_chunks, bucket_name=None, autocreate_account=None, autocreate_container=True, **kwargs): if not account_id or not name: raise BadRequest("Missing account or container") if autocreate_account is None: autocreate_account = self.autocreate if mtime is None: mtime = '0' else: mtime = Timestamp(mtime).normal if dtime is None: dtime = '0' else: dtime = Timestamp(dtime).normal if object_count is None: object_count = 0 if bytes_used is None: bytes_used = 0 if damaged_objects is None: damaged_objects = 0 if missing_chunks is None: missing_chunks = 0 # If no bucket name is provided, set it to '' # (we cannot pass None to the Lua script). bucket_name = bucket_name or '' bucket_lock = self.blockkey(bucket_name) now = Timestamp().normal ckey = AccountBackend.ckey(account_id, name) keys = [ self.akey(account_id), ckey, self.clistkey(account_id), self._bucket_prefix, self.blistkey(account_id) ] args = [ account_id, name, bucket_name, bucket_lock, mtime, dtime, object_count, bytes_used, damaged_objects, missing_chunks, str(autocreate_account), now, EXPIRE_TIME, str(autocreate_container) ] try: self.script_update_container(keys=keys, args=args) except redis.exceptions.ResponseError as exc: if text_type(exc).endswith("no_account"): raise NotFound("Account %s not found" % account_id) if text_type(exc).endswith("no_container"): raise NotFound("Container %s not found" % name) elif text_type(exc).endswith("no_update_needed"): raise Conflict("No update needed, " "event older than last container update") else: raise return name
def _get_timestamp(self): return Timestamp().normal