def test_scope_queue_name(self):
self.assertEqual(utils.scope_queue_name('my-q'), '.my-q')
self.assertEqual(utils.scope_queue_name('my-q', None), '.my-q')
self.assertEqual(utils.scope_queue_name('my-q', '123'), '123.my-q')
self.assertEqual(utils.scope_queue_name(None), '.')
self.assertEqual(utils.scope_queue_name(None, '123'), '123.')
def test_scope_queue_name(self):
self.assertEqual(utils.scope_queue_name('my-q'), '.my-q')
self.assertEqual(utils.scope_queue_name('my-q', None), '.my-q')
self.assertEqual(utils.scope_queue_name('my-q', '123'), '123.my-q')
self.assertEqual(utils.scope_queue_name(None), '.')
self.assertEqual(utils.scope_queue_name(None, '123'), '123.')
def delete(self, name, project=None):
# Note(prashanthr_): Pipelining is used to ensure no race conditions
# occur.
if not self.exists(name, project):
raise errors.QueueDoesNotExist(name, project)
q_id = utils.scope_queue_name(name, project)
qset_id = utils.scope_queue_name(QUEUES_SET_STORE_NAME, project)
pipe = self._pipeline
pipe.zrem(qset_id, q_id)
pipe.delete(q_id)
self.driver.message_controller._delete_queue_messages(name, project)
pipe.execute()
def delete(self, name, project=None):
# Note(prashanthr_): Pipelining is used to ensure no race conditions
# occur.
if not self.exists(name, project):
raise errors.QueueDoesNotExist(name, project)
q_id = utils.scope_queue_name(name, project)
qset_id = utils.scope_queue_name(QUEUES_SET_STORE_NAME, project)
pipe = self._pipeline
pipe.zrem(qset_id, q_id)
pipe.delete(q_id)
self.driver.message_controller._delete_queue_messages(name, project)
pipe.execute()
def get_metadata(self, name, project=None):
if not self.exists(name, project):
raise errors.QueueDoesNotExist(name, project)
q_id = utils.scope_queue_name(name, project)
metadata = self._get_queue_info(q_id, 'm')
return {} if utils.is_metadata_empty(metadata) else metadata
def create(self, name, project=None):
# Note(prashanthr_): Implement as a lua script.
q_id = utils.scope_queue_name(name, project)
qset_id = utils.scope_queue_name(QUEUES_SET_STORE_NAME, project)
pipe = self._pipeline
# Check if the queue already exists.
if self._client.zrank(qset_id, q_id) is not None:
return False
# Pipeline ensures atomic inserts.
q_info = {'c': 1, 'cl': 0, 'm': {}, 't': timeutils.utcnow_ts()}
pipe.zadd(qset_id, 1, q_id).hmset(q_id, q_info)
return all(map(bool, pipe.execute()))
def get_metadata(self, name, project=None):
if not self.exists(name, project):
raise errors.QueueDoesNotExist(name, project)
q_id = utils.scope_queue_name(name, project)
metadata = self._get_queue_info(q_id, 'm')
return {} if utils.is_metadata_empty(metadata) else metadata
def stats(self, name, project=None):
if not self.exists(name, project=project):
raise errors.QueueDoesNotExist(name, project)
q_id = utils.scope_queue_name(name, project)
q_info = self._client.hgetall(q_id)
claimed = int(q_info['cl'])
total = int(q_info['c'])
msg_ctrl = self.driver.message_controller
now = timeutils.utcnow_ts()
message_stats = {
'claimed': claimed,
'free': total - claimed,
'total': total
}
try:
newest = msg_ctrl.first(name, project, 1)
oldest = msg_ctrl.first(name, project, -1)
except errors.QueueIsEmpty:
pass
else:
message_stats['newest'] = utils.stat_message(newest, now)
message_stats['oldest'] = utils.stat_message(oldest, now)
return {'messages': message_stats}
def stats(self, name, project=None):
if not self.exists(name, project=project):
raise errors.QueueDoesNotExist(name, project)
q_id = utils.scope_queue_name(name, project)
q_info = self._client.hgetall(q_id)
claimed = int(q_info['cl'])
total = int(q_info['c'])
msg_ctrl = self.driver.message_controller
now = timeutils.utcnow_ts()
message_stats = {
'claimed': claimed,
'free': total-claimed,
'total': total
}
try:
newest = msg_ctrl.first(name, project, 1)
oldest = msg_ctrl.first(name, project, -1)
except errors.QueueIsEmpty:
pass
else:
message_stats['newest'] = utils.stat_message(newest, now)
message_stats['oldest'] = utils.stat_message(oldest, now)
return {'messages': message_stats}
def test_inc_claimed(self):
queue_name = 'inc-claimed'
q_controller = self.q_controller
q_controller.create(queue_name)
q_controller._inc_claimed(queue_name, None, 10)
scoped_q_name = utils.scope_queue_name(queue_name)
self.assertEqual(self.connection.hgetall(scoped_q_name)['cl'], '10')
def test_inc_claimed(self):
queue_name = 'inc-claimed'
q_controller = self.q_controller
q_controller.create(queue_name)
q_controller._inc_claimed(queue_name, None, 10)
scoped_q_name = utils.scope_queue_name(queue_name)
self.assertEqual(self.connection.hgetall(scoped_q_name)['cl'], '10')
def list(self, project=None, marker=None,
limit=storage.DEFAULT_QUEUES_PER_PAGE, detailed=False):
client = self._client
qset_id = utils.scope_queue_name(QUEUES_SET_STORE_NAME, project)
marker = utils.scope_queue_name(marker, project)
start = client.zrank(qset_id, marker) or 0
cursor = (q for q in client.zrange(qset_id, start,
start + (limit - 1)))
marker_next = {}
def denormalizer(q_info, q_name):
queue = {'name': utils.descope_queue_name(q_name)}
marker_next['next'] = queue['name']
if detailed:
queue['metadata'] = q_info[1]
return queue
yield utils.QueueListCursor(self._client, cursor, denormalizer)
yield marker_next and marker_next['next']
def set_metadata(self, name, metadata, project=None):
if not self.exists(name, project):
raise errors.QueueDoesNotExist(name, project)
q_id = utils.scope_queue_name(name, project)
q_info = {'c': 1, 'cl': 0, 'm': metadata, 't': timeutils.utcnow_ts()}
if self.exists(name, project):
q_info['c'] = self._get_queue_info(q_id, 'c')
q_info['cl'] = self._get_queue_info(q_id, 'cl')
self._client.hmset(q_id, q_info)
def create(self, name, project=None):
# Note(prashanthr_): Implement as a lua script.
q_id = utils.scope_queue_name(name, project)
qset_id = utils.scope_queue_name(QUEUES_SET_STORE_NAME, project)
pipe = self._pipeline
# Check if the queue already exists.
if self._client.zrank(qset_id, q_id) is not None:
return False
# Pipeline ensures atomic inserts.
q_info = {
'c': 1,
'cl': 0,
'm': {},
't': timeutils.utcnow_ts()
}
pipe.zadd(qset_id, 1, q_id).hmset(q_id, q_info)
return all(map(bool, pipe.execute()))
def list(self,
project=None,
marker=None,
limit=storage.DEFAULT_QUEUES_PER_PAGE,
detailed=False):
client = self._client
qset_id = utils.scope_queue_name(QUEUES_SET_STORE_NAME, project)
marker = utils.scope_queue_name(marker, project)
start = client.zrank(qset_id, marker) or 0
cursor = (q
for q in client.zrange(qset_id, start, start + (limit - 1)))
marker_next = {}
def denormalizer(q_info, q_name):
queue = {'name': utils.descope_queue_name(q_name)}
marker_next['next'] = queue['name']
if detailed:
queue['metadata'] = q_info[1]
return queue
yield utils.QueueListCursor(self._client, cursor, denormalizer)
yield marker_next and marker_next['next']
def set_metadata(self, name, metadata, project=None):
if not self.exists(name, project):
raise errors.QueueDoesNotExist(name, project)
q_id = utils.scope_queue_name(name, project)
q_info = {
'c': 1,
'cl': 0,
'm': metadata,
't': timeutils.utcnow_ts()
}
if self.exists(name, project):
q_info['c'] = self._get_queue_info(q_id, 'c')
q_info['cl'] = self._get_queue_info(q_id, 'cl')
self._client.hmset(q_id, q_info)
def _inc_claimed(self, name, project, amount=1):
q_id = utils.scope_queue_name(name, project)
cl_count = self._get_queue_info(q_id, 'cl', int)
self._set_claim_counter(q_id, cl_count + amount)
def exists(self, name, project=None):
# Note(prashanthr_): try to fit in caching if possible.
q_id = utils.scope_queue_name(name, project)
qset_id = utils.scope_queue_name(QUEUES_SET_STORE_NAME, project)
return self._client.zrank(qset_id, q_id) is not None
def exists(self, name, project=None):
# Note(prashanthr_): try to fit in caching if possible.
q_id = utils.scope_queue_name(name, project)
qset_id = utils.scope_queue_name(QUEUES_SET_STORE_NAME, project)
return self._client.zrank(qset_id, q_id) is not None
def _inc_claimed(self, name, project, amount=1):
q_id = utils.scope_queue_name(name, project)
cl_count = self._get_queue_info(q_id, 'cl', int)
self._set_claim_counter(q_id, cl_count+amount)