def get(self, queue, claim_id, project=None):
if not self._exists(queue, claim_id, project):
raise errors.ClaimDoesNotExist(queue, project, claim_id)
claim_msgs_key = utils.scope_claim_messages(claim_id,
CLAIM_MESSAGES_SUFFIX)
# basic_messages
msg_keys = self._get_claimed_message_keys(claim_msgs_key)
claimed_msgs = messages.Message.from_redis_bulk(msg_keys,
self._client)
now = timeutils.utcnow_ts()
basic_messages = [msg.to_basic(now)
for msg in claimed_msgs if msg]
# claim_meta
now = timeutils.utcnow_ts()
expires, ttl = self._get_claim_info(claim_id, [b'e', b't'])
update_time = expires - ttl
age = now - update_time
claim_meta = {
'age': age,
'ttl': ttl,
'id': claim_id,
}
return claim_meta, basic_messages
def test_set_time_override_using_default(self):
now = timeutils.utcnow_ts()
# NOTE(kgriffs): Normally it's bad form to sleep in a unit test,
# but this is the only way to test that set_time_override defaults
# to setting the override to the current time.
time.sleep(1)
timeutils.set_time_override()
overriden_now = timeutils.utcnow_ts()
self.assertThat(now, matchers.LessThan(overriden_now))
def test_utcnow_ts(self):
skynet_self_aware_ts = 872835240
skynet_dt = datetime.datetime.utcfromtimestamp(skynet_self_aware_ts)
self.assertEqual(self.skynet_self_aware_time, skynet_dt)
# NOTE(kgriffs): timeutils.utcnow_ts() uses time.time()
# IFF time override is not set.
with mock.patch('time.time') as time_mock:
time_mock.return_value = skynet_self_aware_ts
ts = timeutils.utcnow_ts()
self.assertEqual(ts, skynet_self_aware_ts)
timeutils.set_time_override(skynet_dt)
ts = timeutils.utcnow_ts()
self.assertEqual(ts, skynet_self_aware_ts)
def test_authorize_object_already_created(self, mock_create):
# the expires time is calculated from the current time and
# a ttl value in the object. Fix the current time so we can
# test expires is calculated correctly as expected
self.addCleanup(timeutils.clear_time_override)
timeutils.set_time_override()
ttl = 10
expires = timeutils.utcnow_ts() + ttl
db_dict = copy.deepcopy(fakes.fake_token_dict)
db_dict['expires'] = expires
mock_create.return_value = db_dict
obj = token_obj.ConsoleAuthToken(
context=self.context,
console_type=fakes.fake_token_dict['console_type'],
host=fakes.fake_token_dict['host'],
port=fakes.fake_token_dict['port'],
internal_access_path=fakes.fake_token_dict['internal_access_path'],
instance_uuid=fakes.fake_token_dict['instance_uuid'],
access_url_base=fakes.fake_token_dict['access_url_base'],
)
obj.authorize(100)
self.assertRaises(exception.ObjectActionError,
obj.authorize,
100)
def _exists(self, queue, claim_id, project):
client = self._client
claims_set_key = utils.scope_claims_set(queue, project,
QUEUE_CLAIMS_SUFFIX)
# In some cases, the queue maybe doesn't exist. So we should check
# whether the queue exists. Return False if no such queue exists.
# Todo(flwang): We should delete all related data after the queue is
# deleted. See the blueprint for more detail:
# https://blueprints.launchpad.net/zaqar/+spec/clear-resources-after-delete-queue
if not self._queue_ctrl._exists(queue, project):
return False
# Return False if no such claim exists
# TODO(prashanthr_): Discuss the feasibility of a bloom filter.
if client.zscore(claims_set_key, claim_id) is None:
return False
expires = self._get_claim_info(claim_id, b'e')[0]
now = timeutils.utcnow_ts()
if expires <= now:
# NOTE(kgriffs): Redis should automatically remove the
# other records in the very near future. This one
# has to be manually deleted, however.
client.zrem(claims_set_key, claim_id)
return False
return True
def _clear(self):
"""Expunges expired keys."""
now = timeutils.utcnow_ts()
for k in list(self.cache):
(_value, timeout) = self.cache[k]
if timeout > 0 and now >= timeout:
del self.cache[k]
def _purge_metering_info(self):
deadline_timestamp = timeutils.utcnow_ts() - self.conf.report_interval
label_ids = [
label_id for label_id, info in self.metering_infos.items() if info["last_update"] < deadline_timestamp
]
for label_id in label_ids:
del self.metering_infos[label_id]
def _create(self, name, metadata=None, project=None):
# TODO(prashanthr_): Implement as a lua script.
queue_key = utils.scope_queue_name(name, project)
qset_key = utils.scope_queue_name(QUEUES_SET_STORE_NAME, project)
# Check if the queue already exists.
if self._exists(name, project):
return False
queue = {
'c': 0,
'cl': 0,
'm': self._packer(metadata or {}),
't': timeutils.utcnow_ts()
}
# Pipeline ensures atomic inserts.
with self._client.pipeline() as pipe:
pipe.zadd(qset_key, 1, queue_key).hmset(queue_key, queue)
try:
pipe.execute()
except redis.exceptions.ResponseError:
return False
return True
def _get_unlocked(self, key, default=None):
now = timeutils.utcnow_ts()
try:
timeout, value = self._cache[key]
except KeyError:
return (0, default)
if timeout and now >= timeout:
# NOTE(flaper87): Record expired,
# remove it from the cache but catch
# KeyError and ValueError in case
# _purge_expired removed this key already.
try:
del self._cache[key]
except KeyError:
pass
try:
# NOTE(flaper87): Keys with ttl == 0
# don't exist in the _keys_expires dict
self._keys_expires[timeout].remove(key)
except (KeyError, ValueError):
pass
return (0, default)
return (timeout, value)
def _exists_unlocked(self, key):
now = timeutils.utcnow_ts()
try:
timeout = self._cache[key][0]
return not timeout or now <= timeout
except KeyError:
return False
def delete(self, queue, claim_id, project=None):
# NOTE(prashanthr_): Return silently when the claim
# does not exist
if not self._exists(queue, claim_id, project):
return
now = timeutils.utcnow_ts()
claim_msgs_key = utils.scope_claim_messages(claim_id,
CLAIM_MESSAGES_SUFFIX)
msg_keys = self._get_claimed_message_keys(claim_msgs_key)
claimed_msgs = messages.MessageEnvelope.from_redis_bulk(msg_keys,
self._client)
# Update the claim id and claim expiration info
# for all the messages.
claims_set_key = utils.scope_claims_set(queue, project,
QUEUE_CLAIMS_SUFFIX)
with self._client.pipeline() as pipe:
pipe.zrem(claims_set_key, claim_id)
pipe.delete(claim_id)
pipe.delete(claim_msgs_key)
for msg in claimed_msgs:
if msg:
msg.claim_id = None
msg.claim_expires = now
# TODO(kgriffs): Rather than writing back the
# entire message, only set the fields that
# have changed.
msg.to_redis(pipe)
pipe.execute()
def destroy_instance(instance_id, environment_id):
unit = db_session.get_session()
instance = unit.query(models.Instance).get(
(environment_id, instance_id))
if instance and not instance.destroyed:
instance.destroyed = timeutils.utcnow_ts()
instance.save(unit)
def _count(self, queue_name, project=None, include_claimed=False):
"""Return total number of messages in a queue.
This method is designed to very quickly count the number
of messages in a given queue. Expired messages are not
counted, of course. If the queue does not exist, the
count will always be 0.
Note: Some expired messages may be included in the count if
they haven't been GC'd yet. This is done for performance.
"""
query = {
# Messages must belong to this queue and project.
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
# NOTE(kgriffs): Messages must be finalized (i.e., must not
# be part of an unfinalized transaction).
#
# See also the note wrt 'tx' within the definition
# of ACTIVE_INDEX_FIELDS.
'tx': None,
}
if not include_claimed:
# Exclude messages that are claimed
query['c.e'] = {'$lte': timeutils.utcnow_ts()}
collection = self._collection(queue_name, project)
return collection.count(filter=query, hint=COUNTING_INDEX_FIELDS)
def post(self, queue, messages, client_uuid, project=None):
msgset_key = utils.msgset_key(queue, project)
counter_key = utils.scope_queue_index(queue, project,
MESSAGE_RANK_COUNTER_SUFFIX)
message_ids = []
now = timeutils.utcnow_ts()
with self._client.pipeline() as pipe:
for msg in messages:
prepared_msg = Message(
ttl=msg['ttl'],
created=now,
client_uuid=client_uuid,
claim_id=None,
claim_expires=now,
body=msg.get('body', {}),
)
prepared_msg.to_redis(pipe)
message_ids.append(prepared_msg.id)
pipe.execute()
# NOTE(kgriffs): If this call fails, we will return
# an error to the client and the messages will be
# orphaned, but Redis will remove them when they
# expire, so we will just pretend they don't exist
# in that case.
self._index_messages(msgset_key, counter_key, message_ids)
return message_ids
def _get_claim(self, message_id):
"""Gets minimal claim doc for a message.
:returns: {'id': cid, 'expires': ts} IFF the message is claimed,
and that claim has not expired.
"""
claim = self._client.hmget(message_id, 'c', 'c.e')
if claim == [None, None]:
# NOTE(kgriffs): message_id was not found
return None
info = {
# NOTE(kgriffs): A "None" claim is serialized as an empty str
'id': encodeutils.safe_decode(claim[0]) or None,
'expires': int(claim[1]),
}
# Is the message claimed?
now = timeutils.utcnow_ts()
if info['id'] and (now < info['expires']):
return info
# Not claimed
return None
def get(self, queue, claim_id, project=None):
message_ctrl = self.driver.message_controller
now = timeutils.utcnow_ts(True)
self._exists(queue, claim_id, project)
container = utils._claim_container(queue, project)
headers, claim_obj = self._client.get_object(container, claim_id)
def g():
for msg_id in jsonutils.loads(claim_obj):
try:
headers, msg = message_ctrl._find_message(queue, msg_id,
project)
except errors.MessageDoesNotExist:
continue
else:
yield utils._message_to_json(msg_id, msg, headers, now)
claim_meta = {
'id': claim_id,
'age': now - float(headers['x-timestamp']),
'ttl': int(headers['x-delete-at']) - math.floor(now),
}
return claim_meta, g()
def _filter_messages(messages, filters, to_basic, marker):
"""Create a filtering iterator over a list of messages.
The function accepts a list of filters to be filtered
before the the message can be included as a part of the reply.
"""
now = timeutils.utcnow_ts()
for msg in messages:
# NOTE(kgriffs): Message may have been deleted, so
# check each value to ensure we got a message back
if msg is None:
continue
# NOTE(kgriffs): Check to see if any of the filters
# indiciate that this message should be skipped.
for should_skip in filters:
if should_skip(msg):
break
else:
marker['next'] = msg.id
if to_basic:
yield msg.to_basic(now)
else:
yield msg
def track_instance(instance_id, environment_id, instance_type,
type_name, type_title=None, unit_count=None):
unit = db_session.get_session()
try:
with unit.begin():
env = unit.query(models.Environment).get(environment_id)
instance = models.Instance()
instance.instance_id = instance_id
instance.environment_id = environment_id
instance.tenant_id = env.tenant_id
instance.instance_type = instance_type
instance.created = timeutils.utcnow_ts()
instance.destroyed = None
instance.type_name = type_name
instance.type_title = type_title
instance.unit_count = unit_count
unit.add(instance)
except exception.DBDuplicateEntry:
unit.execute(
sqlalchemy.update(models.Instance).where(
models.Instance.instance_id == instance_id and
models.Instance.environment_id == environment_id).values(
unit_count=unit_count))
def get(self, queue, claim_id, project=None):
msg_ctrl = self.driver.message_controller
# Base query, always check expire time
now = timeutils.utcnow_ts()
cid = utils.to_oid(claim_id)
if cid is None:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
try:
# Lets get claim's data
# from the first message
# in the iterator
msgs = _messages_iter(msg_ctrl._claimed(queue, cid, now,
project=project))
claim = next(msgs)
update_time = claim['e'] - claim['t']
age = now - update_time
claim_meta = {
'age': int(age),
'ttl': claim['t'],
'id': str(claim['id']),
}
except StopIteration:
raise errors.ClaimDoesNotExist(cid, queue, project)
return claim_meta, msgs
def update(self, queue, subscription_id, project=None, **kwargs):
names = ('subscriber', 'ttl', 'options')
key_transform = lambda x: 'u' if x == 'subscriber' else x[0]
fields = common_utils.fields(kwargs, names,
pred=lambda x: x is not None,
key_transform=key_transform)
assert fields, ('`subscriber`, `ttl`, '
'or `options` not found in kwargs')
new_ttl = fields.get('t', None)
if new_ttl is not None:
now = timeutils.utcnow_ts()
now_dt = datetime.datetime.utcfromtimestamp(now)
expires = now_dt + datetime.timedelta(seconds=new_ttl)
fields['e'] = expires
try:
res = self._collection.update(
{'_id': utils.to_oid(subscription_id),
'p': project},
{'$set': fields},
upsert=False)
except pymongo.errors.DuplicateKeyError:
raise errors.SubscriptionAlreadyExists()
if not res['updatedExisting']:
raise errors.SubscriptionDoesNotExist(subscription_id)
def set(self, key, value, time=0, min_compress_len=0):
"""Sets the value for a key."""
timeout = 0
if time != 0:
timeout = timeutils.utcnow_ts() + time
self.cache[key] = (timeout, value)
return True
def put(self, key, value, time=CACHE_TIMEOUT):
"""Sets the value for a key."""
timeout = 0
if time != 0:
timeout = timeutils.utcnow_ts() + time
self._cache[key] = (timeout, value)
return True
def stats(self, name, project=None):
if not self.queue_controller.exists(name, project=project):
raise errors.QueueDoesNotExist(name, project)
controller = self.message_controller
active = controller._count(name, project=project,
include_claimed=False)
total = controller._count(name, project=project,
include_claimed=True)
message_stats = {
'claimed': total - active,
'free': active,
'total': total,
}
try:
oldest = controller.first(name, project=project, sort=1)
newest = controller.first(name, project=project, sort=-1)
except errors.QueueIsEmpty:
pass
else:
now = timeutils.utcnow_ts()
message_stats['oldest'] = utils.stat_message(oldest, now)
message_stats['newest'] = utils.stat_message(newest, now)
return {'messages': message_stats}
def list(self, queue_name, project=None, marker=None,
limit=storage.DEFAULT_MESSAGES_PER_PAGE,
echo=False, client_uuid=None, include_claimed=False,
include_delayed=False):
if marker is not None:
try:
marker = int(marker)
except ValueError:
yield iter([])
messages = self._list(queue_name, project=project, marker=marker,
client_uuid=client_uuid, echo=echo,
include_claimed=include_claimed,
include_delayed=include_delayed, limit=limit)
marker_id = {}
now = timeutils.utcnow_ts()
# NOTE (kgriffs) @utils.raises_conn_error not needed on this
# function, since utils.HookedCursor already has it.
def denormalizer(msg):
marker_id['next'] = msg['k']
return _basic_message(msg, now)
yield utils.HookedCursor(messages, denormalizer)
yield str(marker_id['next'])
def create(self, queue, subscriber, ttl, options, project=None):
subscription_id = str(uuid.uuid4())
subset_key = utils.scope_subscription_ids_set(queue,
project,
SUBSCRIPTION_IDS_SUFFIX)
source = queue
now = timeutils.utcnow_ts()
ttl = int(ttl)
expires = now + ttl
subscription = {'id': subscription_id,
's': source,
'u': subscriber,
't': ttl,
'e': expires,
'o': options,
'p': project}
if not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
try:
# Pipeline ensures atomic inserts.
with self._client.pipeline() as pipe:
pipe.zadd(subset_key, 1,
subscription_id).hmset(subscription_id,
subscription)
pipe.execute()
return subscription_id
except redis.exceptions.ResponseError:
return None
def _event_transform(self, event, tenant_id, _region):
return dict(
event=1,
meta=dict(
tenantId='0ab1ac0a-2867-402d',
region='useast'),
creation_time=timeutils.utcnow_ts())
def create(self, queue, subscriber, ttl, options, project=None):
subscription_id = uuidutils.generate_uuid()
subset_key = utils.scope_subscription_ids_set(queue,
project,
SUBSCRIPTION_IDS_SUFFIX)
source = queue
now = timeutils.utcnow_ts()
expires = now + ttl
confirmed = 0
subscription = {'id': subscription_id,
's': source,
'u': subscriber,
't': ttl,
'e': expires,
'o': self._packer(options),
'p': project,
'c': confirmed}
try:
# Pipeline ensures atomic inserts.
with self._client.pipeline() as pipe:
if not self._is_duplicated_subscriber(subscriber,
queue,
project):
pipe.zadd(subset_key, {subscription_id: 1}).hmset(
subscription_id, subscription)
pipe.expire(subscription_id, ttl)
pipe.execute()
else:
return None
return subscription_id
except redis.exceptions.ResponseError:
return None
def test_gc(self):
self.queue_controller.create(self.queue_name)
for _ in range(100):
self.message_controller.post(self.queue_name,
[{'ttl': 300, 'body': 'yo gabba'}],
client_uuid=uuidutils.generate_uuid())
now = timeutils.utcnow_ts()
timeutils_utcnow = 'oslo_utils.timeutils.utcnow_ts'
# Test a single claim
with mock.patch(timeutils_utcnow) as mock_utcnow:
mock_utcnow.return_value = now - 1
self.controller.create(self.queue_name, {'ttl': 1, 'grace': 60})
num_removed = self.controller._gc(self.queue_name, None)
self.assertEqual(1, num_removed)
# Test multiple claims
with mock.patch(timeutils_utcnow) as mock_utcnow:
mock_utcnow.return_value = now - 1
for _ in range(5):
self.controller.create(self.queue_name,
{'ttl': 1, 'grace': 60})
# NOTE(kgriffs): These ones should not be cleaned up
self.controller.create(self.queue_name, {'ttl': 60, 'grace': 60})
self.controller.create(self.queue_name, {'ttl': 60, 'grace': 60})
num_removed = self.controller._gc(self.queue_name, None)
self.assertEqual(5, num_removed)
def create(self, queue, metadata, project=None,
limit=storage.DEFAULT_MESSAGES_PER_CLAIM):
claim_ttl = metadata['ttl']
grace = metadata['grace']
now = timeutils.utcnow_ts()
msg_ttl = claim_ttl + grace
claim_expires = now + claim_ttl
msg_expires = claim_expires + grace
claim_id = uuidutils.generate_uuid()
claimed_msgs = []
# NOTE(kgriffs): Claim some messages
msgset_key = utils.msgset_key(queue, project)
claimed_ids = self._claim_messages(msgset_key, now, limit,
claim_id, claim_expires,
msg_ttl, msg_expires)
if claimed_ids:
claimed_msgs = messages.Message.from_redis_bulk(claimed_ids,
self._client)
claimed_msgs = [msg.to_basic(now) for msg in claimed_msgs]
# NOTE(kgriffs): Perist claim records
with self._client.pipeline() as pipe:
claim_msgs_key = utils.scope_claim_messages(
claim_id, CLAIM_MESSAGES_SUFFIX)
for mid in claimed_ids:
pipe.rpush(claim_msgs_key, mid)
pipe.expire(claim_msgs_key, claim_ttl)
claim_info = {
'id': claim_id,
't': claim_ttl,
'e': claim_expires,
'n': len(claimed_ids),
}
pipe.hmset(claim_id, claim_info)
pipe.expire(claim_id, claim_ttl)
# NOTE(kgriffs): Add the claim ID to a set so that
# existence checks can be performed quickly. This
# is also used as a watch key in order to gaurd
# against race conditions.
#
# A sorted set is used to facilitate cleaning
# up the IDs of expired claims.
claims_set_key = utils.scope_claims_set(queue, project,
QUEUE_CLAIMS_SUFFIX)
pipe.zadd(claims_set_key, claim_expires, claim_id)
pipe.execute()
return claim_id, claimed_msgs
def _get(self, queue, message_id, project=None, check_queue=True):
if check_queue and not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue, project)
now = timeutils.utcnow_ts(True)
headers, msg = self._find_message(queue, message_id, project)
return utils._message_to_json(message_id, msg, headers, now)
def first(self, queue_name, project=None, sort=1):
cursor = self._list(queue_name,
project=project,
include_claimed=True,
sort=sort,
limit=1)
try:
message = next(cursor)
except StopIteration:
raise errors.QueueIsEmpty(queue_name, project)
now = timeutils.utcnow_ts()
return _basic_message(message, now)
def _claimed(self,
queue_name,
claim_id,
expires=None,
limit=None,
project=None):
if claim_id is None:
claim_id = {'$ne': None}
query = {
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
'c.id': claim_id,
'c.e': {
'$gt': expires or timeutils.utcnow_ts()
},
}
# NOTE(kgriffs): Claimed messages bust be queried from
# the primary to avoid a race condition caused by the
# multi-phased "create claim" algorithm.
preference = pymongo.read_preferences.ReadPreference.PRIMARY
collection = self._collection(queue_name, project)
msgs = collection.find(
query, sort=[('k', 1)],
read_preference=preference).hint(CLAIMED_INDEX_FIELDS)
if limit is not None:
msgs = msgs.limit(limit)
now = timeutils.utcnow_ts()
def denormalizer(msg):
doc = _basic_message(msg, now)
doc['claim'] = msg['c']
return doc
return utils.HookedCursor(msgs, denormalizer)
def subscription_patching(self, subscription):
"""Restrictions on an update of subscription.
:param subscription: dict of subscription
:raises: ValidationFailed if the subscription is invalid.
"""
if not subscription:
raise ValidationFailed(_(u'No subscription to create.'))
subscriber = subscription.get('subscriber', None)
subscriber_type = None
if subscriber:
parsed_uri = six.moves.urllib_parse.urlparse(subscriber)
subscriber_type = parsed_uri.scheme
if subscriber_type not in self._limits_conf.subscriber_types:
msg = _(u'The subscriber type of subscription must be '
u'supported in the list {0}.')
raise ValidationFailed(msg, self._limits_conf.subscriber_types)
options = subscription.get('options', None)
if options and not isinstance(options, dict):
msg = _(u'Options must be a dict.')
raise ValidationFailed(msg)
ttl = subscription.get('ttl', None)
if ttl:
if not isinstance(ttl, int):
msg = _(u'TTL must be an integer.')
raise ValidationFailed(msg)
if ttl < MIN_SUBSCRIPTION_TTL:
msg = _(u'The TTL for a subscription '
'must be at least {0} seconds long.')
raise ValidationFailed(msg, MIN_SUBSCRIPTION_TTL)
# NOTE(flwang): By this change, technically, user can set a very
# big TTL so as to get a very long subscription.
now = timeutils.utcnow_ts()
now_dt = datetime.datetime.utcfromtimestamp(now)
msg = _(u'The TTL seconds for a subscription plus current time'
' must be less than {0}.')
try:
# NOTE(flwang): If below expression works, then we believe the
# ttl is acceptable otherwise it exceeds the max time of
# python.
now_dt + datetime.timedelta(seconds=ttl)
except OverflowError:
raise ValidationFailed(msg, datetime.datetime.max)
def transform(metrics, tenant_id, region):
transformed_metric = {'metric': {},
'meta': {'tenantId': tenant_id, 'region': region},
'creation_time': timeutils.utcnow_ts()}
if isinstance(metrics, list):
transformed_metrics = []
for metric in metrics:
transformed_metric['metric'] = metric
transformed_metrics.append(rest_utils.as_json(transformed_metric))
return transformed_metrics
else:
transformed_metric['metric'] = metrics
return [rest_utils.as_json(transformed_metric)]
def _get(self, queue, claim_id, project=None):
try:
container = utils._claim_container(queue, project)
headers, claim = self._client.get_object(container, claim_id)
except swiftclient.ClientException as exc:
if exc.http_status != 404:
raise
return
now = timeutils.utcnow_ts(True)
return {
'id': claim_id,
'age': now - float(headers['x-timestamp']),
'ttl': int(headers['x-delete-at']) - math.floor(now),
}
def _gc(self, queue, project):
"""Garbage-collect expired claim data.
Not all claim data can be automatically expired. This method
cleans up the remainder.
:returns: Number of claims removed
"""
claims_set_key = utils.scope_claims_set(queue, project,
QUEUE_CLAIMS_SUFFIX)
now = timeutils.utcnow_ts()
num_removed = self._client.zremrangebyscore(claims_set_key, 0, now)
return num_removed
def _list(self, queue, project=None, marker=None,
limit=storage.DEFAULT_MESSAGES_PER_PAGE,
echo=False, client_uuid=None,
include_claimed=False,
to_basic=True):
if not self._queue_ctrl.exists(queue, project):
raise errors.QueueDoesNotExist(queue,
project)
msgset_key = utils.msgset_key(queue, project)
client = self._client
if not marker and not include_claimed:
# NOTE(kgriffs): Skip claimed messages at the head
# of the queue; otherwise we would just filter them all
# out and likely end up with an empty list to return.
marker = self._find_first_unclaimed(queue, project, limit)
start = client.zrank(msgset_key, marker) or 0
else:
rank = client.zrank(msgset_key, marker)
start = rank + 1 if rank else 0
message_ids = client.zrange(msgset_key, start,
start + (limit - 1))
messages = Message.from_redis_bulk(message_ids, client)
# NOTE(prashanthr_): Build a list of filters for checking
# the following:
#
# 1. Message is expired
# 2. Message is claimed
# 3. Message should not be echoed
#
now = timeutils.utcnow_ts()
filters = [functools.partial(utils.msg_expired_filter, now=now)]
if not include_claimed:
filters.append(functools.partial(utils.msg_claimed_filter,
now=now))
if not echo:
filters.append(functools.partial(utils.msg_echo_filter,
client_uuid=client_uuid))
marker = {}
yield _filter_messages(messages, filters, to_basic, marker)
yield marker['next']
def post(self, queue_name, messages, client_uuid, project=None):
# NOTE(flaper87): This method should be safe to retry on
# autoreconnect, since we've a 2-step insert for messages.
# The worst-case scenario is that we'll increase the counter
# several times and we'd end up with some non-active messages.
if not self._queue_ctrl.exists(queue_name, project):
raise errors.QueueDoesNotExist(queue_name, project)
# NOTE(flaper87): Make sure the counter exists. This method
# is an upsert.
self._get_counter(queue_name, project)
now = timeutils.utcnow_ts()
now_dt = datetime.datetime.utcfromtimestamp(now)
collection = self._collection(queue_name, project)
messages = list(messages)
msgs_n = len(messages)
next_marker = self._inc_counter(queue_name, project,
amount=msgs_n) - msgs_n
prepared_messages = [{
PROJ_QUEUE:
utils.scope_queue_name(queue_name, project),
't':
message['ttl'],
'e':
now_dt + datetime.timedelta(seconds=message['ttl']),
'u':
client_uuid,
'c': {
'id': None,
'e': now,
'c': 0
},
'd':
now + message.get('delay', 0),
'b':
message['body'] if 'body' in message else {},
'k':
next_marker + index,
'tx':
None,
} for index, message in enumerate(messages)]
res = collection.insert_many(prepared_messages,
bypass_document_validation=True)
return [str(id_) for id_ in res.inserted_ids]
def first(self, queue, project=None, sort=1):
if sort not in (1, -1):
raise ValueError(u'sort must be either 1 (ascending) '
u'or -1 (descending)')
message_id = self._get_first_message_id(queue, project, sort)
if not message_id:
raise errors.QueueIsEmpty(queue, project)
message = Message.from_redis(message_id, self._client)
if message is None:
raise errors.QueueIsEmpty(queue, project)
now = timeutils.utcnow_ts()
return message.to_basic(now, include_created=True)
def _add_metering_info(self, label_id, pkts, bytes):
ts = timeutils.utcnow_ts()
info = self.metering_infos.get(label_id, {'bytes': 0,
'pkts': 0,
'time': 0,
'first_update': ts,
'last_update': ts})
info['bytes'] += bytes
info['pkts'] += pkts
info['time'] += ts - info['last_update']
info['last_update'] = ts
self.metering_infos[label_id] = info
return info
def get_aggregated_stats(environment_id):
unit = db_session.get_session()
now = timeutils.utcnow_ts()
query = unit.query(models.Instance.instance_type, func.sum(
func.coalesce(models.Instance.destroyed, now) -
models.Instance.created), func.count()).filter(
models.Instance.environment_id == environment_id)
res = query.group_by(models.Instance.instance_type).all()
return [{
'type': int(record[0]),
'duration': int(record[1]),
'count': int(record[2])
} for record in res]
def stats(self, name, project=None):
if not self._queue_ctrl.exists(name, project=project):
raise errors.QueueDoesNotExist(name, project)
total = 0
claimed = 0
container = utils._message_container(name, project)
try:
_, objects = self._client.get_container(container)
except swiftclient.ClientException as exc:
if exc.http_status == 404:
raise errors.QueueIsEmpty(name, project)
newest = None
oldest = None
now = timeutils.utcnow_ts(True)
for obj in objects:
try:
headers = self._client.head_object(container, obj['name'])
except swiftclient.ClientException as exc:
if exc.http_status != 404:
raise
else:
created = float(headers['x-timestamp'])
created_iso = datetime.datetime.utcfromtimestamp(
created).strftime('%Y-%m-%dT%H:%M:%SZ')
newest = {
'id': obj['name'],
'age': now - created,
'created': created_iso
}
if oldest is None:
oldest = copy.deepcopy(newest)
total += 1
if headers.get('x-object-meta-claimid'):
claimed += 1
msg_stats = {
'claimed': claimed,
'free': total - claimed,
'total': total,
}
if newest is not None:
msg_stats['newest'] = newest
msg_stats['oldest'] = oldest
return {'messages': msg_stats}
def delete(self, queue_name, message_id, project=None, claim=None):
# NOTE(cpp-cabrera): return early - this is an invalid message
# id so we won't be able to find it any way
mid = utils.to_oid(message_id)
if mid is None:
return
collection = self._collection(queue_name, project)
query = {
'_id': mid,
PROJ_QUEUE: utils.scope_queue_name(queue_name, project),
}
cid = utils.to_oid(claim)
if cid is None:
raise errors.ClaimDoesNotExist(claim, queue_name, project)
now = timeutils.utcnow_ts()
cursor = collection.find(query).hint(ID_INDEX_FIELDS)
try:
message = next(cursor)
except StopIteration:
return
if claim is None:
if _is_claimed(message, now):
raise errors.MessageIsClaimed(message_id)
else:
if message['c']['id'] != cid:
kwargs = {}
# NOTE(flaper87): In pymongo 3.0 PRIMARY is the default and
# `read_preference` is read only. We'd need to set it when the
# client is created.
# NOTE(kgriffs): Read from primary in case the message
# was just barely claimed, and claim hasn't made it to
# the secondary.
message = collection.find_one(query, **kwargs)
if message['c']['id'] != cid:
if _is_claimed(message, now):
raise errors.MessageNotClaimedBy(message_id, claim)
raise errors.MessageNotClaimed(message_id)
collection.remove(query['_id'], w=0)
def set_status(self, status, force=False):
"""Use conductor to update the DB app status."""
if force or self.is_installed:
LOG.debug("Casting set_status message to conductor "
"(status is '%s').", status.description)
context = trove_context.TroveContext()
heartbeat = {'service_status': status.description}
conductor_api.API(context).heartbeat(
CONF.guest_id, heartbeat,
sent=timeutils.utcnow_ts(microsecond=True))
LOG.debug("Successfully cast set_status.")
self.status = status
else:
LOG.debug("Prepare has not completed yet, skipping heartbeat.")
def denormalizer(record, sid):
now = timeutils.utcnow_ts()
ttl = int(record[2])
expires = int(record[3])
created = expires - ttl
ret = {
'id': sid,
'source': record[0],
'subscriber': record[1],
'ttl': ttl,
'age': now - created,
'options': self._unpacker(record[4]),
}
marker_next['next'] = sid
return ret
def test_notification_info_update_timestamp(self):
self.member_manager.hashring = mock.Mock()
ctxt, publisher_id, event_type, payload, metadata = fake_notification()
# Set an old timestamp, and insert into members
payload['timestamp'] = self.old_timestamp
self.member_manager.members.append(copy.deepcopy(payload))
# Reset timestamp, and simulate notification, add_node not called
# Timestamp in member manager is updated.
payload['timestamp'] = timeutils.utcnow_ts()
self.assertNotEqual(payload['timestamp'],
self.member_manager.members[0]['timestamp'])
self.member_manager.info(ctxt, publisher_id, event_type, payload,
metadata)
self.member_manager.hashring.add_node.assert_not_called()
self.assertEqual(payload['timestamp'],
self.member_manager.members[0]['timestamp'])
def bulk_get(self, queue, message_ids, project=None):
if not self._queue_ctrl.exists(queue, project):
return iter([])
# NOTE(prashanthr_): Pipelining is used here purely
# for performance.
with self._client.pipeline() as pipe:
for mid in message_ids:
pipe.hgetall(mid)
messages = pipe.execute()
# NOTE(kgriffs): Skip messages that may have been deleted
now = timeutils.utcnow_ts()
return (Message.from_hmap(msg).to_basic(now)
for msg in messages if msg)
def update(self, queue, claim_id, metadata, project=None):
cid = utils.to_oid(claim_id)
if cid is None:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
now = timeutils.utcnow_ts()
grace = metadata['grace']
ttl = metadata['ttl']
claim_expires = now + ttl
claim_expires_dt = datetime.datetime.utcfromtimestamp(claim_expires)
message_ttl = ttl + grace
message_expires = datetime.datetime.utcfromtimestamp(
claim_expires + grace)
msg_ctrl = self.driver.message_controller
claimed = msg_ctrl._claimed(queue, cid, expires=now,
limit=1, project=project)
try:
next(claimed)
except StopIteration:
raise errors.ClaimDoesNotExist(claim_id, queue, project)
meta = {
'id': cid,
't': ttl,
'e': claim_expires,
}
# TODO(kgriffs): Create methods for these so we don't interact
# with the messages collection directly (loose coupling)
scope = utils.scope_queue_name(queue, project)
collection = msg_ctrl._collection(queue, project)
collection.update_many({'p_q': scope, 'c.id': cid},
{'$set': {'c': meta}},
upsert=False)
# NOTE(flaper87): Dirty hack!
# This sets the expiration time to
# `expires` on messages that would
# expire before claim.
collection.update_many({'p_q': scope,
'e': {'$lt': claim_expires_dt},
'c.id': cid},
{'$set': {'e': message_expires,
't': message_ttl}},
upsert=False)
def create_in_section(cls,
checkpoints_section,
indices_section,
bank_lease,
owner_id,
plan,
checkpoint_id=None):
checkpoint_id = checkpoint_id or cls._generate_id()
checkpoint_section = checkpoints_section.get_sub_section(checkpoint_id)
timestamp = timeutils.utcnow_ts()
created_at = timeutils.utcnow().strftime('%Y-%m-%d')
checkpoint_section.create_object(
key=_INDEX_FILE_NAME,
value={
"version": cls.VERSION,
"id": checkpoint_id,
"status": constants.CHECKPOINT_STATUS_PROTECTING,
"owner_id": owner_id,
"provider_id": plan.get("provider_id"),
"project_id": plan.get("project_id"),
"protection_plan": {
"id": plan.get("id"),
"name": plan.get("name"),
"provider_id": plan.get("provider_id"),
"resources": plan.get("resources")
},
"created_at": created_at,
"timestamp": timestamp
})
indices_section.create_object(key="/by-provider/%s@%s" %
(timestamp, checkpoint_id),
value=checkpoint_id)
indices_section.create_object(key="/by-date/%s/%s@%s" %
(created_at, timestamp, checkpoint_id),
value=checkpoint_id)
indices_section.create_object(
key="/by-plan/%s/%s/%s@%s" %
(plan.get("id"), created_at, timestamp, checkpoint_id),
value=checkpoint_id)
return Checkpoint(checkpoint_section, indices_section, bank_lease,
checkpoint_id)
def cleanup_deleted_ports(self):
"""Cleanup the "self._deleted_ports" set based on the current TS
The variable "self._deleted_ports_ts" contains a timestamp
ordered list of tuples (timestamp, port_id). Every port older than the
current timestamp minus "timestamp_delta" will be deleted from
"self._deleted_ports" and "self._deleted_ports_ts".
"""
timestamp_min = timeutils.utcnow_ts() - DELETED_PORT_MAX_AGE
idx = None
for idx, (ts, port_id) in enumerate(self._deleted_ports_ts):
if ts > timestamp_min:
break
self._deleted_ports.remove(port_id)
if idx:
self._deleted_ports_ts = self._deleted_ports_ts[idx:]
def _create_msg(self, queue, msg, client_uuid, project):
slug = str(uuid.uuid1())
now = timeutils.utcnow_ts()
contents = jsonutils.dumps(
{'body': msg.get('body', {}), 'claim_id': None,
'ttl': msg['ttl'], 'claim_count': 0,
'delay_expires': now + msg.get('delay', 0)})
utils._put_or_create_container(
self._client,
utils._message_container(queue, project),
slug,
contents=contents,
content_type='application/json',
headers={
'x-object-meta-clientid': str(client_uuid),
'x-delete-after': msg['ttl']})
return slug
def update(self, queue, subscription_id, project=None, **kwargs):
names = ('subscriber', 'ttl', 'options')
key_transform = lambda x: 'u' if x == 'subscriber' else x[0]
fields = common_utils.fields(kwargs,
names,
pred=lambda x: x is not None,
key_transform=key_transform)
assert fields, ('`subscriber`, `ttl`, '
'or `options` not found in kwargs')
# Let's get our subscription by ID. If it does not exist,
# SubscriptionDoesNotExist error will be raised internally.
subscription_to_update = self.get(queue,
subscription_id,
project=project)
new_subscriber = fields.get('u', None)
# Let's do some checks to prevent subscription duplication.
if new_subscriber:
# Check if 'new_subscriber' is really new for our subscription.
if subscription_to_update['subscriber'] != new_subscriber:
# It's new. We should raise error if this subscriber already
# exists for the queue and project.
if self._is_duplicated_subscriber(new_subscriber, queue,
project):
raise errors.SubscriptionAlreadyExists()
# NOTE(Eva-i): if there are new options, we need to pack them before
# sending to the database.
new_options = fields.get('o', None)
if new_options is not None:
fields['o'] = self._packer(new_options)
new_ttl = fields.get('t', None)
if new_ttl is not None:
now = timeutils.utcnow_ts()
expires = now + new_ttl
fields['e'] = expires
# Pipeline ensures atomic inserts.
with self._client.pipeline() as pipe:
pipe.hmset(subscription_id, fields)
if new_ttl is not None:
pipe.expire(subscription_id, new_ttl)
pipe.execute()
def test_basic_message(self):
now = timeutils.utcnow_ts()
body = {
'msg': 'Hello Earthlings!',
'unicode': u'ab\u00e7',
'bytes': b'ab\xc3\xa7',
b'ab\xc3\xa7': 'one, two, three',
u'ab\u00e7': 'one, two, three',
}
msg = _create_sample_message(now=now, body=body)
basic_msg = msg.to_basic(now + 5)
self.assertEqual(msg.id, basic_msg['id'])
self.assertEqual(5, basic_msg['age'])
self.assertEqual(body, basic_msg['body'])
self.assertEqual(msg.ttl, basic_msg['ttl'])
def _filter_messages(messages, filters, marker, get_object, list_objects,
limit):
"""Create a filtering iterator over a list of messages.
The function accepts a list of filters to be filtered
before the the message can be included as a part of the reply.
"""
now = timeutils.utcnow_ts(True)
for msg in messages:
if msg is None:
continue
marker['next'] = msg['name']
try:
headers, obj = get_object(msg['name'])
except swiftclient.ClientException as exc:
if exc.http_status == 404:
continue
raise
obj = jsonutils.loads(obj)
for should_skip in filters:
if should_skip(obj, headers):
break
else:
limit -= 1
yield {
'id': marker['next'],
'ttl': obj['ttl'],
'client_uuid': headers['x-object-meta-clientid'],
'body': obj['body'],
'age': now - float(headers['x-timestamp']),
'claim_id': obj['claim_id'],
'claim_count': obj.get('claim_count', 0),
}
if limit <= 0:
break
if limit > 0 and marker:
# We haven't reached the limit, let's try to get some more messages
_, objects = list_objects(marker=marker['next'])
if not objects:
return
for msg in _filter_messages(objects, filters, marker, get_object,
list_objects, limit):
yield msg
def test_authorize(self, mock_create):
# the expires time is calculated from the current time and
# a ttl value in the object. Fix the current time so we can
# test expires is calculated correctly as expected
self.addCleanup(timeutils.clear_time_override)
timeutils.set_time_override()
ttl = 10
expires = timeutils.utcnow_ts() + ttl
db_dict = copy.deepcopy(fakes.fake_token_dict)
db_dict['expires'] = expires
mock_create.return_value = db_dict
create_dict = copy.deepcopy(fakes.fake_token_dict)
create_dict['expires'] = expires
del create_dict['id']
del create_dict['created_at']
del create_dict['updated_at']
expected = copy.deepcopy(fakes.fake_token_dict)
del expected['token_hash']
del expected['expires']
expected['token'] = fakes.fake_token
obj = token_obj.ConsoleAuthToken(
context=self.context,
console_type=fakes.fake_token_dict['console_type'],
host=fakes.fake_token_dict['host'],
port=fakes.fake_token_dict['port'],
internal_access_path=fakes.fake_token_dict['internal_access_path'],
instance_uuid=fakes.fake_token_dict['instance_uuid'],
access_url_base=fakes.fake_token_dict['access_url_base'],
)
with mock.patch('uuid.uuid4', return_value=fakes.fake_token):
token = obj.authorize(ttl)
mock_create.assert_called_once_with(self.context, create_dict)
self.assertEqual(token, fakes.fake_token)
self.compare_obj(obj, expected)
url = obj.access_url
expected_url = '%s?token=%s' % (
fakes.fake_token_dict['access_url_base'],
fakes.fake_token)
self.assertEqual(expected_url, url)
def create(self, queue, subscriber, ttl, options, project=None):
source = queue
now = timeutils.utcnow_ts()
now_dt = datetime.datetime.utcfromtimestamp(now)
expires = now_dt + datetime.timedelta(seconds=ttl)
confirmed = False
try:
subscription_id = self._collection.insert({'s': source,
'u': subscriber,
't': ttl,
'e': expires,
'o': options,
'p': project,
'c': confirmed})
return subscription_id
except pymongo.errors.DuplicateKeyError:
return None
def create(self, queue, subscriber, ttl, options, project=None):
source = queue
now = timeutils.utcnow_ts()
ttl = int(ttl)
expires = now + ttl
if not self._queue_ctrl.exists(source, project):
raise errors.QueueDoesNotExist(source, project)
try:
subscription_id = self._collection.insert({'s': source,
'u': subscriber,
't': ttl,
'e': expires,
'o': options,
'p': project})
return subscription_id
except pymongo.errors.DuplicateKeyError:
return None
def authorize(self, ttl):
"""Authorise the console token and store in the database.
:param ttl: time to live in seconds
:returns: an authorized token
The expires value is set for ttl seconds in the future and the token
hash is stored in the database. This function can only succeed if the
token is unique and the object has not already been stored.
"""
if self.obj_attr_is_set('id'):
raise exception.ObjectActionError(
action='authorize',
reason=_('must be a new object to authorize'))
token = uuidutils.generate_uuid()
token_hash = utils.get_sha256_str(token)
expires = timeutils.utcnow_ts() + ttl
updates = self.obj_get_changes()
# NOTE(melwitt): token could be in the updates if authorize() has been
# called twice on the same object. 'token' is not a database column and
# should not be included in the call to create the database record.
if 'token' in updates:
del updates['token']
updates['token_hash'] = token_hash
updates['expires'] = expires
try:
db_obj = db.console_auth_token_create(self._context, updates)
db_obj['token'] = token
self._from_db_object(self._context, self, db_obj)
except DBDuplicateEntry:
# NOTE(PaulMurray) we are generating the token above so this
# should almost never happen - but technically its possible
raise exception.TokenInUse()
LOG.debug(
"Authorized token with expiry %(expires)s for console "
"connection %(console)s", {
'expires': expires,
'console': strutils.mask_password(self)
})
return token
def test_backup_older_timestamp_discarded(self):
old_name = "oldname"
new_name = "renamed"
bkup_id = self._create_backup(old_name)
bkup = self._get_backup(bkup_id)
now = timeutils.utcnow_ts(microsecond=True)
past = now - 60
self.cond_mgr.update_backup(None,
self.instance_id,
bkup_id,
sent=now,
name=old_name)
self.cond_mgr.update_backup(None,
self.instance_id,
bkup_id,
sent=past,
name=new_name)
bkup = self._get_backup(bkup_id)
self.assertEqual(old_name, bkup.name)
def test_backup_newer_timestamp_accepted(self):
old_name = "oldname"
new_name = "renamed"
bkup_id = self._create_backup(old_name)
bkup = self._get_backup(bkup_id)
now = timeutils.utcnow_ts(microsecond=True)
future = now + 60
self.cond_mgr.update_backup(None,
self.instance_id,
bkup_id,
sent=now,
name=old_name)
self.cond_mgr.update_backup(None,
self.instance_id,
bkup_id,
sent=future,
name=new_name)
bkup = self._get_backup(bkup_id)
self.assertEqual(new_name, bkup.name)
