def get(self, queue, claim_id, project=None): if not self._exists(queue, claim_id, project): raise errors.ClaimDoesNotExist(queue, project, claim_id) claim_msgs_key = utils.scope_claim_messages(claim_id, CLAIM_MESSAGES_SUFFIX) # basic_messages msg_keys = self._get_claimed_message_keys(claim_msgs_key) claimed_msgs = messages.Message.from_redis_bulk(msg_keys, self._client) now = timeutils.utcnow_ts() basic_messages = [msg.to_basic(now) for msg in claimed_msgs if msg] # claim_meta now = timeutils.utcnow_ts() expires, ttl = self._get_claim_info(claim_id, [b'e', b't']) update_time = expires - ttl age = now - update_time claim_meta = { 'age': age, 'ttl': ttl, 'id': claim_id, } return claim_meta, basic_messages
def test_set_time_override_using_default(self): now = timeutils.utcnow_ts() # NOTE(kgriffs): Normally it's bad form to sleep in a unit test, # but this is the only way to test that set_time_override defaults # to setting the override to the current time. time.sleep(1) timeutils.set_time_override() overriden_now = timeutils.utcnow_ts() self.assertThat(now, matchers.LessThan(overriden_now))
def test_utcnow_ts(self): skynet_self_aware_ts = 872835240 skynet_dt = datetime.datetime.utcfromtimestamp(skynet_self_aware_ts) self.assertEqual(self.skynet_self_aware_time, skynet_dt) # NOTE(kgriffs): timeutils.utcnow_ts() uses time.time() # IFF time override is not set. with mock.patch('time.time') as time_mock: time_mock.return_value = skynet_self_aware_ts ts = timeutils.utcnow_ts() self.assertEqual(ts, skynet_self_aware_ts) timeutils.set_time_override(skynet_dt) ts = timeutils.utcnow_ts() self.assertEqual(ts, skynet_self_aware_ts)
def test_authorize_object_already_created(self, mock_create): # the expires time is calculated from the current time and # a ttl value in the object. Fix the current time so we can # test expires is calculated correctly as expected self.addCleanup(timeutils.clear_time_override) timeutils.set_time_override() ttl = 10 expires = timeutils.utcnow_ts() + ttl db_dict = copy.deepcopy(fakes.fake_token_dict) db_dict['expires'] = expires mock_create.return_value = db_dict obj = token_obj.ConsoleAuthToken( context=self.context, console_type=fakes.fake_token_dict['console_type'], host=fakes.fake_token_dict['host'], port=fakes.fake_token_dict['port'], internal_access_path=fakes.fake_token_dict['internal_access_path'], instance_uuid=fakes.fake_token_dict['instance_uuid'], access_url_base=fakes.fake_token_dict['access_url_base'], ) obj.authorize(100) self.assertRaises(exception.ObjectActionError, obj.authorize, 100)
def _exists(self, queue, claim_id, project): client = self._client claims_set_key = utils.scope_claims_set(queue, project, QUEUE_CLAIMS_SUFFIX) # In some cases, the queue maybe doesn't exist. So we should check # whether the queue exists. Return False if no such queue exists. # Todo(flwang): We should delete all related data after the queue is # deleted. See the blueprint for more detail: # https://blueprints.launchpad.net/zaqar/+spec/clear-resources-after-delete-queue if not self._queue_ctrl._exists(queue, project): return False # Return False if no such claim exists # TODO(prashanthr_): Discuss the feasibility of a bloom filter. if client.zscore(claims_set_key, claim_id) is None: return False expires = self._get_claim_info(claim_id, b'e')[0] now = timeutils.utcnow_ts() if expires <= now: # NOTE(kgriffs): Redis should automatically remove the # other records in the very near future. This one # has to be manually deleted, however. client.zrem(claims_set_key, claim_id) return False return True
def _clear(self): """Expunges expired keys.""" now = timeutils.utcnow_ts() for k in list(self.cache): (_value, timeout) = self.cache[k] if timeout > 0 and now >= timeout: del self.cache[k]
def _purge_metering_info(self): deadline_timestamp = timeutils.utcnow_ts() - self.conf.report_interval label_ids = [ label_id for label_id, info in self.metering_infos.items() if info["last_update"] < deadline_timestamp ] for label_id in label_ids: del self.metering_infos[label_id]
def _create(self, name, metadata=None, project=None): # TODO(prashanthr_): Implement as a lua script. queue_key = utils.scope_queue_name(name, project) qset_key = utils.scope_queue_name(QUEUES_SET_STORE_NAME, project) # Check if the queue already exists. if self._exists(name, project): return False queue = { 'c': 0, 'cl': 0, 'm': self._packer(metadata or {}), 't': timeutils.utcnow_ts() } # Pipeline ensures atomic inserts. with self._client.pipeline() as pipe: pipe.zadd(qset_key, 1, queue_key).hmset(queue_key, queue) try: pipe.execute() except redis.exceptions.ResponseError: return False return True
def _get_unlocked(self, key, default=None): now = timeutils.utcnow_ts() try: timeout, value = self._cache[key] except KeyError: return (0, default) if timeout and now >= timeout: # NOTE(flaper87): Record expired, # remove it from the cache but catch # KeyError and ValueError in case # _purge_expired removed this key already. try: del self._cache[key] except KeyError: pass try: # NOTE(flaper87): Keys with ttl == 0 # don't exist in the _keys_expires dict self._keys_expires[timeout].remove(key) except (KeyError, ValueError): pass return (0, default) return (timeout, value)
def _exists_unlocked(self, key): now = timeutils.utcnow_ts() try: timeout = self._cache[key][0] return not timeout or now <= timeout except KeyError: return False
def delete(self, queue, claim_id, project=None): # NOTE(prashanthr_): Return silently when the claim # does not exist if not self._exists(queue, claim_id, project): return now = timeutils.utcnow_ts() claim_msgs_key = utils.scope_claim_messages(claim_id, CLAIM_MESSAGES_SUFFIX) msg_keys = self._get_claimed_message_keys(claim_msgs_key) claimed_msgs = messages.MessageEnvelope.from_redis_bulk(msg_keys, self._client) # Update the claim id and claim expiration info # for all the messages. claims_set_key = utils.scope_claims_set(queue, project, QUEUE_CLAIMS_SUFFIX) with self._client.pipeline() as pipe: pipe.zrem(claims_set_key, claim_id) pipe.delete(claim_id) pipe.delete(claim_msgs_key) for msg in claimed_msgs: if msg: msg.claim_id = None msg.claim_expires = now # TODO(kgriffs): Rather than writing back the # entire message, only set the fields that # have changed. msg.to_redis(pipe) pipe.execute()
def destroy_instance(instance_id, environment_id): unit = db_session.get_session() instance = unit.query(models.Instance).get( (environment_id, instance_id)) if instance and not instance.destroyed: instance.destroyed = timeutils.utcnow_ts() instance.save(unit)
def _count(self, queue_name, project=None, include_claimed=False): """Return total number of messages in a queue. This method is designed to very quickly count the number of messages in a given queue. Expired messages are not counted, of course. If the queue does not exist, the count will always be 0. Note: Some expired messages may be included in the count if they haven't been GC'd yet. This is done for performance. """ query = { # Messages must belong to this queue and project. PROJ_QUEUE: utils.scope_queue_name(queue_name, project), # NOTE(kgriffs): Messages must be finalized (i.e., must not # be part of an unfinalized transaction). # # See also the note wrt 'tx' within the definition # of ACTIVE_INDEX_FIELDS. 'tx': None, } if not include_claimed: # Exclude messages that are claimed query['c.e'] = {'$lte': timeutils.utcnow_ts()} collection = self._collection(queue_name, project) return collection.count(filter=query, hint=COUNTING_INDEX_FIELDS)
def post(self, queue, messages, client_uuid, project=None): msgset_key = utils.msgset_key(queue, project) counter_key = utils.scope_queue_index(queue, project, MESSAGE_RANK_COUNTER_SUFFIX) message_ids = [] now = timeutils.utcnow_ts() with self._client.pipeline() as pipe: for msg in messages: prepared_msg = Message( ttl=msg['ttl'], created=now, client_uuid=client_uuid, claim_id=None, claim_expires=now, body=msg.get('body', {}), ) prepared_msg.to_redis(pipe) message_ids.append(prepared_msg.id) pipe.execute() # NOTE(kgriffs): If this call fails, we will return # an error to the client and the messages will be # orphaned, but Redis will remove them when they # expire, so we will just pretend they don't exist # in that case. self._index_messages(msgset_key, counter_key, message_ids) return message_ids
def _get_claim(self, message_id): """Gets minimal claim doc for a message. :returns: {'id': cid, 'expires': ts} IFF the message is claimed, and that claim has not expired. """ claim = self._client.hmget(message_id, 'c', 'c.e') if claim == [None, None]: # NOTE(kgriffs): message_id was not found return None info = { # NOTE(kgriffs): A "None" claim is serialized as an empty str 'id': encodeutils.safe_decode(claim[0]) or None, 'expires': int(claim[1]), } # Is the message claimed? now = timeutils.utcnow_ts() if info['id'] and (now < info['expires']): return info # Not claimed return None
def get(self, queue, claim_id, project=None): message_ctrl = self.driver.message_controller now = timeutils.utcnow_ts(True) self._exists(queue, claim_id, project) container = utils._claim_container(queue, project) headers, claim_obj = self._client.get_object(container, claim_id) def g(): for msg_id in jsonutils.loads(claim_obj): try: headers, msg = message_ctrl._find_message(queue, msg_id, project) except errors.MessageDoesNotExist: continue else: yield utils._message_to_json(msg_id, msg, headers, now) claim_meta = { 'id': claim_id, 'age': now - float(headers['x-timestamp']), 'ttl': int(headers['x-delete-at']) - math.floor(now), } return claim_meta, g()
def _filter_messages(messages, filters, to_basic, marker): """Create a filtering iterator over a list of messages. The function accepts a list of filters to be filtered before the the message can be included as a part of the reply. """ now = timeutils.utcnow_ts() for msg in messages: # NOTE(kgriffs): Message may have been deleted, so # check each value to ensure we got a message back if msg is None: continue # NOTE(kgriffs): Check to see if any of the filters # indiciate that this message should be skipped. for should_skip in filters: if should_skip(msg): break else: marker['next'] = msg.id if to_basic: yield msg.to_basic(now) else: yield msg
def track_instance(instance_id, environment_id, instance_type, type_name, type_title=None, unit_count=None): unit = db_session.get_session() try: with unit.begin(): env = unit.query(models.Environment).get(environment_id) instance = models.Instance() instance.instance_id = instance_id instance.environment_id = environment_id instance.tenant_id = env.tenant_id instance.instance_type = instance_type instance.created = timeutils.utcnow_ts() instance.destroyed = None instance.type_name = type_name instance.type_title = type_title instance.unit_count = unit_count unit.add(instance) except exception.DBDuplicateEntry: unit.execute( sqlalchemy.update(models.Instance).where( models.Instance.instance_id == instance_id and models.Instance.environment_id == environment_id).values( unit_count=unit_count))
def get(self, queue, claim_id, project=None): msg_ctrl = self.driver.message_controller # Base query, always check expire time now = timeutils.utcnow_ts() cid = utils.to_oid(claim_id) if cid is None: raise errors.ClaimDoesNotExist(claim_id, queue, project) try: # Lets get claim's data # from the first message # in the iterator msgs = _messages_iter(msg_ctrl._claimed(queue, cid, now, project=project)) claim = next(msgs) update_time = claim['e'] - claim['t'] age = now - update_time claim_meta = { 'age': int(age), 'ttl': claim['t'], 'id': str(claim['id']), } except StopIteration: raise errors.ClaimDoesNotExist(cid, queue, project) return claim_meta, msgs
def update(self, queue, subscription_id, project=None, **kwargs): names = ('subscriber', 'ttl', 'options') key_transform = lambda x: 'u' if x == 'subscriber' else x[0] fields = common_utils.fields(kwargs, names, pred=lambda x: x is not None, key_transform=key_transform) assert fields, ('`subscriber`, `ttl`, ' 'or `options` not found in kwargs') new_ttl = fields.get('t', None) if new_ttl is not None: now = timeutils.utcnow_ts() now_dt = datetime.datetime.utcfromtimestamp(now) expires = now_dt + datetime.timedelta(seconds=new_ttl) fields['e'] = expires try: res = self._collection.update( {'_id': utils.to_oid(subscription_id), 'p': project}, {'$set': fields}, upsert=False) except pymongo.errors.DuplicateKeyError: raise errors.SubscriptionAlreadyExists() if not res['updatedExisting']: raise errors.SubscriptionDoesNotExist(subscription_id)
def set(self, key, value, time=0, min_compress_len=0): """Sets the value for a key.""" timeout = 0 if time != 0: timeout = timeutils.utcnow_ts() + time self.cache[key] = (timeout, value) return True
def put(self, key, value, time=CACHE_TIMEOUT): """Sets the value for a key.""" timeout = 0 if time != 0: timeout = timeutils.utcnow_ts() + time self._cache[key] = (timeout, value) return True
def stats(self, name, project=None): if not self.queue_controller.exists(name, project=project): raise errors.QueueDoesNotExist(name, project) controller = self.message_controller active = controller._count(name, project=project, include_claimed=False) total = controller._count(name, project=project, include_claimed=True) message_stats = { 'claimed': total - active, 'free': active, 'total': total, } try: oldest = controller.first(name, project=project, sort=1) newest = controller.first(name, project=project, sort=-1) except errors.QueueIsEmpty: pass else: now = timeutils.utcnow_ts() message_stats['oldest'] = utils.stat_message(oldest, now) message_stats['newest'] = utils.stat_message(newest, now) return {'messages': message_stats}
def list(self, queue_name, project=None, marker=None, limit=storage.DEFAULT_MESSAGES_PER_PAGE, echo=False, client_uuid=None, include_claimed=False, include_delayed=False): if marker is not None: try: marker = int(marker) except ValueError: yield iter([]) messages = self._list(queue_name, project=project, marker=marker, client_uuid=client_uuid, echo=echo, include_claimed=include_claimed, include_delayed=include_delayed, limit=limit) marker_id = {} now = timeutils.utcnow_ts() # NOTE (kgriffs) @utils.raises_conn_error not needed on this # function, since utils.HookedCursor already has it. def denormalizer(msg): marker_id['next'] = msg['k'] return _basic_message(msg, now) yield utils.HookedCursor(messages, denormalizer) yield str(marker_id['next'])
def create(self, queue, subscriber, ttl, options, project=None): subscription_id = str(uuid.uuid4()) subset_key = utils.scope_subscription_ids_set(queue, project, SUBSCRIPTION_IDS_SUFFIX) source = queue now = timeutils.utcnow_ts() ttl = int(ttl) expires = now + ttl subscription = {'id': subscription_id, 's': source, 'u': subscriber, 't': ttl, 'e': expires, 'o': options, 'p': project} if not self._queue_ctrl.exists(queue, project): raise errors.QueueDoesNotExist(queue, project) try: # Pipeline ensures atomic inserts. with self._client.pipeline() as pipe: pipe.zadd(subset_key, 1, subscription_id).hmset(subscription_id, subscription) pipe.execute() return subscription_id except redis.exceptions.ResponseError: return None
def _event_transform(self, event, tenant_id, _region): return dict( event=1, meta=dict( tenantId='0ab1ac0a-2867-402d', region='useast'), creation_time=timeutils.utcnow_ts())
def create(self, queue, subscriber, ttl, options, project=None): subscription_id = uuidutils.generate_uuid() subset_key = utils.scope_subscription_ids_set(queue, project, SUBSCRIPTION_IDS_SUFFIX) source = queue now = timeutils.utcnow_ts() expires = now + ttl confirmed = 0 subscription = {'id': subscription_id, 's': source, 'u': subscriber, 't': ttl, 'e': expires, 'o': self._packer(options), 'p': project, 'c': confirmed} try: # Pipeline ensures atomic inserts. with self._client.pipeline() as pipe: if not self._is_duplicated_subscriber(subscriber, queue, project): pipe.zadd(subset_key, {subscription_id: 1}).hmset( subscription_id, subscription) pipe.expire(subscription_id, ttl) pipe.execute() else: return None return subscription_id except redis.exceptions.ResponseError: return None
def test_gc(self): self.queue_controller.create(self.queue_name) for _ in range(100): self.message_controller.post(self.queue_name, [{'ttl': 300, 'body': 'yo gabba'}], client_uuid=uuidutils.generate_uuid()) now = timeutils.utcnow_ts() timeutils_utcnow = 'oslo_utils.timeutils.utcnow_ts' # Test a single claim with mock.patch(timeutils_utcnow) as mock_utcnow: mock_utcnow.return_value = now - 1 self.controller.create(self.queue_name, {'ttl': 1, 'grace': 60}) num_removed = self.controller._gc(self.queue_name, None) self.assertEqual(1, num_removed) # Test multiple claims with mock.patch(timeutils_utcnow) as mock_utcnow: mock_utcnow.return_value = now - 1 for _ in range(5): self.controller.create(self.queue_name, {'ttl': 1, 'grace': 60}) # NOTE(kgriffs): These ones should not be cleaned up self.controller.create(self.queue_name, {'ttl': 60, 'grace': 60}) self.controller.create(self.queue_name, {'ttl': 60, 'grace': 60}) num_removed = self.controller._gc(self.queue_name, None) self.assertEqual(5, num_removed)
def create(self, queue, metadata, project=None, limit=storage.DEFAULT_MESSAGES_PER_CLAIM): claim_ttl = metadata['ttl'] grace = metadata['grace'] now = timeutils.utcnow_ts() msg_ttl = claim_ttl + grace claim_expires = now + claim_ttl msg_expires = claim_expires + grace claim_id = uuidutils.generate_uuid() claimed_msgs = [] # NOTE(kgriffs): Claim some messages msgset_key = utils.msgset_key(queue, project) claimed_ids = self._claim_messages(msgset_key, now, limit, claim_id, claim_expires, msg_ttl, msg_expires) if claimed_ids: claimed_msgs = messages.Message.from_redis_bulk(claimed_ids, self._client) claimed_msgs = [msg.to_basic(now) for msg in claimed_msgs] # NOTE(kgriffs): Perist claim records with self._client.pipeline() as pipe: claim_msgs_key = utils.scope_claim_messages( claim_id, CLAIM_MESSAGES_SUFFIX) for mid in claimed_ids: pipe.rpush(claim_msgs_key, mid) pipe.expire(claim_msgs_key, claim_ttl) claim_info = { 'id': claim_id, 't': claim_ttl, 'e': claim_expires, 'n': len(claimed_ids), } pipe.hmset(claim_id, claim_info) pipe.expire(claim_id, claim_ttl) # NOTE(kgriffs): Add the claim ID to a set so that # existence checks can be performed quickly. This # is also used as a watch key in order to gaurd # against race conditions. # # A sorted set is used to facilitate cleaning # up the IDs of expired claims. claims_set_key = utils.scope_claims_set(queue, project, QUEUE_CLAIMS_SUFFIX) pipe.zadd(claims_set_key, claim_expires, claim_id) pipe.execute() return claim_id, claimed_msgs
def _get(self, queue, message_id, project=None, check_queue=True): if check_queue and not self._queue_ctrl.exists(queue, project): raise errors.QueueDoesNotExist(queue, project) now = timeutils.utcnow_ts(True) headers, msg = self._find_message(queue, message_id, project) return utils._message_to_json(message_id, msg, headers, now)
def first(self, queue_name, project=None, sort=1): cursor = self._list(queue_name, project=project, include_claimed=True, sort=sort, limit=1) try: message = next(cursor) except StopIteration: raise errors.QueueIsEmpty(queue_name, project) now = timeutils.utcnow_ts() return _basic_message(message, now)
def _claimed(self, queue_name, claim_id, expires=None, limit=None, project=None): if claim_id is None: claim_id = {'$ne': None} query = { PROJ_QUEUE: utils.scope_queue_name(queue_name, project), 'c.id': claim_id, 'c.e': { '$gt': expires or timeutils.utcnow_ts() }, } # NOTE(kgriffs): Claimed messages bust be queried from # the primary to avoid a race condition caused by the # multi-phased "create claim" algorithm. preference = pymongo.read_preferences.ReadPreference.PRIMARY collection = self._collection(queue_name, project) msgs = collection.find( query, sort=[('k', 1)], read_preference=preference).hint(CLAIMED_INDEX_FIELDS) if limit is not None: msgs = msgs.limit(limit) now = timeutils.utcnow_ts() def denormalizer(msg): doc = _basic_message(msg, now) doc['claim'] = msg['c'] return doc return utils.HookedCursor(msgs, denormalizer)
def subscription_patching(self, subscription): """Restrictions on an update of subscription. :param subscription: dict of subscription :raises: ValidationFailed if the subscription is invalid. """ if not subscription: raise ValidationFailed(_(u'No subscription to create.')) subscriber = subscription.get('subscriber', None) subscriber_type = None if subscriber: parsed_uri = six.moves.urllib_parse.urlparse(subscriber) subscriber_type = parsed_uri.scheme if subscriber_type not in self._limits_conf.subscriber_types: msg = _(u'The subscriber type of subscription must be ' u'supported in the list {0}.') raise ValidationFailed(msg, self._limits_conf.subscriber_types) options = subscription.get('options', None) if options and not isinstance(options, dict): msg = _(u'Options must be a dict.') raise ValidationFailed(msg) ttl = subscription.get('ttl', None) if ttl: if not isinstance(ttl, int): msg = _(u'TTL must be an integer.') raise ValidationFailed(msg) if ttl < MIN_SUBSCRIPTION_TTL: msg = _(u'The TTL for a subscription ' 'must be at least {0} seconds long.') raise ValidationFailed(msg, MIN_SUBSCRIPTION_TTL) # NOTE(flwang): By this change, technically, user can set a very # big TTL so as to get a very long subscription. now = timeutils.utcnow_ts() now_dt = datetime.datetime.utcfromtimestamp(now) msg = _(u'The TTL seconds for a subscription plus current time' ' must be less than {0}.') try: # NOTE(flwang): If below expression works, then we believe the # ttl is acceptable otherwise it exceeds the max time of # python. now_dt + datetime.timedelta(seconds=ttl) except OverflowError: raise ValidationFailed(msg, datetime.datetime.max)
def transform(metrics, tenant_id, region): transformed_metric = {'metric': {}, 'meta': {'tenantId': tenant_id, 'region': region}, 'creation_time': timeutils.utcnow_ts()} if isinstance(metrics, list): transformed_metrics = [] for metric in metrics: transformed_metric['metric'] = metric transformed_metrics.append(rest_utils.as_json(transformed_metric)) return transformed_metrics else: transformed_metric['metric'] = metrics return [rest_utils.as_json(transformed_metric)]
def _get(self, queue, claim_id, project=None): try: container = utils._claim_container(queue, project) headers, claim = self._client.get_object(container, claim_id) except swiftclient.ClientException as exc: if exc.http_status != 404: raise return now = timeutils.utcnow_ts(True) return { 'id': claim_id, 'age': now - float(headers['x-timestamp']), 'ttl': int(headers['x-delete-at']) - math.floor(now), }
def _gc(self, queue, project): """Garbage-collect expired claim data. Not all claim data can be automatically expired. This method cleans up the remainder. :returns: Number of claims removed """ claims_set_key = utils.scope_claims_set(queue, project, QUEUE_CLAIMS_SUFFIX) now = timeutils.utcnow_ts() num_removed = self._client.zremrangebyscore(claims_set_key, 0, now) return num_removed
def _list(self, queue, project=None, marker=None, limit=storage.DEFAULT_MESSAGES_PER_PAGE, echo=False, client_uuid=None, include_claimed=False, to_basic=True): if not self._queue_ctrl.exists(queue, project): raise errors.QueueDoesNotExist(queue, project) msgset_key = utils.msgset_key(queue, project) client = self._client if not marker and not include_claimed: # NOTE(kgriffs): Skip claimed messages at the head # of the queue; otherwise we would just filter them all # out and likely end up with an empty list to return. marker = self._find_first_unclaimed(queue, project, limit) start = client.zrank(msgset_key, marker) or 0 else: rank = client.zrank(msgset_key, marker) start = rank + 1 if rank else 0 message_ids = client.zrange(msgset_key, start, start + (limit - 1)) messages = Message.from_redis_bulk(message_ids, client) # NOTE(prashanthr_): Build a list of filters for checking # the following: # # 1. Message is expired # 2. Message is claimed # 3. Message should not be echoed # now = timeutils.utcnow_ts() filters = [functools.partial(utils.msg_expired_filter, now=now)] if not include_claimed: filters.append(functools.partial(utils.msg_claimed_filter, now=now)) if not echo: filters.append(functools.partial(utils.msg_echo_filter, client_uuid=client_uuid)) marker = {} yield _filter_messages(messages, filters, to_basic, marker) yield marker['next']
def post(self, queue_name, messages, client_uuid, project=None): # NOTE(flaper87): This method should be safe to retry on # autoreconnect, since we've a 2-step insert for messages. # The worst-case scenario is that we'll increase the counter # several times and we'd end up with some non-active messages. if not self._queue_ctrl.exists(queue_name, project): raise errors.QueueDoesNotExist(queue_name, project) # NOTE(flaper87): Make sure the counter exists. This method # is an upsert. self._get_counter(queue_name, project) now = timeutils.utcnow_ts() now_dt = datetime.datetime.utcfromtimestamp(now) collection = self._collection(queue_name, project) messages = list(messages) msgs_n = len(messages) next_marker = self._inc_counter(queue_name, project, amount=msgs_n) - msgs_n prepared_messages = [{ PROJ_QUEUE: utils.scope_queue_name(queue_name, project), 't': message['ttl'], 'e': now_dt + datetime.timedelta(seconds=message['ttl']), 'u': client_uuid, 'c': { 'id': None, 'e': now, 'c': 0 }, 'd': now + message.get('delay', 0), 'b': message['body'] if 'body' in message else {}, 'k': next_marker + index, 'tx': None, } for index, message in enumerate(messages)] res = collection.insert_many(prepared_messages, bypass_document_validation=True) return [str(id_) for id_ in res.inserted_ids]
def first(self, queue, project=None, sort=1): if sort not in (1, -1): raise ValueError(u'sort must be either 1 (ascending) ' u'or -1 (descending)') message_id = self._get_first_message_id(queue, project, sort) if not message_id: raise errors.QueueIsEmpty(queue, project) message = Message.from_redis(message_id, self._client) if message is None: raise errors.QueueIsEmpty(queue, project) now = timeutils.utcnow_ts() return message.to_basic(now, include_created=True)
def _add_metering_info(self, label_id, pkts, bytes): ts = timeutils.utcnow_ts() info = self.metering_infos.get(label_id, {'bytes': 0, 'pkts': 0, 'time': 0, 'first_update': ts, 'last_update': ts}) info['bytes'] += bytes info['pkts'] += pkts info['time'] += ts - info['last_update'] info['last_update'] = ts self.metering_infos[label_id] = info return info
def get_aggregated_stats(environment_id): unit = db_session.get_session() now = timeutils.utcnow_ts() query = unit.query(models.Instance.instance_type, func.sum( func.coalesce(models.Instance.destroyed, now) - models.Instance.created), func.count()).filter( models.Instance.environment_id == environment_id) res = query.group_by(models.Instance.instance_type).all() return [{ 'type': int(record[0]), 'duration': int(record[1]), 'count': int(record[2]) } for record in res]
def stats(self, name, project=None): if not self._queue_ctrl.exists(name, project=project): raise errors.QueueDoesNotExist(name, project) total = 0 claimed = 0 container = utils._message_container(name, project) try: _, objects = self._client.get_container(container) except swiftclient.ClientException as exc: if exc.http_status == 404: raise errors.QueueIsEmpty(name, project) newest = None oldest = None now = timeutils.utcnow_ts(True) for obj in objects: try: headers = self._client.head_object(container, obj['name']) except swiftclient.ClientException as exc: if exc.http_status != 404: raise else: created = float(headers['x-timestamp']) created_iso = datetime.datetime.utcfromtimestamp( created).strftime('%Y-%m-%dT%H:%M:%SZ') newest = { 'id': obj['name'], 'age': now - created, 'created': created_iso } if oldest is None: oldest = copy.deepcopy(newest) total += 1 if headers.get('x-object-meta-claimid'): claimed += 1 msg_stats = { 'claimed': claimed, 'free': total - claimed, 'total': total, } if newest is not None: msg_stats['newest'] = newest msg_stats['oldest'] = oldest return {'messages': msg_stats}
def delete(self, queue_name, message_id, project=None, claim=None): # NOTE(cpp-cabrera): return early - this is an invalid message # id so we won't be able to find it any way mid = utils.to_oid(message_id) if mid is None: return collection = self._collection(queue_name, project) query = { '_id': mid, PROJ_QUEUE: utils.scope_queue_name(queue_name, project), } cid = utils.to_oid(claim) if cid is None: raise errors.ClaimDoesNotExist(claim, queue_name, project) now = timeutils.utcnow_ts() cursor = collection.find(query).hint(ID_INDEX_FIELDS) try: message = next(cursor) except StopIteration: return if claim is None: if _is_claimed(message, now): raise errors.MessageIsClaimed(message_id) else: if message['c']['id'] != cid: kwargs = {} # NOTE(flaper87): In pymongo 3.0 PRIMARY is the default and # `read_preference` is read only. We'd need to set it when the # client is created. # NOTE(kgriffs): Read from primary in case the message # was just barely claimed, and claim hasn't made it to # the secondary. message = collection.find_one(query, **kwargs) if message['c']['id'] != cid: if _is_claimed(message, now): raise errors.MessageNotClaimedBy(message_id, claim) raise errors.MessageNotClaimed(message_id) collection.remove(query['_id'], w=0)
def set_status(self, status, force=False): """Use conductor to update the DB app status.""" if force or self.is_installed: LOG.debug("Casting set_status message to conductor " "(status is '%s').", status.description) context = trove_context.TroveContext() heartbeat = {'service_status': status.description} conductor_api.API(context).heartbeat( CONF.guest_id, heartbeat, sent=timeutils.utcnow_ts(microsecond=True)) LOG.debug("Successfully cast set_status.") self.status = status else: LOG.debug("Prepare has not completed yet, skipping heartbeat.")
def denormalizer(record, sid): now = timeutils.utcnow_ts() ttl = int(record[2]) expires = int(record[3]) created = expires - ttl ret = { 'id': sid, 'source': record[0], 'subscriber': record[1], 'ttl': ttl, 'age': now - created, 'options': self._unpacker(record[4]), } marker_next['next'] = sid return ret
def test_notification_info_update_timestamp(self): self.member_manager.hashring = mock.Mock() ctxt, publisher_id, event_type, payload, metadata = fake_notification() # Set an old timestamp, and insert into members payload['timestamp'] = self.old_timestamp self.member_manager.members.append(copy.deepcopy(payload)) # Reset timestamp, and simulate notification, add_node not called # Timestamp in member manager is updated. payload['timestamp'] = timeutils.utcnow_ts() self.assertNotEqual(payload['timestamp'], self.member_manager.members[0]['timestamp']) self.member_manager.info(ctxt, publisher_id, event_type, payload, metadata) self.member_manager.hashring.add_node.assert_not_called() self.assertEqual(payload['timestamp'], self.member_manager.members[0]['timestamp'])
def bulk_get(self, queue, message_ids, project=None): if not self._queue_ctrl.exists(queue, project): return iter([]) # NOTE(prashanthr_): Pipelining is used here purely # for performance. with self._client.pipeline() as pipe: for mid in message_ids: pipe.hgetall(mid) messages = pipe.execute() # NOTE(kgriffs): Skip messages that may have been deleted now = timeutils.utcnow_ts() return (Message.from_hmap(msg).to_basic(now) for msg in messages if msg)
def update(self, queue, claim_id, metadata, project=None): cid = utils.to_oid(claim_id) if cid is None: raise errors.ClaimDoesNotExist(claim_id, queue, project) now = timeutils.utcnow_ts() grace = metadata['grace'] ttl = metadata['ttl'] claim_expires = now + ttl claim_expires_dt = datetime.datetime.utcfromtimestamp(claim_expires) message_ttl = ttl + grace message_expires = datetime.datetime.utcfromtimestamp( claim_expires + grace) msg_ctrl = self.driver.message_controller claimed = msg_ctrl._claimed(queue, cid, expires=now, limit=1, project=project) try: next(claimed) except StopIteration: raise errors.ClaimDoesNotExist(claim_id, queue, project) meta = { 'id': cid, 't': ttl, 'e': claim_expires, } # TODO(kgriffs): Create methods for these so we don't interact # with the messages collection directly (loose coupling) scope = utils.scope_queue_name(queue, project) collection = msg_ctrl._collection(queue, project) collection.update_many({'p_q': scope, 'c.id': cid}, {'$set': {'c': meta}}, upsert=False) # NOTE(flaper87): Dirty hack! # This sets the expiration time to # `expires` on messages that would # expire before claim. collection.update_many({'p_q': scope, 'e': {'$lt': claim_expires_dt}, 'c.id': cid}, {'$set': {'e': message_expires, 't': message_ttl}}, upsert=False)
def create_in_section(cls, checkpoints_section, indices_section, bank_lease, owner_id, plan, checkpoint_id=None): checkpoint_id = checkpoint_id or cls._generate_id() checkpoint_section = checkpoints_section.get_sub_section(checkpoint_id) timestamp = timeutils.utcnow_ts() created_at = timeutils.utcnow().strftime('%Y-%m-%d') checkpoint_section.create_object( key=_INDEX_FILE_NAME, value={ "version": cls.VERSION, "id": checkpoint_id, "status": constants.CHECKPOINT_STATUS_PROTECTING, "owner_id": owner_id, "provider_id": plan.get("provider_id"), "project_id": plan.get("project_id"), "protection_plan": { "id": plan.get("id"), "name": plan.get("name"), "provider_id": plan.get("provider_id"), "resources": plan.get("resources") }, "created_at": created_at, "timestamp": timestamp }) indices_section.create_object(key="/by-provider/%s@%s" % (timestamp, checkpoint_id), value=checkpoint_id) indices_section.create_object(key="/by-date/%s/%s@%s" % (created_at, timestamp, checkpoint_id), value=checkpoint_id) indices_section.create_object( key="/by-plan/%s/%s/%s@%s" % (plan.get("id"), created_at, timestamp, checkpoint_id), value=checkpoint_id) return Checkpoint(checkpoint_section, indices_section, bank_lease, checkpoint_id)
def cleanup_deleted_ports(self): """Cleanup the "self._deleted_ports" set based on the current TS The variable "self._deleted_ports_ts" contains a timestamp ordered list of tuples (timestamp, port_id). Every port older than the current timestamp minus "timestamp_delta" will be deleted from "self._deleted_ports" and "self._deleted_ports_ts". """ timestamp_min = timeutils.utcnow_ts() - DELETED_PORT_MAX_AGE idx = None for idx, (ts, port_id) in enumerate(self._deleted_ports_ts): if ts > timestamp_min: break self._deleted_ports.remove(port_id) if idx: self._deleted_ports_ts = self._deleted_ports_ts[idx:]
def _create_msg(self, queue, msg, client_uuid, project): slug = str(uuid.uuid1()) now = timeutils.utcnow_ts() contents = jsonutils.dumps( {'body': msg.get('body', {}), 'claim_id': None, 'ttl': msg['ttl'], 'claim_count': 0, 'delay_expires': now + msg.get('delay', 0)}) utils._put_or_create_container( self._client, utils._message_container(queue, project), slug, contents=contents, content_type='application/json', headers={ 'x-object-meta-clientid': str(client_uuid), 'x-delete-after': msg['ttl']}) return slug
def update(self, queue, subscription_id, project=None, **kwargs): names = ('subscriber', 'ttl', 'options') key_transform = lambda x: 'u' if x == 'subscriber' else x[0] fields = common_utils.fields(kwargs, names, pred=lambda x: x is not None, key_transform=key_transform) assert fields, ('`subscriber`, `ttl`, ' 'or `options` not found in kwargs') # Let's get our subscription by ID. If it does not exist, # SubscriptionDoesNotExist error will be raised internally. subscription_to_update = self.get(queue, subscription_id, project=project) new_subscriber = fields.get('u', None) # Let's do some checks to prevent subscription duplication. if new_subscriber: # Check if 'new_subscriber' is really new for our subscription. if subscription_to_update['subscriber'] != new_subscriber: # It's new. We should raise error if this subscriber already # exists for the queue and project. if self._is_duplicated_subscriber(new_subscriber, queue, project): raise errors.SubscriptionAlreadyExists() # NOTE(Eva-i): if there are new options, we need to pack them before # sending to the database. new_options = fields.get('o', None) if new_options is not None: fields['o'] = self._packer(new_options) new_ttl = fields.get('t', None) if new_ttl is not None: now = timeutils.utcnow_ts() expires = now + new_ttl fields['e'] = expires # Pipeline ensures atomic inserts. with self._client.pipeline() as pipe: pipe.hmset(subscription_id, fields) if new_ttl is not None: pipe.expire(subscription_id, new_ttl) pipe.execute()
def test_basic_message(self): now = timeutils.utcnow_ts() body = { 'msg': 'Hello Earthlings!', 'unicode': u'ab\u00e7', 'bytes': b'ab\xc3\xa7', b'ab\xc3\xa7': 'one, two, three', u'ab\u00e7': 'one, two, three', } msg = _create_sample_message(now=now, body=body) basic_msg = msg.to_basic(now + 5) self.assertEqual(msg.id, basic_msg['id']) self.assertEqual(5, basic_msg['age']) self.assertEqual(body, basic_msg['body']) self.assertEqual(msg.ttl, basic_msg['ttl'])
def _filter_messages(messages, filters, marker, get_object, list_objects, limit): """Create a filtering iterator over a list of messages. The function accepts a list of filters to be filtered before the the message can be included as a part of the reply. """ now = timeutils.utcnow_ts(True) for msg in messages: if msg is None: continue marker['next'] = msg['name'] try: headers, obj = get_object(msg['name']) except swiftclient.ClientException as exc: if exc.http_status == 404: continue raise obj = jsonutils.loads(obj) for should_skip in filters: if should_skip(obj, headers): break else: limit -= 1 yield { 'id': marker['next'], 'ttl': obj['ttl'], 'client_uuid': headers['x-object-meta-clientid'], 'body': obj['body'], 'age': now - float(headers['x-timestamp']), 'claim_id': obj['claim_id'], 'claim_count': obj.get('claim_count', 0), } if limit <= 0: break if limit > 0 and marker: # We haven't reached the limit, let's try to get some more messages _, objects = list_objects(marker=marker['next']) if not objects: return for msg in _filter_messages(objects, filters, marker, get_object, list_objects, limit): yield msg
def test_authorize(self, mock_create): # the expires time is calculated from the current time and # a ttl value in the object. Fix the current time so we can # test expires is calculated correctly as expected self.addCleanup(timeutils.clear_time_override) timeutils.set_time_override() ttl = 10 expires = timeutils.utcnow_ts() + ttl db_dict = copy.deepcopy(fakes.fake_token_dict) db_dict['expires'] = expires mock_create.return_value = db_dict create_dict = copy.deepcopy(fakes.fake_token_dict) create_dict['expires'] = expires del create_dict['id'] del create_dict['created_at'] del create_dict['updated_at'] expected = copy.deepcopy(fakes.fake_token_dict) del expected['token_hash'] del expected['expires'] expected['token'] = fakes.fake_token obj = token_obj.ConsoleAuthToken( context=self.context, console_type=fakes.fake_token_dict['console_type'], host=fakes.fake_token_dict['host'], port=fakes.fake_token_dict['port'], internal_access_path=fakes.fake_token_dict['internal_access_path'], instance_uuid=fakes.fake_token_dict['instance_uuid'], access_url_base=fakes.fake_token_dict['access_url_base'], ) with mock.patch('uuid.uuid4', return_value=fakes.fake_token): token = obj.authorize(ttl) mock_create.assert_called_once_with(self.context, create_dict) self.assertEqual(token, fakes.fake_token) self.compare_obj(obj, expected) url = obj.access_url expected_url = '%s?token=%s' % ( fakes.fake_token_dict['access_url_base'], fakes.fake_token) self.assertEqual(expected_url, url)
def create(self, queue, subscriber, ttl, options, project=None): source = queue now = timeutils.utcnow_ts() now_dt = datetime.datetime.utcfromtimestamp(now) expires = now_dt + datetime.timedelta(seconds=ttl) confirmed = False try: subscription_id = self._collection.insert({'s': source, 'u': subscriber, 't': ttl, 'e': expires, 'o': options, 'p': project, 'c': confirmed}) return subscription_id except pymongo.errors.DuplicateKeyError: return None
def create(self, queue, subscriber, ttl, options, project=None): source = queue now = timeutils.utcnow_ts() ttl = int(ttl) expires = now + ttl if not self._queue_ctrl.exists(source, project): raise errors.QueueDoesNotExist(source, project) try: subscription_id = self._collection.insert({'s': source, 'u': subscriber, 't': ttl, 'e': expires, 'o': options, 'p': project}) return subscription_id except pymongo.errors.DuplicateKeyError: return None
def authorize(self, ttl): """Authorise the console token and store in the database. :param ttl: time to live in seconds :returns: an authorized token The expires value is set for ttl seconds in the future and the token hash is stored in the database. This function can only succeed if the token is unique and the object has not already been stored. """ if self.obj_attr_is_set('id'): raise exception.ObjectActionError( action='authorize', reason=_('must be a new object to authorize')) token = uuidutils.generate_uuid() token_hash = utils.get_sha256_str(token) expires = timeutils.utcnow_ts() + ttl updates = self.obj_get_changes() # NOTE(melwitt): token could be in the updates if authorize() has been # called twice on the same object. 'token' is not a database column and # should not be included in the call to create the database record. if 'token' in updates: del updates['token'] updates['token_hash'] = token_hash updates['expires'] = expires try: db_obj = db.console_auth_token_create(self._context, updates) db_obj['token'] = token self._from_db_object(self._context, self, db_obj) except DBDuplicateEntry: # NOTE(PaulMurray) we are generating the token above so this # should almost never happen - but technically its possible raise exception.TokenInUse() LOG.debug( "Authorized token with expiry %(expires)s for console " "connection %(console)s", { 'expires': expires, 'console': strutils.mask_password(self) }) return token
def test_backup_older_timestamp_discarded(self): old_name = "oldname" new_name = "renamed" bkup_id = self._create_backup(old_name) bkup = self._get_backup(bkup_id) now = timeutils.utcnow_ts(microsecond=True) past = now - 60 self.cond_mgr.update_backup(None, self.instance_id, bkup_id, sent=now, name=old_name) self.cond_mgr.update_backup(None, self.instance_id, bkup_id, sent=past, name=new_name) bkup = self._get_backup(bkup_id) self.assertEqual(old_name, bkup.name)
def test_backup_newer_timestamp_accepted(self): old_name = "oldname" new_name = "renamed" bkup_id = self._create_backup(old_name) bkup = self._get_backup(bkup_id) now = timeutils.utcnow_ts(microsecond=True) future = now + 60 self.cond_mgr.update_backup(None, self.instance_id, bkup_id, sent=now, name=old_name) self.cond_mgr.update_backup(None, self.instance_id, bkup_id, sent=future, name=new_name) bkup = self._get_backup(bkup_id) self.assertEqual(new_name, bkup.name)