def create(self, conjunctive=False, gid=None, loader=None, format=None): self.__lock.acquire() uuid_lock = None cached = False p = r.pipeline(transaction=True) p.multi() try: uuid = shortuuid.uuid() if conjunctive: if 'persist' in app.config['STORE']: g = ConjunctiveGraph('Sleepycat') g.open('store/resources/{}'.format(uuid), create=True) else: g = ConjunctiveGraph() g.store.graph_aware = False self.__graph_dict[g] = uuid self.__uuid_dict[uuid] = g return g else: g = resources_cache.get_context(uuid) try: if gid is not None: st_uuid = r.hget(self.__gids_key, gid) if st_uuid is not None: cached = True uuid = st_uuid uuid_lock = self.uuid_lock(uuid) uuid_lock.acquire() g = self.__uuid_dict[uuid] uuid_lock.release() else: post_ts = dt.now() elapsed = (post_ts - self.__last_creation_ts).total_seconds() throttling = (1.0 / GRAPH_THROTTLING) - elapsed if throttling > 0: sleep(throttling) temp_key = '{}:cache:{}'.format(AGENT_ID, uuid) counter_key = '{}:cnt'.format(temp_key) ttl = MIN_CACHE_TIME + int(10 * random()) ttl_ts = calendar.timegm((dt.now() + datetime.timedelta(ttl)).timetuple()) if st_uuid is None: p.delete(counter_key) p.sadd(self.__cache_key, uuid) p.hset(self.__gids_key, uuid, gid) p.hset(self.__gids_key, gid, uuid) self.__last_creation_ts = dt.now() p.incr(counter_key) p.set(temp_key, ttl_ts) p.expire(temp_key, ttl) uuid_lock = self.uuid_lock(uuid) uuid_lock.acquire() except Exception, e: log.error(e.message) traceback.print_exc() self.__graph_dict[g] = uuid self.__uuid_dict[uuid] = g
def __purge(self): while True: self.__lock.acquire() try: obsolete = filter(lambda x: not r.exists('{}:cache:{}'.format(AGENT_ID, x)), r.smembers(self.__cache_key)) if obsolete: with r.pipeline(transaction=True) as p: p.multi() log.info('Removing {} resouces from cache...'.format(len(obsolete))) for uuid in obsolete: uuid_lock = self.uuid_lock(uuid) uuid_lock.acquire() try: gid = r.hget(self.__gids_key, uuid) counter_key = '{}:cache:{}:cnt'.format(AGENT_ID, uuid) usage_counter = r.get(counter_key) if usage_counter is None or int(usage_counter) <= 0: try: resources_cache.remove_context(resources_cache.get_context(uuid)) p.srem(self.__cache_key, uuid) p.hdel(self.__gids_key, uuid) p.hdel(self.__gids_key, gid) p.delete(counter_key) g = self.__uuid_dict[uuid] del self.__uuid_dict[uuid] del self.__graph_dict[g] except Exception, e: traceback.print_exc() log.error('Purging resource {} with uuid {}'.format(gid, uuid)) p.execute() finally: uuid_lock.release()
def used_channels(): """ Selects all channels that were declared by current requests """ req_channel_keys = r.keys('{}:requests:*:'.format(AGENT_ID)) for rck in req_channel_keys: try: channel = r.hget(rck, 'channel') yield channel except Exception as e: traceback.print_exc() log.warning(e.message)
def _remove(self, pipe): """ Removes data relating to the recovery of a fragment for this request """ fragment_id = r.hget('{}'.format(self._request_key), 'fragment_id') self._fragment_key = self.__f_key_pattern.format(fragment_id) pipe.srem('{}:requests'.format(self._fragment_key), self._request_id) pipe.delete('{}gp'.format(self._request_key)) pipe.delete('{}map'.format(self._request_key)) pipe.delete('{}pl'.format(self._request_key)) pipe.delete('{}filters'.format(self._request_key)) super(FragmentSink, self)._remove(pipe)
def build_response(rid): """ Creates a response instance for a given request id :param rid: Request identifier :return: The response object """ from agora.stoa.actions import get_instance response_class = r.hget('{}:requests:{}:'.format(AGENT_ID, rid), '__response_class') if response_class is None: raise AttributeError('Cannot create a response for {}'.format(rid)) (module_name, class_name) = tuple(response_class.split('.')) return get_instance(module_name, class_name, rid)
def _remove(self, pipe): """ Remove all delivery data """ # If this request is the only one that's using such channel, it is removed channel_b64 = r.hget(self._request_key, 'channel') sharing = channel_sharing(channel_b64) if not sharing: log.info('Removing delivery channel ({}) for request {}'.format(channel_b64, self._request_id)) pipe.delete('{}:channels:{}'.format(AGENT_ID, channel_b64)) else: log.info('Cannot remove delivery channel of request {}. It is being shared with {} another requests'.format( self.request_id, sharing)) super(DeliverySink, self)._remove(pipe) pipe.srem(self.__deliveries_key, self._request_id) pipe.srem(self.__ready_key, self._request_id)
def stream(self): return parse_bool(r.hget('{}'.format(self._request_key), '__stream'))
def delivery(self): return r.hget('{}'.format(self._request_key), 'delivery')