def __purge(self): while True: self.__lock.acquire() try: obsolete = filter(lambda x: not r.exists('{}:cache:{}'.format(AGENT_ID, x)), r.smembers(self.__cache_key)) if obsolete: with r.pipeline(transaction=True) as p: p.multi() log.info('Removing {} resouces from cache...'.format(len(obsolete))) for uuid in obsolete: uuid_lock = self.uuid_lock(uuid) uuid_lock.acquire() try: gid = r.hget(self.__gids_key, uuid) counter_key = '{}:cache:{}:cnt'.format(AGENT_ID, uuid) usage_counter = r.get(counter_key) if usage_counter is None or int(usage_counter) <= 0: try: resources_cache.remove_context(resources_cache.get_context(uuid)) p.srem(self.__cache_key, uuid) p.hdel(self.__gids_key, uuid) p.hdel(self.__gids_key, gid) p.delete(counter_key) g = self.__uuid_dict[uuid] del self.__uuid_dict[uuid] del self.__graph_dict[g] except Exception, e: traceback.print_exc() log.error('Purging resource {} with uuid {}'.format(gid, uuid)) p.execute() finally: uuid_lock.release()
def result_set(self): def extract_fields(result): for r in result: yield r['_id'] if not r.exists('{}:{}:rs'.format(fragments_key, self.sink.fragment_id)): _update_result_set(self.sink.fragment_id, self.sink.fragment_gp) pattern = {} projection = {} mapping = filter(lambda x: x.startswith('?'), self.sink.mapping) for v in mapping: value = self.sink.map(v, fmap=True) if not value.startswith('?'): if value.startswith('"'): value = value.strip('"') else: value = value.lstrip('<').rstrip('>') pattern[v.lstrip('?')] = value elif not value.startswith('?_'): # All those variables that start with '_' won't be projected projection[v.lstrip('?')] = True table = db[self.sink.fragment_id] pipeline = [{"$match": {v: pattern[v] for v in pattern}}, {"$group": {'_id': {v: '$' + v for v in projection}}}] return extract_fields(table.aggregate(pipeline))
def __get_fragment(fid): if not r.sismember('{}:fragments'.format(AGENT_ID), fid): raise NotFound('The fragment {} does not exist'.format(fid)) f_dict = { 'id': fid, 'gp': list(r.smembers('{}:fragments:{}:gp'.format(AGENT_ID, fid))), 'synced': r.exists('{}:fragments:{}:sync'.format(AGENT_ID, fid)), 'requests': list(r.smembers('{}:fragments:{}:requests'.format(AGENT_ID, fid))) } return f_dict
def get_request(rid): if not r.exists('{}:requests:{}:'.format(AGENT_ID, rid)): raise NotFound('The request {} does not exist'.format(rid)) r_dict = filter_hash_attrs('{}:requests:{}:'.format(AGENT_ID, rid), lambda x: not x.startswith('__')) channel = r_dict['channel'] ch_dict = r.hgetall('{}:channels:{}'.format(AGENT_ID, channel)) broker = r_dict['broker'] br_dict = r.hgetall('{}:brokers:{}'.format(AGENT_ID, broker)) r_dict['channel'] = ch_dict r_dict['broker'] = br_dict if 'mapping' in r_dict: r_dict['mapping'] = eval(r_dict['mapping']) return jsonify(r_dict)