def __purge(self):
while True:
self.__lock.acquire()
try:
obsolete = filter(lambda x: not r.exists('{}:cache:{}'.format(AGENT_ID, x)),
r.smembers(self.__cache_key))
if obsolete:
with r.pipeline(transaction=True) as p:
p.multi()
log.info('Removing {} resouces from cache...'.format(len(obsolete)))
for uuid in obsolete:
uuid_lock = self.uuid_lock(uuid)
uuid_lock.acquire()
try:
gid = r.hget(self.__gids_key, uuid)
counter_key = '{}:cache:{}:cnt'.format(AGENT_ID, uuid)
usage_counter = r.get(counter_key)
if usage_counter is None or int(usage_counter) <= 0:
try:
resources_cache.remove_context(resources_cache.get_context(uuid))
p.srem(self.__cache_key, uuid)
p.hdel(self.__gids_key, uuid)
p.hdel(self.__gids_key, gid)
p.delete(counter_key)
g = self.__uuid_dict[uuid]
del self.__uuid_dict[uuid]
del self.__graph_dict[g]
except Exception, e:
traceback.print_exc()
log.error('Purging resource {} with uuid {}'.format(gid, uuid))
p.execute()
finally:
uuid_lock.release()
def result_set(self):
def extract_fields(result):
for r in result:
yield r['_id']
if not r.exists('{}:{}:rs'.format(fragments_key, self.sink.fragment_id)):
_update_result_set(self.sink.fragment_id, self.sink.fragment_gp)
pattern = {}
projection = {}
mapping = filter(lambda x: x.startswith('?'), self.sink.mapping)
for v in mapping:
value = self.sink.map(v, fmap=True)
if not value.startswith('?'):
if value.startswith('"'):
value = value.strip('"')
else:
value = value.lstrip('<').rstrip('>')
pattern[v.lstrip('?')] = value
elif not value.startswith('?_'):
# All those variables that start with '_' won't be projected
projection[v.lstrip('?')] = True
table = db[self.sink.fragment_id]
pipeline = [{"$match": {v: pattern[v] for v in pattern}},
{"$group": {'_id': {v: '$' + v for v in projection}}}]
return extract_fields(table.aggregate(pipeline))
def __get_fragment(fid):
if not r.sismember('{}:fragments'.format(AGENT_ID), fid):
raise NotFound('The fragment {} does not exist'.format(fid))
f_dict = {
'id': fid,
'gp': list(r.smembers('{}:fragments:{}:gp'.format(AGENT_ID, fid))),
'synced': r.exists('{}:fragments:{}:sync'.format(AGENT_ID, fid)),
'requests': list(r.smembers('{}:fragments:{}:requests'.format(AGENT_ID, fid)))
}
return f_dict
def get_request(rid):
if not r.exists('{}:requests:{}:'.format(AGENT_ID, rid)):
raise NotFound('The request {} does not exist'.format(rid))
r_dict = filter_hash_attrs('{}:requests:{}:'.format(AGENT_ID, rid), lambda x: not x.startswith('__'))
channel = r_dict['channel']
ch_dict = r.hgetall('{}:channels:{}'.format(AGENT_ID, channel))
broker = r_dict['broker']
br_dict = r.hgetall('{}:brokers:{}'.format(AGENT_ID, broker))
r_dict['channel'] = ch_dict
r_dict['broker'] = br_dict
if 'mapping' in r_dict:
r_dict['mapping'] = eval(r_dict['mapping'])
return jsonify(r_dict)
