def create(self, conjunctive=False, gid=None, loader=None, format=None):
self.__lock.acquire()
uuid_lock = None
cached = False
p = r.pipeline(transaction=True)
p.multi()
try:
uuid = shortuuid.uuid()
if conjunctive:
if 'persist' in app.config['STORE']:
g = ConjunctiveGraph('Sleepycat')
g.open('store/resources/{}'.format(uuid), create=True)
else:
g = ConjunctiveGraph()
g.store.graph_aware = False
self.__graph_dict[g] = uuid
self.__uuid_dict[uuid] = g
return g
else:
g = resources_cache.get_context(uuid)
try:
if gid is not None:
st_uuid = r.hget(self.__gids_key, gid)
if st_uuid is not None:
cached = True
uuid = st_uuid
uuid_lock = self.uuid_lock(uuid)
uuid_lock.acquire()
g = self.__uuid_dict[uuid]
uuid_lock.release()
else:
post_ts = dt.now()
elapsed = (post_ts - self.__last_creation_ts).total_seconds()
throttling = (1.0 / GRAPH_THROTTLING) - elapsed
if throttling > 0:
sleep(throttling)
temp_key = '{}:cache:{}'.format(AGENT_ID, uuid)
counter_key = '{}:cnt'.format(temp_key)
ttl = MIN_CACHE_TIME + int(10 * random())
ttl_ts = calendar.timegm((dt.now() + datetime.timedelta(ttl)).timetuple())
if st_uuid is None:
p.delete(counter_key)
p.sadd(self.__cache_key, uuid)
p.hset(self.__gids_key, uuid, gid)
p.hset(self.__gids_key, gid, uuid)
self.__last_creation_ts = dt.now()
p.incr(counter_key)
p.set(temp_key, ttl_ts)
p.expire(temp_key, ttl)
uuid_lock = self.uuid_lock(uuid)
uuid_lock.acquire()
except Exception, e:
log.error(e.message)
traceback.print_exc()
self.__graph_dict[g] = uuid
self.__uuid_dict[uuid] = g
def __purge(self):
while True:
self.__lock.acquire()
try:
obsolete = filter(lambda x: not r.exists('{}:cache:{}'.format(AGENT_ID, x)),
r.smembers(self.__cache_key))
if obsolete:
with r.pipeline(transaction=True) as p:
p.multi()
log.info('Removing {} resouces from cache...'.format(len(obsolete)))
for uuid in obsolete:
uuid_lock = self.uuid_lock(uuid)
uuid_lock.acquire()
try:
gid = r.hget(self.__gids_key, uuid)
counter_key = '{}:cache:{}:cnt'.format(AGENT_ID, uuid)
usage_counter = r.get(counter_key)
if usage_counter is None or int(usage_counter) <= 0:
try:
resources_cache.remove_context(resources_cache.get_context(uuid))
p.srem(self.__cache_key, uuid)
p.hdel(self.__gids_key, uuid)
p.hdel(self.__gids_key, gid)
p.delete(counter_key)
g = self.__uuid_dict[uuid]
del self.__uuid_dict[uuid]
del self.__graph_dict[g]
except Exception, e:
traceback.print_exc()
log.error('Purging resource {} with uuid {}'.format(gid, uuid))
p.execute()
finally:
uuid_lock.release()
def used_channels():
"""
Selects all channels that were declared by current requests
"""
req_channel_keys = r.keys('{}:requests:*:'.format(AGENT_ID))
for rck in req_channel_keys:
try:
channel = r.hget(rck, 'channel')
yield channel
except Exception as e:
traceback.print_exc()
log.warning(e.message)
def _remove(self, pipe):
"""
Removes data relating to the recovery of a fragment for this request
"""
fragment_id = r.hget('{}'.format(self._request_key), 'fragment_id')
self._fragment_key = self.__f_key_pattern.format(fragment_id)
pipe.srem('{}:requests'.format(self._fragment_key), self._request_id)
pipe.delete('{}gp'.format(self._request_key))
pipe.delete('{}map'.format(self._request_key))
pipe.delete('{}pl'.format(self._request_key))
pipe.delete('{}filters'.format(self._request_key))
super(FragmentSink, self)._remove(pipe)
def build_response(rid):
"""
Creates a response instance for a given request id
:param rid: Request identifier
:return: The response object
"""
from agora.stoa.actions import get_instance
response_class = r.hget('{}:requests:{}:'.format(AGENT_ID, rid), '__response_class')
if response_class is None:
raise AttributeError('Cannot create a response for {}'.format(rid))
(module_name, class_name) = tuple(response_class.split('.'))
return get_instance(module_name, class_name, rid)
def _remove(self, pipe):
"""
Remove all delivery data
"""
# If this request is the only one that's using such channel, it is removed
channel_b64 = r.hget(self._request_key, 'channel')
sharing = channel_sharing(channel_b64)
if not sharing:
log.info('Removing delivery channel ({}) for request {}'.format(channel_b64, self._request_id))
pipe.delete('{}:channels:{}'.format(AGENT_ID, channel_b64))
else:
log.info('Cannot remove delivery channel of request {}. It is being shared with {} another requests'.format(
self.request_id, sharing))
super(DeliverySink, self)._remove(pipe)
pipe.srem(self.__deliveries_key, self._request_id)
pipe.srem(self.__ready_key, self._request_id)
def stream(self):
return parse_bool(r.hget('{}'.format(self._request_key), '__stream'))
def delivery(self):
return r.hget('{}'.format(self._request_key), 'delivery')
