def _build(self):
"""
This function does not yield anything only when the new state is 'streaming'
:return:
"""
timestamp = calendar.timegm(dt.now().timetuple())
lock = r.lock('fragments:{}:lock'.format(self.sink.fragment_id), lock_class=Lock)
lock.acquire()
fragment = None
try:
fragment, stream = self.fragment(stream=True, timestamp=timestamp)
if stream:
self.sink.stream = True
if fragment:
self.sink.delivery = 'mixing'
else:
self.sink.delivery = 'streaming'
else:
if fragment:
self.sink.delivery = 'pushing'
log.debug('Fragment retrieved from cache for request number {}'.format(self._request_id))
else:
self.sink.delivery = 'sent'
log.debug('Sending end stream signal since there is no fragment and stream is disabled')
yield (), {'state': 'end'}
self.sink.stream = False
except Exception as e:
log.warning(e.message)
self.sink.stream = True
self.sink.delivery = 'streaming'
finally:
lock.release()
if fragment:
log.info('Building a stream result from cache for request number {}...'.format(self._request_id))
for ch in chunks(fragment, 1000):
if ch:
yield [(map_variables(c, self.sink.mapping), s.n3(), p.n3(), o.n3()) for
(c, s, p, o)
in ch], {'source': 'store'}
lock.acquire()
try:
if self.sink.delivery == 'pushing' or (self.sink.delivery == 'mixing' and not self.sink.stream):
self.sink.delivery = 'sent'
log.info(
'The response stream of request {} is completed. Notifying...'.format(self.sink.request_id))
yield (), {'state': 'end'}
elif self.sink.delivery == 'mixing' and self.sink.stream:
self.sink.delivery = 'streaming'
finally:
lock.release()
def build(self):
super(FragmentResponse, self).build()
lock_consume_key = 'fragments:{}:lock:consume'.format(self.sink.fragment_id)
c_lock = r.lock(lock_consume_key, lock_class=Lock)
c_lock.acquire()
generator = self._build()
try:
for response in generator:
yield response
except Exception, e:
traceback.print_exc()
log.error(e.message)
def __remove_fragment(fid):
log.debug("Waiting to remove fragment {}...".format(fid))
lock_key = "fragments:{}:lock".format(fid)
lock = r.lock(lock_key, lock_class=Lock)
lock.acquire()
with r.pipeline(transaction=True) as p:
requests, r_sinks = __load_fragment_requests(fid)
__notify_completion(fid, r_sinks)
fragment_keys = r.keys("fragments:{}*".format(fid))
map(lambda k: p.delete(k), fragment_keys)
p.srem("fragments", fid)
p.execute()
log.info("Fragment {} has been removed".format(fid))
def __pull_fragment(fid):
tps = r.smembers("fragments:{}:gp".format(fid))
requests, r_sinks = __load_fragment_requests(fid)
log.info(
"""Starting collection of fragment {}:
- GP: {}
- Supporting: ({}) {}""".format(
fid, list(tps), len(requests), list(requests)
)
)
start_time = datetime.now()
try:
fgm_gen, _, graph = agora_client.get_fragment_generator(
"{ %s }" % " . ".join(tps), workers=N_COLLECTORS, provider=graph_provider, queue_size=N_COLLECTORS
)
except Exception:
log.error("Agora is not available")
return
# There is no search plan to execute
if not list(graph.subjects(RDF.type, AGORA.SearchTree)):
log.info("There is no search plan for fragment {}. Removing...".format(fid))
# TODO: Send additional headers notifying the reason to end
__notify_completion(fid, r_sinks)
__remove_fragment(fid)
return
triple_patterns = {tpn: __triple_pattern(graph, tpn) for tpn in graph.subjects(RDF.type, AGORA.TriplePattern)}
fragment_contexts = {tpn: (fid, triple_patterns[tpn]) for tpn in triple_patterns}
__bind_prefixes(graph)
lock_key = "fragments:{}:lock".format(fid)
lock = r.lock(lock_key, lock_class=Lock)
lock.acquire()
lock_consume_key = "fragments:{}:lock:consume".format(fid)
c_lock = r.lock(lock_consume_key, lock_class=Lock)
c_lock.acquire()
# Update fragment contexts
with r.pipeline(transaction=True) as p:
p.multi()
p.set("fragments:{}:pulling".format(fid), True)
p.delete("fragments:{}:contexts".format(fid))
for tpn in fragment_contexts.keys():
p.sadd("fragments:{}:contexts".format(fid), fragment_contexts[tpn])
p.execute()
lock.release()
c_lock.release()
n_triples = 0
fragment_weight = 0
fragment_delta = 0
try:
for (c, s, p, o) in fgm_gen:
pre_ts = datetime.now()
triple_weight = len(u"{}{}{}".format(s, p, o))
fragment_weight += triple_weight
fragment_delta += triple_weight
lock.acquire()
if add_stream_triple(fid, triple_patterns[c], (s, p, o)):
__consume_quad(fid, (triple_patterns[c], s, p, o), graph, sinks=r_sinks)
lock.release()
if fragment_delta > 1000:
fragment_delta = 0
log.info("Pulling fragment {} [{} kB]".format(fid, fragment_weight / 1000.0))
if r.scard("fragments:{}:requests".format(fid)) != len(requests):
requests, r_sinks = __load_fragment_requests(fid)
n_triples += 1
post_ts = datetime.now()
elapsed = (post_ts - pre_ts).total_seconds()
excess = (1.0 / COLLECT_THROTTLING) - elapsed
if excess > 0:
sleep(excess)
except Exception, e:
traceback.print_exc()
