def _save(self, action):
super(FragmentSink, self)._save(action)
self._build_graph_pattern(action)
fragment_mapping = self.__check_gp()
exists = fragment_mapping is not None
if not exists:
self._fragment_id = str(uuid())
self._pipe.sadd('fragments', self._fragment_id)
self._pipe.sadd('fragments:{}:gp'.format(self._fragment_id), *self._graph_pattern)
mapping = {str(k): str(k) for k in self._variables_dict.values()}
else:
self._fragment_id, mapping = fragment_mapping
if r.get('fragments:{}:on_demand'.format(self._fragment_id)) is not None:
self._pipe.delete('fragments:{}:sync'.format(self._fragment_id))
self._pipe.hset(self._request_key, 'mapping', mapping)
self._pipe.hset(self._request_key, 'preferred_labels', self._preferred_labels)
self._pipe.sadd('fragments:{}:requests'.format(self._fragment_id), self._request_id)
self._pipe.hset('{}'.format(self._request_key), 'fragment_id', self._fragment_id)
self._pipe.hset('{}'.format(self._request_key), 'pattern', ' . '.join(self._graph_pattern))
self._dict_fields['mapping'] = mapping
if not exists:
log.info('Request {} has started a new fragment collection: {}'.format(self._request_id, self._fragment_id))
else:
log.info('Request {} is going to re-use fragment {}'.format(self._request_id, self._fragment_id))
n_fragment_reqs = r.scard('fragments:{}:requests'.format(self._fragment_id))
log.info('Fragment {} is supporting {} more requests'.format(self._fragment_id, n_fragment_reqs))
def __deliver_responses():
import time
registered_deliveries = r.scard('deliveries')
deliveries_ready = r.scard('deliveries:ready')
log.info("""Delivery daemon started:
- Deliveries: {}
- Ready: {}""".format(registered_deliveries, deliveries_ready))
log.info('Delivery daemon started')
futures = {}
while True:
ready = r.smembers('deliveries:ready')
for rid in ready:
if rid not in futures:
log.info('Response delivery of request {} is ready. Preparing...'.format(rid))
futures[rid] = thp.submit(__deliver_response, rid)
for obsolete_rid in set.difference(set(futures.keys()), ready):
if obsolete_rid in futures and futures[obsolete_rid].done():
del futures[obsolete_rid]
sent = r.smembers('deliveries:sent')
for rid in sent:
r.srem('deliveries:ready', rid)
r.srem('deliveries', rid)
try:
response = build_response(rid)
response.sink.remove()
log.info('Request {} was sent and cleared'.format(rid))
except AttributeError:
log.warning('Request number {} was deleted by other means'.format(rid))
pass
r.delete('deliveries:sent')
time.sleep(1)
def __collect_fragments():
registered_fragments = r.scard("fragments")
synced_fragments = len(r.keys("fragments:*:sync"))
log.info(
"""Collector daemon started:
- Fragments: {}
- Synced: {}""".format(
registered_fragments, synced_fragments
)
)
futures = {}
while True:
for fid in filter(
lambda x: r.get("fragments:{}:sync".format(x)) is None and r.get("fragments:{}:pulling".format(x)) is None,
r.smembers("fragments"),
):
if fid in futures:
if futures[fid].done():
del futures[fid]
if fid not in futures:
futures[fid] = thp.submit(__pull_fragment, fid)
time.sleep(1)
def __pull_fragment(fid):
tps = r.smembers("fragments:{}:gp".format(fid))
requests, r_sinks = __load_fragment_requests(fid)
log.info(
"""Starting collection of fragment {}:
- GP: {}
- Supporting: ({}) {}""".format(
fid, list(tps), len(requests), list(requests)
)
)
start_time = datetime.now()
try:
fgm_gen, _, graph = agora_client.get_fragment_generator(
"{ %s }" % " . ".join(tps), workers=N_COLLECTORS, provider=graph_provider, queue_size=N_COLLECTORS
)
except Exception:
log.error("Agora is not available")
return
# There is no search plan to execute
if not list(graph.subjects(RDF.type, AGORA.SearchTree)):
log.info("There is no search plan for fragment {}. Removing...".format(fid))
# TODO: Send additional headers notifying the reason to end
__notify_completion(fid, r_sinks)
__remove_fragment(fid)
return
triple_patterns = {tpn: __triple_pattern(graph, tpn) for tpn in graph.subjects(RDF.type, AGORA.TriplePattern)}
fragment_contexts = {tpn: (fid, triple_patterns[tpn]) for tpn in triple_patterns}
__bind_prefixes(graph)
lock_key = "fragments:{}:lock".format(fid)
lock = r.lock(lock_key, lock_class=Lock)
lock.acquire()
lock_consume_key = "fragments:{}:lock:consume".format(fid)
c_lock = r.lock(lock_consume_key, lock_class=Lock)
c_lock.acquire()
# Update fragment contexts
with r.pipeline(transaction=True) as p:
p.multi()
p.set("fragments:{}:pulling".format(fid), True)
p.delete("fragments:{}:contexts".format(fid))
for tpn in fragment_contexts.keys():
p.sadd("fragments:{}:contexts".format(fid), fragment_contexts[tpn])
p.execute()
lock.release()
c_lock.release()
n_triples = 0
fragment_weight = 0
fragment_delta = 0
try:
for (c, s, p, o) in fgm_gen:
pre_ts = datetime.now()
triple_weight = len(u"{}{}{}".format(s, p, o))
fragment_weight += triple_weight
fragment_delta += triple_weight
lock.acquire()
if add_stream_triple(fid, triple_patterns[c], (s, p, o)):
__consume_quad(fid, (triple_patterns[c], s, p, o), graph, sinks=r_sinks)
lock.release()
if fragment_delta > 1000:
fragment_delta = 0
log.info("Pulling fragment {} [{} kB]".format(fid, fragment_weight / 1000.0))
if r.scard("fragments:{}:requests".format(fid)) != len(requests):
requests, r_sinks = __load_fragment_requests(fid)
n_triples += 1
post_ts = datetime.now()
elapsed = (post_ts - pre_ts).total_seconds()
excess = (1.0 / COLLECT_THROTTLING) - elapsed
if excess > 0:
sleep(excess)
except Exception, e:
traceback.print_exc()