def _save(self, action): super(FragmentSink, self)._save(action) self._build_graph_pattern(action) fragment_mapping = self.__check_gp() exists = fragment_mapping is not None if not exists: self._fragment_id = str(uuid()) self._pipe.sadd('fragments', self._fragment_id) self._pipe.sadd('fragments:{}:gp'.format(self._fragment_id), *self._graph_pattern) mapping = {str(k): str(k) for k in self._variables_dict.values()} else: self._fragment_id, mapping = fragment_mapping if r.get('fragments:{}:on_demand'.format(self._fragment_id)) is not None: self._pipe.delete('fragments:{}:sync'.format(self._fragment_id)) self._pipe.hset(self._request_key, 'mapping', mapping) self._pipe.hset(self._request_key, 'preferred_labels', self._preferred_labels) self._pipe.sadd('fragments:{}:requests'.format(self._fragment_id), self._request_id) self._pipe.hset('{}'.format(self._request_key), 'fragment_id', self._fragment_id) self._pipe.hset('{}'.format(self._request_key), 'pattern', ' . '.join(self._graph_pattern)) self._dict_fields['mapping'] = mapping if not exists: log.info('Request {} has started a new fragment collection: {}'.format(self._request_id, self._fragment_id)) else: log.info('Request {} is going to re-use fragment {}'.format(self._request_id, self._fragment_id)) n_fragment_reqs = r.scard('fragments:{}:requests'.format(self._fragment_id)) log.info('Fragment {} is supporting {} more requests'.format(self._fragment_id, n_fragment_reqs))
def __deliver_responses(): import time registered_deliveries = r.scard('deliveries') deliveries_ready = r.scard('deliveries:ready') log.info("""Delivery daemon started: - Deliveries: {} - Ready: {}""".format(registered_deliveries, deliveries_ready)) log.info('Delivery daemon started') futures = {} while True: ready = r.smembers('deliveries:ready') for rid in ready: if rid not in futures: log.info('Response delivery of request {} is ready. Preparing...'.format(rid)) futures[rid] = thp.submit(__deliver_response, rid) for obsolete_rid in set.difference(set(futures.keys()), ready): if obsolete_rid in futures and futures[obsolete_rid].done(): del futures[obsolete_rid] sent = r.smembers('deliveries:sent') for rid in sent: r.srem('deliveries:ready', rid) r.srem('deliveries', rid) try: response = build_response(rid) response.sink.remove() log.info('Request {} was sent and cleared'.format(rid)) except AttributeError: log.warning('Request number {} was deleted by other means'.format(rid)) pass r.delete('deliveries:sent') time.sleep(1)
def __collect_fragments(): registered_fragments = r.scard("fragments") synced_fragments = len(r.keys("fragments:*:sync")) log.info( """Collector daemon started: - Fragments: {} - Synced: {}""".format( registered_fragments, synced_fragments ) ) futures = {} while True: for fid in filter( lambda x: r.get("fragments:{}:sync".format(x)) is None and r.get("fragments:{}:pulling".format(x)) is None, r.smembers("fragments"), ): if fid in futures: if futures[fid].done(): del futures[fid] if fid not in futures: futures[fid] = thp.submit(__pull_fragment, fid) time.sleep(1)
def __pull_fragment(fid): tps = r.smembers("fragments:{}:gp".format(fid)) requests, r_sinks = __load_fragment_requests(fid) log.info( """Starting collection of fragment {}: - GP: {} - Supporting: ({}) {}""".format( fid, list(tps), len(requests), list(requests) ) ) start_time = datetime.now() try: fgm_gen, _, graph = agora_client.get_fragment_generator( "{ %s }" % " . ".join(tps), workers=N_COLLECTORS, provider=graph_provider, queue_size=N_COLLECTORS ) except Exception: log.error("Agora is not available") return # There is no search plan to execute if not list(graph.subjects(RDF.type, AGORA.SearchTree)): log.info("There is no search plan for fragment {}. Removing...".format(fid)) # TODO: Send additional headers notifying the reason to end __notify_completion(fid, r_sinks) __remove_fragment(fid) return triple_patterns = {tpn: __triple_pattern(graph, tpn) for tpn in graph.subjects(RDF.type, AGORA.TriplePattern)} fragment_contexts = {tpn: (fid, triple_patterns[tpn]) for tpn in triple_patterns} __bind_prefixes(graph) lock_key = "fragments:{}:lock".format(fid) lock = r.lock(lock_key, lock_class=Lock) lock.acquire() lock_consume_key = "fragments:{}:lock:consume".format(fid) c_lock = r.lock(lock_consume_key, lock_class=Lock) c_lock.acquire() # Update fragment contexts with r.pipeline(transaction=True) as p: p.multi() p.set("fragments:{}:pulling".format(fid), True) p.delete("fragments:{}:contexts".format(fid)) for tpn in fragment_contexts.keys(): p.sadd("fragments:{}:contexts".format(fid), fragment_contexts[tpn]) p.execute() lock.release() c_lock.release() n_triples = 0 fragment_weight = 0 fragment_delta = 0 try: for (c, s, p, o) in fgm_gen: pre_ts = datetime.now() triple_weight = len(u"{}{}{}".format(s, p, o)) fragment_weight += triple_weight fragment_delta += triple_weight lock.acquire() if add_stream_triple(fid, triple_patterns[c], (s, p, o)): __consume_quad(fid, (triple_patterns[c], s, p, o), graph, sinks=r_sinks) lock.release() if fragment_delta > 1000: fragment_delta = 0 log.info("Pulling fragment {} [{} kB]".format(fid, fragment_weight / 1000.0)) if r.scard("fragments:{}:requests".format(fid)) != len(requests): requests, r_sinks = __load_fragment_requests(fid) n_triples += 1 post_ts = datetime.now() elapsed = (post_ts - pre_ts).total_seconds() excess = (1.0 / COLLECT_THROTTLING) - elapsed if excess > 0: sleep(excess) except Exception, e: traceback.print_exc()