def get_fragment(fid): if not r.sismember('fragments', fid): raise NotFound('The fragment {} does not exist'.format(fid)) pulling = r.get('fragments:{}:pulling'.format(fid)) if pulling is None: pulling = 'False' fr_dict = {'id': fid, 'pattern': "{ %s }" % ' . '.join(r.smembers('fragments:{}:gp'.format(fid))), 'updated': r.get('fragments:{}:updated'.format(fid)), 'pulling': eval(pulling), 'requests': list(r.smembers('fragments:{}:requests'.format(fid)))} if fr_dict['pulling']: fr_dict['triples'] = r.zcard('fragments:{}:stream'.format(fid)) return jsonify(fr_dict)
def _save(self, action): super(FragmentSink, self)._save(action) self._build_graph_pattern(action) fragment_mapping = self.__check_gp() exists = fragment_mapping is not None if not exists: self._fragment_id = str(uuid()) self._pipe.sadd('fragments', self._fragment_id) self._pipe.sadd('fragments:{}:gp'.format(self._fragment_id), *self._graph_pattern) mapping = {str(k): str(k) for k in self._variables_dict.values()} else: self._fragment_id, mapping = fragment_mapping if r.get('fragments:{}:on_demand'.format(self._fragment_id)) is not None: self._pipe.delete('fragments:{}:sync'.format(self._fragment_id)) self._pipe.hset(self._request_key, 'mapping', mapping) self._pipe.hset(self._request_key, 'preferred_labels', self._preferred_labels) self._pipe.sadd('fragments:{}:requests'.format(self._fragment_id), self._request_id) self._pipe.hset('{}'.format(self._request_key), 'fragment_id', self._fragment_id) self._pipe.hset('{}'.format(self._request_key), 'pattern', ' . '.join(self._graph_pattern)) self._dict_fields['mapping'] = mapping if not exists: log.info('Request {} has started a new fragment collection: {}'.format(self._request_id, self._fragment_id)) else: log.info('Request {} is going to re-use fragment {}'.format(self._request_id, self._fragment_id)) n_fragment_reqs = r.scard('fragments:{}:requests'.format(self._fragment_id)) log.info('Fragment {} is supporting {} more requests'.format(self._fragment_id, n_fragment_reqs))
def get_fragment_triples(fid): if not r.sismember('fragments', fid): raise NotFound('The fragment {} does not exist'.format(fid)) if r.get('fragments:{}:pulling'.format(fid)) is not None: triples = [u'{} {} {} .'.format(s.n3(), p.n3(), o.n3()) for (c, s, p, o) in load_stream_triples(fid, '+inf')] response = make_response('\n'.join(triples)) response.headers['content-type'] = 'text/n3' return response return 'hello!'
def register_enrichment(pipe, fid, target, links): e_hash = generate_enrichment_hash(target, links) if not r.sismember('enrichments', e_hash): eid = shortuuid.uuid() enrichment_data = EnrichmentData(eid, fid, target, links) enrichment_data.save(pipe) pipe.sadd('enrichments', e_hash) pipe.set('map:enrichments:{}'.format(e_hash), eid) else: eid = r.get('map:enrichments:{}'.format(e_hash)) return eid
def __collect_fragments(): registered_fragments = r.scard("fragments") synced_fragments = len(r.keys("fragments:*:sync")) log.info( """Collector daemon started: - Fragments: {} - Synced: {}""".format( registered_fragments, synced_fragments ) ) futures = {} while True: for fid in filter( lambda x: r.get("fragments:{}:sync".format(x)) is None and r.get("fragments:{}:pulling".format(x)) is None, r.smembers("fragments"), ): if fid in futures: if futures[fid].done(): del futures[fid] if fid not in futures: futures[fid] = thp.submit(__pull_fragment, fid) time.sleep(1)
def is_pulling(self): return r.get('fragments:{}:pulling'.format(self._fragment_id)) is not None
def fragment_on_demand(self): return r.get('fragments:{}:on_demand'.format(self._fragment_id))
def fragment_updated_on(self): return r.get('fragments:{}:updated'.format(self._fragment_id))