def get_fragment(fid):
if not r.sismember('fragments', fid):
raise NotFound('The fragment {} does not exist'.format(fid))
pulling = r.get('fragments:{}:pulling'.format(fid))
if pulling is None:
pulling = 'False'
fr_dict = {'id': fid, 'pattern': "{ %s }" % ' . '.join(r.smembers('fragments:{}:gp'.format(fid))),
'updated': r.get('fragments:{}:updated'.format(fid)),
'pulling': eval(pulling),
'requests': list(r.smembers('fragments:{}:requests'.format(fid)))}
if fr_dict['pulling']:
fr_dict['triples'] = r.zcard('fragments:{}:stream'.format(fid))
return jsonify(fr_dict)
def _save(self, action):
super(FragmentSink, self)._save(action)
self._build_graph_pattern(action)
fragment_mapping = self.__check_gp()
exists = fragment_mapping is not None
if not exists:
self._fragment_id = str(uuid())
self._pipe.sadd('fragments', self._fragment_id)
self._pipe.sadd('fragments:{}:gp'.format(self._fragment_id), *self._graph_pattern)
mapping = {str(k): str(k) for k in self._variables_dict.values()}
else:
self._fragment_id, mapping = fragment_mapping
if r.get('fragments:{}:on_demand'.format(self._fragment_id)) is not None:
self._pipe.delete('fragments:{}:sync'.format(self._fragment_id))
self._pipe.hset(self._request_key, 'mapping', mapping)
self._pipe.hset(self._request_key, 'preferred_labels', self._preferred_labels)
self._pipe.sadd('fragments:{}:requests'.format(self._fragment_id), self._request_id)
self._pipe.hset('{}'.format(self._request_key), 'fragment_id', self._fragment_id)
self._pipe.hset('{}'.format(self._request_key), 'pattern', ' . '.join(self._graph_pattern))
self._dict_fields['mapping'] = mapping
if not exists:
log.info('Request {} has started a new fragment collection: {}'.format(self._request_id, self._fragment_id))
else:
log.info('Request {} is going to re-use fragment {}'.format(self._request_id, self._fragment_id))
n_fragment_reqs = r.scard('fragments:{}:requests'.format(self._fragment_id))
log.info('Fragment {} is supporting {} more requests'.format(self._fragment_id, n_fragment_reqs))
def get_fragment_triples(fid):
if not r.sismember('fragments', fid):
raise NotFound('The fragment {} does not exist'.format(fid))
if r.get('fragments:{}:pulling'.format(fid)) is not None:
triples = [u'{} {} {} .'.format(s.n3(), p.n3(), o.n3()) for (c, s, p, o) in load_stream_triples(fid, '+inf')]
response = make_response('\n'.join(triples))
response.headers['content-type'] = 'text/n3'
return response
return 'hello!'
def register_enrichment(pipe, fid, target, links):
e_hash = generate_enrichment_hash(target, links)
if not r.sismember('enrichments', e_hash):
eid = shortuuid.uuid()
enrichment_data = EnrichmentData(eid, fid, target, links)
enrichment_data.save(pipe)
pipe.sadd('enrichments', e_hash)
pipe.set('map:enrichments:{}'.format(e_hash), eid)
else:
eid = r.get('map:enrichments:{}'.format(e_hash))
return eid
def __collect_fragments():
registered_fragments = r.scard("fragments")
synced_fragments = len(r.keys("fragments:*:sync"))
log.info(
"""Collector daemon started:
- Fragments: {}
- Synced: {}""".format(
registered_fragments, synced_fragments
)
)
futures = {}
while True:
for fid in filter(
lambda x: r.get("fragments:{}:sync".format(x)) is None and r.get("fragments:{}:pulling".format(x)) is None,
r.smembers("fragments"),
):
if fid in futures:
if futures[fid].done():
del futures[fid]
if fid not in futures:
futures[fid] = thp.submit(__pull_fragment, fid)
time.sleep(1)
def is_pulling(self):
return r.get('fragments:{}:pulling'.format(self._fragment_id)) is not None
def fragment_on_demand(self):
return r.get('fragments:{}:on_demand'.format(self._fragment_id))
def fragment_updated_on(self):
return r.get('fragments:{}:updated'.format(self._fragment_id))
