def _load(self):
"""
Loads data relating to the recovery of a fragment for this request
"""
super(FragmentSink, self)._load()
self._graph_pattern = GraphPattern(r.smembers('{}gp'.format(self._request_key)))
self._fragment_pattern = GraphPattern(r.smembers('{}:gp'.format(self._fragment_key)))
self._filter_mapping = r.hgetall('{}filters'.format(self._request_key))
self._dict_fields['mapping'] = r.hgetall('{}map'.format(self._request_key))
self._dict_fields['preferred_labels'] = set(r.smembers('{}pl'.format(self._request_key)))
def __get_fragment(fid):
if not r.sismember('{}:fragments'.format(AGENT_ID), fid):
raise NotFound('The fragment {} does not exist'.format(fid))
f_dict = {
'id': fid,
'gp': list(r.smembers('{}:fragments:{}:gp'.format(AGENT_ID, fid))),
'synced': r.exists('{}:fragments:{}:sync'.format(AGENT_ID, fid)),
'requests': list(r.smembers('{}:fragments:{}:requests'.format(AGENT_ID, fid)))
}
return f_dict
def load(self):
dict_fields = r.hgetall(self._enrichment_key)
self.target = URIRef(dict_fields.get('target', None))
self.fragment_id = dict_fields.get('fragment_id', None)
self.links = map(lambda (link, v): (URIRef(link), v), [eval(pair_str) for pair_str in
r.smembers('{}:links'.format(
self._enrichment_key))])
def __notify_completion(fid, sinks):
"""
Notify the ending of a fragment collection to all registered plugins
:param fid: Fragment id
:param sinks: Set of dependent sinks
:return:
"""
for sink in sinks.values():
if sink.delivery == 'accepted':
sink.delivery = 'ready'
if FragmentPlugin.plugins:
fragment_gp = GraphPattern(r.smembers('{}:fragments:{}:gp'.format(AGENT_ID, fid)))
for plugin in FragmentPlugin.plugins():
try:
filtered_sinks = filter(lambda _: isinstance(sinks[_], plugin.sink_class), sinks)
for rid in filtered_sinks:
sink = sinks[rid]
if plugin.sink_aware:
plugin.complete(fid, sink)
if not plugin.sink_aware:
plugin.complete(fid, fragment_gp)
except Exception as e:
log.warning(e.message)
def __purge(self):
while True:
self.__lock.acquire()
try:
obsolete = filter(lambda x: not r.exists('{}:cache:{}'.format(AGENT_ID, x)),
r.smembers(self.__cache_key))
if obsolete:
with r.pipeline(transaction=True) as p:
p.multi()
log.info('Removing {} resouces from cache...'.format(len(obsolete)))
for uuid in obsolete:
uuid_lock = self.uuid_lock(uuid)
uuid_lock.acquire()
try:
gid = r.hget(self.__gids_key, uuid)
counter_key = '{}:cache:{}:cnt'.format(AGENT_ID, uuid)
usage_counter = r.get(counter_key)
if usage_counter is None or int(usage_counter) <= 0:
try:
resources_cache.remove_context(resources_cache.get_context(uuid))
p.srem(self.__cache_key, uuid)
p.hdel(self.__gids_key, uuid)
p.hdel(self.__gids_key, gid)
p.delete(counter_key)
g = self.__uuid_dict[uuid]
del self.__uuid_dict[uuid]
del self.__graph_dict[g]
except Exception, e:
traceback.print_exc()
log.error('Purging resource {} with uuid {}'.format(gid, uuid))
p.execute()
finally:
uuid_lock.release()
def __deliver_responses():
import time
__log.info('Delivery daemon started')
# Declare in-progress deliveries dictionary
futures = {}
while True:
try:
# Get all ready deliveries
ready = r.smembers(__ready_key)
for rid in ready:
# If the delivery is not in the thread pool, just submit it
if rid not in futures:
__log.info('Response delivery of request {} is ready. Putting it in queue...'.format(rid))
futures[rid] = __thp.submit(__deliver_response, rid)
# Clear futures that have already ceased to be ready
for obsolete_rid in set.difference(set(futures.keys()), ready):
if obsolete_rid in futures and futures[obsolete_rid].done():
del futures[obsolete_rid]
# All those deliveries that are marked as 'sent' are being cleared here along its request data
sent = r.smembers(__sent_key)
for rid in sent:
r.srem(__ready_key, rid)
r.srem(__deliveries_key, rid)
try:
response = build_response(rid)
response.sink.remove() # Its lock is removed too
__log.info('Request {} was sent and cleared'.format(rid))
except AttributeError:
traceback.print_exc()
__log.warning('Request number {} was deleted by other means'.format(rid))
pass
r.srem(__sent_key, rid)
except Exception as e:
__log.error(e.message)
traceback.print_exc()
finally:
time.sleep(0.1)
def __check_gp_mappings(self, gp=None):
"""
Used in _save method. Seeks matches with some fragment already registered
:param gp: By default, _graph_pattern attribute is used when gp is None
:return: The matching fragment id and the mapping dictionary or None if there is no matching
"""
if gp is None:
gp = self._graph_pattern
gp_keys = r.keys('{}:*:gp'.format(self._fragments_key))
for gpk in gp_keys:
stored_gp = GraphPattern(r.smembers(gpk))
mapping = stored_gp.mapping(gp)
if mapping:
return gpk.split(':')[-2], mapping
return None
def __load_fragment_requests(fid):
"""
Load all requests and their sinks that are related to a given fragment id
:param fid: Fragment id
:return: A dictionary of sinks of all fragment requests
"""
sinks_ = {}
fragment_requests_key = '{}:{}:requests'.format(fragments_key, fid)
for rid in r.smembers(fragment_requests_key):
try:
sinks_[rid] = build_response(rid).sink
except Exception, e:
log.warning(e.message)
with r.pipeline(transaction=True) as p:
p.multi()
p.srem(fragment_requests_key, rid)
p.execute()
def __collect_fragments():
registered_fragments = r.scard(fragments_key)
synced_fragments = len(r.keys('{}:*:sync'.format(fragments_key)))
log.info("""Collector daemon started:
- Fragments: {}
- Synced: {}""".format(registered_fragments, synced_fragments))
futures = {}
while True:
for fid in filter(
lambda x: r.get('{}:{}:sync'.format(fragments_key, x)) is None and r.get(
'{}:{}:pulling'.format(fragments_key, x)) is None,
r.smembers(fragments_key)):
if fid in futures:
if futures[fid].done():
del futures[fid]
if fid not in futures:
futures[fid] = thp.submit(__pull_fragment, fid)
time.sleep(1)
def __pull_fragment(fid):
fragment_key = '{}:{}'.format(fragments_key, fid)
tps = r.smembers('{}:gp'.format(fragment_key))
r_sinks = __load_fragment_requests(fid)
log.info("""Starting collection of fragment {}:
- GP: {}
- Supporting: ({}) {}""".format(fid, list(tps), len(r_sinks), list(r_sinks.keys())))
try:
prefixes, gen = get_query_generator(*tps,
broker_host=BROKER['host'],
agora_host=AGORA['host'],
broker_port=BROKER['port'],
agora_port=AGORA['port'], wait=True)
except Exception, e:
traceback.print_exc()
log.error('Scholar is not available')
return
def get_fragment_enrichments(fid):
return [EnrichmentData(eid) for eid in r.smembers('{}:fragments:{}:enrichments'.format(AGENT_ID, fid))]
def fragment_contexts(fid):
return r.smembers('{}:{}:contexts'.format(fragments_key, fid))
def __pull_fragment(fid):
"""
Pull and replace (if needed) a given fragment
:param fid: Fragment id
"""
fragment_key = '{}:{}'.format(fragments_key, fid)
# Load fragment graph pattern
tps = r.smembers('{}:gp'.format(fragment_key))
# Load fragment requests (including their sinks)
r_sinks = __load_fragment_requests(fid)
log.info("""Starting collection of fragment {}:
- GP: {}
- Supporting: ({}) {}""".format(fid, list(tps), len(r_sinks), list(r_sinks)))
# Prepare the corresponding fragment generator and fetch the search plan
start_time = datetime.now()
try:
fgm_gen, _, graph = agora_client.get_fragment_generator('{ %s }' % ' . '.join(tps), workers=N_COLLECTORS,
provider=graph_provider, queue_size=N_COLLECTORS)
except Exception:
log.error('Agora is not available')
return
# In case there is not SearchTree in the plan: notify, remove and abort collection
if not list(graph.subjects(RDF.type, AGORA.SearchTree)):
log.info('There is no search plan for fragment {}. Removing...'.format(fid))
# TODO: Send additional headers notifying the reason to end
__notify_completion(fid, r_sinks)
__remove_fragment(fid)
return
# Update cache graph prefixes
__bind_prefixes(graph)
# Extract triple patterns' dictionary from the search plan
context_tp = {tpn: __extract_tp_from_plan(graph, tpn) for tpn in
graph.subjects(RDF.type, AGORA.TriplePattern)}
frag_contexts = {tpn: (fid, context_tp[tpn]) for tpn in context_tp}
lock = fragment_lock(fid)
lock.acquire()
# Update fragment contexts
with r.pipeline(transaction=True) as p:
p.multi()
p.set('{}:pulling'.format(fragment_key), True)
contexts_key = '{}:contexts'.format(fragment_key)
p.delete(contexts_key)
for tpn in context_tp.keys():
p.sadd(contexts_key, frag_contexts[tpn])
p.execute()
lock.release()
# Init fragment collection counters
n_triples = 0
fragment_weight = 0
fragment_delta = 0
log.info('Collecting fragment {}...'.format(fid))
try:
# Iterate all fragment triples and their contexts
for (c, s, p, o) in fgm_gen:
pre_ts = datetime.now()
# Update weights and counters
triple_weight = len(u'{}{}{}'.format(s, p, o))
fragment_weight += triple_weight
fragment_delta += triple_weight
# Store the triple if it was not obtained before and notify related requests
try:
lock.acquire()
new_triple = add_stream_triple(fid, context_tp[c], (s, p, o))
lock.release()
if new_triple:
__consume_quad(fid, (context_tp[c], s, p, o), graph, sinks=r_sinks)
n_triples += 1
except Exception, e:
log.warning(e.message)
traceback.print_exc()
if fragment_delta > 10000:
fragment_delta = 0
log.info('Pulling fragment {} [{} kB]'.format(fid, fragment_weight / 1000.0))
if n_triples % 100 == 0:
# Update fragment requests
if r.scard('{}:requests'.format(fragment_key)) != len(r_sinks):
r_sinks = __load_fragment_requests(fid)
post_ts = datetime.now()
elapsed = (post_ts - pre_ts).total_seconds()
throttling = THROTTLING_TIME - elapsed
if throttling > 0:
sleep(throttling)
except Exception, e:
log.warning(e.message)
traceback.print_exc()
def __pull_fragment(fid):
"""
Pull and replace (if needed) a given fragment
:param fid: Fragment id
"""
fragment_key = '{}:{}'.format(fragments_key, fid)
on_events = r.get('{}:events'.format(fragment_key))
if on_events == 'True' and not change_in_fragment_resource(fid, int(r.get('{}:ud'.format(fragment_key)))):
with r.pipeline(transaction=True) as p:
p.multi()
sync_key = '{}:sync'.format(fragment_key)
p.set(sync_key, True)
durability = int(r.get('{}:ud'.format(fragment_key)))
p.expire(sync_key, durability)
p.set('{}:updated'.format(fragment_key), calendar.timegm(dt.utcnow().timetuple()))
p.delete('{}:pulling'.format(fragment_key))
p.execute()
return
# Load fragment graph pattern
tps = r.smembers('{}:gp'.format(fragment_key))
# Load fragment requests (including their sinks)
r_sinks = __load_fragment_requests(fid)
log.info("""Starting collection of fragment {}:
- GP: {}
- Supporting: ({}) {}""".format(fid, list(tps), len(r_sinks), list(r_sinks)))
init_fragment_resources(fid)
# Prepare the corresponding fragment generator and fetch the search plan
start_time = datetime.utcnow()
try:
fgm_gen, _, graph = agora_client.get_fragment_generator('{ %s }' % ' . '.join(tps), workers=N_COLLECTORS,
provider=graph_provider, queue_size=N_COLLECTORS*100)
except Exception:
traceback.print_exc()
log.error('Agora is not available')
return
# In case there is not SearchTree in the plan: notify, remove and abort collection
if not list(graph.subjects(RDF.type, AGORA.SearchTree)):
log.info('There is no search plan for fragment {}. Removing...'.format(fid))
# TODO: Send additional headers notifying the reason to end
__notify_completion(fid, r_sinks)
__remove_fragment(fid)
return
# Update cache graph prefixes
__bind_prefixes(graph)
# Extract triple patterns' dictionary from the search plan
context_tp = {tpn: __extract_tp_from_plan(graph, tpn) for tpn in
graph.subjects(RDF.type, AGORA.TriplePattern)}
frag_contexts = {tpn: (fid, context_tp[tpn]) for tpn in context_tp}
lock = fragment_lock(fid)
lock.acquire()
# Update fragment contexts
with r.pipeline(transaction=True) as p:
p.multi()
p.set('{}:pulling'.format(fragment_key), True)
contexts_key = '{}:contexts'.format(fragment_key)
p.delete(contexts_key)
clear_fragment_stream(fid)
for tpn in context_tp.keys():
p.sadd(contexts_key, frag_contexts[tpn])
p.execute()
lock.release()
# Init fragment collection counters
n_triples = 0
fragment_weight = 0
fragment_delta = 0
log.info('Collecting fragment {}...'.format(fid))
try:
# Iterate all fragment triples and their contexts
pre_ts = datetime.utcnow()
for (c, s, p, o) in fgm_gen:
# Update weights and counters
triple_weight = len(u'{}{}{}'.format(s, p, o))
fragment_weight += triple_weight
fragment_delta += triple_weight
# Store the triple if it was not obtained before and notify related requests
try:
lock.acquire()
new_triple = add_stream_triple(fid, context_tp[c], (s, p, o))
lock.release()
if new_triple:
if isinstance(s, URIRef):
if s not in resource_in_fragment:
resource_in_fragment[s] = set([])
resource_in_fragment[s].add(fid)
fragment_resources[fid].add(s)
__consume_quad(fid, (context_tp[c], s, p, o), graph, sinks=r_sinks)
n_triples += 1
except Exception as e:
log.warning(e.message)
traceback.print_exc()
if fragment_delta > 10000:
fragment_delta = 0
log.info('Pulling fragment {} [{} kB]'.format(fid, fragment_weight / 1000.0))
if n_triples % 100 == 0:
# Update fragment requests
if r.scard('{}:requests'.format(fragment_key)) != len(r_sinks):
r_sinks = __load_fragment_requests(fid)
post_ts = datetime.utcnow()
elapsed = (post_ts - pre_ts).total_seconds()
throttling = THROTTLING_TIME - elapsed
if throttling > 0:
sleep(throttling)
pre_ts = datetime.utcnow()
except Exception as e:
log.warning(e.message)
traceback.print_exc()
elapsed = (datetime.utcnow() - start_time).total_seconds()
log.info(
'{} triples retrieved for fragment {} in {} s [{} kB]'.format(n_triples, fid, elapsed,
fragment_weight / 1000.0))
# Update fragment cache and its contexts
lock.acquire()
try:
__update_fragment_cache(fid, tps)
log.info('Fragment {} data has been replaced with the recently collected'.format(fid))
__cache_plan_context(fid, graph)
log.info('BGP context of fragment {} has been cached'.format(fid))
log.info('Updating result set for fragment {}...'.format(fid))
# Calculate sync times and update fragment flags
with r.pipeline(transaction=True) as p:
p.multi()
sync_key = '{}:sync'.format(fragment_key)
demand_key = '{}:on_demand'.format(fragment_key)
# Fragment is now synced
p.set(sync_key, True)
# If the fragment collection time has not exceeded the threshold, switch to on-demand mode
# if elapsed < ON_DEMAND_TH and elapsed * random.random() < ON_DEMAND_TH / 4:
# p.set(demand_key, True)
# log.info('Fragment {} has been switched to on-demand mode'.format(fid))
# else:
p.delete(demand_key)
updated_delay = int(r.get('{}:ud'.format(fragment_key)))
last_requests_ts = map(lambda x: int(x), r.lrange('{}:hist'.format(fragment_key), 0, -1))
print last_requests_ts
current_ts = calendar.timegm(datetime.utcnow().timetuple())
first_collection = r.get('{}:updated'.format(fragment_key)) is None
base_ts = last_requests_ts[:]
if not first_collection:
if current_ts - base_ts[0] <= updated_delay:
current_ts += updated_delay # Force
base_ts = [current_ts] + base_ts
request_intervals = [i - j for i, j in zip(base_ts[:-1], base_ts[1:])]
if request_intervals:
avg_gap = reduce(lambda x, y: x + y, request_intervals) / len(request_intervals)
print avg_gap,
durability = avg_gap - elapsed if avg_gap > updated_delay else updated_delay - elapsed
else:
durability = updated_delay - elapsed
durability = int(max(durability, 1))
print durability
if durability <= updated_delay - elapsed:
p.expire(sync_key, durability)
log.info('Fragment {} is considered synced for {} s'.format(fid, durability))
else:
clear_fragment_stream(fid)
p.delete('{}:updated'.format(fragment_key))
p.delete('{}:hist'.format(fragment_key))
log.info('Fragment {} will no longer be automatically updated'.format(fid))
p.set('{}:updated'.format(fragment_key), calendar.timegm(dt.utcnow().timetuple()))
p.delete('{}:pulling'.format(fragment_key))
p.execute()
__notify_completion(fid, r_sinks)
finally:
lock.release()
log.info('Fragment {} collection is complete!'.format(fid))
def get_fragments():
fragment_ids = list(r.smembers('{}:fragments'.format(AGENT_ID)))
f_list = [{'id': fid, 'gp': list(r.smembers('{}:fragments:{}:gp'.format(AGENT_ID, fid)))} for fid in fragment_ids]
return jsonify(fragments=f_list)
