from agora.stoa.actions.core import AGENT_ID from agora.stoa.actions.core import STOA from agora.stoa.actions.core.fragment import FragmentRequest, FragmentAction, FragmentSink from agora.stoa.actions.core.utils import parse_bool, chunks from agora.stoa.messaging.reply import reply from agora.stoa.store import r from agora.stoa.store.triples import load_stream_triples, fragments_cache __author__ = 'Fernando Serena' log = logging.getLogger('agora.scholar.actions.stream') log.info("'Cleaning stream requests' locks...") request_locks = r.keys('{}:requests:*:lock'.format(AGENT_ID)) for rlk in request_locks: r.delete(rlk) class StreamPlugin(FragmentPlugin): @property def sink_class(self): return StreamSink def consume(self, fid, (c, s, p, o), graph, *args): sink = args[0] sink.lock.acquire() try: # Prevent from consuming a triple when the delivery state says it was completely sent # Anyway, this HAS TO BE REMOVED from here, because the stream flag should be enough if sink.delivery == 'sent': return
fragments_key = '{}:fragments'.format(AGENT_ID) log.info("""Fragment daemon setup: - On-demand threshold: {} - Minimum sync time: {} - Maximum concurrent collectors: {} - Maximum concurrent fragments: {}""".format(ON_DEMAND_TH, MIN_SYNC, N_COLLECTORS, MAX_CONCURRENT_FRAGMENTS)) # Fragment collection threadpool thp = ThreadPoolExecutor(max_workers=min(8, MAX_CONCURRENT_FRAGMENTS)) log.info('Cleaning fragment locks...') fragment_locks = r.keys('{}:*lock*'.format(fragments_key)) for flk in fragment_locks: r.delete(flk) log.info('Cleaning fragment pulling flags...') fragment_pullings = r.keys('{}:*:pulling'.format(fragments_key)) for fpk in fragment_pullings: r.delete(fpk) def fragment_lock(fid): """ :param fid: Fragment id :return: A redis-based lock object for a given fragment """ lock_key = '{}:{}:lock'.format(fragments_key, fid) return r.lock(lock_key, lock_class=Lock)
__author__ = 'Fernando Serena' log = logging.getLogger('agora.stoa.store.triples') pool = ThreadPoolExecutor(max_workers=4) GRAPH_THROTTLING = max(1, int(app.config.get('CACHE', {}).get('graph_throttling', 30))) MIN_CACHE_TIME = max(0, int(app.config.get('CACHE', {}).get('min_cache_time', 10))) log.info("""Triple store setup: - Graph throttling: {} - Minimum cache time: {}""".format(GRAPH_THROTTLING, MIN_CACHE_TIME)) log.info('Cleaning cache...') uuid_locks = r.keys('{}:cache*'.format(AGENT_ID)) for ulk in uuid_locks: r.delete(ulk) def load_stream_triples(fid, until): def __triplify(x): def __extract_lang(v): def __lang_tag_match(strg, search=re.compile(r'[^a-z]').search): return not bool(search(strg)) if '@' in v: try: (v_aux, lang) = tuple(v.split('@')) (v, lang) = (v_aux, lang) if __lang_tag_match(lang) else (v, None) except ValueError: lang = None else: