from agora.stoa.actions.core import AGENT_ID
from agora.stoa.actions.core import STOA
from agora.stoa.actions.core.fragment import FragmentRequest, FragmentAction, FragmentSink
from agora.stoa.actions.core.utils import parse_bool, chunks
from agora.stoa.messaging.reply import reply
from agora.stoa.store import r
from agora.stoa.store.triples import load_stream_triples, fragments_cache
__author__ = 'Fernando Serena'
log = logging.getLogger('agora.scholar.actions.stream')
log.info("'Cleaning stream requests' locks...")
request_locks = r.keys('{}:requests:*:lock'.format(AGENT_ID))
for rlk in request_locks:
r.delete(rlk)
class StreamPlugin(FragmentPlugin):
@property
def sink_class(self):
return StreamSink
def consume(self, fid, (c, s, p, o), graph, *args):
sink = args[0]
sink.lock.acquire()
try:
# Prevent from consuming a triple when the delivery state says it was completely sent
# Anyway, this HAS TO BE REMOVED from here, because the stream flag should be enough
if sink.delivery == 'sent':
return
fragments_key = '{}:fragments'.format(AGENT_ID)
log.info("""Fragment daemon setup:
- On-demand threshold: {}
- Minimum sync time: {}
- Maximum concurrent collectors: {}
- Maximum concurrent fragments: {}""".format(ON_DEMAND_TH, MIN_SYNC, N_COLLECTORS,
MAX_CONCURRENT_FRAGMENTS))
# Fragment collection threadpool
thp = ThreadPoolExecutor(max_workers=min(8, MAX_CONCURRENT_FRAGMENTS))
log.info('Cleaning fragment locks...')
fragment_locks = r.keys('{}:*lock*'.format(fragments_key))
for flk in fragment_locks:
r.delete(flk)
log.info('Cleaning fragment pulling flags...')
fragment_pullings = r.keys('{}:*:pulling'.format(fragments_key))
for fpk in fragment_pullings:
r.delete(fpk)
def fragment_lock(fid):
"""
:param fid: Fragment id
:return: A redis-based lock object for a given fragment
"""
lock_key = '{}:{}:lock'.format(fragments_key, fid)
return r.lock(lock_key, lock_class=Lock)
__author__ = 'Fernando Serena'
log = logging.getLogger('agora.stoa.store.triples')
pool = ThreadPoolExecutor(max_workers=4)
GRAPH_THROTTLING = max(1, int(app.config.get('CACHE', {}).get('graph_throttling', 30)))
MIN_CACHE_TIME = max(0, int(app.config.get('CACHE', {}).get('min_cache_time', 10)))
log.info("""Triple store setup:
- Graph throttling: {}
- Minimum cache time: {}""".format(GRAPH_THROTTLING, MIN_CACHE_TIME))
log.info('Cleaning cache...')
uuid_locks = r.keys('{}:cache*'.format(AGENT_ID))
for ulk in uuid_locks:
r.delete(ulk)
def load_stream_triples(fid, until):
def __triplify(x):
def __extract_lang(v):
def __lang_tag_match(strg, search=re.compile(r'[^a-z]').search):
return not bool(search(strg))
if '@' in v:
try:
(v_aux, lang) = tuple(v.split('@'))
(v, lang) = (v_aux, lang) if __lang_tag_match(lang) else (v, None)
except ValueError:
lang = None
else:
