def used_channels():
"""
Selects all channels that were declared by current requests
"""
req_channel_keys = r.keys('{}:requests:*:'.format(AGENT_ID))
for rck in req_channel_keys:
try:
channel = r.hget(rck, 'channel')
yield channel
except Exception as e:
traceback.print_exc()
log.warning(e.message)
def __check_gp_mappings(self, gp=None):
"""
Used in _save method. Seeks matches with some fragment already registered
:param gp: By default, _graph_pattern attribute is used when gp is None
:return: The matching fragment id and the mapping dictionary or None if there is no matching
"""
if gp is None:
gp = self._graph_pattern
gp_keys = r.keys('{}:*:gp'.format(self._fragments_key))
for gpk in gp_keys:
stored_gp = GraphPattern(r.smembers(gpk))
mapping = stored_gp.mapping(gp)
if mapping:
return gpk.split(':')[-2], mapping
return None
def __collect_fragments():
registered_fragments = r.scard(fragments_key)
synced_fragments = len(r.keys('{}:*:sync'.format(fragments_key)))
log.info("""Collector daemon started:
- Fragments: {}
- Synced: {}""".format(registered_fragments, synced_fragments))
futures = {}
while True:
for fid in filter(
lambda x: r.get('{}:{}:sync'.format(fragments_key, x)) is None and r.get(
'{}:{}:pulling'.format(fragments_key, x)) is None,
r.smembers(fragments_key)):
if fid in futures:
if futures[fid].done():
del futures[fid]
if fid not in futures:
futures[fid] = thp.submit(__pull_fragment, fid)
time.sleep(1)
def __remove_fragment(fid):
"""
Completely remove a fragment from the system after notifying its known consumers
:param fid: Fragment identifier
"""
log.debug('Waiting to remove fragment {}...'.format(fid))
lock = fragment_lock(fid)
lock.acquire()
r_sinks = __load_fragment_requests(fid)
__notify_completion(fid, r_sinks)
fragment_keys = r.keys('{}:{}*'.format(fragments_key, fid))
with r.pipeline(transaction=True) as p:
p.multi()
map(lambda k: p.delete(k), fragment_keys)
p.srem(fragments_key, fid)
p.execute()
# Fragment lock key was just implicitly removed, so it's not necessary to release the lock
# lock.release()
log.info('Fragment {} has been removed'.format(fid))
THROTTLING_TIME = (1.0 / (COLLECT_THROTTLING * 1000))
fragments_key = '{}:fragments'.format(AGENT_ID)
log.info("""Fragment daemon setup:
- On-demand threshold: {}
- Minimum sync time: {}
- Maximum concurrent collectors: {}
- Maximum concurrent fragments: {}""".format(ON_DEMAND_TH, MIN_SYNC, N_COLLECTORS,
MAX_CONCURRENT_FRAGMENTS))
# Fragment collection threadpool
thp = ThreadPoolExecutor(max_workers=min(8, MAX_CONCURRENT_FRAGMENTS))
log.info('Cleaning fragment locks...')
fragment_locks = r.keys('{}:*lock*'.format(fragments_key))
for flk in fragment_locks:
r.delete(flk)
log.info('Cleaning fragment pulling flags...')
fragment_pullings = r.keys('{}:*:pulling'.format(fragments_key))
for fpk in fragment_pullings:
r.delete(fpk)
def fragment_lock(fid):
"""
:param fid: Fragment id
:return: A redis-based lock object for a given fragment
"""
lock_key = '{}:{}:lock'.format(fragments_key, fid)
fragment_contexts
from agora.scholar.daemons.fragment import fragment_lock
from agora.stoa.actions.core import AGENT_ID
from agora.stoa.actions.core import STOA
from agora.stoa.actions.core.fragment import FragmentRequest, FragmentAction, FragmentSink
from agora.stoa.actions.core.utils import parse_bool, chunks
from agora.stoa.messaging.reply import reply
from agora.stoa.store import r
from agora.stoa.store.triples import load_stream_triples, fragments_cache
__author__ = 'Fernando Serena'
log = logging.getLogger('agora.scholar.actions.stream')
log.info("'Cleaning stream requests' locks...")
request_locks = r.keys('{}:requests:*:lock'.format(AGENT_ID))
for rlk in request_locks:
r.delete(rlk)
class StreamPlugin(FragmentPlugin):
@property
def sink_class(self):
return StreamSink
def consume(self, fid, (c, s, p, o), graph, *args):
sink = args[0]
sink.lock.acquire()
try:
# Prevent from consuming a triple when the delivery state says it was completely sent
# Anyway, this HAS TO BE REMOVED from here, because the stream flag should be enough
import re
__author__ = 'Fernando Serena'
log = logging.getLogger('agora.stoa.store.triples')
pool = ThreadPoolExecutor(max_workers=4)
GRAPH_THROTTLING = max(1, int(app.config.get('CACHE', {}).get('graph_throttling', 30)))
MIN_CACHE_TIME = max(0, int(app.config.get('CACHE', {}).get('min_cache_time', 10)))
log.info("""Triple store setup:
- Graph throttling: {}
- Minimum cache time: {}""".format(GRAPH_THROTTLING, MIN_CACHE_TIME))
log.info('Cleaning cache...')
uuid_locks = r.keys('{}:cache*'.format(AGENT_ID))
for ulk in uuid_locks:
r.delete(ulk)
def load_stream_triples(fid, until):
def __triplify(x):
def __extract_lang(v):
def __lang_tag_match(strg, search=re.compile(r'[^a-z]').search):
return not bool(search(strg))
if '@' in v:
try:
(v_aux, lang) = tuple(v.split('@'))
(v, lang) = (v_aux, lang) if __lang_tag_match(lang) else (v, None)
except ValueError:
def get_requests():
requests = [rk.split(':')[1] for rk in r.keys('{}:requests:*:'.format(AGENT_ID))]
return jsonify(requests=requests)