Exemplo n.º 1
0
def used_channels():
    """
    Selects all channels that were declared by current requests
    """
    req_channel_keys = r.keys('{}:requests:*:'.format(AGENT_ID))
    for rck in req_channel_keys:
        try:
            channel = r.hget(rck, 'channel')
            yield channel
        except Exception as e:
            traceback.print_exc()
            log.warning(e.message)
Exemplo n.º 2
0
 def __check_gp_mappings(self, gp=None):
     """
     Used in _save method. Seeks matches with some fragment already registered
     :param gp: By default, _graph_pattern attribute is used when gp is None
     :return: The matching fragment id and the mapping dictionary or None if there is no matching
     """
     if gp is None:
         gp = self._graph_pattern
     gp_keys = r.keys('{}:*:gp'.format(self._fragments_key))
     for gpk in gp_keys:
         stored_gp = GraphPattern(r.smembers(gpk))
         mapping = stored_gp.mapping(gp)
         if mapping:
             return gpk.split(':')[-2], mapping
     return None
Exemplo n.º 3
0
def __collect_fragments():
    registered_fragments = r.scard(fragments_key)
    synced_fragments = len(r.keys('{}:*:sync'.format(fragments_key)))
    log.info("""Collector daemon started:
                    - Fragments: {}
                    - Synced: {}""".format(registered_fragments, synced_fragments))

    futures = {}
    while True:
        for fid in filter(
                lambda x: r.get('{}:{}:sync'.format(fragments_key, x)) is None and r.get(
                    '{}:{}:pulling'.format(fragments_key, x)) is None,
                r.smembers(fragments_key)):
            if fid in futures:
                if futures[fid].done():
                    del futures[fid]
            if fid not in futures:
                futures[fid] = thp.submit(__pull_fragment, fid)
        time.sleep(1)
Exemplo n.º 4
0
def __remove_fragment(fid):
    """
    Completely remove a fragment from the system after notifying its known consumers
    :param fid: Fragment identifier
    """
    log.debug('Waiting to remove fragment {}...'.format(fid))
    lock = fragment_lock(fid)
    lock.acquire()

    r_sinks = __load_fragment_requests(fid)
    __notify_completion(fid, r_sinks)
    fragment_keys = r.keys('{}:{}*'.format(fragments_key, fid))
    with r.pipeline(transaction=True) as p:
        p.multi()
        map(lambda k: p.delete(k), fragment_keys)
        p.srem(fragments_key, fid)
        p.execute()

    # Fragment lock key was just implicitly removed, so it's not necessary to release the lock
    # lock.release()
    log.info('Fragment {} has been removed'.format(fid))
Exemplo n.º 5
0
THROTTLING_TIME = (1.0 / (COLLECT_THROTTLING * 1000))

fragments_key = '{}:fragments'.format(AGENT_ID)

log.info("""Fragment daemon setup:
                    - On-demand threshold: {}
                    - Minimum sync time: {}
                    - Maximum concurrent collectors: {}
                    - Maximum concurrent fragments: {}""".format(ON_DEMAND_TH, MIN_SYNC, N_COLLECTORS,
                                                                 MAX_CONCURRENT_FRAGMENTS))

# Fragment collection threadpool
thp = ThreadPoolExecutor(max_workers=min(8, MAX_CONCURRENT_FRAGMENTS))

log.info('Cleaning fragment locks...')
fragment_locks = r.keys('{}:*lock*'.format(fragments_key))
for flk in fragment_locks:
    r.delete(flk)

log.info('Cleaning fragment pulling flags...')
fragment_pullings = r.keys('{}:*:pulling'.format(fragments_key))
for fpk in fragment_pullings:
    r.delete(fpk)


def fragment_lock(fid):
    """
    :param fid: Fragment id
    :return: A redis-based lock object for a given fragment
    """
    lock_key = '{}:{}:lock'.format(fragments_key, fid)
Exemplo n.º 6
0
    fragment_contexts
from agora.scholar.daemons.fragment import fragment_lock
from agora.stoa.actions.core import AGENT_ID
from agora.stoa.actions.core import STOA
from agora.stoa.actions.core.fragment import FragmentRequest, FragmentAction, FragmentSink
from agora.stoa.actions.core.utils import parse_bool, chunks
from agora.stoa.messaging.reply import reply
from agora.stoa.store import r
from agora.stoa.store.triples import load_stream_triples, fragments_cache

__author__ = 'Fernando Serena'

log = logging.getLogger('agora.scholar.actions.stream')

log.info("'Cleaning stream requests' locks...")
request_locks = r.keys('{}:requests:*:lock'.format(AGENT_ID))
for rlk in request_locks:
    r.delete(rlk)


class StreamPlugin(FragmentPlugin):
    @property
    def sink_class(self):
        return StreamSink

    def consume(self, fid, (c, s, p, o), graph, *args):
        sink = args[0]
        sink.lock.acquire()
        try:
            # Prevent from consuming a triple when the delivery state says it was completely sent
            # Anyway, this HAS TO BE REMOVED from here, because the stream flag should be enough
Exemplo n.º 7
0
import re

__author__ = 'Fernando Serena'

log = logging.getLogger('agora.stoa.store.triples')
pool = ThreadPoolExecutor(max_workers=4)

GRAPH_THROTTLING = max(1, int(app.config.get('CACHE', {}).get('graph_throttling', 30)))
MIN_CACHE_TIME = max(0, int(app.config.get('CACHE', {}).get('min_cache_time', 10)))

log.info("""Triple store setup:
                    - Graph throttling: {}
                    - Minimum cache time: {}""".format(GRAPH_THROTTLING, MIN_CACHE_TIME))

log.info('Cleaning cache...')
uuid_locks = r.keys('{}:cache*'.format(AGENT_ID))
for ulk in uuid_locks:
    r.delete(ulk)


def load_stream_triples(fid, until):
    def __triplify(x):
        def __extract_lang(v):
            def __lang_tag_match(strg, search=re.compile(r'[^a-z]').search):
                return not bool(search(strg))

            if '@' in v:
                try:
                    (v_aux, lang) = tuple(v.split('@'))
                    (v, lang) = (v_aux, lang) if __lang_tag_match(lang) else (v, None)
                except ValueError:
Exemplo n.º 8
0
def get_requests():
    requests = [rk.split(':')[1] for rk in r.keys('{}:requests:*:'.format(AGENT_ID))]
    return jsonify(requests=requests)