Exemple #1
0
 def __call__(self):
     if uwsgi.mule_id() == self.num:
         print " i am the mule"
         while True:
             message = uwsgi.mule_get_msg()
             if message:
                 self.f(message)
Exemple #2
0
def loop():
    """
    Main mule loop. As messages come in data is sent to the endpoint.
    """
    while True:
        environ = json.loads(uwsgi.mule_get_msg())

        uri = environ['PATH_INFO']
        remote_addr = apply_mask_to_ip(environ['REMOTE_ADDR'])
        user_agent = environ['HTTP_USER_AGENT']

        dt = datetime.datetime.now()

        # Use \0 as a delimiter.
        data = "\0".join([
            CONFIG['SANS_USERID'], CONFIG['SANS_KEY'], uri,
            remote_addr, user_agent,
            dt.strftime('%Y-%m-%d'), dt.strftime('%H:%M:%S'),
            CONFIG['IP_MASK']])
        try:
            result = urllib.urlopen(
                CONFIG['ENDPOINT'] % urllib.quote(CONFIG['SANS_USERID']),
                urllib.urlencode({'DATA': base64.encodestring(data)}))
            if int(result.getcode()) != 200:
                raise Exception('Did not get a 200')
        except Exception, ex:
            raise Exception('Could not send data: %s' % ex)
Exemple #3
0
def mule_loop():
    looper = Looper(app_config)
    while True:
        message = uwsgi.mule_get_msg()
        if message == b'start-job-monitoring':
            looper.start_job_check_loop()
        else:
            print('unknown message:')
            print(message)
Exemple #4
0
def main():
    root_repo_config = config["github"]
    python_executable = config["subprocess_python_interpreter"]
    while True:
        api_key = uwsgi.mule_get_msg().decode()
        repo_config = root_repo_config.get(api_key)
        if repo_config is None:
            logging.warning("Failed to find api_key {} in {}", api_key, config)
            continue
        documentation.update_repo(repo_config, static_rebuild_python_executable=python_executable)
Exemple #5
0
def run_mule():
    """Enter mule loop. Receive tasks and executes them"""
    if not UWSGI:
        logger.error("You must be within uwsgi to run mule")
        return

    while True:
        logger.info("Mule running, Waiting for messages..")
        data = uwsgi.mule_get_msg()
        execute_task(data)
Exemple #6
0
 def mule_msg_iter(timeout):
     end = monotonic() + timeout
     while True:
         timeout = max(0, end - monotonic())
         # secret timeout parameter that only cool kids know about!!!
         item = uwsgi.mule_get_msg(timeout=int(ceil(timeout)))
         if item is not None and len(item) > 0:
             yield item
         elif timeout == 0:
             break
Exemple #7
0
def main_loop():
    try:
        import uwsgi
    except Exception as e:
        logger.exception("Mule not running in uwsgi, exiting: {}".format(
            str(e)))
        print("Error, not running in uwsgi")
        return

    print("Running scheduler in uwsgi mule")
    scheduler = Scheduler()
    scheduler.start()

    pmh = PluginManagerHandler()
    pmh.load_plugins()

    try:
        with sqla_session() as session:
            Joblock.clear_locks(session)
    except Exception as e:
        logger.exception(
            "Unable to clear old locks from database at startup: {}".format(
                str(e)))

    while True:
        mule_data = uwsgi.mule_get_msg()
        data: dict = json.loads(mule_data)
        if data['when'] and isinstance(data['when'], int):
            data['run_date'] = datetime.datetime.utcnow() + datetime.timedelta(
                seconds=data['when'])
            del data['when']
        kwargs = {}
        for k, v in data.items():
            if k not in ['func', 'trigger', 'id', 'run_date']:
                kwargs[k] = v
        # Perform pre-schedule job checks
        try:
            if not pre_schedule_checks(scheduler, kwargs):
                continue
        except Exception as e:
            logger.exception(
                "Unable to perform pre-schedule job checks: {}".format(e))

        scheduler.add_job(data['func'],
                          trigger=data['trigger'],
                          kwargs=kwargs,
                          id=data['id'],
                          run_date=data['run_date'],
                          name=data['func'])
Exemple #8
0
def process_requests():
    while True:
        #print("Waiting for messages... yawn.")
        message = uwsgi.mule_get_msg().decode('ascii')
        token, source_id, format = message.split('|')
        source_id = int(source_id)
        client = WAMPClient(token, settings.WAMP_URI, loop=loop)
        try:
            task_id = client.call_no_wait('convert', source_id, format)
        finally:
            client.close()
        if not task_id:
            print("ERROR - no task id")
        else:
            print("Conversion submitted", task_id)
Exemple #9
0
def loop():
    while True:
        key = uwsgi.mule_get_msg()
        key = key.decode('utf-8')
        ids = key.split('_')
        uwsgi.cache_set(key, 'inprogress')
        try:
            result = bfs(ids[0], ids[1])
        except:
            uwsgi.cache_update(key, 'fail')
        else:
            if result:
                uwsgi.cache_update(key, 'found')
                print(key)
            else:
                uwsgi.cache_update(key, 'notfound')
Exemple #10
0
def process_requests():
    delegated_client = None

    while True:
        #print("Waiting for messages... yawn.")
        message = uwsgi.mule_get_msg().decode('ascii')
        user,role, source_id, format = message.split('|')
        source_id=int(source_id)
        try:
            if not delegated_client or not delegated_client.is_active():
                delegated_client= DelegatedClient(settings.DELEGATED_TOKEN, 
                                                  settings.DELEGATED_URI, loop)
        
            task_id=delegated_client.call_no_wait(user, role, 'convert', source_id, format )
        except Exception:
            traceback.print_exc()
        else:
            print ("Conversion submitted", task_id)
Exemple #11
0
def process_requests():
    delegated_client = None

    while True:
        #print("Waiting for messages... yawn.")
        message = uwsgi.mule_get_msg().decode('ascii')
        user, role, source_id, format = message.split('|')
        source_id = int(source_id)
        try:
            if not delegated_client or not delegated_client.is_active():
                delegated_client = DelegatedClient(settings.DELEGATED_TOKEN,
                                                   settings.DELEGATED_URI,
                                                   loop)

            task_id = delegated_client.call_no_wait(user, role, 'convert',
                                                    source_id, format)
        except Exception:
            traceback.print_exc()
        else:
            print("Conversion submitted", task_id)
Exemple #12
0
    def getMessage(self):
        """WebApp message getter

        uses uWsgi message mecanism to recieve messages from the web app

        :returns: expid
        :rtype: string
        :returns: action
        :rtype: string
        """

        while True:
            try:
                message = json.loads(uwsgi.mule_get_msg())
                expid = message['id']
                action = message['action']
                self.logger.info("Got massage: %s, %s" % (expid, action))
            except:
                raise
            if message is not None:
                break
        return expid, action
Exemple #13
0
def main_loop():
    try:
        import uwsgi
    except:
        print("Error, not running in uwsgi")
        return

    print("Running scheduler in uwsgi mule")
    scheduler = Scheduler()
    scheduler.start()

    while True:
        mule_data = uwsgi.mule_get_msg()
        data: dict = json.loads(mule_data)
        if data['when'] and isinstance(data['when'], int):
            data['run_date'] = datetime.datetime.utcnow() + datetime.timedelta(seconds=data['when'])
            del data['when']
        kwargs = {}
        for k, v in data.items():
            if k not in ['func', 'trigger', 'id', 'run_date']:
                kwargs[k] = v
        scheduler.add_job(data['func'], trigger=data['trigger'], kwargs=kwargs,
                          id=data['id'], run_date=data['run_date'])
Exemple #14
0
 def worker():
     print('Offload worker started')
     while True:
         payload = mule_get_msg()
         params = loads(payload)
         offload_cache.offload_helper(params)
Exemple #15
0
 def get_message(self):
     return uwsgi.mule_get_msg()
        if event['drop_runs'] is not None:
            rules.drop_runs = event['drop_runs']
        return rules
    else:
        return None


#######################################

if __name__ == '__main__':
    logging.warning("start recalc scores mule")
    db = get_db()

    while True:
        logging.debug("recalc waiting... mule_id=%r", uwsgi.mule_id())
        uwsgi.mule_get_msg(
        )  # wait on any msg indicating we need to recalc something
        # FIXME consider changing this to uwsgi.signal_wait() so that we can filter on a particular type
        logging.debug("RECALC")

        run_id = 1  # trigger first iteration
        while run_id is not None:
            # find next run to recalc
            with db:
                event = get_event(db)
                run_id = db.query_single(
                    "SELECT run_id FROM runs WHERE event_id=? AND recalc",
                    (event['event_id'], ))
                if run_id is not None:
                    # indicate we are processing run
                    db.update('runs', run_id, recalc=2)
                    rules = get_rules(event)
    if event['drop_runs'] is not None:
      rules.drop_runs = event['drop_runs']
    return rules
  else:
    return None


#######################################

if __name__ == '__main__':
  logging.warning("start recalc scores mule")
  db = get_db()

  while True:
    logging.debug("recalc waiting... mule_id=%r", uwsgi.mule_id())
    uwsgi.mule_get_msg() # wait on any msg indicating we need to recalc something
    # FIXME consider changing this to uwsgi.signal_wait() so that we can filter on a particular type
    logging.debug("RECALC")

    run_id = 1 # trigger first iteration
    while run_id is not None:
      # find next run to recalc
      with db:
        event = get_event(db)
        run_id = db.query_single("SELECT run_id FROM runs WHERE event_id=? AND recalc", (event['event_id'],))
        if run_id is not None:
          # indicate we are processing run
          db.update('runs', run_id, recalc=2)
          rules = get_rules(event)
          rules.recalc_run(db, run_id)
Exemple #18
0
from FancyModel import FancyModel
import uwsgi
import json

if __name__ == '__main__':
    fnc = FancyModel()
    while True:
        uwsgi.mule_get_msg()
        req = uwsgi.queue_pull()
        if req is None:
            continue
        json_in = json.loads(req.decode("utf-8"))
        text = json_in["text"]
        # to store transliterations
        json_out = {"res": fnc.predict(text)}
        uwsgi.cache_update(json_in.get("id"),
                           json.dumps(json_out, ensure_ascii=False), 0,
                           "mcache")
Exemple #19
0
 def __call__(self):
     if uwsgi.mule_id() == self.num:
         while True:
             message = uwsgi.mule_get_msg()
             if message:
                 self.f(message)
Exemple #20
0
#!/usr/bin/env python2.7
# -*- coding: utf-8 -*-
"""We need this mule because we can't uwsgi.spool in a greenlet.

To resolve this problem, we send a message from the greenlet to the mule (that
IS part of the uWSGI stack, not only having the context), that itself transmits
the message on the spool.
"""
import ast
import uwsgi

while True:
    message = uwsgi.mule_get_msg()
    if message:
        message = ast.literal_eval(message)
        uwsgi.spool(message)