def __call__(self):
if uwsgi.mule_id() == self.num:
print " i am the mule"
while True:
message = uwsgi.mule_get_msg()
if message:
self.f(message)
def loop():
"""
Main mule loop. As messages come in data is sent to the endpoint.
"""
while True:
environ = json.loads(uwsgi.mule_get_msg())
uri = environ['PATH_INFO']
remote_addr = apply_mask_to_ip(environ['REMOTE_ADDR'])
user_agent = environ['HTTP_USER_AGENT']
dt = datetime.datetime.now()
# Use \0 as a delimiter.
data = "\0".join([
CONFIG['SANS_USERID'], CONFIG['SANS_KEY'], uri,
remote_addr, user_agent,
dt.strftime('%Y-%m-%d'), dt.strftime('%H:%M:%S'),
CONFIG['IP_MASK']])
try:
result = urllib.urlopen(
CONFIG['ENDPOINT'] % urllib.quote(CONFIG['SANS_USERID']),
urllib.urlencode({'DATA': base64.encodestring(data)}))
if int(result.getcode()) != 200:
raise Exception('Did not get a 200')
except Exception, ex:
raise Exception('Could not send data: %s' % ex)
def mule_loop():
looper = Looper(app_config)
while True:
message = uwsgi.mule_get_msg()
if message == b'start-job-monitoring':
looper.start_job_check_loop()
else:
print('unknown message:')
print(message)
def main():
root_repo_config = config["github"]
python_executable = config["subprocess_python_interpreter"]
while True:
api_key = uwsgi.mule_get_msg().decode()
repo_config = root_repo_config.get(api_key)
if repo_config is None:
logging.warning("Failed to find api_key {} in {}", api_key, config)
continue
documentation.update_repo(repo_config, static_rebuild_python_executable=python_executable)
def run_mule():
"""Enter mule loop. Receive tasks and executes them"""
if not UWSGI:
logger.error("You must be within uwsgi to run mule")
return
while True:
logger.info("Mule running, Waiting for messages..")
data = uwsgi.mule_get_msg()
execute_task(data)
def mule_msg_iter(timeout):
end = monotonic() + timeout
while True:
timeout = max(0, end - monotonic())
# secret timeout parameter that only cool kids know about!!!
item = uwsgi.mule_get_msg(timeout=int(ceil(timeout)))
if item is not None and len(item) > 0:
yield item
elif timeout == 0:
break
def main_loop():
try:
import uwsgi
except Exception as e:
logger.exception("Mule not running in uwsgi, exiting: {}".format(
str(e)))
print("Error, not running in uwsgi")
return
print("Running scheduler in uwsgi mule")
scheduler = Scheduler()
scheduler.start()
pmh = PluginManagerHandler()
pmh.load_plugins()
try:
with sqla_session() as session:
Joblock.clear_locks(session)
except Exception as e:
logger.exception(
"Unable to clear old locks from database at startup: {}".format(
str(e)))
while True:
mule_data = uwsgi.mule_get_msg()
data: dict = json.loads(mule_data)
if data['when'] and isinstance(data['when'], int):
data['run_date'] = datetime.datetime.utcnow() + datetime.timedelta(
seconds=data['when'])
del data['when']
kwargs = {}
for k, v in data.items():
if k not in ['func', 'trigger', 'id', 'run_date']:
kwargs[k] = v
# Perform pre-schedule job checks
try:
if not pre_schedule_checks(scheduler, kwargs):
continue
except Exception as e:
logger.exception(
"Unable to perform pre-schedule job checks: {}".format(e))
scheduler.add_job(data['func'],
trigger=data['trigger'],
kwargs=kwargs,
id=data['id'],
run_date=data['run_date'],
name=data['func'])
def process_requests():
while True:
#print("Waiting for messages... yawn.")
message = uwsgi.mule_get_msg().decode('ascii')
token, source_id, format = message.split('|')
source_id = int(source_id)
client = WAMPClient(token, settings.WAMP_URI, loop=loop)
try:
task_id = client.call_no_wait('convert', source_id, format)
finally:
client.close()
if not task_id:
print("ERROR - no task id")
else:
print("Conversion submitted", task_id)
def loop():
while True:
key = uwsgi.mule_get_msg()
key = key.decode('utf-8')
ids = key.split('_')
uwsgi.cache_set(key, 'inprogress')
try:
result = bfs(ids[0], ids[1])
except:
uwsgi.cache_update(key, 'fail')
else:
if result:
uwsgi.cache_update(key, 'found')
print(key)
else:
uwsgi.cache_update(key, 'notfound')
def process_requests():
delegated_client = None
while True:
#print("Waiting for messages... yawn.")
message = uwsgi.mule_get_msg().decode('ascii')
user,role, source_id, format = message.split('|')
source_id=int(source_id)
try:
if not delegated_client or not delegated_client.is_active():
delegated_client= DelegatedClient(settings.DELEGATED_TOKEN,
settings.DELEGATED_URI, loop)
task_id=delegated_client.call_no_wait(user, role, 'convert', source_id, format )
except Exception:
traceback.print_exc()
else:
print ("Conversion submitted", task_id)
def process_requests():
delegated_client = None
while True:
#print("Waiting for messages... yawn.")
message = uwsgi.mule_get_msg().decode('ascii')
user, role, source_id, format = message.split('|')
source_id = int(source_id)
try:
if not delegated_client or not delegated_client.is_active():
delegated_client = DelegatedClient(settings.DELEGATED_TOKEN,
settings.DELEGATED_URI,
loop)
task_id = delegated_client.call_no_wait(user, role, 'convert',
source_id, format)
except Exception:
traceback.print_exc()
else:
print("Conversion submitted", task_id)
def getMessage(self):
"""WebApp message getter
uses uWsgi message mecanism to recieve messages from the web app
:returns: expid
:rtype: string
:returns: action
:rtype: string
"""
while True:
try:
message = json.loads(uwsgi.mule_get_msg())
expid = message['id']
action = message['action']
self.logger.info("Got massage: %s, %s" % (expid, action))
except:
raise
if message is not None:
break
return expid, action
def main_loop():
try:
import uwsgi
except:
print("Error, not running in uwsgi")
return
print("Running scheduler in uwsgi mule")
scheduler = Scheduler()
scheduler.start()
while True:
mule_data = uwsgi.mule_get_msg()
data: dict = json.loads(mule_data)
if data['when'] and isinstance(data['when'], int):
data['run_date'] = datetime.datetime.utcnow() + datetime.timedelta(seconds=data['when'])
del data['when']
kwargs = {}
for k, v in data.items():
if k not in ['func', 'trigger', 'id', 'run_date']:
kwargs[k] = v
scheduler.add_job(data['func'], trigger=data['trigger'], kwargs=kwargs,
id=data['id'], run_date=data['run_date'])
def worker():
print('Offload worker started')
while True:
payload = mule_get_msg()
params = loads(payload)
offload_cache.offload_helper(params)
def get_message(self):
return uwsgi.mule_get_msg()
if event['drop_runs'] is not None:
rules.drop_runs = event['drop_runs']
return rules
else:
return None
#######################################
if __name__ == '__main__':
logging.warning("start recalc scores mule")
db = get_db()
while True:
logging.debug("recalc waiting... mule_id=%r", uwsgi.mule_id())
uwsgi.mule_get_msg(
) # wait on any msg indicating we need to recalc something
# FIXME consider changing this to uwsgi.signal_wait() so that we can filter on a particular type
logging.debug("RECALC")
run_id = 1 # trigger first iteration
while run_id is not None:
# find next run to recalc
with db:
event = get_event(db)
run_id = db.query_single(
"SELECT run_id FROM runs WHERE event_id=? AND recalc",
(event['event_id'], ))
if run_id is not None:
# indicate we are processing run
db.update('runs', run_id, recalc=2)
rules = get_rules(event)
if event['drop_runs'] is not None:
rules.drop_runs = event['drop_runs']
return rules
else:
return None
#######################################
if __name__ == '__main__':
logging.warning("start recalc scores mule")
db = get_db()
while True:
logging.debug("recalc waiting... mule_id=%r", uwsgi.mule_id())
uwsgi.mule_get_msg() # wait on any msg indicating we need to recalc something
# FIXME consider changing this to uwsgi.signal_wait() so that we can filter on a particular type
logging.debug("RECALC")
run_id = 1 # trigger first iteration
while run_id is not None:
# find next run to recalc
with db:
event = get_event(db)
run_id = db.query_single("SELECT run_id FROM runs WHERE event_id=? AND recalc", (event['event_id'],))
if run_id is not None:
# indicate we are processing run
db.update('runs', run_id, recalc=2)
rules = get_rules(event)
rules.recalc_run(db, run_id)
from FancyModel import FancyModel
import uwsgi
import json
if __name__ == '__main__':
fnc = FancyModel()
while True:
uwsgi.mule_get_msg()
req = uwsgi.queue_pull()
if req is None:
continue
json_in = json.loads(req.decode("utf-8"))
text = json_in["text"]
# to store transliterations
json_out = {"res": fnc.predict(text)}
uwsgi.cache_update(json_in.get("id"),
json.dumps(json_out, ensure_ascii=False), 0,
"mcache")
def __call__(self):
if uwsgi.mule_id() == self.num:
while True:
message = uwsgi.mule_get_msg()
if message:
self.f(message)
#!/usr/bin/env python2.7
# -*- coding: utf-8 -*-
"""We need this mule because we can't uwsgi.spool in a greenlet.
To resolve this problem, we send a message from the greenlet to the mule (that
IS part of the uWSGI stack, not only having the context), that itself transmits
the message on the spool.
"""
import ast
import uwsgi
while True:
message = uwsgi.mule_get_msg()
if message:
message = ast.literal_eval(message)
uwsgi.spool(message)
