def setUp(self):
self.addCleanup(cleanTasks)
for priority, name in constants.tasks:
task = {'name': name, 'at': int(time.time() + 10)}
if priority is not None:
task['priority'] = str(priority)
uwsgi.spool(task)
def sendmail(self, subject, body, thread, comment, to=None):
if uwsgi:
uwsgi.spool({b"subject": subject.encode("utf-8"),
b"body": body.encode("utf-8"),
b"to": to})
else:
start_new_thread(self._retry, (subject, body, to))
def get_recipes_without_images(*args):
app = create_app()
import os
if len(os.listdir('/home/ubuntu/eaterator/spool')) > 300:
return
with app.app_context():
default = -1
recipes = Recipe.query.\
filter(
not_(
Recipe.pk.in_(
db.session.query(func.distinct(RecipeImage.recipe))
)
),
Recipe.title.isnot(None)
).limit(55).all()
if len(recipes) <= 0:
app.logger.debug(
"CLICKR CRON | Added reicpes from failed searches")
default = -2
recipes = Recipe.query.filter(
Recipe.pk.in_(
db.session.query(RecipeImage.recipe).filter(
RecipeImage.secret == 'default',
RecipeImage.farm_id != '-2'))).limit(55).all()
for recipe in recipes:
if recipe.title:
uwsgi.spool({
b'pk': str(recipe.pk).encode('utf-8'),
b'title': recipe.title.encode('utf-8'),
b'default': str(default).encode('utf-8')
})
db.session.close()
db.session.remove()
return
def notify(self, thread, comment):
body = self.format(thread, comment)
if uwsgi:
uwsgi.spool({b"subject": thread["title"].encode("utf-8"), b"body": body.encode("utf-8")})
else:
start_new_thread(self._retry, (thread["title"], body))
def sendmail(self, subject, body, thread, comment, to=None):
to = to or self.conf.get("to")
if uwsgi:
uwsgi.spool({b"subject": subject.encode("utf-8"),
b"body": body.encode("utf-8"),
b"to": to.encode("utf-8")})
else:
start_new_thread(self._retry, (subject, body, to))
def notify(self, thread, comment):
body = self.format(thread, comment)
if uwsgi:
uwsgi.spool({b"subject": thread["title"].encode("utf-8"),
b"body": body.encode("utf-8")})
else:
start_new_thread(self._retry, (thread["title"], body))
def start_processing(cls, att):
try:
import uwsgi
uwsgi.spool({b"processor": cls.__name__.encode(), b"id": str(att.id).encode()})
except Exception as exc:
print("Not running in uWSGI env. Executing in foreground: {}".format(exc))
proc = cls(att)
return proc.process()
def delay(cls, *args, **kwargs):
id = str(uuid.uuid4())
kwargs['language'] = translation.get_language()
import uwsgi
uwsgi.spool({
'task': pickle.dumps(cls(id)),
'args': pickle.dumps(args),
'kwargs': pickle.dumps(kwargs),
})
return TaskStatus(id)
def delay(cls, *args, **kwargs):
id = str(uuid.uuid4())
kwargs['language'] = translation.get_language()
import uwsgi
uwsgi.spool({
'task': pickle.dumps(cls(id)),
'args': pickle.dumps(args),
'kwargs': pickle.dumps(kwargs),
})
return TaskStatus(id)
def spool():
logger.debug(f'uwsgi.spool({arg})')
try:
uwsgi.spool(arg)
except Exception:
tt, value, tb = sys.exc_info()
call.exception = '\n'.join(
traceback.format_exception(tt, value, tb))
call.save_status('unspoolable')
logger.exception(f'{self} -> Call(id={call.pk}).spool():'
f' uwsgi.spool exception !')
def spool(self):
self.set_status('spooled')
transaction.commit()
if uwsgi:
print('SPOOL')
uwsgi.spool({b'uuid': str(self.pk).encode('ascii')})
else:
self.execute()
return self
def sendmail(self, subject, body, thread, comment, to=None):
to = to or self.conf.get("to")
if not subject:
# Fallback, just in case as an empty subject does not work
subject = 'isso notification'
if uwsgi:
uwsgi.spool({b"subject": subject.encode("utf-8"),
b"body": body.encode("utf-8"),
b"to": to.encode("utf-8")})
else:
start_new_thread(self._retry, (subject, body, to))
def application(env, start_response):
body = env['wsgi.input'].read()
qs = cgi.parse_qs(env['QUERY_STRING'])
entityID = int(qs['entityID'][0])
uwsgi.spool(body=body,
entityID=entityID,
name=qs['name'][0],
worldID=qs['worldID'][0],
sdkType=qs['sdkType'][0],
fbID=qs['fbID'][0])
start_response('200 OK', [])
return 'OK'
def get(self, id):
""" Request a validation for a submission, without needing to update the receipt """
# TODO: should confirm submission exists & has receipt
if 'uwsgi' in sys.modules:
uwsgi.spool({'submission_id': id})
else:
logging.debug(
"UWSGI not available; skipping validation of submission {}.".
format(id))
return make_response(
jsonify(message="Submission {} queued for validation".format(id)),
200)
def sendmail(self, subject, body, thread, comment, to=None):
if uwsgi:
uwsgi.spool({
b"subject": subject.encode("utf-8"),
b"body": body.encode("utf-8"),
b"to": to
})
elif self.mailQ.empty():
self.mailQ.put((subject, body, to))
pdb.set_trace()
start_new_thread(self._retry, ())
else:
self.mailQ.put((subject, body, to))
def monitor_changes(db):
previous_seq = 0
count = 0
while True:
stream = ChangesStream(db, feed="continuous",
heartbeat=True, since=previous_seq)
log.debug("pull from changes")
for c in stream:
previous_seq = c['seq']
count += 1
if count % 100 == 0:
log.debug("updating views")
try:
uwsgi.spool({"action": _UPDATE_VIEW, "uri": db.uri})
except:
pass
def __call__(self, *args, **kwargs):
arguments = self.base_dict
if len(args) > 0:
arguments.update(args[0])
if kwargs:
arguments.update(kwargs)
return uwsgi.spool(arguments)
def schedule(self, task, args, kwargs, **spool_kwargs):
body = {}
for body_param in task.body_params:
if body_param not in kwargs:
continue
body[body_param] = kwargs.pop(body_param)
job = {self.identifier: task.name}
if args:
job[b'args'] = pickle.dumps(args)
if kwargs:
job[b'kwargs'] = pickle.dumps(kwargs)
if body:
job[b'body'] = pickle.dumps(body)
for key, value in spool_kwargs.items():
job[key.encode('utf8')] = str(value).encode('utf8')
uwsgi.spool(job)
def spool(self, *args, **kwargs):
arguments = self.base_dict
if len(args) > 0:
arguments.update(args[0])
if kwargs:
arguments.update(kwargs)
return uwsgi.spool(arguments)
def spool(self, *args, **kwargs):
arguments = self.base_dict
arguments['ud_spool_ret'] = str(uwsgi.SPOOL_RETRY)
if len(args) > 0:
arguments.update(args[0])
if kwargs:
arguments.update(kwargs)
return uwsgi.spool(arguments)
def application(env, start_response):
name = uwsgi.spool({'Hello':'World', 'I am a':'long running task'})
print("spooled as %s" % name)
start_response('200 Ok', [('Content-Type','text/plain'),('uWSGI-Status', 'spooled')])
return "task spooled"
def add_task(queue, data):
"""
:param :
:return:
:rtype:
"""
body = json.dumps(data).encode(encoding='utf8')
spooler = os.path.join(settings.SPOOLER_DIR, queue).encode(encoding='utf8')
try:
uwsgi.spool({
b'body': body,
b'spooler': spooler,
})
return True
except Exception as e:
return False
def spool(self, *args, **kwargs):
arguments = self.base_dict
arguments["ud_spool_ret"] = str(uwsgi.SPOOL_RETRY)
if len(args) > 0:
arguments.update(args[0])
if kwargs:
arguments.update(kwargs)
return uwsgi.spool(arguments)
def hello():
try:
# NOTE: simulate a failure.
if randint(1, 5) == 1:
raise Exception("this is the end")
mkey = mclient.get("sleep")
if mkey:
return "Hello World from cache :: {} !".format(mkey)
# NOTE: simulate a long operation.
print("sleep my friend !")
time.sleep(5)
mclient.set("sleep", "Hearth of the Swarm")
return "Hello World !"
except Exception as e:
uwsgi.spool({'name': 'Wings of liberty'})
return "Hello World with failure !"
def notify(self, thread, comment):
body = self.format(thread, comment)
if thread["title"] is None:
thread["title"] = "New comment"
if self.conf.get("to") is not None:
to_addr = self.conf.get("to")
if uwsgi:
uwsgi.spool({
b"subject": thread["title"].encode("utf-8"),
b"body": body.encode("utf-8"),
b"to": to_addr.encode("utf-8")
})
else:
start_new_thread(self._retry, (thread["title"], body, to_addr))
def put(self, id):
""" Edit a submission """
submission = Submission.query.get(id)
if submission:
submission.receipt = request.get_json().get(
"receipt", submission.receipt)
submission.status = "received"
submission.modified = datetime.datetime.utcnow()
db.session.commit()
logging.info("Edited submission {}".format(id))
# Asynchronously kick off the validate if available
# The spooler callback will fetch the receipt and pass to the validator.
if 'uwsgi' in sys.modules:
uwsgi.spool({'submission_id': id})
else:
logging.debug("UWSGI not available; skipping validation.")
return jsonify(submission=submission.to_dict())
else:
return make_response(
jsonify(message="Submission {} does not exist".format(id)),
404)
def __call__(self, *args, **kwargs):
arguments = self.base_dict
if not self.pass_arguments:
if len(args) > 0:
arguments.update(args[0])
if kwargs:
arguments.update(kwargs)
else:
spooler_args = {}
for key in ('message_dict', 'spooler', 'priority', 'at', 'body'):
if key in kwargs:
spooler_args.update({key: kwargs.pop(key)})
arguments.update(spooler_args)
arguments.update({'args': pickle.dumps(args), 'kwargs': pickle.dumps(kwargs)})
return uwsgi.spool(arguments)
def __call__(self, *args, **kwargs):
arguments = self.base_dict
if not self.pass_arguments:
if len(args) > 0:
arguments.update(args[0])
if kwargs:
arguments.update(kwargs)
else:
spooler_args = {}
for key in ('message_dict', 'spooler', 'priority', 'at', 'body'):
if key in kwargs:
spooler_args.update({key: kwargs.pop(key)})
arguments.update(spooler_args)
arguments.update({'args': pickle.dumps(args), 'kwargs': pickle.dumps(kwargs)})
return uwsgi.spool(arguments)
def __call__(self, *args, **kwargs):
arguments = self.base_dict
if not self.pass_arguments:
if len(args) > 0:
arguments.update(args[0])
if kwargs:
arguments.update(kwargs)
else:
spooler_args = {}
for key in ("message_dict", "spooler", "priority", "at", "body"):
if key in kwargs:
spooler_args.update({key: kwargs.pop(key)})
arguments.update(spooler_args)
arguments.update({"args": pickle.dumps(args), "kwargs": pickle.dumps(kwargs)})
return uwsgi.spool(arguments)
def __call__(self, *args, **kwargs):
arguments = self.base_dict.copy()
if not self.pass_arguments:
if len(args) > 0:
arguments.update(args[0])
if kwargs:
arguments.update(kwargs)
else:
spooler_args = {}
for key in ("message_dict", "spooler", "priority", "at", "body"):
if key in kwargs:
spooler_args.update({key: kwargs.pop(key)})
arguments.update(spooler_args)
arguments.update({
"args": pickle.dumps(args),
"kwargs": pickle.dumps(kwargs)
})
return uwsgi.spool(_encode_to_spooler(arguments))
def setUp(self):
for priority, name in spooler_priority_constants.tasks:
task = {'name': name, 'at': int(time.time() + 10)}
if priority is not None:
task['priority'] = str(priority)
uwsgi.spool(task)
def execute_async(self):
return uwsgi.spool(self.get_message_content())
def notify(self, subject, body, retries=5):
uwsgi.spool({"subject": subject.encode('utf-8'), "body": body.encode('utf-8')})
def producer():
uwsgi.spool(ud_spool_func="consumer", dest=random.choice(projects))
time.sleep(2)
#!/usr/bin/env python2.7
# -*- coding: utf-8 -*-
"""We need this mule because we can't uwsgi.spool in a greenlet.
To resolve this problem, we send a message from the greenlet to the mule (that
IS part of the uWSGI stack, not only having the context), that itself transmits
the message on the spool.
"""
import ast
import uwsgi
while True:
message = uwsgi.mule_get_msg()
if message:
message = ast.literal_eval(message)
uwsgi.spool(message)
def producer():
uwsgi.spool(ud_spool_func="consumer", dest=random.choice(projects))
time.sleep(2)
def setUp(self):
for priority, name in spooler_priority_constants.tasks:
task = {'name': name, 'at': int(time.time() + 10)}
if priority is not None:
task['priority'] = str(priority)
uwsgi.spool(task)
def execute_async(self):
return uwsgi.spool(self.get_message_content())
# uwsgi --master --plugins=python27 --spooler=/var/spool/uwsgi/ --spooler-import spooler_dir.py
import uwsgi
def spooler_func(env):
print(uwsgi.spooler_dir())
return uwsgi.SPOOL_RETRY
uwsgi.spooler = spooler_func
uwsgi.spool({"foo": "bar"})
# uwsgi --master --plugins=python27 --spooler=/var/spool/uwsgi/ --spooler-import spooler_dir.py
import uwsgi
def spooler_func(env):
print(uwsgi.spooler_dir())
return uwsgi.SPOOL_RETRY
uwsgi.spooler = spooler_func
uwsgi.spool({"foo": "bar"})
