def _func(*args, **kw):
after = kw.pop('__ztask_after', 0)
if setting('ZTASKD_DISABLED', False):
try:
socket.send_pyobj(('ztask_log', (
'Would have called but ZTASKD_DISABLED is True',
function_name,
), None, 0))
except:
log.info(
'Would have sent %s but ZTASKD_DISABLED is'
'True' % function_name
)
return
elif setting('ZTASKD_ALWAYS_EAGER', False):
log.info(
'Running %s in ZTASKD_ALWAYS_EAGER mode' % function_name
)
if after > 0:
log.info(
'Ignoring timeout of %d seconds because '
'ZTASKD_ALWAYS_EAGER is set' % after
)
func(*args, **kw)
else:
try:
socket.send_pyobj((function_name, args, kw, after))
except:
if after > 0:
log.info(
'Ignoring timeout of %s seconds because function is'
' being run in-process' % after
)
func(*args, **kw)
def _on_load(self):
ztaskd_on_load = setting("ZTASKD_ON_LOAD", ())
for callable_name in ztaskd_on_load:
log.info("ON_LOAD calling %s" % callable_name)
parts = callable_name.split(".")
module_name = ".".join(parts[:-1])
member_name = parts[-1]
if not module_name in sys.modules:
importlib.import_module(module_name)
callable_fn = getattr(sys.modules[module_name], member_name)
callable_fn()
def _handle(self, use_reloader, replay_failed):
ztask_url = setting("ZTASKD_URL", "tcp://127.0.0.1:5555")
log.info("%sServer starting on %s." % ("Development " if use_reloader else "", ztask_url))
self._on_load()
socket = context.socket(PULL)
socket.bind(ztask_url)
def _queue_handler(socket, *args, **kw):
try:
function_name, args, kw, after = socket.recv_pyobj()
if function_name == "ztask_log":
log.warn("%s: %s" % (args[0], args[1]))
return
task = Task.objects.create(
function_name=function_name,
args=pickle.dumps(args),
kwargs=pickle.dumps(kw),
retry_count=setting("ZTASKD_RETRY_COUNT", 5),
next_attempt=time.time() + after,
)
if after:
ioloop.DelayedCallback(
lambda: self._call_function(task.pk, function_name=function_name, args=args, kw=kw),
after * 1000,
io_loop=self.io_loop,
).start()
else:
self._call_function(task.pk, function_name=function_name, args=args, kw=kw)
except:
log.exception("Error setting up function")
# Reload tasks if necessary
if replay_failed:
replay_tasks = Task.objects.all().order_by("created")
else:
replay_tasks = Task.objects.filter(retry_count__gt=0).order_by("created")
for task in replay_tasks:
if task.next_attempt < time.time():
ioloop.DelayedCallback(lambda: self._call_function(task.pk), 5000, io_loop=self.io_loop).start()
else:
after = task.next_attempt - time.time()
ioloop.DelayedCallback(lambda: self._call_function(task.pk), after * 1000, io_loop=self.io_loop).start()
self.io_loop = ioloop.IOLoop.instance()
self.io_loop.add_handler(socket, _queue_handler, self.io_loop.READ)
self.io_loop.start()
def _call_function(self, task_id, function_name=None, args=None, kw=None):
try:
if not function_name:
try:
task = Task.objects.get(pk=task_id)
function_name = task.function_name
args = pickle.loads(str(task.args))
kw = pickle.loads(str(task.kwargs))
except:
log.exception("Count not get task id %s" % task_id)
return None
log.info("Calling %s" % function_name)
try:
function = self.func_cache[function_name]
except KeyError:
parts = function_name.split(".")
module_name = ".".join(parts[:-1])
member_name = parts[-1]
if not module_name in sys.modules:
importlib.import_module(module_name)
function = getattr(sys.modules[module_name], member_name)
self.func_cache[function_name] = function
function(*args, **kw)
log.info("Called %s successfully" % function_name)
Task.objects.get(pk=task_id).delete()
except Exception, e:
log.exception("Error calling %s" % function_name)
try:
retry_count = setting("ZTASKD_RETRY_COUNT", 5)
task = Task.objects.get(pk=task_id)
if task.retry_count > 0:
task.retry_count = task.retry_count - 1
task.next_attempt = time.time() + retry_count
ioloop.DelayedCallback(
lambda: self._call_function(task.pk), retry_count * 1000, io_loop=self.io_loop
).start()
task.failed = datetime.datetime.utcnow()
task.last_exception = "%s" % e
task.save()
except:
log.exception("Error capturing exception in _call_function")
def _queue_handler(socket, *args, **kw):
try:
function_name, args, kw, after = socket.recv_pyobj()
if function_name == "ztask_log":
log.warn("%s: %s" % (args[0], args[1]))
return
task = Task.objects.create(
function_name=function_name,
args=pickle.dumps(args),
kwargs=pickle.dumps(kw),
retry_count=setting("ZTASKD_RETRY_COUNT", 5),
next_attempt=time.time() + after,
)
if after:
ioloop.DelayedCallback(
lambda: self._call_function(task.pk, function_name=function_name, args=args, kw=kw),
after * 1000,
io_loop=self.io_loop,
).start()
else:
self._call_function(task.pk, function_name=function_name, args=args, kw=kw)
except:
log.exception("Error setting up function")
from django.utils.log import getLogger
from django.utils import autoreload, importlib
from django.core.management.base import BaseCommand
try:
from zmq import PULL
except:
from zmq import UPSTREAM as PULL
from zmq.eventloop import ioloop
from django_ztask.models import Task
from django_ztask.utils import setting
from django_ztask.context import shared_context as context
log = getLogger(setting("ZTASKD_LOGGER", "django_ztaskd"))
class Command(BaseCommand):
option_list = BaseCommand.option_list + (
make_option(
"--noreload",
action="store_false",
dest="use_reloader",
default=True,
help="Tells Django to NOT use the auto-reloader",
),
make_option(
"--replayfailed",
action="store_true",
def wrapper(func):
function_name = '%s.%s' % (func.__module__, func.__name__)
log.info('Registered task: %s' % function_name)
socket = context.socket(PUSH)
socket.connect(setting('ZTASKD_URL', 'tcp://127.0.0.1:5555'))
@wraps(func)
def _func(*args, **kw):
after = kw.pop('__ztask_after', 0)
if setting('ZTASKD_DISABLED', False):
try:
socket.send_pyobj(('ztask_log', (
'Would have called but ZTASKD_DISABLED is True',
function_name,
), None, 0))
except:
log.info(
'Would have sent %s but ZTASKD_DISABLED is'
'True' % function_name
)
return
elif setting('ZTASKD_ALWAYS_EAGER', False):
log.info(
'Running %s in ZTASKD_ALWAYS_EAGER mode' % function_name
)
if after > 0:
log.info(
'Ignoring timeout of %d seconds because '
'ZTASKD_ALWAYS_EAGER is set' % after
)
func(*args, **kw)
else:
try:
socket.send_pyobj((function_name, args, kw, after))
except:
if after > 0:
log.info(
'Ignoring timeout of %s seconds because function is'
' being run in-process' % after
)
func(*args, **kw)
def _func_delay(*args, **kw):
try:
socket.send_pyobj(('ztask_log', (
'.delay is deprecated... use.async instead', function_name
), None, 0))
except:
pass
_func(*args, **kw)
def _func_after(*args, **kw):
try:
after = args[0]
if type(after) != types.IntType:
raise TypeError(
'The first argument of .after must be an integer '
'representing seconds to wait'
)
kw['__ztask_after'] = after
_func(*args[1:], **kw)
except:
log.exception('Error adding delayed task:\n%s')
setattr(func, 'async', _func)
setattr(func, 'delay', _func_delay)
setattr(func, 'after', _func_after)
return func
# -*- coding: utf-8 -*-
import types
from functools import wraps
from django.utils.log import getLogger
from django_ztask.utils import setting
try:
from zmq import PUSH
except:
from zmq import DOWNSTREAM as PUSH
from django_ztask.context import shared_context as context
log = getLogger(setting('ZTASKD_LOGGER', 'django_ztaskd'))
def task():
def wrapper(func):
function_name = '%s.%s' % (func.__module__, func.__name__)
log.info('Registered task: %s' % function_name)
socket = context.socket(PUSH)
socket.connect(setting('ZTASKD_URL', 'tcp://127.0.0.1:5555'))
@wraps(func)
def _func(*args, **kw):
after = kw.pop('__ztask_after', 0)
if setting('ZTASKD_DISABLED', False):
try:
socket.send_pyobj(('ztask_log', (
'Would have called but ZTASKD_DISABLED is True',
