def _func(*args, **kw): after = kw.pop('__ztask_after', 0) if setting('ZTASKD_DISABLED', False): try: socket.send_pyobj(('ztask_log', ( 'Would have called but ZTASKD_DISABLED is True', function_name, ), None, 0)) except: log.info( 'Would have sent %s but ZTASKD_DISABLED is' 'True' % function_name ) return elif setting('ZTASKD_ALWAYS_EAGER', False): log.info( 'Running %s in ZTASKD_ALWAYS_EAGER mode' % function_name ) if after > 0: log.info( 'Ignoring timeout of %d seconds because ' 'ZTASKD_ALWAYS_EAGER is set' % after ) func(*args, **kw) else: try: socket.send_pyobj((function_name, args, kw, after)) except: if after > 0: log.info( 'Ignoring timeout of %s seconds because function is' ' being run in-process' % after ) func(*args, **kw)
def _on_load(self): ztaskd_on_load = setting("ZTASKD_ON_LOAD", ()) for callable_name in ztaskd_on_load: log.info("ON_LOAD calling %s" % callable_name) parts = callable_name.split(".") module_name = ".".join(parts[:-1]) member_name = parts[-1] if not module_name in sys.modules: importlib.import_module(module_name) callable_fn = getattr(sys.modules[module_name], member_name) callable_fn()
def _handle(self, use_reloader, replay_failed): ztask_url = setting("ZTASKD_URL", "tcp://127.0.0.1:5555") log.info("%sServer starting on %s." % ("Development " if use_reloader else "", ztask_url)) self._on_load() socket = context.socket(PULL) socket.bind(ztask_url) def _queue_handler(socket, *args, **kw): try: function_name, args, kw, after = socket.recv_pyobj() if function_name == "ztask_log": log.warn("%s: %s" % (args[0], args[1])) return task = Task.objects.create( function_name=function_name, args=pickle.dumps(args), kwargs=pickle.dumps(kw), retry_count=setting("ZTASKD_RETRY_COUNT", 5), next_attempt=time.time() + after, ) if after: ioloop.DelayedCallback( lambda: self._call_function(task.pk, function_name=function_name, args=args, kw=kw), after * 1000, io_loop=self.io_loop, ).start() else: self._call_function(task.pk, function_name=function_name, args=args, kw=kw) except: log.exception("Error setting up function") # Reload tasks if necessary if replay_failed: replay_tasks = Task.objects.all().order_by("created") else: replay_tasks = Task.objects.filter(retry_count__gt=0).order_by("created") for task in replay_tasks: if task.next_attempt < time.time(): ioloop.DelayedCallback(lambda: self._call_function(task.pk), 5000, io_loop=self.io_loop).start() else: after = task.next_attempt - time.time() ioloop.DelayedCallback(lambda: self._call_function(task.pk), after * 1000, io_loop=self.io_loop).start() self.io_loop = ioloop.IOLoop.instance() self.io_loop.add_handler(socket, _queue_handler, self.io_loop.READ) self.io_loop.start()
def _call_function(self, task_id, function_name=None, args=None, kw=None): try: if not function_name: try: task = Task.objects.get(pk=task_id) function_name = task.function_name args = pickle.loads(str(task.args)) kw = pickle.loads(str(task.kwargs)) except: log.exception("Count not get task id %s" % task_id) return None log.info("Calling %s" % function_name) try: function = self.func_cache[function_name] except KeyError: parts = function_name.split(".") module_name = ".".join(parts[:-1]) member_name = parts[-1] if not module_name in sys.modules: importlib.import_module(module_name) function = getattr(sys.modules[module_name], member_name) self.func_cache[function_name] = function function(*args, **kw) log.info("Called %s successfully" % function_name) Task.objects.get(pk=task_id).delete() except Exception, e: log.exception("Error calling %s" % function_name) try: retry_count = setting("ZTASKD_RETRY_COUNT", 5) task = Task.objects.get(pk=task_id) if task.retry_count > 0: task.retry_count = task.retry_count - 1 task.next_attempt = time.time() + retry_count ioloop.DelayedCallback( lambda: self._call_function(task.pk), retry_count * 1000, io_loop=self.io_loop ).start() task.failed = datetime.datetime.utcnow() task.last_exception = "%s" % e task.save() except: log.exception("Error capturing exception in _call_function")
def _queue_handler(socket, *args, **kw): try: function_name, args, kw, after = socket.recv_pyobj() if function_name == "ztask_log": log.warn("%s: %s" % (args[0], args[1])) return task = Task.objects.create( function_name=function_name, args=pickle.dumps(args), kwargs=pickle.dumps(kw), retry_count=setting("ZTASKD_RETRY_COUNT", 5), next_attempt=time.time() + after, ) if after: ioloop.DelayedCallback( lambda: self._call_function(task.pk, function_name=function_name, args=args, kw=kw), after * 1000, io_loop=self.io_loop, ).start() else: self._call_function(task.pk, function_name=function_name, args=args, kw=kw) except: log.exception("Error setting up function")
from django.utils.log import getLogger from django.utils import autoreload, importlib from django.core.management.base import BaseCommand try: from zmq import PULL except: from zmq import UPSTREAM as PULL from zmq.eventloop import ioloop from django_ztask.models import Task from django_ztask.utils import setting from django_ztask.context import shared_context as context log = getLogger(setting("ZTASKD_LOGGER", "django_ztaskd")) class Command(BaseCommand): option_list = BaseCommand.option_list + ( make_option( "--noreload", action="store_false", dest="use_reloader", default=True, help="Tells Django to NOT use the auto-reloader", ), make_option( "--replayfailed", action="store_true",
def wrapper(func): function_name = '%s.%s' % (func.__module__, func.__name__) log.info('Registered task: %s' % function_name) socket = context.socket(PUSH) socket.connect(setting('ZTASKD_URL', 'tcp://127.0.0.1:5555')) @wraps(func) def _func(*args, **kw): after = kw.pop('__ztask_after', 0) if setting('ZTASKD_DISABLED', False): try: socket.send_pyobj(('ztask_log', ( 'Would have called but ZTASKD_DISABLED is True', function_name, ), None, 0)) except: log.info( 'Would have sent %s but ZTASKD_DISABLED is' 'True' % function_name ) return elif setting('ZTASKD_ALWAYS_EAGER', False): log.info( 'Running %s in ZTASKD_ALWAYS_EAGER mode' % function_name ) if after > 0: log.info( 'Ignoring timeout of %d seconds because ' 'ZTASKD_ALWAYS_EAGER is set' % after ) func(*args, **kw) else: try: socket.send_pyobj((function_name, args, kw, after)) except: if after > 0: log.info( 'Ignoring timeout of %s seconds because function is' ' being run in-process' % after ) func(*args, **kw) def _func_delay(*args, **kw): try: socket.send_pyobj(('ztask_log', ( '.delay is deprecated... use.async instead', function_name ), None, 0)) except: pass _func(*args, **kw) def _func_after(*args, **kw): try: after = args[0] if type(after) != types.IntType: raise TypeError( 'The first argument of .after must be an integer ' 'representing seconds to wait' ) kw['__ztask_after'] = after _func(*args[1:], **kw) except: log.exception('Error adding delayed task:\n%s') setattr(func, 'async', _func) setattr(func, 'delay', _func_delay) setattr(func, 'after', _func_after) return func
# -*- coding: utf-8 -*- import types from functools import wraps from django.utils.log import getLogger from django_ztask.utils import setting try: from zmq import PUSH except: from zmq import DOWNSTREAM as PUSH from django_ztask.context import shared_context as context log = getLogger(setting('ZTASKD_LOGGER', 'django_ztaskd')) def task(): def wrapper(func): function_name = '%s.%s' % (func.__module__, func.__name__) log.info('Registered task: %s' % function_name) socket = context.socket(PUSH) socket.connect(setting('ZTASKD_URL', 'tcp://127.0.0.1:5555')) @wraps(func) def _func(*args, **kw): after = kw.pop('__ztask_after', 0) if setting('ZTASKD_DISABLED', False): try: socket.send_pyobj(('ztask_log', ( 'Would have called but ZTASKD_DISABLED is True',