def _handle(self, use_reloader, replay_failed): self.logger.info("%sServer starting on %s." % ('Development ' if use_reloader else '', settings.ZTASKD_URL)) self._on_load() socket = context.socket(PULL) socket.bind(settings.ZTASKD_URL) def _queue_handler(socket, *args, **kwargs): try: function_name, args, kwargs, after = socket.recv_pyobj() if function_name == 'ztask_log': self.logger.warn('%s: %s' % (args[0], args[1])) return task = Task.objects.create( function_name=function_name, args=pickle.dumps(args), kwargs=pickle.dumps(kwargs), retry_count=settings.ZTASKD_RETRY_COUNT, next_attempt=time.time() + after ) if after: ioloop.DelayedCallback(lambda: self._call_function(task.pk, function_name=function_name, args=args, kwargs=kwargs), after * 1000, io_loop=self.io_loop).start() else: self._call_function(task.pk, function_name=function_name, args=args, kwargs=kwargs) except Exception, e: self.logger.error('Error setting up function. Details:\n%s' % e) traceback.print_exc(e)
def wrapper(func): function_name = '%s.%s' % (func.__module__, func.__name__) logger = logging.getLogger('ztaskd') logger.info('Registered task: %s' % function_name) from django_ztask.context import shared_context as context socket = context.socket(PUSH) socket.connect(settings.ZTASKD_URL) @wraps(func) def _func(*args, **kwargs): after = kwargs.pop('__ztask_after', 0) if settings.ZTASKD_DISABLED: try: socket.send_pyobj(('ztask_log', ('Would have called but ZTASKD_DISABLED is True', function_name), None, 0)) except: logger.info('Would have sent %s but ZTASKD_DISABLED is True' % function_name) return elif settings.ZTASKD_ALWAYS_EAGER: logger.info('Running %s in ZTASKD_ALWAYS_EAGER mode' % function_name) if after > 0: logger.info('Ignoring timeout of %d seconds because ZTASKD_ALWAYS_EAGER is set' % after) func(*args, **kwargs) else: try: socket.send_pyobj((function_name, args, kwargs, after)) except Exception, e: if after > 0: logger.info('Ignoring timeout of %s seconds because function is being run in-process' % after) func(*args, **kwargs)
def _handle(self): logger.info("Worker listening on %s." % (settings.ZTASK_WORKER_URL,)) socket = context.socket(PULL) socket.connect(settings.ZTASK_WORKER_URL) while True: task_id = socket.recv_pyobj() logger.info('Worker received task (%s)' % (str(task_id),))
def wrapper(func): function_name = '%s.%s' % (func.__module__, func.__name__) logger = logging.getLogger('ztask') logger.setLevel(logging.INFO) handler = logging.StreamHandler() formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') handler.setFormatter(formatter) logger.addHandler(handler) #logger.info('Registered task: %s' % function_name) from django_ztask.context import shared_context as context socket = context.socket(zmq.PUSH) socket.connect(settings.ZTASKD_URL) @wraps(func) def _func(*args, **kwargs): after = kwargs.pop('__ztask_after', 0) # check if the server is alive alive_socket = context.socket(zmq.REQ) alive_socket.connect(settings.ZTASKD_ALIVE_URL) poll = zmq.Poller() poll.register(alive_socket, zmq.POLLIN) request = 'ping' alive_socket.send(request) socks = dict(poll.poll(settings.ZTASKD_ALIVE_TIMEOUT)) if socks.get(alive_socket) == zmq.POLLIN: reply = alive_socket.recv() else: # consider the server dead alive_socket.setsockopt(zmq.LINGER, 0) alive_socket.close() poll.unregister(alive_socket) logger.info('Running %s in-process because the ztaskd server is not responding' % function_name) func(*args, **kwargs) return if settings.ZTASKD_DISABLED: try: socket.send_pyobj(('ztask_log', ('Would have called but ZTASKD_DISABLED is True', function_name), None, 0)) except: logger.info('Would have sent %s but ZTASKD_DISABLED is True' % function_name) return elif settings.ZTASKD_ALWAYS_EAGER: logger.info('Running %s in ZTASKD_ALWAYS_EAGER mode' % function_name) if after > 0: logger.info('Ignoring timeout of %d seconds because ZTASKD_ALWAYS_EAGER is set' % after) func(*args, **kwargs) else: try: socket.send_pyobj((function_name, args, kwargs, after)) except Exception, e: if after > 0: logger.info('Ignoring timeout of %s seconds because function is being run in-process' % after) func(*args, **kwargs)
def build_job_distribution_socket(): """ Set up a push connection to all of the worker nodes so we can send jobs. """ global __socket__ if not __socket__: from django_ztask.context import shared_context as context __socket__ = context.socket(PUSH) for worker_url in settings.ZTASKD_WORKER_URL_LIST: __socket__.connect(worker_url) return __socket__
def _handle(self, use_reloader, replay_failed): ztask_url = setting("ZTASKD_URL", "tcp://127.0.0.1:5555") log.info("%sServer starting on %s." % ("Development " if use_reloader else "", ztask_url)) self._on_load() socket = context.socket(PULL) socket.bind(ztask_url) def _queue_handler(socket, *args, **kw): try: function_name, args, kw, after = socket.recv_pyobj() if function_name == "ztask_log": log.warn("%s: %s" % (args[0], args[1])) return task = Task.objects.create( function_name=function_name, args=pickle.dumps(args), kwargs=pickle.dumps(kw), retry_count=setting("ZTASKD_RETRY_COUNT", 5), next_attempt=time.time() + after, ) if after: ioloop.DelayedCallback( lambda: self._call_function(task.pk, function_name=function_name, args=args, kw=kw), after * 1000, io_loop=self.io_loop, ).start() else: self._call_function(task.pk, function_name=function_name, args=args, kw=kw) except: log.exception("Error setting up function") # Reload tasks if necessary if replay_failed: replay_tasks = Task.objects.all().order_by("created") else: replay_tasks = Task.objects.filter(retry_count__gt=0).order_by("created") for task in replay_tasks: if task.next_attempt < time.time(): ioloop.DelayedCallback(lambda: self._call_function(task.pk), 5000, io_loop=self.io_loop).start() else: after = task.next_attempt - time.time() ioloop.DelayedCallback(lambda: self._call_function(task.pk), after * 1000, io_loop=self.io_loop).start() self.io_loop = ioloop.IOLoop.instance() self.io_loop.add_handler(socket, _queue_handler, self.io_loop.READ) self.io_loop.start()
def _func(*args, **kwargs): after = kwargs.pop('__ztask_after', 0) # check if the server is alive alive_socket = context.socket(zmq.REQ) alive_socket.connect(settings.ZTASKD_ALIVE_URL) poll = zmq.Poller() poll.register(alive_socket, zmq.POLLIN) request = 'ping' alive_socket.send(request) socks = dict(poll.poll(settings.ZTASKD_ALIVE_TIMEOUT)) if socks.get(alive_socket) == zmq.POLLIN: reply = alive_socket.recv() else: # consider the server dead alive_socket.setsockopt(zmq.LINGER, 0) alive_socket.close() poll.unregister(alive_socket) logger.info('Running %s in-process because the ztaskd server is not responding' % function_name) func(*args, **kwargs) return if settings.ZTASKD_DISABLED: try: socket.send_pyobj(('ztask_log', ('Would have called but ZTASKD_DISABLED is True', function_name), None, 0)) except: logger.info('Would have sent %s but ZTASKD_DISABLED is True' % function_name) return elif settings.ZTASKD_ALWAYS_EAGER: logger.info('Running %s in ZTASKD_ALWAYS_EAGER mode' % function_name) if after > 0: logger.info('Ignoring timeout of %d seconds because ZTASKD_ALWAYS_EAGER is set' % after) func(*args, **kwargs) else: try: socket.send_pyobj((function_name, args, kwargs, after)) except Exception, e: if after > 0: logger.info('Ignoring timeout of %s seconds because function is being run in-process' % after) func(*args, **kwargs)
def wrapper(func): function_name = '%s.%s' % (func.__module__, func.__name__) log.info('Registered task: %s' % function_name) socket = context.socket(PUSH) socket.connect(setting('ZTASKD_URL', 'tcp://127.0.0.1:5555')) @wraps(func) def _func(*args, **kw): after = kw.pop('__ztask_after', 0) if setting('ZTASKD_DISABLED', False): try: socket.send_pyobj(('ztask_log', ( 'Would have called but ZTASKD_DISABLED is True', function_name, ), None, 0)) except: log.info( 'Would have sent %s but ZTASKD_DISABLED is' 'True' % function_name ) return elif setting('ZTASKD_ALWAYS_EAGER', False): log.info( 'Running %s in ZTASKD_ALWAYS_EAGER mode' % function_name ) if after > 0: log.info( 'Ignoring timeout of %d seconds because ' 'ZTASKD_ALWAYS_EAGER is set' % after ) func(*args, **kw) else: try: socket.send_pyobj((function_name, args, kw, after)) except: if after > 0: log.info( 'Ignoring timeout of %s seconds because function is' ' being run in-process' % after ) func(*args, **kw) def _func_delay(*args, **kw): try: socket.send_pyobj(('ztask_log', ( '.delay is deprecated... use.async instead', function_name ), None, 0)) except: pass _func(*args, **kw) def _func_after(*args, **kw): try: after = args[0] if type(after) != types.IntType: raise TypeError( 'The first argument of .after must be an integer ' 'representing seconds to wait' ) kw['__ztask_after'] = after _func(*args[1:], **kw) except: log.exception('Error adding delayed task:\n%s') setattr(func, 'async', _func) setattr(func, 'delay', _func_delay) setattr(func, 'after', _func_after) return func