def worker(task_queue, result_queue, timer, timeout=Conf.TIMEOUT):
"""
Takes a task from the task queue, tries to execute it and puts the result back in the result queue
:type task_queue: multiprocessing.Queue
:type result_queue: multiprocessing.Queue
:type timer: multiprocessing.Value
"""
name = current_process().name
logger.info(
_('{} ready for work at {}').format(name,
current_process().pid))
task_count = 0
# Start reading the task queue
for task in iter(task_queue.get, 'STOP'):
result = None
timer.value = -1 # Idle
task_count += 1
# Get the function from the task
logger.info(_('{} processing [{}]').format(name, task['name']))
f = task['func']
# if it's not an instance try to get it from the string
if not callable(task['func']):
try:
module, func = f.rsplit('.', 1)
m = importlib.import_module(module)
f = getattr(m, func)
except (ValueError, ImportError, AttributeError) as e:
result = (e, False)
if error_reporter:
error_reporter.report()
if rollbar:
rollbar.report_exc_info()
# We're still going
if not result:
db.close_old_connections()
timer_value = task['kwargs'].pop('timeout', timeout or 0)
# signal execution
pre_execute.send(sender="django_q", func=f, task=task)
# execute the payload
timer.value = timer_value # Busy
try:
res = f(*task['args'], **task['kwargs'])
result = (res, True)
except Exception as e:
result = ('{}'.format(e), False)
if error_reporter:
error_reporter.report()
if rollbar:
rollbar.report_exc_info()
# Process result
task['result'] = result[0]
task['success'] = result[1]
task['stopped'] = timezone.now()
result_queue.put(task)
timer.value = -1 # Idle
# Recycle
if task_count == Conf.RECYCLE:
timer.value = -2 # Recycled
break
logger.info(_('{} stopped doing work').format(name))
def worker(task_queue, result_queue, timer, timeout=Conf.TIMEOUT):
"""
Takes a task from the task queue, tries to execute it and puts the result back in the result queue
:type task_queue: multiprocessing.Queue
:type result_queue: multiprocessing.Queue
:type timer: multiprocessing.Value
"""
name = current_process().name
logger.info(_('{} ready for work at {}').format(name, current_process().pid))
task_count = 0
# Start reading the task queue
for task in iter(task_queue.get, 'STOP'):
result = None
timer.value = -1 # Idle
task_count += 1
# Get the function from the task
logger.info(_('{} processing [{}]').format(name, task['name']))
f = task['func']
# if it's not an instance try to get it from the string
if not callable(task['func']):
try:
module, func = f.rsplit('.', 1)
m = importlib.import_module(module)
f = getattr(m, func)
except (ValueError, ImportError, AttributeError) as e:
result = (e, False)
if error_reporter:
error_reporter.report()
if rollbar:
rollbar.report_exc_info()
# We're still going
if not result:
db.close_old_connections()
timer_value = task['kwargs'].pop('timeout', timeout or 0)
# signal execution
pre_execute.send(sender="django_q", func=f, task=task)
# execute the payload
timer.value = timer_value # Busy
try:
res = f(*task['args'], **task['kwargs'])
result = (res, True)
except Exception as e:
result = ('{}'.format(e), False)
if error_reporter:
error_reporter.report()
if rollbar:
rollbar.report_exc_info()
# Process result
task['result'] = result[0]
task['success'] = result[1]
task['stopped'] = timezone.now()
result_queue.put(task)
timer.value = -1 # Idle
# Recycle
if task_count == Conf.RECYCLE:
timer.value = -2 # Recycled
break
logger.info(_('{} stopped doing work').format(name))
def worker(task_queue, result_queue, timer, timeout=Conf.TIMEOUT):
"""
Takes a task from the task queue, tries to execute it and puts the result back in the result queue
:type task_queue: multiprocessing.Queue
:type result_queue: multiprocessing.Queue
:type timer: multiprocessing.Value
"""
name = current_process().name
logger.info(
_('{} ready for work at {}').format(name,
current_process().pid))
task_count = 0
# Start reading the task queue
for task in iter(task_queue.get, 'STOP'):
result = None
timer.value = -1 # Idle
task_count += 1
# record the current worker's PID, so we can kill it later
task['worker_process_pid'] = os.getpid()
logger.info("Got job for worker {}".format(os.getpid()))
Task.objects.filter(id=task['id']).update(
worker_process_pid=task['worker_process_pid'])
# mark the task as being run now.
Task.objects.filter(id=task['id']).update(task_status=Task.INPROGRESS)
# Get the function from the task
logger.info(_('{} processing [{}]').format(name, task['name']))
f = task['func']
# if it's not an instance try to get it from the string
if not callable(task['func']):
try:
module, func = f.rsplit('.', 1)
m = importlib.import_module(module)
f = getattr(m, func)
except (ValueError, ImportError, AttributeError) as e:
result = (e, False, Task.FAILED)
if rollbar:
rollbar.report_exc_info()
# We're still going
if not result:
db.close_old_connections()
if task['is_progress_updating']:
task['kwargs']['update_state'] = partial(
update_task_progress, task)
# execute the payload
timer.value = task['kwargs'].pop('timeout', timeout or 0) # Busy
try:
res = f(*task['args'], **task['kwargs'])
result = (res, True, Task.SUCCESS)
except Exception as e:
e.traceback = traceback.format_exc()
result = (e, False, Task.FAILED)
if rollbar:
rollbar.report_exc_info()
# make sure to remove the update_state func before shuffling across
# process boundaries (through the result_queue), since its globals()
# contains multiprocessing.Queue objects, which are unpickleable
task['kwargs'].pop('update_state', None)
# Process result
task['result'] = result[0]
task['success'] = result[1]
task['task_status'] = result[2]
task['stopped'] = timezone.now()
result_queue.put(task)
timer.value = -1 # Idle
# Recycle
if task_count == Conf.RECYCLE:
timer.value = -2 # Recycled
break
logger.info(_('{} stopped doing work').format(name))
def worker(task_queue, result_queue, timer, timeout=Conf.TIMEOUT):
"""
Takes a task from the task queue, tries to execute it and puts the result back in the result queue
:type task_queue: multiprocessing.Queue
:type result_queue: multiprocessing.Queue
:type timer: multiprocessing.Value
"""
name = current_process().name
logger.info(_('{} ready for work at {}').format(name, current_process().pid))
task_count = 0
# Start reading the task queue
for task in iter(task_queue.get, 'STOP'):
result = None
timer.value = -1 # Idle
task_count += 1
# record the current worker's PID, so we can kill it later
task['worker_process_pid'] = os.getpid()
logger.info("Got job for worker {}".format(os.getpid()))
Task.objects.filter(id=task['id']).update(worker_process_pid=task['worker_process_pid'])
# mark the task as being run now.
Task.objects.filter(id=task['id']).update(task_status=Task.INPROGRESS)
# Get the function from the task
logger.info(_('{} processing [{}]').format(name, task['name']))
f = task['func']
# if it's not an instance try to get it from the string
if not callable(task['func']):
try:
module, func = f.rsplit('.', 1)
m = importlib.import_module(module)
f = getattr(m, func)
except (ValueError, ImportError, AttributeError) as e:
result = (e, False, Task.FAILED)
if rollbar:
rollbar.report_exc_info()
# We're still going
if not result:
db.close_old_connections()
timer_value = task['kwargs'].pop('timeout', timeout or 0)
# signal execution
pre_execute.send(sender="django_q", func=f, task=task)
if task['is_progress_updating']:
task['kwargs']['update_state'] = partial(update_task_progress, task)
# execute the payload
timer.value = timer_value # Busy
try:
res = f(*task['args'], **task['kwargs'])
result = (res, True, Task.SUCCESS)
except Exception as e:
e.traceback = traceback.format_exc()
result = (e, False, Task.FAILED)
if rollbar:
rollbar.report_exc_info()
# make sure to remove the update_state func before shuffling across
# process boundaries (through the result_queue), since its globals()
# contains multiprocessing.Queue objects, which are unpickleable
task['kwargs'].pop('update_state', None)
# Process result
task['result'] = result[0]
task['success'] = result[1]
task['task_status'] = result[2]
task['stopped'] = timezone.now()
result_queue.put(task)
timer.value = -1 # Idle
# Recycle
if task_count == Conf.RECYCLE:
timer.value = -2 # Recycled
break
logger.info(_('{} stopped doing work').format(name))