def parse_url(url):
port = path = auth = userid = password = None
scheme = urlparse(url).scheme
parts = urlparse(url.replace("%s://" % (scheme, ), "http://"))
# The first pymongo.Connection() argument (host) can be
# a mongodb connection URI. If this is the case, don't
# use port but let pymongo get the port(s) from the URI instead.
# This enables the use of replica sets and sharding.
# See pymongo.Connection() for more info.
if scheme != 'mongodb':
netloc = parts.netloc
if '@' in netloc:
auth, _, netloc = partition(parts.netloc, '@')
userid, _, password = partition(auth, ':')
hostname, _, port = partition(netloc, ':')
path = parts.path or ""
if path and path[0] == '/':
path = path[1:]
port = port and int(port) or port
else:
# strip the scheme since it is appended automatically
hostname = url[len('mongodb://'):]
return dict(
{
"hostname": hostname,
"port": port or None,
"userid": userid or None,
"password": password or None,
"transport": scheme,
"virtual_host": path or None
}, **kwdict(dict(parse_qsl(parts.query))))
def trace_task(uuid, args, kwargs, request=None):
R = I = None
kwargs = kwdict(kwargs)
try:
_tls.current_task = task
update_request(request or {}, args=args,
called_directly=False, kwargs=kwargs)
try:
# -*- PRE -*-
send_prerun(sender=task, task_id=uuid, task=task,
args=args, kwargs=kwargs)
loader_task_init(uuid, task)
if track_started:
store_result(uuid, {"pid": pid,
"hostname": hostname}, STARTED)
# -*- TRACE -*-
try:
R = retval = fun(*args, **kwargs)
state = SUCCESS
except RetryTaskError, exc:
I = Info(RETRY, exc)
state, retval = I.state, I.retval
R = I.handle_error_state(task, eager=eager)
except Exception, exc:
if propagate:
raise
I = Info(FAILURE, exc)
state, retval = I.state, I.retval
R = I.handle_error_state(task, eager=eager)
[subtask(errback).apply_async((uuid, ))
for errback in task_request.errbacks or []]
except BaseException, exc:
raise
def trace_task(uuid, args, kwargs, request=None):
R = I = None
kwargs = kwdict(kwargs)
try:
_tls.current_task = task
update_request(request or {}, args=args,
called_directly=False, kwargs=kwargs)
try:
# -*- PRE -*-
send_prerun(sender=task, task_id=uuid, task=task,
args=args, kwargs=kwargs)
loader_task_init(uuid, task)
if track_started:
store_result(uuid, {"pid": pid,
"hostname": hostname}, STARTED)
# -*- TRACE -*-
try:
R = retval = fun(*args, **kwargs)
state, einfo = SUCCESS, None
except RetryTaskError, exc:
I = Info(RETRY, exc, sys.exc_info())
state, retval, einfo = I.state, I.retval, I.exc_info
R = I.handle_error_state(task, eager=eager)
except Exception, exc:
if propagate:
raise
I = Info(FAILURE, exc, sys.exc_info())
state, retval, einfo = I.state, I.retval, I.exc_info
R = I.handle_error_state(task, eager=eager)
except BaseException, exc:
raise
def from_dict(self, d, app=None):
tasks = d["kwargs"]["tasks"]
if d["args"] and tasks:
# partial args passed on to all tasks in the group (Issue #1057).
for task in tasks:
task["args"] = task._merge(d["args"])[0]
return group(tasks, app=app, **kwdict(d["options"]))
def parse_url(url):
port = path = auth = userid = password = None
scheme = urlparse(url).scheme
parts = urlparse(url.replace("%s://" % (scheme, ), "http://"))
# The first pymongo.Connection() argument (host) can be
# a mongodb connection URI. If this is the case, don't
# use port but let pymongo get the port(s) from the URI instead.
# This enables the use of replica sets and sharding.
# See pymongo.Connection() for more info.
if scheme != 'mongodb':
netloc = parts.netloc
if '@' in netloc:
auth, _, netloc = partition(parts.netloc, '@')
userid, _, password = partition(auth, ':')
hostname, _, port = partition(netloc, ':')
path = parts.path or ""
if path and path[0] == '/':
path = path[1:]
port = port and int(port) or port
else:
# strip the scheme since it is appended automatically
hostname = url[len('mongodb://'):]
return dict({"hostname": hostname,
"port": port or None,
"userid": userid or None,
"password": password or None,
"transport": scheme,
"virtual_host": path or None},
**kwdict(dict(parse_qsl(parts.query))))
def from_dict(self, d, app=None):
tasks = [maybe_signature(t, app=app) for t in d['kwargs']['tasks']]
if d['args'] and tasks:
# partial args passed on to all tasks in the group (Issue #1057).
for task in tasks:
task['args'] = task._merge(d['args'])[0]
return _upgrade(d, group(tasks, app=app, **kwdict(d['options'])))
def from_dict(self, d, app=None):
tasks = [maybe_signature(t, app=app) for t in d['kwargs']['tasks']]
if d['args'] and tasks:
# partial args passed on to all tasks in the group (Issue #1057).
for task in tasks:
task['args'] = task._merge(d['args'])[0]
return _upgrade(d, group(tasks, app=app, **kwdict(d['options'])))
def __init__(self,
body,
on_ack=noop,
hostname=None,
eventer=None,
app=None,
connection_errors=None,
request_dict=None,
delivery_info=None,
task=None,
**opts):
self.app = app or app_or_default(app)
name = self.name = body['task']
self.id = body['id']
self.args = body.get('args', [])
self.kwargs = body.get('kwargs', {})
try:
self.kwargs.items
except AttributeError:
raise exceptions.InvalidTaskError(
'Task keyword arguments is not a mapping')
if NEEDS_KWDICT:
self.kwargs = kwdict(self.kwargs)
eta = body.get('eta')
expires = body.get('expires')
utc = body.get('utc', False)
self.on_ack = on_ack
self.hostname = hostname or socket.gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[name]
self.acknowledged = self._already_revoked = False
self.time_start = self.worker_pid = self._terminate_on_ack = None
self._tzlocal = None
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
if eta is not None:
tz = tz_utc if utc else self.tzlocal
self.eta = tz_to_local(maybe_iso8601(eta), self.tzlocal, tz)
else:
self.eta = None
if expires is not None:
tz = tz_utc if utc else self.tzlocal
self.expires = tz_to_local(maybe_iso8601(expires), self.tzlocal,
tz)
else:
self.expires = None
delivery_info = {} if delivery_info is None else delivery_info
self.delivery_info = {
'exchange': delivery_info.get('exchange'),
'routing_key': delivery_info.get('routing_key'),
'priority': delivery_info.get('priority'),
}
# amqplib transport adds the channel here for some reason, so need
# to remove it.
self.delivery_info.pop('channel', None)
self.request_dict = body
def from_dict(self, d, app=None):
tasks = d['kwargs']['tasks']
if d['args'] and tasks:
# partial args passed on to all tasks in the group (Issue #1057).
for task in tasks:
task['args'] = task._merge(d['args'])[0]
return group(tasks, app=app, **kwdict(d['options']))
def _dispatch_event(self, event):
self.event_count += 1
event = kwdict(event)
group, _, type = event.pop('type').partition('-')
self.group_handlers[group](type, event)
if self.event_callback:
self.event_callback(self, event)
def __init__(
self,
body,
on_ack=noop,
hostname=None,
eventer=None,
app=None,
connection_errors=None,
request_dict=None,
delivery_info=None,
task=None,
**opts
):
self.app = app or app_or_default(app)
name = self.name = body["task"]
self.id = body["id"]
self.args = body.get("args", [])
self.kwargs = body.get("kwargs", {})
try:
self.kwargs.items
except AttributeError:
raise exceptions.InvalidTaskError("Task keyword arguments is not a mapping")
if NEEDS_KWDICT:
self.kwargs = kwdict(self.kwargs)
eta = body.get("eta")
expires = body.get("expires")
utc = body.get("utc", False)
self.on_ack = on_ack
self.hostname = hostname or socket.gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[name]
self.acknowledged = self._already_revoked = False
self.time_start = self.worker_pid = self._terminate_on_ack = None
self._tzlocal = None
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
if eta is not None:
tz = tz_utc if utc else self.tzlocal
self.eta = tz_to_local(maybe_iso8601(eta), self.tzlocal, tz)
else:
self.eta = None
if expires is not None:
tz = tz_utc if utc else self.tzlocal
self.expires = tz_to_local(maybe_iso8601(expires), self.tzlocal, tz)
else:
self.expires = None
delivery_info = {} if delivery_info is None else delivery_info
self.delivery_info = {
"exchange": delivery_info.get("exchange"),
"routing_key": delivery_info.get("routing_key"),
"priority": delivery_info.get("priority"),
}
# amqplib transport adds the channel here for some reason, so need
# to remove it.
self.delivery_info.pop("channel", None)
self.request_dict = body
def __init__(self,
body,
on_ack=noop,
hostname=None,
eventer=None,
app=None,
connection_errors=None,
request_dict=None,
delivery_info=None,
task=None,
**opts):
self.app = app or app_or_default(app)
name = self.name = body["task"]
self.id = body["id"]
self.args = body.get("args", [])
self.kwargs = body.get("kwargs", {})
try:
self.kwargs.items
except AttributeError:
raise exceptions.InvalidTaskError(
"Task keyword arguments is not a mapping")
if NEEDS_KWDICT:
self.kwargs = kwdict(self.kwargs)
eta = body.get("eta")
expires = body.get("expires")
utc = body.get("utc", False)
self.on_ack = on_ack
self.hostname = hostname or socket.gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[name]
self.acknowledged = self._already_revoked = False
self.time_start = self.worker_pid = self._terminate_on_ack = None
self._tzlocal = None
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
if eta is not None:
tz = tz_utc if utc else self.tzlocal
self.eta = tz_to_local(maybe_iso8601(eta), self.tzlocal, tz)
else:
self.eta = None
if expires is not None:
tz = tz_utc if utc else self.tzlocal
self.expires = tz_to_local(maybe_iso8601(expires), self.tzlocal,
tz)
else:
self.expires = None
delivery_info = {} if delivery_info is None else delivery_info
self.delivery_info = {
"exchange": delivery_info.get("exchange"),
"routing_key": delivery_info.get("routing_key"),
}
## shortcuts
self._does_debug = logger.isEnabledFor(logging.DEBUG)
self._does_info = logger.isEnabledFor(logging.INFO)
self.request_dict = body
def from_dict(self, d):
tasks = d['kwargs']['tasks']
if d['args'] and tasks:
# partial args passed on to all tasks in the group (Issue #1057).
for task in tasks:
task['args'] = d['args'] + task['args']
return group(tasks, **kwdict(d['options']))
def from_dict(self, d):
tasks = d['kwargs']['tasks']
if d['args'] and tasks:
# partial args passed on to all tasks in the group (Issue #1057).
for task in tasks:
task['args'] = d['args'] + task['args']
return group(tasks, **kwdict(d['options']))
def from_dict(self, d, app=None):
tasks = d['kwargs']['tasks']
if d['args'] and tasks:
# partial args passed on to all tasks in the group (Issue #1057).
for task in tasks:
task['args'] = task._merge(d['args'])[0]
return group(tasks, app=app, **kwdict(d['options']))
def _dispatch_event(self, event):
self.event_count += 1
event = kwdict(event)
group, _, type = event.pop('type').partition('-')
self.group_handlers[group](type, event)
if self.event_callback:
self.event_callback(self, event)
def trace_task(uuid, args, kwargs, request=None):
R = I = None
kwargs = kwdict(kwargs)
try:
push_task(task)
task_request = Context(request or {}, args=args, called_directly=False, kwargs=kwargs)
push_request(task_request)
try:
# -*- PRE -*-
if prerun_receivers:
send_prerun(sender=task, task_id=uuid, task=task, args=args, kwargs=kwargs)
loader_task_init(uuid, task)
if track_started:
store_result(uuid, {"pid": pid, "hostname": hostname}, STARTED)
# -*- TRACE -*-
try:
R = retval = fun(*args, **kwargs)
state = SUCCESS
except Ignore, exc:
I, R = Info(IGNORED, exc), ExceptionInfo(internal=True)
state, retval = I.state, I.retval
except RetryTaskError, exc:
I = Info(RETRY, exc)
state, retval = I.state, I.retval
R = I.handle_error_state(task, eager=eager)
except Exception, exc:
if propagate:
raise
I = Info(FAILURE, exc)
state, retval = I.state, I.retval
R = I.handle_error_state(task, eager=eager)
[subtask(errback).apply_async((uuid,)) for errback in task_request.errbacks or []]
def _dispatch_event(self, event):
self.event_count += 1
event = kwdict(event)
group, _, subject = event.pop("type").partition("-")
self.group_handlers[group](subject, event)
if self.event_callback:
self.event_callback(self, event)
def __init__(self, body, on_ack=noop,
hostname=None, eventer=None, app=None,
connection_errors=None, request_dict=None,
delivery_info=None, task=None, on_reject=noop, **opts):
self.app = app
name = self.name = body['task']
self.id = body['id']
self.args = body.get('args', [])
self.kwargs = body.get('kwargs', {})
try:
self.kwargs.items
except AttributeError:
raise InvalidTaskError(
'Task keyword arguments is not a mapping')
if NEEDS_KWDICT:
self.kwargs = kwdict(self.kwargs)
eta = body.get('eta')
expires = body.get('expires')
utc = self.utc = body.get('utc', False)
self.on_ack = on_ack
self.on_reject = on_reject
self.hostname = hostname or socket.gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[name]
self.acknowledged = self._already_revoked = False
self.time_start = self.worker_pid = self._terminate_on_ack = None
self._tzlocal = None
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
if eta is not None:
try:
self.eta = maybe_iso8601(eta)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError(
'invalid eta value {0!r}: {1}'.format(eta, exc))
if utc:
self.eta = maybe_make_aware(self.eta, self.tzlocal)
else:
self.eta = None
if expires is not None:
try:
self.expires = maybe_iso8601(expires)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError(
'invalid expires value {0!r}: {1}'.format(expires, exc))
if utc:
self.expires = maybe_make_aware(self.expires, self.tzlocal)
else:
self.expires = None
delivery_info = {} if delivery_info is None else delivery_info
self.delivery_info = {
'exchange': delivery_info.get('exchange'),
'routing_key': delivery_info.get('routing_key'),
'priority': delivery_info.get('priority'),
'redelivered': delivery_info.get('redelivered'),
}
self.request_dict = body
def _dispatch_event(self, event):
self.event_count += 1
event = kwdict(event)
group, _, subject = event.pop('type').partition('-')
getattr(self, group + '_event')(subject, event)
if self.event_callback:
self.event_callback(self, event)
def from_dict(self, d):
tasks = d["kwargs"]["tasks"]
if d["args"] and tasks:
# partial args passed on to all tasks in the group (Issue #1057).
for task in tasks:
task["args"] = d["args"] + task["args"]
return group(tasks, **kwdict(d["options"]))
def test_kwdict(self):
def f(**kwargs):
return kwargs
kw = {u"foo": "foo",
u"bar": "bar"}
self.assertTrue(f(**utils.kwdict(kw)))
def test_kwdict(self):
def f(**kwargs):
return kwargs
kw = {'foo': 'foo',
'bar': 'bar'}
self.assertTrue(f(**utils.kwdict(kw)))
def test_kwdict(self):
def f(**kwargs):
return kwargs
kw = {u"foo": "foo",
u"bar": "bar"}
self.assertTrue(f(**utils.kwdict(kw)))
def process_task(self, body, message):
fun = body["fun"]
args = body["args"]
kwargs = body["kwargs"]
print ("Got task: %s on %s" % (reprcall(fun.__name__, args, kwargs), message.delivery_info))
try:
fun(*args, **kwdict(kwargs))
except Exception, exc:
print ("task raised exception: %r" % exc)
def process_task(self, body, message):
func = body["func"]
args = body["args"]
kwargs = body["kwargs"]
self.info("Got task: %s", reprcall(func.__name__, args, kwargs))
try:
func(*args, **kwdict(kwargs))
except Exception, exc:
self.error("task raised exception: %r", exc)
def process_task(self, body, message):
func = body["func"]
args = body["args"]
kwargs = body["kwargs"]
self.info("Got task: %s", reprcall(func.__name__, args, kwargs))
try:
func(*args, **kwdict(kwargs))
except Exception, exc:
self.error("task raised exception: %r", exc)
def dispatch(self, method, arguments=None, reply_to=None):
arguments = arguments or {}
handle = reply_to and self.handle_call or self.handle_cast
try:
reply = handle(method, kwdict(arguments))
except SystemExit:
raise
except Exception, exc:
reply = {"error": repr(exc)}
def process_task(self, body, message):
fun = body['fun']
args = body['args']
kwargs = body['kwargs']
self.info('Got task: %s', reprcall(fun.__name__, args, kwargs))
try:
fun(*args, **kwdict(kwargs))
except Exception, exc:
self.error('task raised exception: %r', exc)
def process_task(self, body, message):
fun = body['fun']
args = body['args']
kwargs = body['kwargs']
logger.info('Got task: %s', reprcall(fun.__name__, args, kwargs))
try:
fun(*args, **kwdict(kwargs))
except Exception, exc:
logger.error('task raised exception: %r', exc)
def process_task(self, body, message):
func = body['func']
args = body['args']
kwargs = body['kwargs']
logger.info('Got task: %s', reprcall(func.__name__, args, kwargs))
try:
func(args, **kwdict(kwargs))
except Exception as exc:
logger.error('task raised exception: %r', exc)
message.ack()
def _dispatch_event(self, event, kwdict=kwdict):
self.event_count += 1
event = kwdict(event)
group, _, subject = event['type'].partition('-')
try:
self._get_handler(group)(subject, event)
except KeyError:
pass
if self.event_callback:
self.event_callback(self, event)
def process_task(self, body, message):
fun = body["fun"]
args = body["args"]
kwargs = body["kwargs"]
logger.info("Got task: %s", reprcall(fun.__name__, args, kwargs))
try:
fun(*args, **kwdict(kwargs))
except Exception as exc:
logger.error("task raised exception: %r", exc)
message.ack()
def _dispatch_event(self, event, kwdict=kwdict):
self.event_count += 1
event = kwdict(event)
group, _, subject = event['type'].partition('-')
try:
self._get_handler(group)(subject, event)
except KeyError:
pass
if self.event_callback:
self.event_callback(self, event)
def __init__(self, body, on_ack=noop,
hostname=None, eventer=None, app=None,
connection_errors=None, request_dict=None,
delivery_info=None, task=None, **opts):
self.app = app or app_or_default(app)
name = self.name = body['task']
self.id = body['id']
self.args = body.get('args', [])
self.kwargs = body.get('kwargs', {})
try:
self.kwargs.items
except AttributeError:
raise exceptions.InvalidTaskError(
'Task keyword arguments is not a mapping')
if NEEDS_KWDICT:
self.kwargs = kwdict(self.kwargs)
eta = body.get('eta')
expires = body.get('expires')
utc = self.utc = body.get('utc', False)
self.on_ack = on_ack
self.hostname = hostname or socket.gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[name]
self.acknowledged = self._already_revoked = False
self.time_start = self.worker_pid = self._terminate_on_ack = None
self._tzlocal = None
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
if eta is not None:
self.eta = maybe_iso8601(eta)
if utc:
self.eta = maybe_make_aware(self.eta, self.tzlocal)
else:
self.eta = None
if expires is not None:
self.expires = maybe_iso8601(expires)
if utc:
self.expires = maybe_make_aware(self.expires, self.tzlocal)
else:
self.expires = None
delivery_info = {} if delivery_info is None else delivery_info
self.delivery_info = {
'exchange': delivery_info.get('exchange'),
'routing_key': delivery_info.get('routing_key'),
'priority': delivery_info.get('priority'),
}
# amqplib transport adds the channel here for some reason, so need
# to remove it.
self.delivery_info.pop('channel', None)
self.request_dict = body
def process_task(self, body, message):
fun = body['fun']
args = body['args']
kwargs = body['kwargs']
print('Got task: %s on %s' %
(reprcall(fun.__name__, args, kwargs),
message.delivery_info))
try:
fun(*args, **kwdict(kwargs))
except Exception, exc:
print('task raised exception: %r' % exc)
def worker_event(self, type, fields):
"""Process worker event."""
try:
hostname = fields['hostname']
except KeyError:
pass
else:
worker, created = self.get_or_create_worker(hostname)
handler = getattr(worker, 'on_' + type, None)
if handler:
handler(**(fields if CAN_KWDICT else kwdict(fields)))
return worker, created
def process_task(body, message):
fun = body['fun']
args = body['args']
kwargs = body['kwargs']
#logger.info('Got task: %s', reprcall(fun.__name__, args, kwargs))
logit("Got task : %s" % reprcall(fun.__name__, args, kwargs))
try:
fun(*args, **kwdict(kwargs))
except Exception as exc:
#logger.error('task raised exception: %r', exc)
logit("task raised exception: %r" % exc)
message.ack()
def execute_bare(task, uuid, args, kwargs, request=None, Info=TraceInfo):
R = I = None
kwargs = kwdict(kwargs)
try:
try:
R = retval = task(*args, **kwargs)
state = SUCCESS
except Exception, exc:
I = Info(FAILURE, exc)
state, retval = I.state, I.retval
R = I.handle_error_state(task)
except BaseException, exc:
raise
def __init__(self, body, on_ack=noop,
hostname=None, eventer=None, app=None,
connection_errors=None, request_dict=None,
delivery_info=None, task=None, **opts):
self.app = app or app_or_default(app)
name = self.name = body['task']
self.id = body['id']
self.args = body.get('args', [])
self.kwargs = body.get('kwargs', {})
try:
self.kwargs.items
except AttributeError:
raise exceptions.InvalidTaskError(
'Task keyword arguments is not a mapping')
if NEEDS_KWDICT:
self.kwargs = kwdict(self.kwargs)
eta = body.get('eta')
expires = body.get('expires')
utc = body.get('utc', False)
self.on_ack = on_ack
self.hostname = hostname or socket.gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[name]
self.acknowledged = self._already_revoked = False
self.time_start = self.worker_pid = self._terminate_on_ack = None
self._tzlocal = None
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
if eta is not None:
tz = tz_utc if utc else self.tzlocal
self.eta = tz_to_local(maybe_iso8601(eta), self.tzlocal, tz)
else:
self.eta = None
if expires is not None:
tz = tz_utc if utc else self.tzlocal
self.expires = tz_to_local(maybe_iso8601(expires),
self.tzlocal, tz)
else:
self.expires = None
self.delivery_info = delivery_info or {}
# amqplib transport adds the channel here for some reason, so need
# to remove it. sqs also adds additional unpickleable attributes
# than need to be removed.
for key in ['channel', 'sqs_message', 'sqs_queue']:
self.delivery_info.pop(key, None)
self.request_dict = body
def process_task(self, body, message):
fun = self.taskMap[body['fun']]
args = body['args']
kwargs = body['kwargs']
print 'Got task:', body['fun']#, args, kwargs
try:
result = fun(*args, **kwdict(kwargs))
if result:
print 'Result:'#print fun.__name__,result
for k,v in result.iteritems():
print "\t",k,'||',v
except Exception, exc:
#re-queue the request
#send_as_task(self.connection,body['fun'],args,kwargs,priority='itch')
print "err", 'task raised exception:', exc
def __init__(self,
body,
on_ack=noop,
hostname=None,
eventer=None,
app=None,
connection_errors=None,
request_dict=None,
delivery_info=None,
task=None,
**opts):
self.app = app or app_or_default(app)
name = self.name = body['task']
self.id = body['id']
self.args = body.get('args', [])
self.kwargs = body.get('kwargs', {})
try:
self.kwargs.items
except AttributeError:
raise InvalidTaskError('Task keyword arguments is not a mapping')
if NEEDS_KWDICT:
self.kwargs = kwdict(self.kwargs)
eta = body.get('eta')
expires = body.get('expires')
utc = self.utc = body.get('utc', False)
self.on_ack = on_ack
self.hostname = hostname or socket.gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[name]
self.acknowledged = self._already_revoked = False
self.time_start = self.worker_pid = self._terminate_on_ack = None
self._tzlocal = None
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
if eta is not None:
try:
self.eta = maybe_iso8601(eta)
except (AttributeError, ValueError), exc:
raise InvalidTaskError('invalid eta value %r: %s' % (
eta,
exc,
))
if utc:
self.eta = maybe_make_aware(self.eta, self.tzlocal)
def process_task(self, body, mesage):
fun = body['fun']
args = body['args']
kwargs = body['kwargs']
logger.info(str.format(
'[~] Got task: {}',
reprcall(fun.__name__, args, kwargs)
))
try:
fun(*args, **kwdict(kwargs))
except Exception as e:
logger.error(str.format(
'[#] Task raised exception: {}', e
))
mesage.ack()
def _DISPATCH(self, body, ticket=None):
"""Dispatch message to the appropriate method
in :attr:`state`, handle possible exceptions,
and return a response suitable to be used in a reply.
To protect from calling special methods it does not dispatch
method names starting with underscore (``_``).
This returns the return value or exception error
with defaults fields in a suitable format to be used
as a reply.
The exceptions :exc:`SystemExit` and :exc:`KeyboardInterrupt`
will not be handled, and will propagate.
In the case of a successful call the return value will
be::
{'ok': return_value, **default_fields}
If the method raised an exception the return value
will be::
{'nok': [repr exc, str traceback], **default_fields}
:raises KeyError: if the method specified is unknown
or is a special method (name starting with underscore).
"""
if ticket:
sticket = '%s' % (shortuuid(ticket), )
else:
ticket = sticket = str(self.next_anon_ticket())
try:
method, args = itemgetter('method', 'args')(body)
self.log.info('#%s --> %s', sticket, self._reprcall(method, args))
act = self.lookup_action(method)
r = {'ok': act(**kwdict(args or {}))}
self.log.info('#%s <-- %s', sticket, reprkwargs(r))
except self.Next:
raise
except Exception as exc:
einfo = sys.exc_info()
r = {'nok': [safe_repr(exc), self._get_traceback(einfo)]}
self.log.error('#%s <-- nok=%r', sticket, exc)
return dict(self._default_fields, **r)
def __init__(self, channel, payload, **kwargs):
self._raw = payload
properties = payload['properties']
body = payload.get('body')
if body:
body = channel.decode_body(body, properties.get('body_encoding'))
kwargs.update({
'body': body,
'delivery_tag': properties['delivery_tag'],
'content_type': payload.get('content-type'),
'content_encoding': payload.get('content-encoding'),
'headers': payload.get('headers'),
'properties': properties,
'delivery_info': properties.get('delivery_info'),
'postencode': 'utf-8',
})
super(Message, self).__init__(channel, **kwdict(kwargs))
def trace_task(uuid, args, kwargs, request=None):
R = I = None
kwargs = kwdict(kwargs)
try:
push_task(task)
task_request = Context(request or {},
args=args,
called_directly=False,
kwargs=kwargs)
push_request(task_request)
try:
# -*- PRE -*-
if prerun_receivers:
send_prerun(sender=task,
task_id=uuid,
task=task,
args=args,
kwargs=kwargs)
loader_task_init(uuid, task)
if track_started:
store_result(uuid, {
'pid': pid,
'hostname': hostname
}, STARTED)
# -*- TRACE -*-
try:
R = retval = fun(*args, **kwargs)
state = SUCCESS
except Ignore, exc:
I, R = Info(IGNORED, exc), ExceptionInfo(internal=True)
state, retval = I.state, I.retval
except RetryTaskError, exc:
I = Info(RETRY, exc)
state, retval = I.state, I.retval
R = I.handle_error_state(task, eager=eager)
except Exception, exc:
if propagate:
raise
I = Info(FAILURE, exc)
state, retval = I.state, I.retval
R = I.handle_error_state(task, eager=eager)
[
subtask(errback).apply_async((uuid, ))
for errback in task_request.errbacks or []
]
def __init__(self, channel, payload, **kwargs):
self._raw = payload
properties = payload['properties']
body = payload.get('body')
if body:
body = channel.decode_body(body, properties.get('body_encoding'))
kwargs.update({
'body': body,
'delivery_tag': properties['delivery_tag'],
'content_type': payload.get('content-type'),
'content_encoding': payload.get('content-encoding'),
'headers': payload.get('headers'),
'properties': properties,
'delivery_info': properties.get('delivery_info'),
'postencode': 'utf-8',
})
super(Message, self).__init__(channel, **kwdict(kwargs))
def parse_url(url):
auth = userid = password = None
scheme = urlparse(url).scheme
parts = urlparse(url.replace("%s://" % (scheme, ), "http://"))
netloc = parts.netloc
if '@' in netloc:
auth, _, netloc = partition(parts.netloc, '@')
userid, _, password = partition(auth, ':')
hostname, _, port = partition(netloc, ':')
path = parts.path or ""
if path and path[0] == '/':
path = path[path.index('/') + 1:]
return dict({"hostname": hostname,
"port": port and int(port) or None,
"userid": userid or None,
"password": password or None,
"transport": scheme,
"virtual_host": path or None},
**kwdict(dict(parse_qsl(parts.query))))
def trace_task(uuid, args, kwargs, request=None):
# R - is the possibly prepared return value.
# I - is the Info object.
# retval - is the always unmodified return value.
# state - is the resulting task state.
# This function is very long because we have unrolled all the calls
# for performance reasons, and because the function is so long
# we want the main variables (I, and R) to stand out visually from the
# the rest of the variables, so breaking PEP8 is worth it ;)
R = I = retval = state = None
kwargs = kwdict(kwargs)
try:
push_task(task)
task_request = Context(request or {},
args=args,
called_directly=False,
kwargs=kwargs)
push_request(task_request)
try:
# -*- PRE -*-
if prerun_receivers:
send_prerun(sender=task,
task_id=uuid,
task=task,
args=args,
kwargs=kwargs)
loader_task_init(uuid, task)
if track_started:
store_result(
uuid,
{
'pid': pid,
'hostname': hostname
},
STARTED,
request=task_request,
)
# -*- TRACE -*-
try:
R = retval = fun(*args, **kwargs)
state = SUCCESS
except Reject as exc:
I, R = Info(REJECTED, exc), ExceptionInfo(internal=True)
state, retval = I.state, I.retval
except Ignore as exc:
I, R = Info(IGNORED, exc), ExceptionInfo(internal=True)
state, retval = I.state, I.retval
except Retry as exc:
I, R, state, retval = on_error(
task_request,
exc,
uuid,
RETRY,
call_errbacks=False,
)
except Exception as exc:
I, R, state, retval = on_error(task_request, exc, uuid)
except BaseException as exc:
raise
else:
try:
# callback tasks must be applied before the result is
# stored, so that result.children is populated.
# groups are called inline and will store trail
# separately, so need to call them separately
# so that the trail's not added multiple times :(
# (Issue #1936)
callbacks = task.request.callbacks
if callbacks:
if len(task.request.callbacks) > 1:
sigs, groups = [], []
for sig in callbacks:
sig = signature(sig, app=app)
if isinstance(sig, group):
groups.append(sig)
else:
sigs.append(sig)
for group_ in groups:
group_.apply_async((retval, ))
if sigs:
group(sigs).apply_async((retval, ))
else:
signature(callbacks[0], app=app).delay(retval)
if publish_result:
store_result(
uuid,
retval,
SUCCESS,
request=task_request,
)
except EncodeError as exc:
I, R, state, retval = on_error(task_request, exc, uuid)
else:
if task_on_success:
task_on_success(retval, uuid, args, kwargs)
if success_receivers:
send_success(sender=task, result=retval)
# -* POST *-
if state not in IGNORE_STATES:
if task_request.chord:
on_chord_part_return(task, state, R)
if task_after_return:
task_after_return(
state,
retval,
uuid,
args,
kwargs,
None,
)
finally:
try:
if postrun_receivers:
send_postrun(sender=task,
task_id=uuid,
task=task,
args=args,
kwargs=kwargs,
retval=retval,
state=state)
finally:
pop_task()
pop_request()
if not eager:
try:
backend_cleanup()
loader_cleanup()
except (KeyboardInterrupt, SystemExit, MemoryError):
raise
except Exception as exc:
_logger.error('Process cleanup failed: %r',
exc,
exc_info=True)
except MemoryError:
raise
except Exception as exc:
if eager:
raise
R = report_internal_error(task, exc)
return R, I
def __init__(self,
body,
on_ack=noop,
hostname=None,
eventer=None,
app=None,
connection_errors=None,
request_dict=None,
message=None,
task=None,
on_reject=noop,
**opts):
self.app = app
name = self.name = body['task']
self.id = body['id']
self.args = body.get('args', [])
self.kwargs = body.get('kwargs', {})
try:
self.kwargs.items
except AttributeError:
raise InvalidTaskError('Task keyword arguments is not a mapping')
if NEEDS_KWDICT:
self.kwargs = kwdict(self.kwargs)
eta = body.get('eta')
expires = body.get('expires')
utc = self.utc = body.get('utc', False)
self.on_ack = on_ack
self.on_reject = on_reject
self.hostname = hostname or socket.gethostname()
self.eventer = eventer
self.connection_errors = connection_errors or ()
self.task = task or self.app.tasks[name]
self.acknowledged = self._already_revoked = False
self.time_start = self.worker_pid = self._terminate_on_ack = None
self._apply_result = None
self._tzlocal = None
# timezone means the message is timezone-aware, and the only timezone
# supported at this point is UTC.
if eta is not None:
try:
self.eta = maybe_iso8601(eta)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError('invalid eta value {0!r}: {1}'.format(
eta, exc))
if utc:
self.eta = maybe_make_aware(self.eta, self.tzlocal)
else:
self.eta = None
if expires is not None:
try:
self.expires = maybe_iso8601(expires)
except (AttributeError, ValueError, TypeError) as exc:
raise InvalidTaskError(
'invalid expires value {0!r}: {1}'.format(expires, exc))
if utc:
self.expires = maybe_make_aware(self.expires, self.tzlocal)
else:
self.expires = None
if message:
delivery_info = message.delivery_info or {}
properties = message.properties or {}
body.update({
'headers': message.headers,
'reply_to': properties.get('reply_to'),
'correlation_id': properties.get('correlation_id'),
'delivery_info': {
'exchange':
delivery_info.get('exchange'),
'routing_key':
delivery_info.get('routing_key'),
'priority':
properties.get('priority', delivery_info.get('priority')),
'redelivered':
delivery_info.get('redelivered'),
}
})
else:
body.update(DEFAULT_FIELDS)
self.request_dict = body
def from_dict(self, d):
return group(d["kwargs"]["tasks"], **kwdict(d["options"]))
def from_dict(self, d):
kwargs = d["kwargs"]
return chord(kwargs["header"], kwargs.get("body"),
**kwdict(d["options"]))
def from_dict(self, d):
return chain(*d["kwargs"]["tasks"], **kwdict(d["options"]))
def from_dict(self, d, app=None):
typ = d.get('subtask_type')
if typ:
return self.TYPES[typ].from_dict(kwdict(d), app=app)
return Signature(d, app=app)
def from_dict(self, d, app=None):
tasks = d['kwargs']['tasks']
if d['args'] and tasks:
# partial args passed on to first task in chain (Issue #1057).
tasks[0]['args'] = tasks[0]._merge(d['args'])[0]
return chain(*d['kwargs']['tasks'], app=app, **kwdict(d['options']))
def from_dict(self, d, app=None):
args, d['kwargs'] = self._unpack_args(**kwdict(d['kwargs']))
return self(*args, app=app, **kwdict(d))
def trace_task(uuid, args, kwargs, request=None):
R = I = None
kwargs = kwdict(kwargs)
try:
push_task(task)
task_request = Context(request or {},
args=args,
called_directly=False,
kwargs=kwargs)
push_request(task_request)
try:
# -*- PRE -*-
if prerun_receivers:
send_prerun(sender=task,
task_id=uuid,
task=task,
args=args,
kwargs=kwargs)
loader_task_init(uuid, task)
if track_started:
store_result(uuid, {
'pid': pid,
'hostname': hostname
}, STARTED)
# -*- TRACE -*-
try:
R = retval = fun(*args, **kwargs)
state = SUCCESS
except Ignore as exc:
I, R = Info(IGNORED, exc), ExceptionInfo(internal=True)
except RetryTaskError as exc:
I = Info(RETRY, exc)
state, retval = I.state, I.retval
R = I.handle_error_state(task, eager=eager)
except Exception as exc:
if propagate:
raise
I = Info(FAILURE, exc)
state, retval = I.state, I.retval
R = I.handle_error_state(task, eager=eager)
[
subtask(errback).apply_async((uuid, ))
for errback in task_request.errbacks or []
]
except BaseException as exc:
raise
else:
# callback tasks must be applied before the result is
# stored, so that result.children is populated.
[
subtask(callback).apply_async((retval, ))
for callback in task_request.callbacks or []
]
if publish_result:
store_result(uuid, retval, SUCCESS)
if task_on_success:
task_on_success(retval, uuid, args, kwargs)
if success_receivers:
send_success(sender=task, result=retval)
# -* POST *-
if task_request.chord:
on_chord_part_return(task)
if task_after_return:
task_after_return(state, retval, uuid, args, kwargs, None)
if postrun_receivers:
send_postrun(sender=task,
task_id=uuid,
task=task,
args=args,
kwargs=kwargs,
retval=retval,
state=state)
finally:
pop_task()
pop_request()
if not eager:
try:
backend_cleanup()
loader_cleanup()
except (KeyboardInterrupt, SystemExit, MemoryError):
raise
except Exception as exc:
_logger.error('Process cleanup failed: %r',
exc,
exc_info=True)
except Exception as exc:
if eager:
raise
R = report_internal_error(task, exc)
return R, I