def test_unregister(self):
with self.assertRaises(SerializerNotInstalled):
unregister('nonexisting')
dumps('foo', serializer='pickle')
unregister('pickle')
with self.assertRaises(SerializerNotInstalled):
dumps('foo', serializer='pickle')
register_pickle()
def test_content_type_encoding(self):
# Using the 'raw' serializer
assert (dumps(unicode_string, serializer='raw')[-1] ==
unicode_string_as_utf8)
assert (dumps(latin_string, serializer='raw')[-1] ==
latin_string_as_utf8)
# And again w/o a specific serializer to check the
# code where we force unicode objects into a string.
assert dumps(unicode_string)[-1] == unicode_string_as_utf8
assert dumps(latin_string)[-1] == latin_string_as_utf8
def serialize(self, data):
content_type, content_encoding, body = dumps(
data, serializer=self._serializer)
if content_encoding != self._content_encoding:
raise ValueError("Content encoding of inner serializer {!r} must "
"match ({!r} != {!r})"
.format(self._serializer, content_encoding,
self._content_encoding))
body = self._ensure_bytes(body)
if len(body) > 0 and body[0] in self.whitespace:
raise ValueError("Inner data may not begin with the following "
"characters {!r}"
.format(str(self.whitespace)))
message = self._signing_key.sign(body)
signature = self.key_codec.encode(message.signature).decode('ascii')
header = {
'signature': signature,
'signer': self._signer,
'content_type': content_type,
'content_encoding': content_encoding,
}
buffer = io.BytesIO()
wrapper = io.TextIOWrapper(buffer, self._content_encoding,
write_through=True)
with wrapper:
json.dump(header, wrapper)
buffer.write(b"\n")
buffer.write(message.message)
return buffer.getvalue()
def test_content_type_encoding(self):
# Using the 'raw' serializer
self.assertEqual(
unicode_string_as_utf8,
dumps(unicode_string, serializer='raw')[-1],
)
self.assertEqual(
latin_string_as_utf8,
dumps(latin_string, serializer='raw')[-1],
)
# And again w/o a specific serializer to check the
# code where we force unicode objects into a string.
self.assertEqual(
unicode_string_as_utf8,
dumps(unicode_string)[-1],
)
self.assertEqual(
latin_string_as_utf8,
dumps(latin_string)[-1],
)
def TaskMessage(name, id=None, args=(), kwargs={}, callbacks=None, errbacks=None, chain=None, **options):
from celery import uuid
from kombu.serialization import dumps
id = id or uuid()
message = Mock(name="TaskMessage-{0}".format(id))
message.headers = {"id": id, "task": name}
embed = {"callbacks": callbacks, "errbacks": errbacks, "chain": chain}
message.headers.update(options)
message.content_type, message.content_encoding, message.body = dumps((args, kwargs, embed), serializer="json")
message.payload = (args, kwargs, embed)
return message
def test_json_dumps(self):
a = loads(
dumps(py_data, serializer='json')[-1],
content_type='application/json',
content_encoding='utf-8',
)
b = loads(
json_data,
content_type='application/json',
content_encoding='utf-8',
)
assert a == b
def test_json_dumps(self):
self.assertEqual(
loads(
dumps(py_data, serializer='json')[-1],
content_type='application/json',
content_encoding='utf-8',
),
loads(
json_data,
content_type='application/json',
content_encoding='utf-8',
),
)
def test_yaml_dumps(self):
register_yaml()
a = loads(
dumps(py_data, serializer='yaml')[-1],
content_type='application/x-yaml',
content_encoding='utf-8',
)
b = loads(
yaml_data,
content_type='application/x-yaml',
content_encoding='utf-8',
)
assert a == b
def test_msgpack_dumps(self):
register_msgpack()
a = loads(
dumps(msgpack_py_data, serializer='msgpack')[-1],
content_type='application/x-msgpack',
content_encoding='binary',
)
b = loads(
msgpack_data,
content_type='application/x-msgpack',
content_encoding='binary',
)
assert a == b
def TaskMessage(name, id=None, args=(), kwargs={}, **options):
from celery import uuid
from kombu.serialization import dumps
id = id or uuid()
message = Mock(name='TaskMessage-{0}'.format(id))
message.headers = {
'id': id,
'task': name,
}
message.headers.update(options)
message.content_type, message.content_encoding, message.body = dumps(
(args, kwargs), serializer='json',
)
message.payload = (args, kwargs)
return message
def _serialize_args_and_kwargs_for_eager_mode(
self, args=None, kwargs=None, **options):
producer = options.get('producer')
with app.producer_or_acquire(producer) as eager_producer:
serializer = options.get(
'serializer', eager_producer.serializer
)
body = args, kwargs
content_type, content_encoding, data = serialization.dumps(
body, serializer
)
args, kwargs = serialization.loads(
data, content_type, content_encoding
)
return args, kwargs
def serialize(self, data):
"""serialize data structure into string"""
assert self._key is not None
assert self._cert is not None
with reraise_errors('Unable to serialize: {0!r}', (Exception,)):
content_type, content_encoding, body = dumps(
bytes_to_str(data), serializer=self._serializer)
# What we sign is the serialized body, not the body itself.
# this way the receiver doesn't have to decode the contents
# to verify the signature (and thus avoiding potential flaws
# in the decoding step).
body = ensure_bytes(body)
return self._pack(body, content_type, content_encoding,
signature=self._key.sign(body, self._digest),
signer=self._cert.get_id())
def __init__(self, exc, serializer=DEFAULT_EXC_SERIALIZER):
"""
:param exc: Exception instance or RemoteException.args
:type exc: BaseException subclass, list or tuple
:param serializer: CELERY_RESULT_SERIALIZER for celery_rpc app
:type serializer: str
"""
if isinstance(exc, BaseException):
cls = exc.__class__
exc_args = exc.args
args = (cls.__module__, cls.__name__, exc_args)
args = [dumps(args, serializer=serializer)[2]]
elif isinstance(exc, (list, tuple)):
args = exc
else:
raise ValueError("Need a BaseException object")
super(RemoteException, self).__init__(*args)
def TaskMessage(name, id=None, args=(), kwargs={}, callbacks=None,
errbacks=None, chain=None, shadow=None, utc=None, **options):
from celery import uuid
from kombu.serialization import dumps
id = id or uuid()
message = Mock(name='TaskMessage-{0}'.format(id))
message.headers = {
'id': id,
'task': name,
'shadow': shadow,
}
embed = {'callbacks': callbacks, 'errbacks': errbacks, 'chain': chain}
message.headers.update(options)
message.content_type, message.content_encoding, message.body = dumps(
(args, kwargs, embed), serializer='json',
)
message.payload = (args, kwargs, embed)
return message
def TaskMessage1(name, id=None, args=(), kwargs={}, callbacks=None, errbacks=None, chain=None, **options):
from celery import uuid
from kombu.serialization import dumps
id = id or uuid()
message = Mock(name="TaskMessage-{0}".format(id))
message.headers = {}
message.payload = {
"task": name,
"id": id,
"args": args,
"kwargs": kwargs,
"callbacks": callbacks,
"errbacks": errbacks,
}
message.payload.update(options)
message.content_type, message.content_encoding, message.body = dumps(message.payload)
return message
def TaskMessage1(name, id=None, args=(), kwargs={}, callbacks=None,
errbacks=None, chain=None, **options):
from celery import uuid
from kombu.serialization import dumps
id = id or uuid()
message = Mock(name='TaskMessage-{0}'.format(id))
message.headers = {}
message.payload = {
'task': name,
'id': id,
'args': args,
'kwargs': kwargs,
'callbacks': callbacks,
'errbacks': errbacks,
}
message.payload.update(options)
message.content_type, message.content_encoding, message.body = dumps(
message.payload,
)
return message
def TaskMessage(name, id=None, args=(), kwargs={}, callbacks=None,
errbacks=None, chain=None, shadow=None, utc=None, **options):
# type: (str, str, Sequence, Mapping, Sequence[Signature],
# Sequence[Signature], Sequence[Signature],
# str, bool, **Any) -> Any
"""Create task message in protocol 2 format."""
from celery import uuid
from kombu.serialization import dumps
id = id or uuid()
message = Mock(name='TaskMessage-{0}'.format(id))
message.headers = {
'id': id,
'task': name,
'shadow': shadow,
}
embed = {'callbacks': callbacks, 'errbacks': errbacks, 'chain': chain}
message.headers.update(options)
message.content_type, message.content_encoding, message.body = dumps(
(args, kwargs, embed), serializer='json',
)
message.payload = (args, kwargs, embed)
return message
def TaskMessage1(name, id=None, args=(), kwargs={}, callbacks=None,
errbacks=None, chain=None, **options):
# type: (str, str, Sequence, Mapping, Sequence[Signature],
# Sequence[Signature], Sequence[Signature]) -> Any
"""Create task message in protocol 1 format."""
from celery import uuid
from kombu.serialization import dumps
id = id or uuid()
message = Mock(name='TaskMessage-{0}'.format(id))
message.headers = {}
message.payload = {
'task': name,
'id': id,
'args': args,
'kwargs': kwargs,
'callbacks': callbacks,
'errbacks': errbacks,
}
message.payload.update(options)
message.content_type, message.content_encoding, message.body = dumps(
message.payload,
)
return message
def _encode(self, data):
return dumps(data, serializer=self.serializer)
def test_dumps__no_serializer(self):
ctyp, cenc, data = dumps(str_to_bytes('foo'))
self.assertEqual(ctyp, 'application/data')
self.assertEqual(cenc, 'binary')
def test_dumps_missing(self):
with self.assertRaises(SerializerNotInstalled):
dumps('foo', serializer='nonexisting')
def apply_async(self, args=None, kwargs=None, task_id=None, producer=None,
link=None, link_error=None, shadow=None, **options):
"""Apply tasks asynchronously by sending a message.
Arguments:
args (Tuple): The positional arguments to pass on to the task.
kwargs (Dict): The keyword arguments to pass on to the task.
countdown (float): Number of seconds into the future that the
task should execute. Defaults to immediate execution.
eta (~datetime.datetime): Absolute time and date of when the task
should be executed. May not be specified if `countdown`
is also supplied.
expires (float, ~datetime.datetime): Datetime or
seconds in the future for the task should expire.
The task won't be executed after the expiration time.
shadow (str): Override task name used in logs/monitoring.
Default is retrieved from :meth:`shadow_name`.
connection (kombu.Connection): Re-use existing broker connection
instead of acquiring one from the connection pool.
retry (bool): If enabled sending of the task message will be
retried in the event of connection loss or failure.
Default is taken from the :setting:`task_publish_retry`
setting. Note that you need to handle the
producer/connection manually for this to work.
retry_policy (Mapping): Override the retry policy used.
See the :setting:`task_publish_retry_policy` setting.
queue (str, kombu.Queue): The queue to route the task to.
This must be a key present in :setting:`task_queues`, or
:setting:`task_create_missing_queues` must be
enabled. See :ref:`guide-routing` for more
information.
exchange (str, kombu.Exchange): Named custom exchange to send the
task to. Usually not used in combination with the ``queue``
argument.
routing_key (str): Custom routing key used to route the task to a
worker server. If in combination with a ``queue`` argument
only used to specify custom routing keys to topic exchanges.
priority (int): The task priority, a number between 0 and 9.
Defaults to the :attr:`priority` attribute.
serializer (str): Serialization method to use.
Can be `pickle`, `json`, `yaml`, `msgpack` or any custom
serialization method that's been registered
with :mod:`kombu.serialization.registry`.
Defaults to the :attr:`serializer` attribute.
compression (str): Optional compression method
to use. Can be one of ``zlib``, ``bzip2``,
or any custom compression methods registered with
:func:`kombu.compression.register`.
Defaults to the :setting:`task_compression` setting.
link (Signature): A single, or a list of tasks signatures
to apply if the task returns successfully.
link_error (Signature): A single, or a list of task signatures
to apply if an error occurs while executing the task.
producer (kombu.Producer): custom producer to use when publishing
the task.
add_to_parent (bool): If set to True (default) and the task
is applied while executing another task, then the result
will be appended to the parent tasks ``request.children``
attribute. Trailing can also be disabled by default using the
:attr:`trail` attribute
publisher (kombu.Producer): Deprecated alias to ``producer``.
headers (Dict): Message headers to be included in the message.
Returns:
celery.result.AsyncResult: Promise of future evaluation.
Raises:
TypeError: If not enough arguments are passed, or too many
arguments are passed. Note that signature checks may
be disabled by specifying ``@task(typing=False)``.
kombu.exceptions.OperationalError: If a connection to the
transport cannot be made, or if the connection is lost.
Note:
Also supports all keyword arguments supported by
:meth:`kombu.Producer.publish`.
"""
if self.typing:
try:
check_arguments = self.__header__
except AttributeError: # pragma: no cover
pass
else:
check_arguments(*(args or ()), **(kwargs or {}))
app = self._get_app()
if app.conf.task_always_eager:
with app.producer_or_acquire(producer) as eager_producer:
serializer = options.get(
'serializer', eager_producer.serializer
)
body = args, kwargs
content_type, content_encoding, data = serialization.dumps(
body, serializer
)
args, kwargs = serialization.loads(
data, content_type, content_encoding
)
with denied_join_result():
return self.apply(args, kwargs, task_id=task_id or uuid(),
link=link, link_error=link_error, **options)
if self.__v2_compat__:
shadow = shadow or self.shadow_name(self(), args, kwargs, options)
else:
shadow = shadow or self.shadow_name(args, kwargs, options)
preopts = self._get_exec_options()
options = dict(preopts, **options) if options else preopts
options.setdefault('ignore_result', self.ignore_result)
return app.send_task(
self.name, args, kwargs, task_id=task_id, producer=producer,
link=link, link_error=link_error, result_cls=self.AsyncResult,
shadow=shadow, task_type=self,
**options
)
def _encode(self, data):
return dumps(data, serializer=self.serializer)
def test_reraises_EncodeError(self):
with self.assertRaises(EncodeError):
dumps([object()], serializer='json')
def serialize(self, data):
content_type, content_encoding, body = dumps(
bytes_to_str(data), serializer=self._serializer)
return b64encode(self.encrypt(ensure_bytes(body)))
def _future_encode(self, data):
_encode = getattr(self, '_encode', None)
if _encode is None:
return dumps(data, serializer=self.serializer)
return _encode(data)
def test_dumps__no_serializer(self):
ctyp, cenc, data = dumps(str_to_bytes('foo'))
self.assertEqual(ctyp, 'application/data')
self.assertEqual(cenc, 'binary')
def test_dumps_missing(self):
with self.assertRaises(SerializerNotInstalled):
dumps('foo', serializer='nonexisting')
def test_utf8(self):
b = BibRecord(bibcode=u'\u01b5')
ctype, enc, data = serialization.dumps(b)
o = serialization.loads(data, 'application/x-adsmsg', 'utf-8')
self.assertTrue(isinstance(o, BibRecord))
self.assertEqual(o.bibcode, u'\u01b5')
def test_pickle_dumps(self):
self.assertEqual(
pickle.loads(pickle_data),
pickle.loads(dumps(py_data, serializer='pickle')[-1]),
)
def apply_async(self,
args=None,
kwargs=None,
task_id=None,
producer=None,
link=None,
link_error=None,
shadow=None,
**options):
"""Apply tasks asynchronously by sending a message.
Arguments:
args (Tuple): The positional arguments to pass on to the task.
kwargs (Dict): The keyword arguments to pass on to the task.
countdown (float): Number of seconds into the future that the
task should execute. Defaults to immediate execution.
eta (~datetime.datetime): Absolute time and date of when the task
should be executed. May not be specified if `countdown`
is also supplied.
expires (float, ~datetime.datetime): Datetime or
seconds in the future for the task should expire.
The task won't be executed after the expiration time.
shadow (str): Override task name used in logs/monitoring.
Default is retrieved from :meth:`shadow_name`.
connection (kombu.Connection): Re-use existing broker connection
instead of acquiring one from the connection pool.
retry (bool): If enabled sending of the task message will be
retried in the event of connection loss or failure.
Default is taken from the :setting:`task_publish_retry`
setting. Note that you need to handle the
producer/connection manually for this to work.
retry_policy (Mapping): Override the retry policy used.
See the :setting:`task_publish_retry_policy` setting.
queue (str, kombu.Queue): The queue to route the task to.
This must be a key present in :setting:`task_queues`, or
:setting:`task_create_missing_queues` must be
enabled. See :ref:`guide-routing` for more
information.
exchange (str, kombu.Exchange): Named custom exchange to send the
task to. Usually not used in combination with the ``queue``
argument.
routing_key (str): Custom routing key used to route the task to a
worker server. If in combination with a ``queue`` argument
only used to specify custom routing keys to topic exchanges.
priority (int): The task priority, a number between 0 and 9.
Defaults to the :attr:`priority` attribute.
serializer (str): Serialization method to use.
Can be `pickle`, `json`, `yaml`, `msgpack` or any custom
serialization method that's been registered
with :mod:`kombu.serialization.registry`.
Defaults to the :attr:`serializer` attribute.
compression (str): Optional compression method
to use. Can be one of ``zlib``, ``bzip2``,
or any custom compression methods registered with
:func:`kombu.compression.register`.
Defaults to the :setting:`task_compression` setting.
link (Signature): A single, or a list of tasks signatures
to apply if the task returns successfully.
link_error (Signature): A single, or a list of task signatures
to apply if an error occurs while executing the task.
producer (kombu.Producer): custom producer to use when publishing
the task.
add_to_parent (bool): If set to True (default) and the task
is applied while executing another task, then the result
will be appended to the parent tasks ``request.children``
attribute. Trailing can also be disabled by default using the
:attr:`trail` attribute
publisher (kombu.Producer): Deprecated alias to ``producer``.
headers (Dict): Message headers to be included in the message.
Returns:
celery.result.AsyncResult: Promise of future evaluation.
Raises:
TypeError: If not enough arguments are passed, or too many
arguments are passed. Note that signature checks may
be disabled by specifying ``@task(typing=False)``.
kombu.exceptions.OperationalError: If a connection to the
transport cannot be made, or if the connection is lost.
Note:
Also supports all keyword arguments supported by
:meth:`kombu.Producer.publish`.
"""
if self.typing:
try:
check_arguments = self.__header__
except AttributeError: # pragma: no cover
pass
else:
check_arguments(*(args or ()), **(kwargs or {}))
if self.__v2_compat__:
shadow = shadow or self.shadow_name(self(), args, kwargs, options)
else:
shadow = shadow or self.shadow_name(args, kwargs, options)
preopts = self._get_exec_options()
options = dict(preopts, **options) if options else preopts
options.setdefault('ignore_result', self.ignore_result)
if self.priority:
options.setdefault('priority', self.priority)
app = self._get_app()
if app.conf.task_always_eager:
with app.producer_or_acquire(producer) as eager_producer:
serializer = options.get('serializer')
if serializer is None:
if eager_producer.serializer:
serializer = eager_producer.serializer
else:
serializer = app.conf.task_serializer
body = args, kwargs
content_type, content_encoding, data = serialization.dumps(
body,
serializer,
)
args, kwargs = serialization.loads(data,
content_type,
content_encoding,
accept=[content_type])
with denied_join_result():
return self.apply(args,
kwargs,
task_id=task_id or uuid(),
link=link,
link_error=link_error,
**options)
else:
return app.send_task(self.name,
args,
kwargs,
task_id=task_id,
producer=producer,
link=link,
link_error=link_error,
result_cls=self.AsyncResult,
shadow=shadow,
task_type=self,
**options)
def encode(self, data):
_, _, payload = dumps(data, serializer=self.serializer)
return payload
def test_reraises_EncodeError(self):
with pytest.raises(EncodeError):
dumps([object()], serializer='json')
def testContentType(self):
""" Encode with correct content-type.
"""
serialized = serialization.dumps(None, 'x-rpc-json')
self.assertEqual('application/json+celery-rpc:v1', serialized[0])
def test_dumps__no_serializer(self):
ctyp, cenc, data = dumps(str_to_bytes('foo'))
assert ctyp == 'application/data'
assert cenc == 'binary'
def test_pickle_dumps(self):
a = pickle.loads(pickle_data),
b = pickle.loads(dumps(py_data, serializer='pickle')[-1]),
assert a == b
def test_dumps__no_serializer(self):
ctyp, cenc, data = dumps(str_to_bytes('foo'))
assert ctyp == 'application/data'
assert cenc == 'binary'
def test_pickle_dumps(self):
a = pickle.loads(pickle_data),
b = pickle.loads(dumps(py_data, serializer='pickle')[-1]),
assert a == b
def test_pickle_dumps(self):
self.assertEqual(
pickle.loads(pickle_data),
pickle.loads(dumps(py_data, serializer='pickle')[-1]),
)
def encode(self, data):
_, _, payload = dumps(data, serializer=self.serializer)
return payload
def serialize(self, data):
content_type, content_encoding, body = dumps(
bytes_to_str(data), serializer=self._serializer)
return b64encode(self.encrypt(ensure_bytes(body)))
