def test_unregister(self): with self.assertRaises(SerializerNotInstalled): unregister('nonexisting') dumps('foo', serializer='pickle') unregister('pickle') with self.assertRaises(SerializerNotInstalled): dumps('foo', serializer='pickle') register_pickle()
def test_content_type_encoding(self): # Using the 'raw' serializer assert (dumps(unicode_string, serializer='raw')[-1] == unicode_string_as_utf8) assert (dumps(latin_string, serializer='raw')[-1] == latin_string_as_utf8) # And again w/o a specific serializer to check the # code where we force unicode objects into a string. assert dumps(unicode_string)[-1] == unicode_string_as_utf8 assert dumps(latin_string)[-1] == latin_string_as_utf8
def serialize(self, data): content_type, content_encoding, body = dumps( data, serializer=self._serializer) if content_encoding != self._content_encoding: raise ValueError("Content encoding of inner serializer {!r} must " "match ({!r} != {!r})" .format(self._serializer, content_encoding, self._content_encoding)) body = self._ensure_bytes(body) if len(body) > 0 and body[0] in self.whitespace: raise ValueError("Inner data may not begin with the following " "characters {!r}" .format(str(self.whitespace))) message = self._signing_key.sign(body) signature = self.key_codec.encode(message.signature).decode('ascii') header = { 'signature': signature, 'signer': self._signer, 'content_type': content_type, 'content_encoding': content_encoding, } buffer = io.BytesIO() wrapper = io.TextIOWrapper(buffer, self._content_encoding, write_through=True) with wrapper: json.dump(header, wrapper) buffer.write(b"\n") buffer.write(message.message) return buffer.getvalue()
def test_content_type_encoding(self): # Using the 'raw' serializer self.assertEqual( unicode_string_as_utf8, dumps(unicode_string, serializer='raw')[-1], ) self.assertEqual( latin_string_as_utf8, dumps(latin_string, serializer='raw')[-1], ) # And again w/o a specific serializer to check the # code where we force unicode objects into a string. self.assertEqual( unicode_string_as_utf8, dumps(unicode_string)[-1], ) self.assertEqual( latin_string_as_utf8, dumps(latin_string)[-1], )
def TaskMessage(name, id=None, args=(), kwargs={}, callbacks=None, errbacks=None, chain=None, **options): from celery import uuid from kombu.serialization import dumps id = id or uuid() message = Mock(name="TaskMessage-{0}".format(id)) message.headers = {"id": id, "task": name} embed = {"callbacks": callbacks, "errbacks": errbacks, "chain": chain} message.headers.update(options) message.content_type, message.content_encoding, message.body = dumps((args, kwargs, embed), serializer="json") message.payload = (args, kwargs, embed) return message
def test_json_dumps(self): a = loads( dumps(py_data, serializer='json')[-1], content_type='application/json', content_encoding='utf-8', ) b = loads( json_data, content_type='application/json', content_encoding='utf-8', ) assert a == b
def test_json_dumps(self): self.assertEqual( loads( dumps(py_data, serializer='json')[-1], content_type='application/json', content_encoding='utf-8', ), loads( json_data, content_type='application/json', content_encoding='utf-8', ), )
def test_yaml_dumps(self): register_yaml() a = loads( dumps(py_data, serializer='yaml')[-1], content_type='application/x-yaml', content_encoding='utf-8', ) b = loads( yaml_data, content_type='application/x-yaml', content_encoding='utf-8', ) assert a == b
def test_msgpack_dumps(self): register_msgpack() a = loads( dumps(msgpack_py_data, serializer='msgpack')[-1], content_type='application/x-msgpack', content_encoding='binary', ) b = loads( msgpack_data, content_type='application/x-msgpack', content_encoding='binary', ) assert a == b
def TaskMessage(name, id=None, args=(), kwargs={}, **options): from celery import uuid from kombu.serialization import dumps id = id or uuid() message = Mock(name='TaskMessage-{0}'.format(id)) message.headers = { 'id': id, 'task': name, } message.headers.update(options) message.content_type, message.content_encoding, message.body = dumps( (args, kwargs), serializer='json', ) message.payload = (args, kwargs) return message
def _serialize_args_and_kwargs_for_eager_mode( self, args=None, kwargs=None, **options): producer = options.get('producer') with app.producer_or_acquire(producer) as eager_producer: serializer = options.get( 'serializer', eager_producer.serializer ) body = args, kwargs content_type, content_encoding, data = serialization.dumps( body, serializer ) args, kwargs = serialization.loads( data, content_type, content_encoding ) return args, kwargs
def serialize(self, data): """serialize data structure into string""" assert self._key is not None assert self._cert is not None with reraise_errors('Unable to serialize: {0!r}', (Exception,)): content_type, content_encoding, body = dumps( bytes_to_str(data), serializer=self._serializer) # What we sign is the serialized body, not the body itself. # this way the receiver doesn't have to decode the contents # to verify the signature (and thus avoiding potential flaws # in the decoding step). body = ensure_bytes(body) return self._pack(body, content_type, content_encoding, signature=self._key.sign(body, self._digest), signer=self._cert.get_id())
def __init__(self, exc, serializer=DEFAULT_EXC_SERIALIZER): """ :param exc: Exception instance or RemoteException.args :type exc: BaseException subclass, list or tuple :param serializer: CELERY_RESULT_SERIALIZER for celery_rpc app :type serializer: str """ if isinstance(exc, BaseException): cls = exc.__class__ exc_args = exc.args args = (cls.__module__, cls.__name__, exc_args) args = [dumps(args, serializer=serializer)[2]] elif isinstance(exc, (list, tuple)): args = exc else: raise ValueError("Need a BaseException object") super(RemoteException, self).__init__(*args)
def TaskMessage(name, id=None, args=(), kwargs={}, callbacks=None, errbacks=None, chain=None, shadow=None, utc=None, **options): from celery import uuid from kombu.serialization import dumps id = id or uuid() message = Mock(name='TaskMessage-{0}'.format(id)) message.headers = { 'id': id, 'task': name, 'shadow': shadow, } embed = {'callbacks': callbacks, 'errbacks': errbacks, 'chain': chain} message.headers.update(options) message.content_type, message.content_encoding, message.body = dumps( (args, kwargs, embed), serializer='json', ) message.payload = (args, kwargs, embed) return message
def TaskMessage1(name, id=None, args=(), kwargs={}, callbacks=None, errbacks=None, chain=None, **options): from celery import uuid from kombu.serialization import dumps id = id or uuid() message = Mock(name="TaskMessage-{0}".format(id)) message.headers = {} message.payload = { "task": name, "id": id, "args": args, "kwargs": kwargs, "callbacks": callbacks, "errbacks": errbacks, } message.payload.update(options) message.content_type, message.content_encoding, message.body = dumps(message.payload) return message
def TaskMessage1(name, id=None, args=(), kwargs={}, callbacks=None, errbacks=None, chain=None, **options): from celery import uuid from kombu.serialization import dumps id = id or uuid() message = Mock(name='TaskMessage-{0}'.format(id)) message.headers = {} message.payload = { 'task': name, 'id': id, 'args': args, 'kwargs': kwargs, 'callbacks': callbacks, 'errbacks': errbacks, } message.payload.update(options) message.content_type, message.content_encoding, message.body = dumps( message.payload, ) return message
def TaskMessage(name, id=None, args=(), kwargs={}, callbacks=None, errbacks=None, chain=None, shadow=None, utc=None, **options): # type: (str, str, Sequence, Mapping, Sequence[Signature], # Sequence[Signature], Sequence[Signature], # str, bool, **Any) -> Any """Create task message in protocol 2 format.""" from celery import uuid from kombu.serialization import dumps id = id or uuid() message = Mock(name='TaskMessage-{0}'.format(id)) message.headers = { 'id': id, 'task': name, 'shadow': shadow, } embed = {'callbacks': callbacks, 'errbacks': errbacks, 'chain': chain} message.headers.update(options) message.content_type, message.content_encoding, message.body = dumps( (args, kwargs, embed), serializer='json', ) message.payload = (args, kwargs, embed) return message
def TaskMessage1(name, id=None, args=(), kwargs={}, callbacks=None, errbacks=None, chain=None, **options): # type: (str, str, Sequence, Mapping, Sequence[Signature], # Sequence[Signature], Sequence[Signature]) -> Any """Create task message in protocol 1 format.""" from celery import uuid from kombu.serialization import dumps id = id or uuid() message = Mock(name='TaskMessage-{0}'.format(id)) message.headers = {} message.payload = { 'task': name, 'id': id, 'args': args, 'kwargs': kwargs, 'callbacks': callbacks, 'errbacks': errbacks, } message.payload.update(options) message.content_type, message.content_encoding, message.body = dumps( message.payload, ) return message
def _encode(self, data): return dumps(data, serializer=self.serializer)
def test_dumps__no_serializer(self): ctyp, cenc, data = dumps(str_to_bytes('foo')) self.assertEqual(ctyp, 'application/data') self.assertEqual(cenc, 'binary')
def test_dumps_missing(self): with self.assertRaises(SerializerNotInstalled): dumps('foo', serializer='nonexisting')
def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, link=None, link_error=None, shadow=None, **options): """Apply tasks asynchronously by sending a message. Arguments: args (Tuple): The positional arguments to pass on to the task. kwargs (Dict): The keyword arguments to pass on to the task. countdown (float): Number of seconds into the future that the task should execute. Defaults to immediate execution. eta (~datetime.datetime): Absolute time and date of when the task should be executed. May not be specified if `countdown` is also supplied. expires (float, ~datetime.datetime): Datetime or seconds in the future for the task should expire. The task won't be executed after the expiration time. shadow (str): Override task name used in logs/monitoring. Default is retrieved from :meth:`shadow_name`. connection (kombu.Connection): Re-use existing broker connection instead of acquiring one from the connection pool. retry (bool): If enabled sending of the task message will be retried in the event of connection loss or failure. Default is taken from the :setting:`task_publish_retry` setting. Note that you need to handle the producer/connection manually for this to work. retry_policy (Mapping): Override the retry policy used. See the :setting:`task_publish_retry_policy` setting. queue (str, kombu.Queue): The queue to route the task to. This must be a key present in :setting:`task_queues`, or :setting:`task_create_missing_queues` must be enabled. See :ref:`guide-routing` for more information. exchange (str, kombu.Exchange): Named custom exchange to send the task to. Usually not used in combination with the ``queue`` argument. routing_key (str): Custom routing key used to route the task to a worker server. If in combination with a ``queue`` argument only used to specify custom routing keys to topic exchanges. priority (int): The task priority, a number between 0 and 9. Defaults to the :attr:`priority` attribute. serializer (str): Serialization method to use. Can be `pickle`, `json`, `yaml`, `msgpack` or any custom serialization method that's been registered with :mod:`kombu.serialization.registry`. Defaults to the :attr:`serializer` attribute. compression (str): Optional compression method to use. Can be one of ``zlib``, ``bzip2``, or any custom compression methods registered with :func:`kombu.compression.register`. Defaults to the :setting:`task_compression` setting. link (Signature): A single, or a list of tasks signatures to apply if the task returns successfully. link_error (Signature): A single, or a list of task signatures to apply if an error occurs while executing the task. producer (kombu.Producer): custom producer to use when publishing the task. add_to_parent (bool): If set to True (default) and the task is applied while executing another task, then the result will be appended to the parent tasks ``request.children`` attribute. Trailing can also be disabled by default using the :attr:`trail` attribute publisher (kombu.Producer): Deprecated alias to ``producer``. headers (Dict): Message headers to be included in the message. Returns: celery.result.AsyncResult: Promise of future evaluation. Raises: TypeError: If not enough arguments are passed, or too many arguments are passed. Note that signature checks may be disabled by specifying ``@task(typing=False)``. kombu.exceptions.OperationalError: If a connection to the transport cannot be made, or if the connection is lost. Note: Also supports all keyword arguments supported by :meth:`kombu.Producer.publish`. """ if self.typing: try: check_arguments = self.__header__ except AttributeError: # pragma: no cover pass else: check_arguments(*(args or ()), **(kwargs or {})) app = self._get_app() if app.conf.task_always_eager: with app.producer_or_acquire(producer) as eager_producer: serializer = options.get( 'serializer', eager_producer.serializer ) body = args, kwargs content_type, content_encoding, data = serialization.dumps( body, serializer ) args, kwargs = serialization.loads( data, content_type, content_encoding ) with denied_join_result(): return self.apply(args, kwargs, task_id=task_id or uuid(), link=link, link_error=link_error, **options) if self.__v2_compat__: shadow = shadow or self.shadow_name(self(), args, kwargs, options) else: shadow = shadow or self.shadow_name(args, kwargs, options) preopts = self._get_exec_options() options = dict(preopts, **options) if options else preopts options.setdefault('ignore_result', self.ignore_result) return app.send_task( self.name, args, kwargs, task_id=task_id, producer=producer, link=link, link_error=link_error, result_cls=self.AsyncResult, shadow=shadow, task_type=self, **options )
def test_reraises_EncodeError(self): with self.assertRaises(EncodeError): dumps([object()], serializer='json')
def serialize(self, data): content_type, content_encoding, body = dumps( bytes_to_str(data), serializer=self._serializer) return b64encode(self.encrypt(ensure_bytes(body)))
def _future_encode(self, data): _encode = getattr(self, '_encode', None) if _encode is None: return dumps(data, serializer=self.serializer) return _encode(data)
def test_utf8(self): b = BibRecord(bibcode=u'\u01b5') ctype, enc, data = serialization.dumps(b) o = serialization.loads(data, 'application/x-adsmsg', 'utf-8') self.assertTrue(isinstance(o, BibRecord)) self.assertEqual(o.bibcode, u'\u01b5')
def test_pickle_dumps(self): self.assertEqual( pickle.loads(pickle_data), pickle.loads(dumps(py_data, serializer='pickle')[-1]), )
def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, link=None, link_error=None, shadow=None, **options): """Apply tasks asynchronously by sending a message. Arguments: args (Tuple): The positional arguments to pass on to the task. kwargs (Dict): The keyword arguments to pass on to the task. countdown (float): Number of seconds into the future that the task should execute. Defaults to immediate execution. eta (~datetime.datetime): Absolute time and date of when the task should be executed. May not be specified if `countdown` is also supplied. expires (float, ~datetime.datetime): Datetime or seconds in the future for the task should expire. The task won't be executed after the expiration time. shadow (str): Override task name used in logs/monitoring. Default is retrieved from :meth:`shadow_name`. connection (kombu.Connection): Re-use existing broker connection instead of acquiring one from the connection pool. retry (bool): If enabled sending of the task message will be retried in the event of connection loss or failure. Default is taken from the :setting:`task_publish_retry` setting. Note that you need to handle the producer/connection manually for this to work. retry_policy (Mapping): Override the retry policy used. See the :setting:`task_publish_retry_policy` setting. queue (str, kombu.Queue): The queue to route the task to. This must be a key present in :setting:`task_queues`, or :setting:`task_create_missing_queues` must be enabled. See :ref:`guide-routing` for more information. exchange (str, kombu.Exchange): Named custom exchange to send the task to. Usually not used in combination with the ``queue`` argument. routing_key (str): Custom routing key used to route the task to a worker server. If in combination with a ``queue`` argument only used to specify custom routing keys to topic exchanges. priority (int): The task priority, a number between 0 and 9. Defaults to the :attr:`priority` attribute. serializer (str): Serialization method to use. Can be `pickle`, `json`, `yaml`, `msgpack` or any custom serialization method that's been registered with :mod:`kombu.serialization.registry`. Defaults to the :attr:`serializer` attribute. compression (str): Optional compression method to use. Can be one of ``zlib``, ``bzip2``, or any custom compression methods registered with :func:`kombu.compression.register`. Defaults to the :setting:`task_compression` setting. link (Signature): A single, or a list of tasks signatures to apply if the task returns successfully. link_error (Signature): A single, or a list of task signatures to apply if an error occurs while executing the task. producer (kombu.Producer): custom producer to use when publishing the task. add_to_parent (bool): If set to True (default) and the task is applied while executing another task, then the result will be appended to the parent tasks ``request.children`` attribute. Trailing can also be disabled by default using the :attr:`trail` attribute publisher (kombu.Producer): Deprecated alias to ``producer``. headers (Dict): Message headers to be included in the message. Returns: celery.result.AsyncResult: Promise of future evaluation. Raises: TypeError: If not enough arguments are passed, or too many arguments are passed. Note that signature checks may be disabled by specifying ``@task(typing=False)``. kombu.exceptions.OperationalError: If a connection to the transport cannot be made, or if the connection is lost. Note: Also supports all keyword arguments supported by :meth:`kombu.Producer.publish`. """ if self.typing: try: check_arguments = self.__header__ except AttributeError: # pragma: no cover pass else: check_arguments(*(args or ()), **(kwargs or {})) if self.__v2_compat__: shadow = shadow or self.shadow_name(self(), args, kwargs, options) else: shadow = shadow or self.shadow_name(args, kwargs, options) preopts = self._get_exec_options() options = dict(preopts, **options) if options else preopts options.setdefault('ignore_result', self.ignore_result) if self.priority: options.setdefault('priority', self.priority) app = self._get_app() if app.conf.task_always_eager: with app.producer_or_acquire(producer) as eager_producer: serializer = options.get('serializer') if serializer is None: if eager_producer.serializer: serializer = eager_producer.serializer else: serializer = app.conf.task_serializer body = args, kwargs content_type, content_encoding, data = serialization.dumps( body, serializer, ) args, kwargs = serialization.loads(data, content_type, content_encoding, accept=[content_type]) with denied_join_result(): return self.apply(args, kwargs, task_id=task_id or uuid(), link=link, link_error=link_error, **options) else: return app.send_task(self.name, args, kwargs, task_id=task_id, producer=producer, link=link, link_error=link_error, result_cls=self.AsyncResult, shadow=shadow, task_type=self, **options)
def encode(self, data): _, _, payload = dumps(data, serializer=self.serializer) return payload
def test_reraises_EncodeError(self): with pytest.raises(EncodeError): dumps([object()], serializer='json')
def testContentType(self): """ Encode with correct content-type. """ serialized = serialization.dumps(None, 'x-rpc-json') self.assertEqual('application/json+celery-rpc:v1', serialized[0])
def test_dumps__no_serializer(self): ctyp, cenc, data = dumps(str_to_bytes('foo')) assert ctyp == 'application/data' assert cenc == 'binary'
def test_pickle_dumps(self): a = pickle.loads(pickle_data), b = pickle.loads(dumps(py_data, serializer='pickle')[-1]), assert a == b