def test_loads__not_accepted(self): with pytest.raises(ContentDisallowed): loads('tainted', 'application/x-evil', 'binary', accept=[]) with pytest.raises(ContentDisallowed): loads('tainted', 'application/x-evil', 'binary', accept=['application/x-json']) assert loads('tainted', 'application/x-doomsday', 'binary', accept=['application/x-doomsday'])
def test_content_type_decoding(self): assert loads( unicode_string_as_utf8, content_type='plain/text', content_encoding='utf-8') == unicode_string assert loads( latin_string_as_latin1, content_type='application/data', content_encoding='latin-1') == latin_string
def test_content_type_binary(self): assert isinstance( loads(unicode_string_as_utf8, content_type='application/data', content_encoding='binary'), bytes_t) assert loads( unicode_string_as_utf8, content_type='application/data', content_encoding='binary') == unicode_string_as_utf8
def test_content_type_decoding(self): self.assertEqual( unicode_string, loads(unicode_string_as_utf8, content_type='plain/text', content_encoding='utf-8'), ) self.assertEqual( latin_string, loads(latin_string_as_latin1, content_type='application/data', content_encoding='latin-1'), )
def test_json_dumps(self): a = loads( dumps(py_data, serializer='json')[-1], content_type='application/json', content_encoding='utf-8', ) b = loads( json_data, content_type='application/json', content_encoding='utf-8', ) assert a == b
def test_content_type_binary(self): self.assertIsInstance( loads(unicode_string_as_utf8, content_type='application/data', content_encoding='binary'), bytes_t, ) self.assertEqual( unicode_string_as_utf8, loads(unicode_string_as_utf8, content_type='application/data', content_encoding='binary'), )
def test_loads_when_disabled(self): disabled = registry._disabled_content_types try: registry.disable('testS') with self.assertRaises(SerializerNotInstalled): loads('xxd', 'application/testS', 'utf-8', force=False) ret = loads('xxd', 'application/testS', 'utf-8', force=True) self.assertEqual(ret, 'decoded') finally: disabled.clear()
def test_yaml_dumps(self): register_yaml() a = loads( dumps(py_data, serializer='yaml')[-1], content_type='application/x-yaml', content_encoding='utf-8', ) b = loads( yaml_data, content_type='application/x-yaml', content_encoding='utf-8', ) assert a == b
def test_msgpack_dumps(self): register_msgpack() a = loads( dumps(msgpack_py_data, serializer='msgpack')[-1], content_type='application/x-msgpack', content_encoding='binary', ) b = loads( msgpack_data, content_type='application/x-msgpack', content_encoding='binary', ) assert a == b
def test_json_dumps(self): self.assertEqual( loads( dumps(py_data, serializer='json')[-1], content_type='application/json', content_encoding='utf-8', ), loads( json_data, content_type='application/json', content_encoding='utf-8', ), )
def deserialize(self, data): data = self._ensure_bytes(data) header, end = self.parse_header(data) # Skip whitespace length = len(data) while end < length and data[end] in self.whitespace: end += 1 header, body = header, data[end:] signer, signature, content_type, content_encoding = ( header['signer'], header['signature'], header['content_type'], header['content_encoding'] ) signature = self.key_codec.decode(signature) if content_encoding != self._content_encoding: raise ValueError("Invalid inner content encoding ({!r} != {!r})" .format(content_encoding, self._content_encoding)) try: verify_key = self._verify_keys[signer] except KeyError: raise ValueError("Unknown signer {!r}".format(signer)) from None verify_key.verify(body, signature) return loads(bytes_to_str(body), content_type, content_encoding, force=True)
def test_pickle_loads(self): self.assertEqual( py_data, loads(pickle_data, content_type='application/x-python-serialize', content_encoding='binary'), )
def deserialize(self, data): """Deserialize data structure from string.""" assert self._cert_store is not None with reraise_errors("Unable to deserialize: {0!r}", (Exception,)): payload = self._unpack(data) signature, signer, body = (payload["signature"], payload["signer"], payload["body"]) self._cert_store[signer].verify(body, signature, self._digest) return loads(bytes_to_str(body), payload["content_type"], payload["content_encoding"], force=True)
def test_yaml_loads(self): register_yaml() self.assertEqual( py_data, loads(yaml_data, content_type='application/x-yaml', content_encoding='utf-8'), )
def deserialize(self, data): decrypted = self.decrypt(b64decode(data)) body = loads( decrypted, self._serializer_content_type, self._serializer_content_encoding, force=True, ) return bytes_to_str(body)
def test_msgpack_loads(self): register_msgpack() res = loads(msgpack_data, content_type='application/x-msgpack', content_encoding='binary') if sys.version_info[0] < 3: for k, v in res.items(): if isinstance(v, text_t): res[k] = v.encode() if isinstance(v, (list, tuple)): res[k] = [i.encode() for i in v] assert res == msgpack_py_data
def _serialize_args_and_kwargs_for_eager_mode( self, args=None, kwargs=None, **options): producer = options.get('producer') with app.producer_or_acquire(producer) as eager_producer: serializer = options.get( 'serializer', eager_producer.serializer ) body = args, kwargs content_type, content_encoding, data = serialization.dumps( body, serializer ) args, kwargs = serialization.loads( data, content_type, content_encoding ) return args, kwargs
def unpack_exception(self, data, serializer): """ Instantiates exception stub for original exception :param module: module name for original exception :param name: class name for original exception :param args: RemoteException.args :return: new constructed exception :rtype: self.RemoteError subclass """ try: # unpacking RemoteException args content_type, content_encoding, dumps = registry._encoders[serializer] data = loads(data, content_type, content_encoding) module, name, args = data try: # trying to import original exception original = symbol_by_name("%s.%s" % (module, name)) # creating parent class for original error and self.RemoteError class_name = from_utf8("Remote" + name) parent = type(class_name, (original, self.RemoteError), {'__module__': module}) except (AttributeError, ImportError): # alternative way for unknown errors parent = self.RemoteError # create and cache exception stub class if name not in self.__registry: self.__registry[name] = create_exception_cls( from_utf8(name), module, parent=parent) exc_class = self.__registry[name] return exc_class(*args) except (ValueError, ContentDisallowed): # loads error return None
def test_register_msgpack__no_msgpack(self): register_msgpack() with self.assertRaises(SerializerNotInstalled): loads('foo', 'application/x-msgpack', 'utf-8')
def test_register_yaml__no_yaml(self): register_yaml() with self.assertRaises(SerializerNotInstalled): loads('foo', 'application/x-yaml', 'utf-8')
def test_loads__trusted_content(self): loads('tainted', 'application/data', 'binary', accept=[]) loads('tainted', 'application/text', 'utf-8', accept=[])
def test_utf8(self): b = BibRecord(bibcode=u'\u01b5') ctype, enc, data = serialization.dumps(b) o = serialization.loads(data, 'application/x-adsmsg', 'utf-8') self.assertTrue(isinstance(o, BibRecord)) self.assertEqual(o.bibcode, u'\u01b5')
def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, link=None, link_error=None, shadow=None, **options): """Apply tasks asynchronously by sending a message. Arguments: args (Tuple): The positional arguments to pass on to the task. kwargs (Dict): The keyword arguments to pass on to the task. countdown (float): Number of seconds into the future that the task should execute. Defaults to immediate execution. eta (~datetime.datetime): Absolute time and date of when the task should be executed. May not be specified if `countdown` is also supplied. expires (float, ~datetime.datetime): Datetime or seconds in the future for the task should expire. The task won't be executed after the expiration time. shadow (str): Override task name used in logs/monitoring. Default is retrieved from :meth:`shadow_name`. connection (kombu.Connection): Re-use existing broker connection instead of acquiring one from the connection pool. retry (bool): If enabled sending of the task message will be retried in the event of connection loss or failure. Default is taken from the :setting:`task_publish_retry` setting. Note that you need to handle the producer/connection manually for this to work. retry_policy (Mapping): Override the retry policy used. See the :setting:`task_publish_retry_policy` setting. queue (str, kombu.Queue): The queue to route the task to. This must be a key present in :setting:`task_queues`, or :setting:`task_create_missing_queues` must be enabled. See :ref:`guide-routing` for more information. exchange (str, kombu.Exchange): Named custom exchange to send the task to. Usually not used in combination with the ``queue`` argument. routing_key (str): Custom routing key used to route the task to a worker server. If in combination with a ``queue`` argument only used to specify custom routing keys to topic exchanges. priority (int): The task priority, a number between 0 and 9. Defaults to the :attr:`priority` attribute. serializer (str): Serialization method to use. Can be `pickle`, `json`, `yaml`, `msgpack` or any custom serialization method that's been registered with :mod:`kombu.serialization.registry`. Defaults to the :attr:`serializer` attribute. compression (str): Optional compression method to use. Can be one of ``zlib``, ``bzip2``, or any custom compression methods registered with :func:`kombu.compression.register`. Defaults to the :setting:`task_compression` setting. link (Signature): A single, or a list of tasks signatures to apply if the task returns successfully. link_error (Signature): A single, or a list of task signatures to apply if an error occurs while executing the task. producer (kombu.Producer): custom producer to use when publishing the task. add_to_parent (bool): If set to True (default) and the task is applied while executing another task, then the result will be appended to the parent tasks ``request.children`` attribute. Trailing can also be disabled by default using the :attr:`trail` attribute publisher (kombu.Producer): Deprecated alias to ``producer``. headers (Dict): Message headers to be included in the message. Returns: celery.result.AsyncResult: Promise of future evaluation. Raises: TypeError: If not enough arguments are passed, or too many arguments are passed. Note that signature checks may be disabled by specifying ``@task(typing=False)``. kombu.exceptions.OperationalError: If a connection to the transport cannot be made, or if the connection is lost. Note: Also supports all keyword arguments supported by :meth:`kombu.Producer.publish`. """ if self.typing: try: check_arguments = self.__header__ except AttributeError: # pragma: no cover pass else: check_arguments(*(args or ()), **(kwargs or {})) app = self._get_app() if app.conf.task_always_eager: with app.producer_or_acquire(producer) as eager_producer: serializer = options.get( 'serializer', eager_producer.serializer ) body = args, kwargs content_type, content_encoding, data = serialization.dumps( body, serializer ) args, kwargs = serialization.loads( data, content_type, content_encoding ) with denied_join_result(): return self.apply(args, kwargs, task_id=task_id or uuid(), link=link, link_error=link_error, **options) if self.__v2_compat__: shadow = shadow or self.shadow_name(self(), args, kwargs, options) else: shadow = shadow or self.shadow_name(args, kwargs, options) preopts = self._get_exec_options() options = dict(preopts, **options) if options else preopts options.setdefault('ignore_result', self.ignore_result) return app.send_task( self.name, args, kwargs, task_id=task_id, producer=producer, link=link, link_error=link_error, result_cls=self.AsyncResult, shadow=shadow, task_type=self, **options )
def test_json_loads(self): assert loads(json_data, content_type='application/json', content_encoding='utf-8') == py_data
def test_register_yaml__no_yaml(self, mask_modules): register_yaml() with pytest.raises(SerializerNotInstalled): loads('foo', 'application/x-yaml', 'utf-8')
def decode(self, payload): payload = PY3 and payload or str(payload) return loads(payload, content_type=self.content_type, content_encoding=self.content_encoding, accept=self.accept)
def test_pickle_loads(self): assert loads(pickle_data, content_type='application/x-python-serialize', content_encoding='binary') == py_data
def test_loads_when_data_is_None(self): loads(None, 'application/testS', 'utf-8')
def apply_async(self, args=None, kwargs=None, task_id=None, producer=None, link=None, link_error=None, shadow=None, **options): """Apply tasks asynchronously by sending a message. Arguments: args (Tuple): The positional arguments to pass on to the task. kwargs (Dict): The keyword arguments to pass on to the task. countdown (float): Number of seconds into the future that the task should execute. Defaults to immediate execution. eta (~datetime.datetime): Absolute time and date of when the task should be executed. May not be specified if `countdown` is also supplied. expires (float, ~datetime.datetime): Datetime or seconds in the future for the task should expire. The task won't be executed after the expiration time. shadow (str): Override task name used in logs/monitoring. Default is retrieved from :meth:`shadow_name`. connection (kombu.Connection): Re-use existing broker connection instead of acquiring one from the connection pool. retry (bool): If enabled sending of the task message will be retried in the event of connection loss or failure. Default is taken from the :setting:`task_publish_retry` setting. Note that you need to handle the producer/connection manually for this to work. retry_policy (Mapping): Override the retry policy used. See the :setting:`task_publish_retry_policy` setting. queue (str, kombu.Queue): The queue to route the task to. This must be a key present in :setting:`task_queues`, or :setting:`task_create_missing_queues` must be enabled. See :ref:`guide-routing` for more information. exchange (str, kombu.Exchange): Named custom exchange to send the task to. Usually not used in combination with the ``queue`` argument. routing_key (str): Custom routing key used to route the task to a worker server. If in combination with a ``queue`` argument only used to specify custom routing keys to topic exchanges. priority (int): The task priority, a number between 0 and 9. Defaults to the :attr:`priority` attribute. serializer (str): Serialization method to use. Can be `pickle`, `json`, `yaml`, `msgpack` or any custom serialization method that's been registered with :mod:`kombu.serialization.registry`. Defaults to the :attr:`serializer` attribute. compression (str): Optional compression method to use. Can be one of ``zlib``, ``bzip2``, or any custom compression methods registered with :func:`kombu.compression.register`. Defaults to the :setting:`task_compression` setting. link (Signature): A single, or a list of tasks signatures to apply if the task returns successfully. link_error (Signature): A single, or a list of task signatures to apply if an error occurs while executing the task. producer (kombu.Producer): custom producer to use when publishing the task. add_to_parent (bool): If set to True (default) and the task is applied while executing another task, then the result will be appended to the parent tasks ``request.children`` attribute. Trailing can also be disabled by default using the :attr:`trail` attribute publisher (kombu.Producer): Deprecated alias to ``producer``. headers (Dict): Message headers to be included in the message. Returns: celery.result.AsyncResult: Promise of future evaluation. Raises: TypeError: If not enough arguments are passed, or too many arguments are passed. Note that signature checks may be disabled by specifying ``@task(typing=False)``. kombu.exceptions.OperationalError: If a connection to the transport cannot be made, or if the connection is lost. Note: Also supports all keyword arguments supported by :meth:`kombu.Producer.publish`. """ if self.typing: try: check_arguments = self.__header__ except AttributeError: # pragma: no cover pass else: check_arguments(*(args or ()), **(kwargs or {})) app = self._get_app() if app.conf.task_always_eager: with app.producer_or_acquire(producer) as eager_producer: serializer = options.get( 'serializer', (eager_producer.serializer if eager_producer.serializer else app.conf.task_serializer)) body = args, kwargs content_type, content_encoding, data = serialization.dumps( body, serializer, ) args, kwargs = serialization.loads(data, content_type, content_encoding, accept=[content_type]) with denied_join_result(): return self.apply(args, kwargs, task_id=task_id or uuid(), link=link, link_error=link_error, **options) if self.__v2_compat__: shadow = shadow or self.shadow_name(self(), args, kwargs, options) else: shadow = shadow or self.shadow_name(args, kwargs, options) preopts = self._get_exec_options() options = dict(preopts, **options) if options else preopts options.setdefault('ignore_result', self.ignore_result) if self.priority: options.setdefault('priority', self.priority) return app.send_task(self.name, args, kwargs, task_id=task_id, producer=producer, link=link, link_error=link_error, result_cls=self.AsyncResult, shadow=shadow, task_type=self, **options)
def test_reraises_DecodeError(self): with pytest.raises(DecodeError): loads(object(), content_type='application/json', content_encoding='utf-8')
def test_json_loads(self): self.assertEqual( py_data, loads(json_data, content_type='application/json', content_encoding='utf-8'), )
def test_yaml_loads(self): pytest.importorskip('yaml') register_yaml() assert loads(yaml_data, content_type='application/x-yaml', content_encoding='utf-8') == py_data
def offline_orphaned_jobs(es_url, dry_run=False): """Set jobs with job-queued or job-started state to job-offline if not synced with celery task state.""" # get redis connection set_redis_pool() global POOL rd = StrictRedis(connection_pool=POOL) # get celery task result serializer content_type, content_encoding, encoder = registry._encoders[ app.conf.CELERY_RESULT_SERIALIZER] accept = prepare_accept_content(app.conf.CELERY_ACCEPT_CONTENT) logging.info("content_type: {}".format(content_type)) logging.info("content_encoding: {}".format(content_encoding)) logging.info("encoder: {}".format(encoder)) logging.info("accept: {}".format(accept)) # query query = { "query": { "bool": { "must": [ { "terms": { "status": ["job-started", "job-queued"] } } ] } }, "_source": ["status", "tags", "uuid"] } url_tmpl = "{}/job_status-current/_search?search_type=scan&scroll=10m&size=100" r = requests.post(url_tmpl.format(es_url), data=json.dumps(query)) if r.status_code != 200: logging.error("Failed to query ES. Got status code %d:\n%s" % (r.status_code, json.dumps(query, indent=2))) r.raise_for_status() scan_result = r.json() count = scan_result['hits']['total'] scroll_id = scan_result['_scroll_id'] # get list of results results = [] while True: r = requests.post('%s/_search/scroll?scroll=10m' % es_url, data=scroll_id) res = r.json() scroll_id = res['_scroll_id'] if len(res['hits']['hits']) == 0: break for hit in res['hits']['hits']: results.append(hit) # check for celery state for res in results: id = res['_id'] src = res.get('_source', {}) status = src['status'] tags = src.get('tags', []) task_id = src['uuid'] # check celery task status in ES task_query = { "query": { "term": { "_id": task_id } }, "_source": ["status"] } r = requests.post('%s/task_status-current/task/_search' % es_url, data=json.dumps(task_query)) if r.status_code != 200: logging.error("Failed to query ES. Got status code %d:\n%s" % (r.status_code, json.dumps(task_query, indent=2))) continue task_res = r.json() if task_res['hits']['total'] > 0: task_info = task_res['hits']['hits'][0] if task_info['_source']['status'] == 'task-failed': updated_status = 'job-failed' elif task_info['_source']['status'] == 'task-succeeded': updated_status = 'job-completed' elif task_info['_source']['status'] in ('task-sent', 'task-started'): continue else: logging.error("Cannot handle task status %s for %s." % (task_info['_source']['status'], task_id)) continue if dry_run: logging.info("Would've update job status to %s for %s." % (updated_status, task_id)) else: new_doc = { "doc": {"status": updated_status}, "doc_as_upsert": True } r = requests.post('%s/job_status-current/job/%s/_update' % (es_url, id), data=json.dumps(new_doc)) result = r.json() if r.status_code != 200: logging.error("Failed to update tags for %s. Got status code %d:\n%s" % (id, r.status_code, json.dumps(result, indent=2))) r.raise_for_status() logging.info("Set job %s to %s." % (id, updated_status)) continue # get celery task metadata in redis task_meta = loads(rd.get('celery-task-meta-%s' % task_id), content_type=content_type, content_encoding=content_encoding, accept=accept) if task_meta is None: updated_status = 'job-offline' if dry_run: logging.info("Would've update job status to %s for %s." % (updated_status, task_id)) else: new_doc = { "doc": {"status": updated_status}, "doc_as_upsert": True } r = requests.post('%s/job_status-current/job/%s/_update' % (es_url, id), data=json.dumps(new_doc)) result = r.json() if r.status_code != 200: logging.error("Failed to update tags for %s. Got status code %d:\n%s" % (id, r.status_code, json.dumps(result, indent=2))) r.raise_for_status() logging.info("Set job %s to %s." % (id, updated_status)) continue
def test_register_msgpack__no_msgpack(self, mask_modules): register_msgpack() with pytest.raises(SerializerNotInstalled): loads('foo', 'application/x-msgpack', 'utf-8')
def test_reraises_DecodeError(self): with self.assertRaises(DecodeError): loads(object(), content_type='application/json', content_encoding='utf-8')
def test_yaml_loads(self): register_yaml() assert loads( yaml_data, content_type='application/x-yaml', content_encoding='utf-8') == py_data