def register_codecs(): from kombu.serialization import registry registry.register('x-rpc-json', x_rpc_json_dumps, x_rpc_json_loads, 'application/json+celery-rpc:v1', 'utf-8') # XXX: Compatibility for ver <= 0.16 registry.register('x-json', x_json_dumps, x_json_loads, 'application/json', 'utf-8')
def register_paloma(): ''' Experimental Celery Serializer ''' def encode(task_body): ''' Encoder ''' _encode = registry._encoders['pickle'][2] task_body['args'] = tuple( [to_model_signature(arg) for arg in task_body['args']]) task_body['kwargs'] = dict([(k, to_model_signature(v)) for (k, v) in task_body['kwargs'].items()]) ret = _encode(task_body) return ret def decode(message): ''' Decoder ''' _decode = registry._decoders[registry._encoders['pickle'][0]] task_body = _decode(message) task_body['args'] = tuple( [from_model_signature(arg) for arg in task_body['args']]) task_body['kwargs'] = dict([(k, from_model_signature(v)) for (k, v) in task_body['kwargs'].items()]) return task_body #: registration registry.register('paloma', encode, decode, content_type='application/x-python-serialize-paloma', content_encoding='binary')
def register_paloma(): ''' Experimental Celery Serializer ''' def encode(task_body): ''' Encoder ''' _encode = registry._encoders['pickle'][2] task_body['args'] = tuple( [ to_model_signature(arg) for arg in task_body['args'] ] ) task_body['kwargs'] =dict( [(k,to_model_signature(v)) for (k,v) in task_body['kwargs'].items()] ) ret = _encode(task_body) return ret def decode(message): ''' Decoder ''' _decode =registry._decoders[ registry._encoders['pickle'][0] ] task_body = _decode(message) task_body['args'] = tuple( [ from_model_signature(arg) for arg in task_body['args'] ] ) task_body['kwargs'] = dict( [(k,from_model_signature(v)) for (k,v) in task_body['kwargs'].items()] ) return task_body #: registration registry.register('paloma', encode, decode, content_type='application/x-python-serialize-paloma', content_encoding='binary')
def register_codecs(): from kombu.serialization import registry registry.register('x-rpc-json', x_rpc_json_dumps, x_rpc_json_loads, 'application/json+celery-rpc:v1', 'utf-8') # XXX: Compatibility for ver <= 0.16 registry.register('x-json', x_json_dumps, x_json_loads, 'application/json', 'utf-8')
def register_auth(key=None, cert=None, store=None): """register security serializer""" s = SecureSerializer(key and PrivateKey(key), cert and Certificate(cert), store and FSCertStore(store), anyjson.serialize, anyjson.deserialize) registry.register("auth", s.serialize, s.deserialize, content_type="application/data", content_encoding="utf-8")
def register_auth(key=None, cert=None, store=None, digest='sha1', serializer='json'): """register security serializer""" s = SecureSerializer(key and PrivateKey(key), cert and Certificate(cert), store and FSCertStore(store), digest=digest, serializer=serializer) registry.register('auth', s.serialize, s.deserialize, content_type='application/data', content_encoding='utf-8')
def register_auth(key=None, cert=None, store=None, digest="sha1", serializer="json"): """Register security serializer.""" s = SecureSerializer( key and PrivateKey(key), cert and Certificate(cert), store and FSCertStore(store), digest=digest, serializer=serializer, ) registry.register("auth", s.serialize, s.deserialize, content_type="application/data", content_encoding="utf-8")
def register_auth(key=None, cert=None, store=None, digest='sha1', serializer='json'): """Register security serializer.""" s = SecureSerializer(key and PrivateKey(key), cert and Certificate(cert), store and FSCertStore(store), digest=digest, serializer=serializer) registry.register('auth', s.serialize, s.deserialize, content_type='application/data', content_encoding='utf-8')
def setup_encrypted_serializer(key=None, serializer='pickle'): encrypted_serializer = EncryptedSerializer(key=key, serializer=serializer) name = "encrypted_{0}".format(serializer) registry.register( "encrypted_{0}".format(serializer), encrypted_serializer.serialize, encrypted_serializer.deserialize, content_type="application/x-encrypted-{0}".format(serializer), content_encoding='utf-8', ) return name
def register_dill(): def encode(obj, dumper=dill.dumps): return dumper(obj, protocol=pickle_protocol) def decode(s): return pickle_loads(str_to_bytes(s), load=dill.load) registry.register(name='dill', encoder=encode, decoder=decode, content_type='application/x-python-serialize', content_encoding='binary')
def register_auth(key=None, cert=None, store=None, digest="sha1", serializer="json"): """register security serializer""" s = SecureSerializer(key and PrivateKey(key), cert and Certificate(cert), store and FSCertStore(store), digest=digest, serializer=serializer) registry.register("auth", s.serialize, s.deserialize, content_type="application/data", content_encoding="utf-8")
def register_auth(key=None, key_password=None, cert=None, store=None, digest=DEFAULT_SECURITY_DIGEST, serializer='json'): """Register security serializer.""" s = SecureSerializer(key and PrivateKey(key, password=key_password), cert and Certificate(cert), store and FSCertStore(store), digest, serializer=serializer) registry.register('auth', s.serialize, s.deserialize, content_type='application/data', content_encoding='utf-8')
def register_auth(key=None, cert=None, store=None, digest=DEFAULT_SECURITY_DIGEST, serializer="json"): """Register security serializer.""" s = SecureSerializer( key and PrivateKey(key), cert and Certificate(cert), store and FSCertStore(store), digest, serializer=serializer, ) registry.register( "auth", s.serialize, s.deserialize, content_type="application/data", content_encoding="utf-8", )
def __init__(self, app_name, app_module): # Add support for memoryview object (used in BinaryField) in pickle copyreg.pickle(memoryview, lambda b: (memoryview, (bytes(b),))) # Monkey patching cascade deletions of related objects # so they are chunked if there are more than 10000 objects. Collector._original_init = Collector.__init__ Collector.__init__ = Collector__init Collector.collect = Collector__collect Collector.delete = Collector__delete # Register rapidjson for task serialization. registry.register('json', json.dumps, partial(_json.loads, _loads=json.loads), content_type='application/json', content_encoding='utf-8') # Disable request.body caching in raven middleware. DjangoRestFrameworkCompatMiddleware.process_request = lambda x, y: None super().__init__(app_name, app_module)
msgpack_py_data = dict(py_data) # Unicode chars are lost in transmit :( msgpack_py_data['unicode'] = 'Th quick brown fox jumps over th lazy dog' msgpack_data = b64decode(str_to_bytes("""\ haNpbnQKpWZsb2F0y0AJIftTyNTxpGxpc3SUpmdlb3JnZaVqZXJyeaZlbGFpbmWlY29zbW+mc3Rya\ W5n2gArVGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIHRoZSBsYXp5IGRvZ6d1bmljb2Rl2g\ ApVGggcXVpY2sgYnJvd24gZm94IGp1bXBzIG92ZXIgdGggbGF6eSBkb2c=\ """)) def say(m): sys.stderr.write('%s\n' % (m, )) registry.register('testS', lambda s: s, lambda s: 'decoded', 'application/testS', 'utf-8') class test_Serialization(TestCase): def test_disable(self): disabled = registry._disabled_content_types try: registry.disable('testS') self.assertIn('application/testS', disabled) disabled.clear() registry.disable('application/testS') self.assertIn('application/testS', disabled) finally: disabled.clear()
def register_codecs(): from kombu.serialization import registry registry.register('x-rpc-json', x_rpc_json_dumps, x_rpc_json_loads, 'application/json+django-rpc-backend:v1', 'utf-8')
msgpack_py_data = dict(py_data) # msgpack only supports tuples msgpack_py_data['list'] = tuple(msgpack_py_data['list']) # Unicode chars are lost in transmit :( msgpack_py_data['unicode'] = 'Th quick brown fox jumps over th lazy dog' msgpack_data = ('\x85\xa3int\n\xa5float\xcb@\t!\xfbS\xc8\xd4\xf1\xa4list' '\x94\xa6george\xa5jerry\xa6elaine\xa5cosmo\xa6string\xda' '\x00+The quick brown fox jumps over the lazy dog\xa7unicode' '\xda\x00)Th quick brown fox jumps over th lazy dog') def say(m): sys.stderr.write('%s\n' % (m, )) registry.register('testS', lambda s: s, lambda s: 'decoded', 'application/testS', 'utf-8') class test_Serialization(TestCase): def test_disable(self): disabled = registry._disabled_content_types try: registry.disable('testS') self.assertIn('application/testS', disabled) disabled.clear() registry.disable('application/testS') self.assertIn('application/testS', disabled) finally: disabled.clear()
import logging from kombu import Connection, Exchange, Queue from kombu.serialization import registry from datawire.core import app from datawire.util import JSONEncoder, queue_loads log = logging.getLogger(__name__) registry.unregister('json') registry.register('json', JSONEncoder().encode, queue_loads, 'application/json') exchange = Exchange(app.config.get('INSTANCE', 'dwre'), 'topic', durable=True) inbound_queue = Queue('inbound', exchange=exchange, routing_key='inbound.#') matching_queue = Queue('matching', exchange=exchange, routing_key='matching.#') entity_queue = Queue('entity', exchange=exchange, routing_key='entity.#') indexing_queue = Queue('indexing', exchange=exchange, routing_key='indexing.#') def connect(): return Connection(app.config.get('AMQP_QUEUE_URI')) def make_consumer(connection, queue, callback): consumer = connection.Consumer([queue], callbacks=[callback]) consumer.consume() log.info("Listening on: %s", queue)
msgpack_py_data = dict(py_data) # msgpack only supports tuples msgpack_py_data["list"] = tuple(msgpack_py_data["list"]) # Unicode chars are lost in transmit :( msgpack_py_data["unicode"] = 'Th quick brown fox jumps over th lazy dog' msgpack_data = ('\x85\xa3int\n\xa5float\xcb@\t!\xfbS\xc8\xd4\xf1\xa4list' '\x94\xa6george\xa5jerry\xa6elaine\xa5cosmo\xa6string\xda' '\x00+The quick brown fox jumps over the lazy dog\xa7unicode' '\xda\x00)Th quick brown fox jumps over th lazy dog') def say(m): sys.stderr.write("%s\n" % (m, )) registry.register('testS', lambda s: s, lambda s: "decoded", "application/testS", "utf-8") class test_Serialization(TestCase): def test_disable(self): disabled = registry._disabled_content_types try: registry.disable("testS") self.assertIn("application/testS", disabled) disabled.clear() registry.disable("application/testS") self.assertIn("application/testS", disabled) finally: disabled.clear()
import logging from kombu import Connection, Exchange, Queue from kombu.serialization import registry from datawire.core import app from datawire.util import JSONEncoder, queue_loads log = logging.getLogger(__name__) registry.unregister('json') registry.register('json', JSONEncoder().encode, queue_loads, 'application/json') exchange = Exchange(app.config.get('INSTANCE', 'dwre'), 'topic', durable=True) inbound_queue = Queue('inbound', exchange=exchange, routing_key='inbound.#') matching_queue = Queue('matching', exchange=exchange, routing_key='matching.#') entity_queue = Queue('entity', exchange=exchange, routing_key='entity.#') indexing_queue = Queue('indexing', exchange=exchange, routing_key='indexing.#') def connect(): return Connection(app.config.get('AMQP_QUEUE_URI')) def make_consumer(connection, queue, callback): consumer = connection.Consumer([queue], callbacks=[callback]) consumer.consume() log.info("Listening on: %s", queue)
# coding: utf-8 from __future__ import absolute_import import json from django.conf import settings as _settings from kombu.serialization import registry from .encoders import XJSONEncoder # Register enhanced json encoder def _json_dumps(obj): return json.dumps(obj, cls=XJSONEncoder) registry.register('x-json', _json_dumps, json.loads, 'application/json', 'utf-8') # Default limit for results of filter call FILTER_LIMIT = 1000 # Default timeout for getting results GET_RESULT_TIMEOUT = 10 # See Celery configuration parameters at # http://docs.celeryproject.org/en/latest/configuration.html # Some reasonable defaults are defined below CELERY_RESULT_BACKEND = 'cache+memory://' CELERY_DEFAULT_QUEUE = 'celery_rpc.requests' CELERY_DEFAULT_EXCHANGE = 'celery_rpc' CELERY_DEFAULT_ROUTING_KEY = 'celery_rpc'
__all__ = ['CallbackQueueDispatcher'] # use a custom JSON serializer so we can properly handle !unsafe and !vault # objects that may exist in events emitted by the callback plugin # see: https://github.com/ansible/ansible/pull/38759 class AnsibleJSONEncoder(json.JSONEncoder): def default(self, o): if getattr(o, 'yaml_tag', None) == '!vault': return o.data return super(AnsibleJSONEncoder, self).default(o) registry.register('json-ansible', lambda obj: json.dumps(obj, cls=AnsibleJSONEncoder), lambda obj: json.loads(obj), content_type='application/json', content_encoding='utf-8') class CallbackQueueDispatcher(object): def __init__(self): self.callback_connection = getattr(settings, 'BROKER_URL', None) self.connection_queue = getattr(settings, 'CALLBACK_QUEUE', '') self.connection = None self.exchange = None self.logger = logging.getLogger( 'awx.main.queue.CallbackQueueDispatcher') def dispatch(self, obj): if not self.callback_connection or not self.connection_queue:
## # To override the following settings, create a separate settings module. # Import this module, override what you need to and set the environment # variable DJANGO_SETTINGS_MODULE to your module. See Django docs for details ## if not DEBUG: # These security settings are annoying while debugging CSRF_COOKIE_SECURE = True SESSION_COOKIE_SECURE = True ## Celery options # load custom kombu encoder registry.register('json_date', jsondate.dumps, jsondate.loads, content_type='application/json+date', content_encoding='utf-8', ) CELERY_SEND_TASK_ERROR_EMAILS = True CELERY_RESULT_BACKEND = BROKER_URL CELERY_ACCEPT_CONTENT = ['json', 'json_date'] CELERY_TASK_SERIALIZER = "json_date" CELERY_RESULT_SERIALIZER = "json_date" CELERY_QUEUES = ( Queue('default', Exchange('default'), routing_key='default'), Broadcast('broadcast_tasks'), ) CELERY_ROUTES = {'queue.tasks.force_garbage_collection': {'queue': 'broadcast_tasks'}} CELERY_DEFAULT_QUEUE = 'default'
msgpack_py_data = dict(py_data) # msgpack only supports tuples msgpack_py_data["list"] = tuple(msgpack_py_data["list"]) # Unicode chars are lost in transmit :( msgpack_py_data["unicode"] = 'Th quick brown fox jumps over th lazy dog' msgpack_data = ('\x85\xa3int\n\xa5float\xcb@\t!\xfbS\xc8\xd4\xf1\xa4list' '\x94\xa6george\xa5jerry\xa6elaine\xa5cosmo\xa6string\xda' '\x00+The quick brown fox jumps over the lazy dog\xa7unicode' '\xda\x00)Th quick brown fox jumps over th lazy dog') def say(m): sys.stderr.write("%s\n" % (m, )) registry.register('testS', lambda s: s, lambda s: "decoded", "application/testS", "utf-8") class test_Serialization(TestCase): def test_disable(self): disabled = registry._disabled_content_types try: registry.disable("testS") self.assertIn("application/testS", disabled) disabled.clear() registry.disable("application/testS") self.assertIn("application/testS", disabled) finally: disabled.clear()