def register_codecs():
from kombu.serialization import registry
registry.register('x-rpc-json', x_rpc_json_dumps, x_rpc_json_loads,
'application/json+celery-rpc:v1', 'utf-8')
# XXX: Compatibility for ver <= 0.16
registry.register('x-json', x_json_dumps, x_json_loads,
'application/json', 'utf-8')
def register_paloma():
''' Experimental Celery Serializer '''
def encode(task_body):
''' Encoder '''
_encode = registry._encoders['pickle'][2]
task_body['args'] = tuple(
[to_model_signature(arg) for arg in task_body['args']])
task_body['kwargs'] = dict([(k, to_model_signature(v))
for (k, v) in task_body['kwargs'].items()])
ret = _encode(task_body)
return ret
def decode(message):
''' Decoder '''
_decode = registry._decoders[registry._encoders['pickle'][0]]
task_body = _decode(message)
task_body['args'] = tuple(
[from_model_signature(arg) for arg in task_body['args']])
task_body['kwargs'] = dict([(k, from_model_signature(v))
for (k, v) in task_body['kwargs'].items()])
return task_body
#: registration
registry.register('paloma',
encode,
decode,
content_type='application/x-python-serialize-paloma',
content_encoding='binary')
def register_paloma():
''' Experimental Celery Serializer '''
def encode(task_body):
''' Encoder '''
_encode = registry._encoders['pickle'][2]
task_body['args'] = tuple(
[ to_model_signature(arg) for arg in task_body['args'] ]
)
task_body['kwargs'] =dict(
[(k,to_model_signature(v)) for (k,v) in task_body['kwargs'].items()]
)
ret = _encode(task_body)
return ret
def decode(message):
''' Decoder '''
_decode =registry._decoders[ registry._encoders['pickle'][0] ]
task_body = _decode(message)
task_body['args'] = tuple(
[ from_model_signature(arg) for arg in task_body['args'] ]
)
task_body['kwargs'] = dict(
[(k,from_model_signature(v)) for (k,v) in task_body['kwargs'].items()]
)
return task_body
#: registration
registry.register('paloma',
encode, decode,
content_type='application/x-python-serialize-paloma',
content_encoding='binary')
def register_codecs():
from kombu.serialization import registry
registry.register('x-rpc-json', x_rpc_json_dumps, x_rpc_json_loads,
'application/json+celery-rpc:v1', 'utf-8')
# XXX: Compatibility for ver <= 0.16
registry.register('x-json', x_json_dumps, x_json_loads, 'application/json',
'utf-8')
def register_auth(key=None, cert=None, store=None):
"""register security serializer"""
s = SecureSerializer(key and PrivateKey(key),
cert and Certificate(cert),
store and FSCertStore(store),
anyjson.serialize, anyjson.deserialize)
registry.register("auth", s.serialize, s.deserialize,
content_type="application/data",
content_encoding="utf-8")
def register_auth(key=None, cert=None, store=None, digest='sha1',
serializer='json'):
"""register security serializer"""
s = SecureSerializer(key and PrivateKey(key),
cert and Certificate(cert),
store and FSCertStore(store),
digest=digest, serializer=serializer)
registry.register('auth', s.serialize, s.deserialize,
content_type='application/data',
content_encoding='utf-8')
def register_auth(key=None, cert=None, store=None, digest="sha1", serializer="json"):
"""Register security serializer."""
s = SecureSerializer(
key and PrivateKey(key),
cert and Certificate(cert),
store and FSCertStore(store),
digest=digest,
serializer=serializer,
)
registry.register("auth", s.serialize, s.deserialize, content_type="application/data", content_encoding="utf-8")
def register_auth(key=None, cert=None, store=None, digest='sha1',
serializer='json'):
"""Register security serializer."""
s = SecureSerializer(key and PrivateKey(key),
cert and Certificate(cert),
store and FSCertStore(store),
digest=digest, serializer=serializer)
registry.register('auth', s.serialize, s.deserialize,
content_type='application/data',
content_encoding='utf-8')
def setup_encrypted_serializer(key=None, serializer='pickle'):
encrypted_serializer = EncryptedSerializer(key=key, serializer=serializer)
name = "encrypted_{0}".format(serializer)
registry.register(
"encrypted_{0}".format(serializer),
encrypted_serializer.serialize,
encrypted_serializer.deserialize,
content_type="application/x-encrypted-{0}".format(serializer),
content_encoding='utf-8',
)
return name
def register_dill():
def encode(obj, dumper=dill.dumps):
return dumper(obj, protocol=pickle_protocol)
def decode(s):
return pickle_loads(str_to_bytes(s), load=dill.load)
registry.register(name='dill',
encoder=encode,
decoder=decode,
content_type='application/x-python-serialize',
content_encoding='binary')
def register_auth(key=None,
cert=None,
store=None,
digest="sha1",
serializer="json"):
"""register security serializer"""
s = SecureSerializer(key and PrivateKey(key),
cert and Certificate(cert),
store and FSCertStore(store),
digest=digest,
serializer=serializer)
registry.register("auth",
s.serialize,
s.deserialize,
content_type="application/data",
content_encoding="utf-8")
def register_auth(key=None,
key_password=None,
cert=None,
store=None,
digest=DEFAULT_SECURITY_DIGEST,
serializer='json'):
"""Register security serializer."""
s = SecureSerializer(key and PrivateKey(key, password=key_password),
cert and Certificate(cert),
store and FSCertStore(store),
digest,
serializer=serializer)
registry.register('auth',
s.serialize,
s.deserialize,
content_type='application/data',
content_encoding='utf-8')
def register_auth(key=None,
cert=None,
store=None,
digest=DEFAULT_SECURITY_DIGEST,
serializer="json"):
"""Register security serializer."""
s = SecureSerializer(
key and PrivateKey(key),
cert and Certificate(cert),
store and FSCertStore(store),
digest,
serializer=serializer,
)
registry.register(
"auth",
s.serialize,
s.deserialize,
content_type="application/data",
content_encoding="utf-8",
)
def __init__(self, app_name, app_module):
# Add support for memoryview object (used in BinaryField) in pickle
copyreg.pickle(memoryview, lambda b: (memoryview, (bytes(b),)))
# Monkey patching cascade deletions of related objects
# so they are chunked if there are more than 10000 objects.
Collector._original_init = Collector.__init__
Collector.__init__ = Collector__init
Collector.collect = Collector__collect
Collector.delete = Collector__delete
# Register rapidjson for task serialization.
registry.register('json',
json.dumps,
partial(_json.loads, _loads=json.loads),
content_type='application/json',
content_encoding='utf-8')
# Disable request.body caching in raven middleware.
DjangoRestFrameworkCompatMiddleware.process_request = lambda x, y: None
super().__init__(app_name, app_module)
msgpack_py_data = dict(py_data)
# Unicode chars are lost in transmit :(
msgpack_py_data['unicode'] = 'Th quick brown fox jumps over th lazy dog'
msgpack_data = b64decode(str_to_bytes("""\
haNpbnQKpWZsb2F0y0AJIftTyNTxpGxpc3SUpmdlb3JnZaVqZXJyeaZlbGFpbmWlY29zbW+mc3Rya\
W5n2gArVGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIHRoZSBsYXp5IGRvZ6d1bmljb2Rl2g\
ApVGggcXVpY2sgYnJvd24gZm94IGp1bXBzIG92ZXIgdGggbGF6eSBkb2c=\
"""))
def say(m):
sys.stderr.write('%s\n' % (m, ))
registry.register('testS', lambda s: s, lambda s: 'decoded',
'application/testS', 'utf-8')
class test_Serialization(TestCase):
def test_disable(self):
disabled = registry._disabled_content_types
try:
registry.disable('testS')
self.assertIn('application/testS', disabled)
disabled.clear()
registry.disable('application/testS')
self.assertIn('application/testS', disabled)
finally:
disabled.clear()
def register_codecs():
from kombu.serialization import registry
registry.register('x-rpc-json', x_rpc_json_dumps, x_rpc_json_loads,
'application/json+django-rpc-backend:v1', 'utf-8')
msgpack_py_data = dict(py_data)
# msgpack only supports tuples
msgpack_py_data['list'] = tuple(msgpack_py_data['list'])
# Unicode chars are lost in transmit :(
msgpack_py_data['unicode'] = 'Th quick brown fox jumps over th lazy dog'
msgpack_data = ('\x85\xa3int\n\xa5float\xcb@\t!\xfbS\xc8\xd4\xf1\xa4list'
'\x94\xa6george\xa5jerry\xa6elaine\xa5cosmo\xa6string\xda'
'\x00+The quick brown fox jumps over the lazy dog\xa7unicode'
'\xda\x00)Th quick brown fox jumps over th lazy dog')
def say(m):
sys.stderr.write('%s\n' % (m, ))
registry.register('testS', lambda s: s, lambda s: 'decoded',
'application/testS', 'utf-8')
class test_Serialization(TestCase):
def test_disable(self):
disabled = registry._disabled_content_types
try:
registry.disable('testS')
self.assertIn('application/testS', disabled)
disabled.clear()
registry.disable('application/testS')
self.assertIn('application/testS', disabled)
finally:
disabled.clear()
import logging
from kombu import Connection, Exchange, Queue
from kombu.serialization import registry
from datawire.core import app
from datawire.util import JSONEncoder, queue_loads
log = logging.getLogger(__name__)
registry.unregister('json')
registry.register('json',
JSONEncoder().encode, queue_loads, 'application/json')
exchange = Exchange(app.config.get('INSTANCE', 'dwre'), 'topic', durable=True)
inbound_queue = Queue('inbound', exchange=exchange, routing_key='inbound.#')
matching_queue = Queue('matching', exchange=exchange, routing_key='matching.#')
entity_queue = Queue('entity', exchange=exchange, routing_key='entity.#')
indexing_queue = Queue('indexing', exchange=exchange, routing_key='indexing.#')
def connect():
return Connection(app.config.get('AMQP_QUEUE_URI'))
def make_consumer(connection, queue, callback):
consumer = connection.Consumer([queue], callbacks=[callback])
consumer.consume()
log.info("Listening on: %s", queue)
msgpack_py_data = dict(py_data)
# msgpack only supports tuples
msgpack_py_data["list"] = tuple(msgpack_py_data["list"])
# Unicode chars are lost in transmit :(
msgpack_py_data["unicode"] = 'Th quick brown fox jumps over th lazy dog'
msgpack_data = ('\x85\xa3int\n\xa5float\xcb@\t!\xfbS\xc8\xd4\xf1\xa4list'
'\x94\xa6george\xa5jerry\xa6elaine\xa5cosmo\xa6string\xda'
'\x00+The quick brown fox jumps over the lazy dog\xa7unicode'
'\xda\x00)Th quick brown fox jumps over th lazy dog')
def say(m):
sys.stderr.write("%s\n" % (m, ))
registry.register('testS', lambda s: s, lambda s: "decoded",
"application/testS", "utf-8")
class test_Serialization(TestCase):
def test_disable(self):
disabled = registry._disabled_content_types
try:
registry.disable("testS")
self.assertIn("application/testS", disabled)
disabled.clear()
registry.disable("application/testS")
self.assertIn("application/testS", disabled)
finally:
disabled.clear()
import logging
from kombu import Connection, Exchange, Queue
from kombu.serialization import registry
from datawire.core import app
from datawire.util import JSONEncoder, queue_loads
log = logging.getLogger(__name__)
registry.unregister('json')
registry.register('json', JSONEncoder().encode, queue_loads, 'application/json')
exchange = Exchange(app.config.get('INSTANCE', 'dwre'),
'topic', durable=True)
inbound_queue = Queue('inbound', exchange=exchange, routing_key='inbound.#')
matching_queue = Queue('matching', exchange=exchange, routing_key='matching.#')
entity_queue = Queue('entity', exchange=exchange, routing_key='entity.#')
indexing_queue = Queue('indexing', exchange=exchange, routing_key='indexing.#')
def connect():
return Connection(app.config.get('AMQP_QUEUE_URI'))
def make_consumer(connection, queue, callback):
consumer = connection.Consumer([queue], callbacks=[callback])
consumer.consume()
log.info("Listening on: %s", queue)
# coding: utf-8
from __future__ import absolute_import
import json
from django.conf import settings as _settings
from kombu.serialization import registry
from .encoders import XJSONEncoder
# Register enhanced json encoder
def _json_dumps(obj):
return json.dumps(obj, cls=XJSONEncoder)
registry.register('x-json', _json_dumps, json.loads, 'application/json', 'utf-8')
# Default limit for results of filter call
FILTER_LIMIT = 1000
# Default timeout for getting results
GET_RESULT_TIMEOUT = 10
# See Celery configuration parameters at
# http://docs.celeryproject.org/en/latest/configuration.html
# Some reasonable defaults are defined below
CELERY_RESULT_BACKEND = 'cache+memory://'
CELERY_DEFAULT_QUEUE = 'celery_rpc.requests'
CELERY_DEFAULT_EXCHANGE = 'celery_rpc'
CELERY_DEFAULT_ROUTING_KEY = 'celery_rpc'
__all__ = ['CallbackQueueDispatcher']
# use a custom JSON serializer so we can properly handle !unsafe and !vault
# objects that may exist in events emitted by the callback plugin
# see: https://github.com/ansible/ansible/pull/38759
class AnsibleJSONEncoder(json.JSONEncoder):
def default(self, o):
if getattr(o, 'yaml_tag', None) == '!vault':
return o.data
return super(AnsibleJSONEncoder, self).default(o)
registry.register('json-ansible',
lambda obj: json.dumps(obj, cls=AnsibleJSONEncoder),
lambda obj: json.loads(obj),
content_type='application/json',
content_encoding='utf-8')
class CallbackQueueDispatcher(object):
def __init__(self):
self.callback_connection = getattr(settings, 'BROKER_URL', None)
self.connection_queue = getattr(settings, 'CALLBACK_QUEUE', '')
self.connection = None
self.exchange = None
self.logger = logging.getLogger(
'awx.main.queue.CallbackQueueDispatcher')
def dispatch(self, obj):
if not self.callback_connection or not self.connection_queue:
##
# To override the following settings, create a separate settings module.
# Import this module, override what you need to and set the environment
# variable DJANGO_SETTINGS_MODULE to your module. See Django docs for details
##
if not DEBUG:
# These security settings are annoying while debugging
CSRF_COOKIE_SECURE = True
SESSION_COOKIE_SECURE = True
## Celery options
# load custom kombu encoder
registry.register('json_date', jsondate.dumps, jsondate.loads,
content_type='application/json+date',
content_encoding='utf-8',
)
CELERY_SEND_TASK_ERROR_EMAILS = True
CELERY_RESULT_BACKEND = BROKER_URL
CELERY_ACCEPT_CONTENT = ['json', 'json_date']
CELERY_TASK_SERIALIZER = "json_date"
CELERY_RESULT_SERIALIZER = "json_date"
CELERY_QUEUES = (
Queue('default', Exchange('default'), routing_key='default'),
Broadcast('broadcast_tasks'),
)
CELERY_ROUTES = {'queue.tasks.force_garbage_collection': {'queue': 'broadcast_tasks'}}
CELERY_DEFAULT_QUEUE = 'default'
msgpack_py_data = dict(py_data)
# msgpack only supports tuples
msgpack_py_data["list"] = tuple(msgpack_py_data["list"])
# Unicode chars are lost in transmit :(
msgpack_py_data["unicode"] = 'Th quick brown fox jumps over th lazy dog'
msgpack_data = ('\x85\xa3int\n\xa5float\xcb@\t!\xfbS\xc8\xd4\xf1\xa4list'
'\x94\xa6george\xa5jerry\xa6elaine\xa5cosmo\xa6string\xda'
'\x00+The quick brown fox jumps over the lazy dog\xa7unicode'
'\xda\x00)Th quick brown fox jumps over th lazy dog')
def say(m):
sys.stderr.write("%s\n" % (m, ))
registry.register('testS', lambda s: s, lambda s: "decoded",
"application/testS", "utf-8")
class test_Serialization(TestCase):
def test_disable(self):
disabled = registry._disabled_content_types
try:
registry.disable("testS")
self.assertIn("application/testS", disabled)
disabled.clear()
registry.disable("application/testS")
self.assertIn("application/testS", disabled)
finally:
disabled.clear()
