def test_custom_serializer(container_factory, rabbit_config,
sniffer_queue_factory):
def encode(value):
value = json.dumps(value)
return value.upper()
def decode(value):
value = value.lower()
return json.loads(value)
register("upperjson", encode, decode, "application/x-upper-json", "utf-8")
class Service(object):
name = "service"
@rpc
def echo(self, arg):
return arg
rabbit_config[SERIALIZER_CONFIG_KEY] = "upperjson"
container = container_factory(Service, rabbit_config)
container.start()
get_messages = sniffer_queue_factory('nameko-rpc')
# verify RPC works end-to-end
with ServiceRpcProxy('service', rabbit_config) as proxy:
assert proxy.echo("hello") == "hello"
# verify sniffed messages serialized as expected
msg = get_messages()[0]
assert '"RESULT": "HELLO"' in msg['payload']
assert msg['properties']['content_type'] == "application/x-upper-json"
def make_celery(app, celery):
""" From http://flask.pocoo.org/docs/0.10/patterns/celery/ """
# Register our custom serializer type before updating the configuration.
from kombu.serialization import register
from doorman.celery_serializer import djson_dumps, djson_loads
register(
'djson', djson_dumps, djson_loads,
content_type='application/x-djson',
content_encoding='utf-8'
)
# Actually update the config
celery.config_from_object(app.config)
# Register Sentry client
if 'SENTRY_DSN' in app.config and app.config['SENTRY_DSN']:
client = Client(app.config['SENTRY_DSN'])
# register a custom filter to filter out duplicate logs
register_logger_signal(client)
# hook into the Celery error handler
register_signal(client)
TaskBase = celery.Task
class ContextTask(TaskBase):
abstract = True
def __call__(self, *args, **kwargs):
with app.app_context():
return TaskBase.__call__(self, *args, **kwargs)
celery.Task = ContextTask
return celery
def make_celery(app, celery):
""" From http://flask.pocoo.org/docs/0.10/patterns/celery/ """
# Register our custom serializer type before updating the configuration.
from kombu.serialization import register
from doorman.celery_serializer import djson_dumps, djson_loads
register(
'djson', djson_dumps, djson_loads,
content_type='application/x-djson',
content_encoding='utf-8'
)
# Actually update the config
celery.config_from_object(app.config)
TaskBase = celery.Task
class ContextTask(TaskBase):
abstract = True
def __call__(self, *args, **kwargs):
with app.app_context():
return TaskBase.__call__(self, *args, **kwargs)
celery.Task = ContextTask
return celery
def register_kombu_serializers():
"""
Register our custom pickle serializer which knows how to handle UTF-8 (non
ascii) messages.
Default kombu pickle de-serializer calls .encode() on the bytes object without providing an
encoding. This means it default to "ascii" and fail with UnicodeDecode error.
https://github.com/celery/kombu/blob/3.0/kombu/utils/encoding.py#L47
"""
def pickle_dumps(obj, dumper=pickle.dumps):
return dumper(obj, protocol=pickle_protocol)
if six.PY3:
def str_to_bytes(s):
if isinstance(s, str):
return s.encode('utf-8')
return s
def unpickle(s):
return pickle_loads(str_to_bytes(s))
else:
def str_to_bytes(s): # noqa
if isinstance(s, unicode):
return s.encode('utf-8')
return s
unpickle = pickle_loads # noqa
register('pickle', pickle_dumps, unpickle,
content_type='application/x-python-serialize',
content_encoding='binary')
def register_kombu_custom_serializer():
register(
'customjson',
custom_dumps,
custom_loads,
content_type='application/x-customjson',
content_encoding='utf-8'
)
def _register_mistral_serialization():
"""Adds mistral serializer to available serializers in kombu.
:return: None
"""
serialization.register(
'mistral_serialization',
encoder=serializers.KombuSerializer.serialize,
decoder=serializers.KombuSerializer.deserialize,
content_type='application/json'
)
def register_serializer():
"""
This is needed for the Kombu entry point to load encoders and decoders
to the registry. Celery depends on Kombu for low level serialization
from task.
"""
from kombu.serialization import register
register(
'tryson',
functools.partial(json.dumps, cls=JSONEncoder),
functools.partial(json.loads, object_hook=JSONDecoder()),
content_type='application/x-tryson',
content_encoding='binary',
)
def __init__(self, name, connection, deployment, durable, queue_arguments,
exchange, topics):
self.connection = connection
self.deployment = deployment
self.durable = durable
self.queue_arguments = queue_arguments
self.name = name
self.last_time = None
self.pmi = None
self.processed = 0
self.total_processed = 0
self.topics = topics
self.exchange = exchange
signal.signal(signal.SIGTERM, self._shutdown)
register('bufferjson', self.loads, anyjson.dumps,
content_type='application/json',
content_encoding='binary')
def register_changes_json():
from kombu.serialization import register
from kombu.utils.encoding import bytes_t
from json import dumps, loads
from uuid import UUID
def _loads(obj):
if isinstance(obj, UUID):
obj = obj.hex
elif isinstance(obj, bytes_t):
obj = obj.decode()
elif isinstance(obj, buffer):
obj = bytes(obj).decode()
return loads(obj)
register('changes_json', dumps, _loads,
content_type='application/json',
content_encoding='utf-8')
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# vim: set et sw=4 ts=4 sts=4 ff=unix fenc=utf8:
# Author: Binux<*****@*****.**>
# http://binux.me
# Created on 2015-05-22 20:54:01
import time
import umsgpack
from kombu import Connection, enable_insecure_serializers
from kombu.serialization import register
from kombu.exceptions import ChannelError
from six.moves import queue as BaseQueue
register('umsgpack', umsgpack.packb, umsgpack.unpackb, 'application/x-msgpack')
enable_insecure_serializers(['umsgpack'])
class KombuQueue(object):
"""
kombu is a high-level interface for multiple message queue backends.
KombuQueue is built on top of kombu API.
"""
Empty = BaseQueue.Empty
Full = BaseQueue.Full
max_timeout = 0.3
def __init__(self, name, url="amqp://", maxsize=0, lazy_limit=True):
"""
from django.conf import settings
app = Celery('phase')
# We need a custom serializer to handle date and datetime objects.
class JSONSerializer(json.JSONEncoder):
def default(self, data):
if isinstance(data, (date, datetime)):
return data.isoformat()
elif isinstance(data, Decimal):
return float(data)
raise TypeError("Unable to serialize %r (type: %s)" %
(data, type(data)))
def my_dumps(obj):
return json.dumps(obj, cls=JSONSerializer)
register('betterjson',
my_dumps,
json.loads,
content_type='application/x-myjson',
content_encoding='utf-8')
# Using a string here means the worker will not have to
# pickle the object when using Windows.
app.config_from_object('django.conf:settings')
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
from celery.signals import task_revoked, user_preload_options
from kombu.serialization import register
from pyramid.paster import bootstrap
from pyramid.request import Request
from pyramid.scripting import prepare
from pyramid.settings import asbool
from pyramid.threadlocal import get_current_request
from testscaffold.celery.encoders import json_dumps, json_loads
log = logging.getLogger(__name__)
register(
"date_json",
json_dumps,
json_loads,
content_type="application/x-date_json",
content_encoding="utf-8",
)
CELERY_CONFIG = {
"CELERY_IMPORTS": ("testscaffold.celery.tasks",),
"CELERYD_TASK_TIME_LIMIT": 300,
"CELERYD_MAX_TASKS_PER_CHILD": 1000,
"CELERY_IGNORE_RESULT": True,
"CELERY_ACCEPT_CONTENT": ("date_json",),
"CELERY_TASK_SERIALIZER": "date_json",
"CELERY_RESULT_SERIALIZER": "date_json",
"BROKER_URL": None,
"CELERYD_CONCURRENCY": None,
"CELERY_TIMEZONE": None,
PROCESS_STATES_LOADING_MUTATION = """
mutation updateProcessStates($name: String, $loading: Boolean) {
update_view_processes(where: {name: {_like: $name}}, _set: {loading: $loading}) {
affected_rows
returning {
name
loading
}
}
}
"""
serialization.register(
"ujson",
json.dumps,
json.loads,
content_type="application/x-ujson",
content_encoding="utf-8",
)
class TLSSMTPHandler(SMTPHandler):
def emit(self, record):
"""
Emit a record.
Format the record and send it to the specified addressees.
"""
try:
import smtplib
try:
from kombu.serialization import register
from bson.json_util import dumps, loads
from celery import signals
from fame.common.config import fame_config
from fame.core import fame_init
register('json_util',
dumps,
loads,
content_type='application/json',
content_encoding='utf-8')
MONGO = 'mongodb://{}:{}/{}'.format(fame_config.mongo_host,
fame_config.mongo_port,
fame_config.mongo_db)
if fame_config.mongo_user and fame_config.mongo_password:
MONGO = 'mongodb://{}:{}@{}:{}/{}'.format(fame_config.mongo_user,
fame_config.mongo_password,
fame_config.mongo_host,
fame_config.mongo_port,
fame_config.mongo_db)
BROKER_URL = MONGO
CELERY_RESULT_BACKEND = MONGO
CELERY_ACCEPT_CONTENT = ['json_util']
CELERY_TASK_SERIALIZER = 'json_util'
CELERY_IMPORTS = ('fame.core.analysis', 'fame.core.repository')
return Website(obj['url'])
return obj
def w_dumps(obj):
return json.dumps(obj, cls=WEncoder)
def w_loads(obj):
return json.loads(obj, object_hook=w_decoder)
register(
name='myjson',
encoder=w_dumps,
decoder=w_loads,
content_type='application/x-myjson',
content_encoding='utf-8'
)
def loads(s):
return s
def dumps(s):
return s
register(
'mem_serializer', dumps, loads,
def register_uuid_json():
"""Register a encoder/decoder for UUID compatable JSON serialization."""
register('uuid_json', _dumps, _loads,
content_type='application/json',
content_encoding='utf-8')
import os
# Register custom serializer for Celery that allows for encoding and decoding
# Python datetime objects (and potentially other ones)
from kombu.serialization import register
from serializers import encoder, decoder
register('ocd_serializer',
encoder,
decoder,
content_encoding='binary',
content_type='application/ocd-msgpack')
CELERY_CONFIG = {
'BROKER_URL': 'redis://127.0.0.1:6379/0',
'CELERY_ACCEPT_CONTENT': ['ocd_serializer'],
'CELERY_TASK_SERIALIZER': 'ocd_serializer',
'CELERY_RESULT_SERIALIZER': 'ocd_serializer',
'CELERY_RESULT_BACKEND':
'ocd_backend.result_backends:OCDRedisBackend+redis://127.0.0.1:6379/0',
'CELERY_IGNORE_RESULT': True,
'CELERY_DISABLE_RATE_LIMITS': True,
# Expire results after 30 minutes; otherwise Redis will keep
# claiming memory for a day
'CELERY_TASK_RESULT_EXPIRES': 1800
}
LOGGING = {
'version': 1,
'formatters': {
'console': {
return o
klass = o[TYPE]
if klass == DATETIME:
return date_parser.parse(o[VALUE])
return o
@staticmethod
def loads(text):
return json.loads(text, cls=JSONDecoder)
class StaticFilesStorage(ManifestStaticFilesStorage):
"""
Exactly the same as ManifestStaticFilesStorage from the Django contrib
package, except this one optionally changes the manifest file name
based on the value of STATICFILES_MANIFEST in settings.
"""
manifest_name = getattr(settings, "STATICFILES_MANIFEST",
"staticfiles.json")
# register serializers for JSON that handle UUIDs and datetime objects
register(
name=getattr(settings, "EDD_SERIALIZE_NAME", "edd-json"),
encoder=JSONEncoder.dumps,
decoder=JSONDecoder.loads,
content_type="application/x-edd-json",
content_encoding="UTF-8",
)
import logging.config
import os
import pickle
from bugsnag.handlers import BugsnagHandler
from kombu.serialization import register
from pythonjsonlogger import jsonlogger
from version import __version__, __version_info__
register('ocd_serializer', pickle.dumps, pickle.loads,
content_encoding='binary',
content_type='application/x-pickle2')
APP_VERSION = __version__
MAJOR_VERSION = __version_info__[0]
MINOR_VERSION = __version_info__[1]
BUGSNAG_APIKEY = os.getenv('BUGSNAG_APIKEY')
RELEASE_STAGE = os.getenv('RELEASE_STAGE', 'production')
REDIS_HOST = os.getenv('REDIS_HOST', "redis")
REDIS_PORT = os.getenv('REDIS_PORT', "6379")
REDIS_URL = 'redis://%s:%s/0' % (REDIS_HOST, REDIS_PORT)
ROOT_PATH = os.path.dirname(os.path.abspath(__file__))
PROJECT_PATH = os.path.dirname(ROOT_PATH)
LOCAL_DUMPS_DIR = os.path.join(PROJECT_PATH, 'local_dumps')
DUMPS_DIR = os.path.join(PROJECT_PATH, 'dumps')
# Use this timezone as default for timezone unaware dates
return CashFlowStatement(**o["kwargs"])
return o
def dumps(o):
return json.dumps(o, cls=Encoder)
def loads(s):
return json.loads(s, object_hook=decoder)
register(
"fiberjson",
dumps,
loads,
content_type="application/x-fiberjson",
content_encoding="utf-8",
)
app = Celery("tasks",
backend="amqp://redis",
broker="amqp://rabbitmq",
include="example.tasks")
app.conf.update(
accept_content=["fiberjson"],
task_serializer="fiberjson",
result_serializer="fiberjson",
)
#
# Copyright (C) Zenoss, Inc. 2009-2019 all rights reserved.
#
# This content is made available according to terms specified in
# License.zenoss under the directory where your Zenoss product is installed.
#
##############################################################################
from __future__ import absolute_import
from celery import Celery
from kombu.serialization import register
from .serialization import without_unicode
# Register custom serializer
register(
"without-unicode",
without_unicode.dump,
without_unicode.load,
content_type="application/x-without-unicode",
content_encoding="utf-8",
)
app = Celery(
"zenjobs",
config_source="Products.Jobber.config:Celery",
task_cls="Products.Jobber.task:ZenTask",
)
def setUp(self):
unittest.TestCase.setUp(self)
serialization.register('adsmsg', *serializer.register_args)
# Register your new serializer methods into kombu
from kombu.serialization import register
import openpathsampling.netcdfplus.dictify as dfy
_ops_to = dfy.CachedUUIDObjectJSON()
# _ops_from = dfy.CachedUUIDObjectJSON()
register('opsjson',
_ops_to.to_json,
_ops_to.from_json,
content_type='application/x-opsjson',
content_encoding='utf-8')
# Tell celery to use your new serializer:
CELERY_ACCEPT_CONTENT = ['opsjson']
CELERY_TASK_SERIALIZER = 'opsjson'
CELERY_RESULT_SERIALIZER = 'opsjson'
from ichnaea.log import configure_raven, configure_stats
CELERY_QUEUES = (
Queue("celery_default", routing_key="celery_default"),
Queue("celery_export", routing_key="celery_export"),
Queue("celery_incoming", routing_key="celery_incoming"),
Queue("celery_insert", routing_key="celery_insert"),
Queue("celery_monitor", routing_key="celery_monitor"),
Queue("celery_reports", routing_key="celery_reports"),
Queue("celery_upload", routing_key="celery_upload"),
) #: List of :class:`kombu.Queue` instances.
register(
"internal_json",
customjson.kombu_dumps,
customjson.kombu_loads,
content_type="application/x-internaljson",
content_encoding="utf-8",
)
def configure_celery(celery_app):
"""
Configure the celery app stored in :data:`ichnaea.async.app.celery_app`.
This is executed both inside the master worker process and once in
each forked worker process.
This parses the application ini and reads in the
:mod:`ichnaea.async.settings`.
"""
from .vpp_mock import (
activate_izitru_mock, activate_tineye_mock, activate_incandescent_mock
)
from .incandescent import (
get_incandescent_results, get_incandescent_results_callback
)
from .tineye import get_tineye_results
from .izitru import get_izitru_results
# @TODO: for debug purpose
from pprint import pprint # noqa
from .logging import debug, print_task_exception # noqa
register('dill', dill.dumps, dill.loads, content_type='application/x-binary-data', content_encoding='binary')
def get_original_image(item, resource):
if 'renditions' in item:
driver = app.data.mongo
px = driver.current_mongo_prefix(resource)
_fs = GridFS(driver.pymongo(prefix=px).db)
for k, v in item['renditions'].items():
if k == 'original':
_file = _fs.get(ObjectId(v['media']))
content = _file.read()
href = v['href']
return (href, content)
raise ImageNotFoundException()
import os
from uuid import uuid4
from celery import Celery
from celery.exceptions import TaskRevokedError
from celery.states import SUCCESS
from kombu.serialization import register
from flagger.json import dumps, loads
register('flagger-json',
dumps,
loads,
content_type='application/x-flagger-json',
content_encoding='utf-8')
app = Celery('flagger.workers.workers')
config = {
'accept_content': ['flagger-json'],
'broker_url':
os.environ.get('broker_url'),
'result_backend':
os.environ.get('result_backend', 'db+sqlite:///../flagger.sqlite'),
'result_serializer':
'flagger-json',
'task_serializer':
'flagger-json',
# 'worker_concurrency': os.environ.get('worker_concurrency', 16),
}
app.config_from_object(config)
default=encode_datetime,
)
def unpack(s):
return msgpack.unpackb(
s,
encoding="utf-8",
unicode_errors="ignore",
object_hook=decode_datetime,
)
register(
"unicode-msgpack-with-dates",
pack,
unpack,
content_type=MESSAGE_CONTENT_TYPE,
content_encoding=MESSAGE_CONTENT_ENCODING,
)
# This is around for compatibility reasons (so that we're able to decode any messages
# that are already in queues, with the old/non-date-aware content_type).
register(
'unicode-msgpack',
pack,
unpack,
content_type='application/x-unicode-msgpack',
content_encoding='binary'
)
def register_datecompatible_serializer(): # pragma: nocover
from kombu.serialization import register
register('nameko-serializer', *register_args)
CELERY_QUEUES = (
Queue('celery_cell', routing_key='celery_cell'),
Queue('celery_default', routing_key='celery_default'),
Queue('celery_export', routing_key='celery_export'),
Queue('celery_incoming', routing_key='celery_incoming'),
Queue('celery_monitor', routing_key='celery_monitor'),
Queue('celery_ocid', routing_key='celery_ocid'),
Queue('celery_reports', routing_key='celery_reports'),
Queue('celery_upload', routing_key='celery_upload'),
Queue('celery_wifi', routing_key='celery_wifi'),
) #: List of :class:`kombu.Queue` instances.
register('internal_json',
internaljson.internal_dumps,
internaljson.internal_loads,
content_type='application/x-internaljson',
content_encoding='utf-8')
def configure_celery(celery_app):
"""
Configure the celery app stored in :data:`ichnaea.async.app.celery_app`.
This is executed both inside the master worker process and once in
each forked worker process.
This parses the application ini and reads in the
:mod:`ichnaea.async.settings`.
"""
conf = read_config()
class JSONDecoder(json.JSONDecoder):
"""
Complement of JSONEncoder, translates encoded datetime objects back to real datetime.
"""
def __init__(self, *args, **kwargs):
super(JSONDecoder, self).__init__(object_hook=self.object_hook,
*args,
**kwargs)
def object_hook(self, o):
if TYPE not in o:
return o
klass = o[TYPE]
if klass == 'datetime':
return parser.parse(o[VALUE])
return o
@staticmethod
def loads(text):
return json.loads(text, cls=JSONDecoder)
# register serializers for JSON that handle UUIDs and datetime objects
register(
name=getattr(settings, 'EDD_SERIALIZE_NAME', 'edd-json'),
encoder=JSONEncoder.dumps,
decoder=JSONDecoder.loads,
content_type='application/x-edd-json',
content_encoding='UTF-8',
)
SERVER_EMAIL = '*****@*****.**'
CELERYD_MAX_TASKS_PER_CHILD = 1
# Default queue
CELERY_DEFAULT_QUEUE = 'seed-common'
CELERY_QUEUES = (
Queue(
CELERY_DEFAULT_QUEUE,
Exchange(CELERY_DEFAULT_QUEUE),
routing_key=CELERY_DEFAULT_QUEUE
),
)
# Register our custom JSON serializer so we can serialize datetime objects in celery.
register('seed_json', CeleryDatetimeSerializer.seed_dumps, CeleryDatetimeSerializer.seed_loads,
content_type='application/json', content_encoding='utf-8')
CELERY_ACCEPT_CONTENT = ['seed_json']
CELERY_TASK_SERIALIZER = 'seed_json'
CELERY_RESULT_SERIALIZER = 'seed_json'
CELERY_TASK_RESULT_EXPIRES = 18000 # 5 hours
CELERY_MESSAGE_COMPRESSION = 'gzip'
BROKER_URL = 'amqp://*****:*****@localhost:5672//'
LOG_FILE = join(SITE_ROOT, '../logs/py.log/')
# Set translation languages for i18n
LANGUAGES = (
('en', 'English'),
)
def test_register(self):
register(None, None, None, None)
configure_stats,
)
CELERY_QUEUES = (
Queue('celery_default', routing_key='celery_default'),
Queue('celery_export', routing_key='celery_export'),
Queue('celery_incoming', routing_key='celery_incoming'),
Queue('celery_insert', routing_key='celery_insert'),
Queue('celery_monitor', routing_key='celery_monitor'),
Queue('celery_reports', routing_key='celery_reports'),
Queue('celery_upload', routing_key='celery_upload'),
)
EXPORT_QUEUE_PREFIX = 'queue_export_'
register('internal_json', customjson.kombu_dumps, customjson.kombu_loads,
content_type='application/x-internaljson',
content_encoding='utf-8')
def configure_celery(celery_app):
conf = read_config()
if conf.has_section('celery'):
section = conf.get_map('celery')
else: # pragma: no cover
# happens while building docs locally and on rtfd.org
return
# testing settings
always_eager = bool(os.environ.get('CELERY_ALWAYS_EAGER', False))
redis_uri = os.environ.get('REDIS_URI', 'redis://localhost:6379/1')
class NumpyKombuJSONEncoder(_json.JSONEncoder):
""" Special json encoder for numpy types """
def default(self, obj):
res = convert_simple_numpy_type(obj)
if res is not None:
return res
return super(NumpyKombuJSONEncoder, self).default(obj)
# Encoder function
def my_dumps(obj):
return _json.dumps(obj, cls=NumpyKombuJSONEncoder)
# Register your new serializer methods into kombu
from kombu.serialization import register
register('myjson', my_dumps, _json.loads,
content_type='application/json',
content_encoding='utf-8')
# Tell celery to use your new serializer:
#celeryApp.conf.accept_content = ['myjson']
celeryApp.conf.task_serializer = 'myjson'
celeryApp.conf.result_serializer = 'myjson'
# try monkey patch startup timeout since we take longer than 4.0 seconds to startup
from celery.concurrency import asynpool
asynpool.PROC_ALIVE_TIMEOUT = 60.0
from . import commonconfig, commandconfig, monitorconfig
from kombu.serialization import register
import json
def oid_safe_dumps(obj):
return json.dumps(obj, default=str)
def oid_safe_loads(obj):
return json.loads(obj)
register('oid_safe_json',
oid_safe_dumps,
oid_safe_loads,
content_type='application/x-oid_safe_json',
content_encoding='utf-8')
_includes = [
'cumulus.ansible.tasks.cluster', 'cumulus.ansible.tasks.volume',
'cumulus.tasks.cluster', 'cumulus.tasks.job', 'cumulus.ssh.tasks.key',
'cumulus.aws.ec2.tasks.key'
]
taskflow_modules = find_taskflow_modules()
_includes += taskflow_modules
# Route short tasks to their own queue
_routes = {
'cumulus.tasks.job.monitor_job': {
def register(cls):
from kombu.serialization import register
register(cls.name, cls.model_encode, cls.model_decode, 'application/json', 'utf-8')
import json
from flask import Flask
from flask.ext.login import LoginManager
from kombu.serialization import register
from timetracker.utils import mongo_encoder
register('mongo_json', mongo_encoder, json.loads, 'application/json')
app = Flask(__name__)
app.config['SECRET_KEY'] = '123456790'
app.config['MONGODB_SETTINGS'] = {'DB': 'timetracker'}
app.config['CELERY_BROKER_URL'] = 'amqp://'
app.config['UPLOAD_PATH'] = '/tmp/'
login_manager = LoginManager()
login_manager.init_app(app)
import timetracker.models
import timetracker.api
import timetracker.admin
import os
from os import walk
from os.path import join, dirname, abspath
import sys
from dive.base.serialization import pjson_dumps, pjson_loads
from kombu.serialization import register, registry
env = os.environ.get
base_dir_path = lambda x: abspath(join(dirname(__file__), x))
# Register custom PJSON to celery
register('pjson',
pjson_dumps,
pjson_loads,
content_type='application/x-pjson',
content_encoding='utf-8')
registry.enable('application/x-pjson')
class BaseConfig(object):
# General
SITE_URL = 'localhost::8081'
SITE_TITLE = 'dive'
SECRET_KEY = 'dive'
PREFERRED_URL_SCHEME = 'http'
SECURITY_PASSWORD_SALT = 'nacl'
# Flask
HOST = '0.0.0.0'
DEBUG = True
PORT = 8081
from __future__ import absolute_import
from celery.schedules import crontab
from kombu.serialization import register
from datetime import timedelta
import os
from futuschedule import tasks
from futuschedule.myjson import my_dumps, my_loads
register('myjson', my_dumps, my_loads,
content_type='application/x-myjson',
content_encoding='utf-8')
BROKER_URL = os.getenv('BROKER_URL', "redis://127.0.0.1:6379/0")
CELERY_RESULT_BACKEND = os.getenv('RESULT_BACKEND', "redis://127.0.0.1/0")
CELERYD_PREFETCH_MULTIPLIER = 6
CELERY_ACCEPT_CONTENT = ['myjson']
CELERY_TASK_SERIALIZER = 'myjson'
CELERY_RESULT_SERIALIZER = CELERY_TASK_SERIALIZER
CELERY_ACCEPT_CONTENT = [CELERY_TASK_SERIALIZER, ]
CELERYBEAT_SCHEDULE = {
'refresh-users': {
'task': 'futuschedule.tasks.refresh_users',
'schedule': crontab(minute=0),
},
'update-meeting-rooms': {
'task': 'futuschedule.tasks.update_meeting_rooms',
'schedule': crontab(minute=0),
}
from kombu.serialization import register
from .decoders import data_dumps, data_loads
register('myjson', data_dumps, data_loads,
content_type='application/x-myjson',
content_encoding='utf-8')
# Tell celery to use your new serializer:
accept_content = ['myjson']
task_serializer = 'myjson'
result_serializer = 'myjson'
if isinstance(v, list):
kwargs[k] = [try_cast(val) for val in v]
if isinstance(v, dict):
cast_item(v)
return o
def dumps(o):
with superdesk.app.app_context():
return MongoJSONEncoder().encode(o)
register("eve/json", dumps, loads, content_type="application/json")
class AppContextTask(TaskBase):
abstract = True
serializer = "eve/json"
def __call__(self, *args, **kwargs):
with superdesk.app.app_context():
try:
return super().__call__(*args, **kwargs)
except werkzeug.exceptions.InternalServerError as e:
superdesk.app.sentry.captureException()
logger.exception(e)
def on_failure(self, exc, task_id, args, kwargs, einfo):
"""Configure Celery."""
import os
from celery.utils.log import get_task_logger
from json_tricks.nonp import dumps
from json_tricks.nonp import loads
from kombu.serialization import register
logger = get_task_logger(__name__)
# Register json-tricks as json encoder
register(
'json_tricks.nonp',
lambda obj: dumps(obj, conv_str_byte=True),
lambda obj: loads(obj, conv_str_byte=True),
content_type='application/x-json-tricks',
content_encoding='utf-8',
)
# Global configuration.
accept_content = ['application/x-json-tricks', 'application/json']
imports = ('api.celery.tasks', )
timezone = 'America/Chicago'
# Beat configuration.
beat_max_loop_interval = 5
# Broker configuration.
broker_url = os.environ.get('CELERY_BROKER_URL', 'rpc://')
# Result configuration.
import os
from os import walk
from os.path import join, dirname, abspath
import sys
from dive.base.serialization import pjson_dumps, pjson_loads
from kombu.serialization import register, registry
env = os.environ.get
base_dir_path = lambda x: abspath(join(dirname(__file__), x))
# Register custom PJSON to celery
register('pjson', pjson_dumps, pjson_loads,
content_type='application/x-pjson',
content_encoding='utf-8')
registry.enable('application/x-pjson')
class BaseConfig(object):
# General
SITE_URL = 'localhost:3009'
SITE_TITLE = 'dive'
SECRET_KEY = 'dive'
PREFERRED_URL_SCHEME = 'http'
SECURITY_PASSWORD_SALT = 'nacl'
# Flask
HOST = '0.0.0.0'
DEBUG = True
PORT = 8081
COMPRESS = True
"""Configure Celery."""
import os
from celery.schedules import crontab
from json_tricks.nonp import dumps
from json_tricks.nonp import loads
from kombu.serialization import register
# Register json-tricks as json encoder
register('json_tricks.nonp', dumps, loads, content_type='application/x-json-tricks', content_encoding='utf-8')
# Global configuration.
accept_content = ['application/json', 'application/x-json-tricks']
imports = ('{{ cookiecutter.project_name }}.celery.tasks', )
timezone = 'America/Chicago'
# Beat configuration.
beat_max_loop_interval = 5
# Broker configuration.
broker_url = os.environ.get('CELERY_BROKER_URL', 'rpc://')
# Result configuration.
result_backend = os.environ.get('CELERY_RESULT_BACKEND', 'redis://')
result_serializer = 'json_tricks.nonp'
# Task configuration.
task_serializer = 'json_tricks.nonp'
# Worker condiguration.
worker_concurrency = 1
import logging.config
import os
import pickle
from bugsnag.handlers import BugsnagHandler
from kombu.serialization import register
from pythonjsonlogger import jsonlogger
from version import __version__, __version_info__
register('ocd_serializer', pickle.dumps, pickle.loads,
content_encoding='binary',
content_type='application/x-pickle2')
APP_VERSION = __version__
MAJOR_VERSION = __version_info__[0]
MINOR_VERSION = __version_info__[1]
BUGSNAG_APIKEY = os.getenv('BUGSNAG_APIKEY')
RELEASE_STAGE = os.getenv('RELEASE_STAGE', 'development')
REDIS_HOST = os.getenv('REDIS_SERVER_HOST', "redis")
REDIS_PORT = os.getenv('REDIS_SERVER_PORT', "6379")
REDIS_URL = 'redis://%s:%s/0' % (REDIS_HOST, REDIS_PORT)
ROOT_PATH = os.path.dirname(os.path.abspath(__file__))
PROJECT_PATH = os.path.dirname(ROOT_PATH)
LOCAL_DUMPS_DIR = os.path.join(PROJECT_PATH, 'local_dumps')
DUMPS_DIR = os.path.join(PROJECT_PATH, 'dumps')
# Use this timezone as default for timezone unaware dates
CELERYD_MAX_TASKS_PER_CHILD = 1
# Default queue
CELERY_DEFAULT_QUEUE = 'seed-common'
CELERY_QUEUES = (
Queue(
CELERY_DEFAULT_QUEUE,
Exchange(CELERY_DEFAULT_QUEUE),
routing_key=CELERY_DEFAULT_QUEUE
),
)
# Register our custom JSON serializer so we can serialize datetime objects in celery.
register('seed_json', CeleryDatetimeSerializer.seed_dumps,
CeleryDatetimeSerializer.seed_loads,
content_type='application/json', content_encoding='utf-8')
CELERY_ACCEPT_CONTENT = ['seed_json']
CELERY_TASK_SERIALIZER = 'seed_json'
CELERY_RESULT_SERIALIZER = 'seed_json'
CELERY_TASK_RESULT_EXPIRES = 18000 # 5 hours
CELERY_MESSAGE_COMPRESSION = 'gzip'
BROKER_URL = 'amqp://*****:*****@localhost:5672//'
LOG_FILE = join(SITE_ROOT, '../logs/py.log/')
# Set translation languages for i18n
LANGUAGES = (
('en', 'English'),
from flask.ext.babel import Babel
from flask.ext.pymongo import PyMongo
from flask_socketio import SocketIO
from kombu.serialization import register
from kombu import serialization
from .utils import memt_dumps, memt_loads
# Flask extensions
babel = Babel()
mongo = PyMongo()
socketio = SocketIO()
# Celery config
celery = Celery(__name__,
broker=Config.CELERY_BROKER_URL,
backend=Config.CELERY_RESULT_BACKEND)
register('memtjson', memt_dumps, memt_loads,
content_type='application/x-memtjson',
content_encoding='utf-8')
serialization.registry._decoders.pop("application/x-python-serialize")
if not os.environ.get('PRODUCTION'):
from flask_debugtoolbar import DebugToolbarExtension
toolbar = DebugToolbarExtension()
import os
# Register custom serializer for Celery that allows for encoding and decoding
# Python datetime objects (and potentially other ones)
from kombu.serialization import register
from serializers import encoder, decoder
register('ocd_serializer', encoder, decoder, content_encoding='binary',
content_type='application/ocd-msgpack')
CELERY_CONFIG = {
'BROKER_URL': 'redis://127.0.0.1:6379/0',
'CELERY_ACCEPT_CONTENT': ['ocd_serializer'],
'CELERY_TASK_SERIALIZER': 'ocd_serializer',
'CELERY_RESULT_SERIALIZER': 'ocd_serializer',
'CELERY_RESULT_BACKEND': 'ocd_backend.result_backends:OCDRedisBackend+redis://127.0.0.1:6379/0',
'CELERY_IGNORE_RESULT': True,
'CELERY_DISABLE_RATE_LIMITS': True,
# Expire results after 30 minutes; otherwise Redis will keep
# claiming memory for a day
'CELERY_TASK_RESULT_EXPIRES': 1800
}
LOGGING = {
'version': 1,
'formatters': {
'console': {
'format': '[%(asctime)s] [%(name)s] [%(levelname)s] - %(message)s',
'datefmt': '%Y-%m-%d %H:%M:%S'
}
},
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# vim: set et sw=4 ts=4 sts=4 ff=unix fenc=utf8:
# Author: Binux<*****@*****.**>
# http://binux.me
# Created on 2015-05-22 20:54:01
import time
import umsgpack
from kombu import Connection, enable_insecure_serializers
from kombu.serialization import register
from kombu.exceptions import ChannelError
from pyspider.libs.queue import Queue as BaseQueue
register('umsgpack', umsgpack.packb, umsgpack.unpackb, 'application/x-msgpack')
enable_insecure_serializers(['umsgpack'])
class KombuQueue(object):
"""
kombu is a high-level interface for multiple message queue backends.
KombuQueue is built on top of kombu API.
"""
Empty = BaseQueue.Empty
Full = BaseQueue.Full
max_timeout = 0.3
def __init__(self, name, url="amqp://", maxsize=0, lazy_limit=True):
typ = obj.pop('__type__')
mod = import_module(obj.pop('__module__'))
klass = getattr(mod, typ)
return klass(**obj)
except KeyError:
pass
return obj
return simplejson.loads(string, object_hook=decode)
# Register a custom serializer. We do this so we can conveniently
# transfer objects without resorting to pickling.
serialization.register('custom_json',
custom_dumps,
custom_loads,
content_type='application/x-bioconda-json',
content_encoding='utf8')
# Instantiate Celery app, setting our AsyncTask as default
# task class and loading the tasks from tasks.py
capp = Celery( # pylint: disable=invalid-name
task_cls=AsyncTask,
include=['bioconda_utils.bot.tasks'])
# Celery must be configured at module level to catch worker as well
# Settings are suggestions from CloudAMPQ
capp.conf.update(
# Set the URL to the AMQP broker using environment variable
broker_url=os.environ.get('CLOUDAMQP_URL'),
# KPIit is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.
"""Default configuration for KPIit."""
import os
from celery.schedules import crontab
from kombu.serialization import register
from kpiit.config import config
from kpiit.json import metric_dumps, metric_loads
# Register new JSON serializer
register(config['celery']['serializer'],
metric_dumps,
metric_loads,
content_type='application/x-metricjson',
content_encoding='utf-8')
# Default schedule crontabs
SCHEDULE_DOI_MONTHLY = crontab(**config['celery']['schedules']['doi'])
SCHEDULE_REPO_DAILY = crontab(**config['celery']['schedules']['repo'])
#: URL of message broker for Celery (default is Redis).
broker_url = config['celery']['broker_url']
#: URL of backend for result storage (default is Redis).
result_backend = config['celery']['result_backend']
#: List of modules to import when the Celery worker starts.
imports = ['kpiit.tasks']
#: Scheduled tasks configuration (aka cronjobs).
def register_custom_serializer() -> None:
from kombu.serialization import register
register('customjson', custom_json_dumps, json.loads,
content_type='application/x-customjson',
content_encoding='utf-8')
__author__ = "Nils Tobias Schmidt"
__email__ = "schmidt89 at informatik.uni-marburg.de"
import pickle
from celery import Celery
from kombu.serialization import register
from androlyze.Constants import PROJECT_NAME
from androlyze.celery.celerysettings import settings
from androlyze.settings import *
from androlyze.util import Util
from androlyze.analyze.distributed.tasks.AnalyzeTask import AnalyzeTask
from celery.registry import tasks
# worker has to import androguard too
Util.set_androguard_path(settings)
# set pickle to specific protocol
register('pickle', lambda s: pickle.dumps(s, 2), lambda s: pickle.loads(s),
content_type='application/x-python-serialize',
content_encoding='binary')
app = Celery(PROJECT_NAME)
# load config
app.config_from_object(CELERY_CONF)
if __name__ == '__main__':
app.start()
worker_sent_task_events = True
# Prefetching is only necessary with high-latency brokers, and apparently can cause problems
worker_prefetch_multiplier = 1
app = Celery(task_cls=CredentialedGirderTask, config_source=CeleryAppConfig)
@app.on_after_configure.connect
def setupPeriodicTasks(sender, **kwargs):
from isic_archive.tasks import maybeSendIngestionNotifications
sender.add_periodic_task(
30,
maybeSendIngestionNotifications.s(),
name='Send any necessary notifications for ingested batches.')
@worker_process_init.connect
def addMailTemplates(sender, **kwargs):
"""Perform the necessary steps from IsicArchive.load."""
mail_utils.addTemplateDirectory(pkg_resources.resource_filename(
'isic_archive', 'mail_templates'),
prepend=True)
register('jsonpickle',
jsonpickle.encode,
jsonpickle.decode,
content_type='application/json',
content_encoding='utf-8')
Queue('celery_blue', routing_key='celery_blue'),
Queue('celery_cell', routing_key='celery_cell'),
Queue('celery_content', routing_key='celery_content'),
Queue('celery_default', routing_key='celery_default'),
Queue('celery_export', routing_key='celery_export'),
Queue('celery_incoming', routing_key='celery_incoming'),
Queue('celery_monitor', routing_key='celery_monitor'),
Queue('celery_ocid', routing_key='celery_ocid'),
Queue('celery_reports', routing_key='celery_reports'),
Queue('celery_upload', routing_key='celery_upload'), # BBB
Queue('celery_wifi', routing_key='celery_wifi'),
) #: List of :class:`kombu.Queue` instances.
register('internal_json',
simplejson.dumps,
simplejson.loads,
content_type='application/x-internaljson',
content_encoding='utf-8')
def configure_celery(celery_app):
"""
Configure the celery app stored in :data:`ichnaea.async.app.celery_app`.
This is executed both inside the master worker process and once in
each forked worker process.
This parses the application ini and reads in the
:mod:`ichnaea.async.settings`.
"""
# This happens at module import time and depends on a properly
if isinstance(v, list):
kwargs[k] = [try_cast(val) for val in v]
if isinstance(v, dict):
cast_item(v)
return o
def dumps(o):
with superdesk.app.app_context():
return MongoJSONEncoder().encode(o)
register('eve/json', dumps, loads, content_type='application/json')
def handle_exception(exc):
"""Log exception to logger and sentry."""
logger.exception(exc)
superdesk.app.sentry.captureException()
class AppContextTask(TaskBase):
abstract = True
serializer = 'eve/json'
app_errors = (
SuperdeskError,
werkzeug.exceptions.InternalServerError, # mongo layer err
)
def test_register(self):
register(None, None, None, None)
return data
# Encoder function
def _dumps(obj):
return json.dumps(obj, cls=NautobotKombuJSONEncoder)
# Decoder function
def _loads(obj):
return json.loads(obj, object_hook=nautobot_kombu_json_loads_hook)
# Register the custom serialization type
register("nautobot_json",
_dumps,
_loads,
content_type="application/x-nautobot-json",
content_encoding="utf-8")
#
# nautobot_task
#
# By exposing `shared_task` within our own namespace, we leave the door open to
# extending and expanding the usage and meaning of shared_task without having
# to undergo further refactoring of task's decorators. We could also transparently
# swap out shared_task to a custom base task.
#
nautobot_task = shared_task
if isinstance(v, list):
kwargs[k] = [try_cast(val) for val in v]
if isinstance(v, dict):
cast_item(v)
return o
def dumps(o):
with superdesk.app.app_context():
return MongoJSONEncoder().encode(o)
register('eve/json', dumps, loads, content_type='application/json')
def handle_exception(exc):
"""Log exception to logger and sentry."""
logger.exception(exc)
superdesk.app.sentry.captureException()
class AppContextTask(TaskBase):
abstract = True
serializer = 'eve/json'
app_errors = (
SuperdeskError,
werkzeug.exceptions.InternalServerError, # mongo layer err
)
def filter(self, rec):
if not hij_log_filter:
return True
for filter_entry in hij_log_filter:
for filter_entry_key in filter_entry:
if rec.__dict__[filter_entry_key] == filter_entry[filter_entry_key]:
return True
return False
mail_log.addFilter(HijackLogFilter())
hij_log.addFilter(HijackLogFilter())
# additional serializer for pg-amqp messages
serialization.register(
"txtjson", json.dumps, json.loads, content_type="text", content_encoding="utf-8"
)
class Detection:
"""
Detection Service.
"""
def __init__(self):
self.worker = None
signal.signal(signal.SIGTERM, self.exit)
signal.signal(signal.SIGINT, self.exit)
signal.signal(signal.SIGCHLD, signal.SIG_IGN)
def run(self) -> NoReturn:
