def __init__(self, app):
Celery.__init__(self,
'app',
backend=app.config['CELERY_RESULT_BACKEND'],
broker=app.config['CELERY_BROKER_URL'])
self.make_celery(app)
self.process = self.make_celery_processing()
def __init__(self, app_name, *args, **kwargs):
Celery.__init__(self, *args, **kwargs)
self._config = adsputils.load_config()
self._session = None
self._engine = None
self._app_name = app_name
self.logger = adsputils.setup_logging(app_name) #default logger
def __init__(self, *args, **kwargs):
global current_flask_app
Celery.__init__(self, *args, **kwargs)
flask_app = kwargs.pop('flask_app', current_flask_app)
if flask_app:
self.init_app(flask_app)
else:
self.flask_app = None
def init_app(self: 'FlaskCelery', app: Flask) -> None:
app.config.setdefault('CELERY_ALWAYS_EAGER', False) # important so it doesn't get executed locally!
app.config.setdefault('CELERY_TASK_SERIALIZER', 'msgpack')
app.config.setdefault('CELERY_RESULT_SERIALIZER', 'msgpack')
app.config.setdefault('CELERY_ACCEPT_CONTENT', ['msgpack'])
app.config.setdefault('CELERY_IGNORE_RESULT', False)
app.config.setdefault('CELERY_TRACK_STARTED', True)
config = get_config()
Celery.__init__(cast(Celery, self), app.import_name,
backend=config.get('CELERY', 'backend'),
broker=config.get('CELERY', 'broker'))
self.conf.update(app.config)
self._monkey_patch(app)
def __init__(self, app_name, *args, **kwargs):
"""
:param: app_name - string, name of the application (can be anything)
:keyword: local_config - dict, configuration that should be applied
over the default config (that is loaded from config.py and local_config.py)
"""
local_config = None
self._config = load_config(extra_frames=1)
if 'local_config' in kwargs and kwargs['local_config']:
local_config = kwargs.pop('local_config')
self._config.update(local_config) #our config
self.logger = setup_logging(app_name,
level=self._config.get(
'LOGGING_LEVEL', 'INFO'))
# make sure that few important params are set for celery
if 'broker' not in kwargs:
kwargs['broker'] = self._config.get('CELERY_BROKER', 'pyamqp://'),
if 'include' not in kwargs:
cm = None
if 'CELERY_INCLUDE' not in self._config:
cm = self._get_callers_module()
parts = cm.split('.')
parts[-1] = 'tasks'
cm = '.'.join(parts)
if '.tasks' not in cm:
self.logger.debug(
'It seems like you are not importing from \'.tasks\': %s',
cm)
self.logger.warn(
'CELERY_INCLUDE is empty, we have to guess it (correct???): %s',
cm)
kwargs['include'] = self._config.get('CELERY_INCLUDE', [cm])
Celery.__init__(self, *args, **kwargs)
self._set_serializer()
self.conf.update(
self._config
) #celery's config (devs should be careful to avoid clashes)
self._engine = self._session = None
if self._config.get('SQLALCHEMY_URL', None):
self._engine = create_engine(
self._config.get('SQLALCHEMY_URL', 'sqlite:///'),
echo=self._config.get('SQLALCHEMY_ECHO', False))
self._session_factory = sessionmaker()
self._session = scoped_session(self._session_factory)
self._session.configure(bind=self._engine)
if self._config.get('CELERY_DEFAULT_EXCHANGE_TYPE',
'topic') != 'topic':
self.logger.warn('The exchange type is not "topic" - ' \
'are you sure CELERY_DEFAULT_EXCHANGE_TYPE is set properly? (%s)',
self._config.get('CELERY_DEFAULT_EXCHANGE_TYPE', ''))
self.exchange = Exchange(self._config.get('CELERY_DEFAULT_EXCHANGE',
'ads-pipeline'),
type=self._config.get(
'CELERY_DEFAULT_EXCHANGE_TYPE', 'topic'))
self.forwarding_connection = None
if self._config.get('OUTPUT_CELERY_BROKER', None):
# kombu connection is lazy loaded, so it's ok to create now
self.forwarding_connection = BrokerConnection(
self._config['OUTPUT_CELERY_BROKER'])
if self.conf.get('OUTPUT_TASKNAME', None):
@self.task(name=self._config['OUTPUT_TASKNAME'],
exchange=self._config.get('OUTPUT_EXCHANGE',
'ads-pipeline'),
queue=self._config.get('OUTPUT_QUEUE',
'update-record'),
routing_key=self._config.get(
'OUTPUT_QUEUE', 'update-record'))
def _forward_message(self, *args, **kwargs):
"""A handler that can be used to forward stuff out of our
queue. It does nothing (it doesn't process data)"""
self.logger.error('We should have never been called directly! %s' % \
(args, kwargs))
self._forward_message = _forward_message
def __init__(self, app_name, *args, **kwargs):
"""
:param: app_name - string, name of the application (can be anything)
:keyword: local_config - dict, configuration that should be applied
over the default config (that is loaded from config.py and local_config.py)
"""
proj_home = None
if 'proj_home' in kwargs:
proj_home = kwargs.pop('proj_home')
self._config = load_config(extra_frames=1,
proj_home=proj_home,
app_name=app_name)
local_config = None
if 'local_config' in kwargs and kwargs['local_config']:
local_config = kwargs.pop('local_config')
self._config.update(local_config) # our config
if not proj_home:
proj_home = self._config.get('PROJ_HOME', None)
self.logger = setup_logging(
app_name,
proj_home=proj_home,
level=self._config.get('LOGGING_LEVEL', 'INFO'),
attach_stdout=self._config.get('LOG_STDOUT', False))
# make sure that few important params are set for celery
if 'broker' not in kwargs:
kwargs['broker'] = self._config.get('CELERY_BROKER', 'pyamqp://'),
if 'include' not in kwargs:
cm = None
if 'CELERY_INCLUDE' not in self._config:
cm = self._get_callers_module()
parts = cm.split('.')
parts[-1] = 'tasks'
cm = '.'.join(parts)
if '.tasks' not in cm:
self.logger.debug(
'It seems like you are not importing from \'.tasks\': %s',
cm)
self.logger.warning(
'CELERY_INCLUDE is empty, we have to guess it (correct???): %s',
cm)
kwargs['include'] = self._config.get('CELERY_INCLUDE', [cm])
Celery.__init__(self, *args, **kwargs)
self._set_serializer()
self.conf.update(
self._config
) # celery's config (devs should be careful to avoid clashes)
self._engine = self._session = None
if self._config.get('SQLALCHEMY_URL', None):
self._engine = create_engine(
self._config.get('SQLALCHEMY_URL', 'sqlite:///'),
echo=self._config.get('SQLALCHEMY_ECHO', False))
self._session_factory = sessionmaker()
self._session = scoped_session(self._session_factory)
self._session.configure(bind=self._engine)
register_after_fork(self._engine, self._engine.dispose)
if self._config.get('CELERY_DEFAULT_EXCHANGE_TYPE',
'topic') != 'topic':
self.logger.warn('The exchange type is not "topic" - ' \
'are you sure CELERY_DEFAULT_EXCHANGE_TYPE is set properly? (%s)',
self._config.get('CELERY_DEFAULT_EXCHANGE_TYPE', ''))
self.exchange = Exchange(self._config.get('CELERY_DEFAULT_EXCHANGE',
'ads-pipeline'),
type=self._config.get(
'CELERY_DEFAULT_EXCHANGE_TYPE', 'topic'))
self.forwarding_connection = None
if self._config.get('OUTPUT_CELERY_BROKER', None):
# kombu connection is lazy loaded, so it's ok to create now
self.forwarding_connection = BrokerConnection(
self._config['OUTPUT_CELERY_BROKER'])
if self.conf.get('OUTPUT_TASKNAME', None):
@self.task(name=self._config['OUTPUT_TASKNAME'],
exchange=self._config.get('OUTPUT_EXCHANGE',
'ads-pipeline'),
queue=self._config.get('OUTPUT_QUEUE',
'update-record'),
routing_key=self._config.get(
'OUTPUT_QUEUE', 'update-record'))
def _forward_message(self, *args, **kwargs):
"""A handler that can be used to forward stuff out of our
queue. It does nothing (it doesn't process data)"""
self.logger.error('We should have never been called directly! %s' % \
(args, kwargs))
self._forward_message = _forward_message
# HTTP connection pool
# - The maximum number of retries each connection should attempt: this
# applies only to failed DNS lookups, socket connections and connection timeouts,
# never to requests where data has made it to the server. By default,
# requests does not retry failed connections.
# http://docs.python-requests.org/en/latest/api/?highlight=max_retries#requests.adapters.HTTPAdapter
self.client = requests.Session()
http_adapter = requests.adapters.HTTPAdapter(
pool_connections=self._config.get(u'REQUESTS_POOL_CONNECTIONS',
10),
pool_maxsize=self._config.get(u'REQUESTS_POOL_MAXSIZE', 1000),
max_retries=self._config.get(u'REQUESTS_POOL_RETRIES', 3),
pool_block=False)
self.client.mount(u'http://', http_adapter)
