def __init__(self, app): Celery.__init__(self, 'app', backend=app.config['CELERY_RESULT_BACKEND'], broker=app.config['CELERY_BROKER_URL']) self.make_celery(app) self.process = self.make_celery_processing()
def __init__(self, app_name, *args, **kwargs): Celery.__init__(self, *args, **kwargs) self._config = adsputils.load_config() self._session = None self._engine = None self._app_name = app_name self.logger = adsputils.setup_logging(app_name) #default logger
def __init__(self, *args, **kwargs): global current_flask_app Celery.__init__(self, *args, **kwargs) flask_app = kwargs.pop('flask_app', current_flask_app) if flask_app: self.init_app(flask_app) else: self.flask_app = None
def init_app(self: 'FlaskCelery', app: Flask) -> None: app.config.setdefault('CELERY_ALWAYS_EAGER', False) # important so it doesn't get executed locally! app.config.setdefault('CELERY_TASK_SERIALIZER', 'msgpack') app.config.setdefault('CELERY_RESULT_SERIALIZER', 'msgpack') app.config.setdefault('CELERY_ACCEPT_CONTENT', ['msgpack']) app.config.setdefault('CELERY_IGNORE_RESULT', False) app.config.setdefault('CELERY_TRACK_STARTED', True) config = get_config() Celery.__init__(cast(Celery, self), app.import_name, backend=config.get('CELERY', 'backend'), broker=config.get('CELERY', 'broker')) self.conf.update(app.config) self._monkey_patch(app)
def __init__(self, app_name, *args, **kwargs): """ :param: app_name - string, name of the application (can be anything) :keyword: local_config - dict, configuration that should be applied over the default config (that is loaded from config.py and local_config.py) """ local_config = None self._config = load_config(extra_frames=1) if 'local_config' in kwargs and kwargs['local_config']: local_config = kwargs.pop('local_config') self._config.update(local_config) #our config self.logger = setup_logging(app_name, level=self._config.get( 'LOGGING_LEVEL', 'INFO')) # make sure that few important params are set for celery if 'broker' not in kwargs: kwargs['broker'] = self._config.get('CELERY_BROKER', 'pyamqp://'), if 'include' not in kwargs: cm = None if 'CELERY_INCLUDE' not in self._config: cm = self._get_callers_module() parts = cm.split('.') parts[-1] = 'tasks' cm = '.'.join(parts) if '.tasks' not in cm: self.logger.debug( 'It seems like you are not importing from \'.tasks\': %s', cm) self.logger.warn( 'CELERY_INCLUDE is empty, we have to guess it (correct???): %s', cm) kwargs['include'] = self._config.get('CELERY_INCLUDE', [cm]) Celery.__init__(self, *args, **kwargs) self._set_serializer() self.conf.update( self._config ) #celery's config (devs should be careful to avoid clashes) self._engine = self._session = None if self._config.get('SQLALCHEMY_URL', None): self._engine = create_engine( self._config.get('SQLALCHEMY_URL', 'sqlite:///'), echo=self._config.get('SQLALCHEMY_ECHO', False)) self._session_factory = sessionmaker() self._session = scoped_session(self._session_factory) self._session.configure(bind=self._engine) if self._config.get('CELERY_DEFAULT_EXCHANGE_TYPE', 'topic') != 'topic': self.logger.warn('The exchange type is not "topic" - ' \ 'are you sure CELERY_DEFAULT_EXCHANGE_TYPE is set properly? (%s)', self._config.get('CELERY_DEFAULT_EXCHANGE_TYPE', '')) self.exchange = Exchange(self._config.get('CELERY_DEFAULT_EXCHANGE', 'ads-pipeline'), type=self._config.get( 'CELERY_DEFAULT_EXCHANGE_TYPE', 'topic')) self.forwarding_connection = None if self._config.get('OUTPUT_CELERY_BROKER', None): # kombu connection is lazy loaded, so it's ok to create now self.forwarding_connection = BrokerConnection( self._config['OUTPUT_CELERY_BROKER']) if self.conf.get('OUTPUT_TASKNAME', None): @self.task(name=self._config['OUTPUT_TASKNAME'], exchange=self._config.get('OUTPUT_EXCHANGE', 'ads-pipeline'), queue=self._config.get('OUTPUT_QUEUE', 'update-record'), routing_key=self._config.get( 'OUTPUT_QUEUE', 'update-record')) def _forward_message(self, *args, **kwargs): """A handler that can be used to forward stuff out of our queue. It does nothing (it doesn't process data)""" self.logger.error('We should have never been called directly! %s' % \ (args, kwargs)) self._forward_message = _forward_message
def __init__(self, app_name, *args, **kwargs): """ :param: app_name - string, name of the application (can be anything) :keyword: local_config - dict, configuration that should be applied over the default config (that is loaded from config.py and local_config.py) """ proj_home = None if 'proj_home' in kwargs: proj_home = kwargs.pop('proj_home') self._config = load_config(extra_frames=1, proj_home=proj_home, app_name=app_name) local_config = None if 'local_config' in kwargs and kwargs['local_config']: local_config = kwargs.pop('local_config') self._config.update(local_config) # our config if not proj_home: proj_home = self._config.get('PROJ_HOME', None) self.logger = setup_logging( app_name, proj_home=proj_home, level=self._config.get('LOGGING_LEVEL', 'INFO'), attach_stdout=self._config.get('LOG_STDOUT', False)) # make sure that few important params are set for celery if 'broker' not in kwargs: kwargs['broker'] = self._config.get('CELERY_BROKER', 'pyamqp://'), if 'include' not in kwargs: cm = None if 'CELERY_INCLUDE' not in self._config: cm = self._get_callers_module() parts = cm.split('.') parts[-1] = 'tasks' cm = '.'.join(parts) if '.tasks' not in cm: self.logger.debug( 'It seems like you are not importing from \'.tasks\': %s', cm) self.logger.warning( 'CELERY_INCLUDE is empty, we have to guess it (correct???): %s', cm) kwargs['include'] = self._config.get('CELERY_INCLUDE', [cm]) Celery.__init__(self, *args, **kwargs) self._set_serializer() self.conf.update( self._config ) # celery's config (devs should be careful to avoid clashes) self._engine = self._session = None if self._config.get('SQLALCHEMY_URL', None): self._engine = create_engine( self._config.get('SQLALCHEMY_URL', 'sqlite:///'), echo=self._config.get('SQLALCHEMY_ECHO', False)) self._session_factory = sessionmaker() self._session = scoped_session(self._session_factory) self._session.configure(bind=self._engine) register_after_fork(self._engine, self._engine.dispose) if self._config.get('CELERY_DEFAULT_EXCHANGE_TYPE', 'topic') != 'topic': self.logger.warn('The exchange type is not "topic" - ' \ 'are you sure CELERY_DEFAULT_EXCHANGE_TYPE is set properly? (%s)', self._config.get('CELERY_DEFAULT_EXCHANGE_TYPE', '')) self.exchange = Exchange(self._config.get('CELERY_DEFAULT_EXCHANGE', 'ads-pipeline'), type=self._config.get( 'CELERY_DEFAULT_EXCHANGE_TYPE', 'topic')) self.forwarding_connection = None if self._config.get('OUTPUT_CELERY_BROKER', None): # kombu connection is lazy loaded, so it's ok to create now self.forwarding_connection = BrokerConnection( self._config['OUTPUT_CELERY_BROKER']) if self.conf.get('OUTPUT_TASKNAME', None): @self.task(name=self._config['OUTPUT_TASKNAME'], exchange=self._config.get('OUTPUT_EXCHANGE', 'ads-pipeline'), queue=self._config.get('OUTPUT_QUEUE', 'update-record'), routing_key=self._config.get( 'OUTPUT_QUEUE', 'update-record')) def _forward_message(self, *args, **kwargs): """A handler that can be used to forward stuff out of our queue. It does nothing (it doesn't process data)""" self.logger.error('We should have never been called directly! %s' % \ (args, kwargs)) self._forward_message = _forward_message # HTTP connection pool # - The maximum number of retries each connection should attempt: this # applies only to failed DNS lookups, socket connections and connection timeouts, # never to requests where data has made it to the server. By default, # requests does not retry failed connections. # http://docs.python-requests.org/en/latest/api/?highlight=max_retries#requests.adapters.HTTPAdapter self.client = requests.Session() http_adapter = requests.adapters.HTTPAdapter( pool_connections=self._config.get(u'REQUESTS_POOL_CONNECTIONS', 10), pool_maxsize=self._config.get(u'REQUESTS_POOL_MAXSIZE', 1000), max_retries=self._config.get(u'REQUESTS_POOL_RETRIES', 3), pool_block=False) self.client.mount(u'http://', http_adapter)