def _connect_signals(self): """Connect callbacks to celery signals. Since we are creating partials here, the weak arg must be False. """ # Beat if self.options["show_beat"]: beat_init.connect( slack_beat_init(**self.options), weak=False ) # Celery if self.options["show_startup"]: celeryd_init.connect( slack_celery_startup(**self.options), weak=False ) if self.options["show_shutdown"]: worker_shutdown.connect( slack_celery_shutdown(**self.options), weak=False ) # Task task_prerun.connect( slack_task_prerun(**self.options), weak=False )
def prepare_celery(): """Runs the djcelery loader which is required to set the correct backend/scheduler Then adds some signal callbacks to ensure all tasks are registered correctly. """ djcelery.setup_loader() beat_init.connect(beat_update_schedule) worker_init.connect(celeryd_discover_tasks)
def install(cls): if hasattr(cls, "_instance"): return self = cls() # Make sure to keep a reference alive so this is not garbage collected. cls._instance = self worker_ready.connect(self.on_worker_ready) beat_init.connect(self.on_beat_init) before_task_publish.connect(self.on_before_task_publish) after_task_publish.connect(self.on_after_task_publish)
try: cassandra_host = connections['default'].settings_dict['HOST'].split(',') keyspace = connections['default'].settings_dict['NAME'] user = connections['default'].settings_dict['USER'] password = connections['default'].settings_dict['PASSWORD'] auth_provider = PlainTextAuthProvider(username=user, password=password) if cql_cluster is not None: cql_cluster.shutdown() if cql_session is not None: cql_session.shutdown() connection.setup(cassandra_host, keyspace, auth_provider=auth_provider) except NoHostAvailable: pass # Initialize worker context for both standard and periodic tasks, just in the # case the 'default' connection is defined # set the default Django settings module for the 'celery' program. worker_process_init.connect(cassandra_init) beat_init.connect(cassandra_init) app = Celery('scraper_module') # Using a string here means the worker will not have to # pickle the object when using Windows. app.config_from_object('django.conf:settings') # set the default Django settings module for the 'celery' program. os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings') app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
from flask import session from snms.core import signals from snms.database import tsdb from .core import SnmsCelery __all__ = ('celery', ) def cassandra_init(**kwargs): """ Initialize a clean Cassandra connection. """ tsdb.restart() worker_process_init.connect(cassandra_init) beat_init.connect(cassandra_init) #: The Celery instance for all SNMS tasks celery = SnmsCelery('snms') @signals.app_created.connect def _load_default_modules(app, **kwargs): celery.loader.import_default_modules() # load all tasks @import_modules.connect def _import_modules(*args, **kwargs): import snms.tasks signals.import_tasks.send()
plugin = plugin.lower() try: __import__("kral.plugins." + plugin + ".tasks", fromlist=["*"]) except ImportError: raise ImportError("Module %s does not exist." % plugin) def kral_init(**kwargs): from djcelery.models import PeriodicTask, PeriodicTasks PeriodicTask.objects.all().delete() PeriodicTasks.objects.all().delete() cache.clear() beat_init.connect(kral_init) def fixurl(url): domain, query = urllib.splitquery(url) new_url = None if query: query = re.sub("utm_(source|medium|campaign)\=([^&]+)&?", "", query) new_url = urlparse.urljoin(domain, "?" + query) else: new_url = domain return new_url @task def url_expand(url, query, n=1, original_url=None, **kwargs):