def _connect_signals(self):
"""Connect callbacks to celery signals.
Since we are creating partials here, the weak arg must be False.
"""
# Beat
if self.options["show_beat"]:
beat_init.connect(
slack_beat_init(**self.options),
weak=False
)
# Celery
if self.options["show_startup"]:
celeryd_init.connect(
slack_celery_startup(**self.options),
weak=False
)
if self.options["show_shutdown"]:
worker_shutdown.connect(
slack_celery_shutdown(**self.options),
weak=False
)
# Task
task_prerun.connect(
slack_task_prerun(**self.options),
weak=False
)
def prepare_celery():
"""Runs the djcelery loader which is required to set the correct backend/scheduler
Then adds some signal callbacks to ensure all tasks are registered correctly.
"""
djcelery.setup_loader()
beat_init.connect(beat_update_schedule)
worker_init.connect(celeryd_discover_tasks)
def install(cls):
if hasattr(cls, "_instance"):
return
self = cls()
# Make sure to keep a reference alive so this is not garbage collected.
cls._instance = self
worker_ready.connect(self.on_worker_ready)
beat_init.connect(self.on_beat_init)
before_task_publish.connect(self.on_before_task_publish)
after_task_publish.connect(self.on_after_task_publish)
try:
cassandra_host = connections['default'].settings_dict['HOST'].split(',')
keyspace = connections['default'].settings_dict['NAME']
user = connections['default'].settings_dict['USER']
password = connections['default'].settings_dict['PASSWORD']
auth_provider = PlainTextAuthProvider(username=user, password=password)
if cql_cluster is not None:
cql_cluster.shutdown()
if cql_session is not None:
cql_session.shutdown()
connection.setup(cassandra_host, keyspace, auth_provider=auth_provider)
except NoHostAvailable:
pass
# Initialize worker context for both standard and periodic tasks, just in the
# case the 'default' connection is defined
# set the default Django settings module for the 'celery' program.
worker_process_init.connect(cassandra_init)
beat_init.connect(cassandra_init)
app = Celery('scraper_module')
# Using a string here means the worker will not have to
# pickle the object when using Windows.
app.config_from_object('django.conf:settings')
# set the default Django settings module for the 'celery' program.
os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'settings')
app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
from flask import session
from snms.core import signals
from snms.database import tsdb
from .core import SnmsCelery
__all__ = ('celery', )
def cassandra_init(**kwargs):
""" Initialize a clean Cassandra connection. """
tsdb.restart()
worker_process_init.connect(cassandra_init)
beat_init.connect(cassandra_init)
#: The Celery instance for all SNMS tasks
celery = SnmsCelery('snms')
@signals.app_created.connect
def _load_default_modules(app, **kwargs):
celery.loader.import_default_modules() # load all tasks
@import_modules.connect
def _import_modules(*args, **kwargs):
import snms.tasks
signals.import_tasks.send()
plugin = plugin.lower()
try:
__import__("kral.plugins." + plugin + ".tasks", fromlist=["*"])
except ImportError:
raise ImportError("Module %s does not exist." % plugin)
def kral_init(**kwargs):
from djcelery.models import PeriodicTask, PeriodicTasks
PeriodicTask.objects.all().delete()
PeriodicTasks.objects.all().delete()
cache.clear()
beat_init.connect(kral_init)
def fixurl(url):
domain, query = urllib.splitquery(url)
new_url = None
if query:
query = re.sub("utm_(source|medium|campaign)\=([^&]+)&?", "", query)
new_url = urlparse.urljoin(domain, "?" + query)
else:
new_url = domain
return new_url
@task
def url_expand(url, query, n=1, original_url=None, **kwargs):
