def create_task(mapper, connection, target):
assert target.id is not None
# TODO(boertel) somehow target isn't the object from the database and the
# deserialization of schedule isn't done
job = Job.query.filter_by(id=target.id).first()
if job:
celery_app = celery.get_celery_app()
task_name = 'cuckoo.tasks.webhook.webhook'
args = [
str(job.id),
str(job.application_id),
]
kwargs = {
'url': job.url,
'data': job.params,
}
entry = RedBeatSchedulerEntry(
str(job.id),
task_name,
job.schedule,
args=args,
kwargs=kwargs,
app=celery_app,
)
entry.save()
def run_job(job):
""" Run this job immediately if this job doesn't have a schedule,
If this job has a schedule, schedule it to run.
"""
spider = Spider.get(Spider.name == job.spider_name)
project_name = spider.project_name
if job.schedule:
minute, hour, day_of_week, day_of_month, month_of_year = job.schedule.split(
' ')
utc_hour = crontab_hour_to_utc(hour, config.CRONTAB_TIMEZONE)
interval = crontab(minute, utc_hour, day_of_week, day_of_month,
month_of_year)
entry = RedBeatSchedulerEntry(job.job_id,
'server.tasks.spiders.crawl',
interval,
kwargs={
'project': project_name,
'spider': job.spider_name,
'job_id': job.job_id,
'parameters': job.parameters,
},
app=app)
entry.save().reschedule()
else:
crawl.delay(project_name, job.spider_name, job.job_id, job.parameters)
def create_schedule(self, app):
data = {
'classname': self.classname,
'params': self.kwargs
}
interval = schedules.schedule(run_every=self.interval)
entry = RedBeatSchedulerEntry(
self.taskname,
'kryptobot.workers.harvester.tasks.launch_harvester',
interval,
kwargs=data,
app=app
)
entry.save()
def start_timelapse_task(header: str, run_every: float, expire_at: str,
data: dict) -> str:
try:
interval = celery.schedules.schedule(run_every=run_every) # seconds
entry = RedBeatSchedulerEntry('timelapse',
'cam_task.capture',
interval,
args=[header, data],
app=app)
entry.save()
return entry.key
except Exception as e:
logger.error(traceback.format_exc())
raise TaskError(e)
def stop_timelapse_task(key: str) -> bool:
try:
entry = RedBeatSchedulerEntry.from_key(key, app=app)
entry.delete()
return True
except Exception as e:
logger.error(traceback.format_exc())
raise TaskError(e)
def _handle_timer(timer):
""" Define a cron-based task
"""
print(
f"\tDevice #{timer['device_id']} runs on <{timer['cron']}> => <{timer['execute']}>"
)
c = timer["cron"].split()
schedule = crontab(minute=c[0],
hour=c[1],
day_of_month=c[2],
month_of_year=c[3],
day_of_week=c[4])
e = Entry(timer['slug'],
'hqcontrol.tasks.execute_script',
schedule,
args=[timer["device_id"], timer["execute"]],
app=app)
e.save()
_key_to_cache(e.key.strip())
return e
def test_delete(self):
initial = self.create_entry()
initial.save()
e = RedBeatSchedulerEntry.from_key(initial.key, app=self.app)
e.delete()
exists = self.app.redbeat_redis.exists(initial.key)
self.assertFalse(exists)
score = self.app.redbeat_redis.zrank(self.app.conf.REDBEAT_SCHEDULE_KEY, initial.key)
self.assertIsNone(score)
def test_delete(self):
initial = self.create_entry()
initial.save()
e = RedBeatSchedulerEntry.from_key(initial.key, app=self.app)
e.delete()
exists = self.app.redbeat_redis.exists(initial.key)
self.assertFalse(exists)
score = self.app.redbeat_redis.zrank(self.app.redbeat_conf.schedule_key, initial.key)
self.assertIsNone(score)
def doAction(args):
if args.add:
url = args.url
key = args.key
if key is None:
print("require key")
return -1
timeout = args.timeout
repeat = args.repeat
if url is None:
print('url is required')
return -1
entry = Entry(f'urlCheck_{key}',
'tasks.urlSpeed',
repeat,
args=['GET', url, timeout, key],
app=tasks.app)
entry.save()
print(f"url added, key={entry.key}, store key for deletion")
elif args.delete:
key = args.key
if key is None:
print("require key")
return -1
entry = Entry.from_key(key, app=tasks.app)
entry.delete()
print("task deleted")
def check_tasks(self, services: list, create: bool = True):
for service_ in services:
service = Service(service_)
try:
entry = Entry.from_key(redbeat_key_prefix +
BASE_KEY.format(service.name),
app=tasks.app)
# todo: check entry existence
except:
if create:
if service.type == 1:
entry = Entry(BASE_KEY.format(service.name),
f'tasks.{service.type_name}',
service.repeat_period,
args=[
service.metadata['url'],
service.metadata['method'],
service.metadata['timeout'],
service.metadata.get('body', None)
],
app=tasks.app)
entry.save()
print(entry.key)
return 0
def __init__(self, worker, **options):
"""Initialize class instance.
Passes User Options defined by the worker to :class:`GraphApiTask`
base class and schedules periodic data extraction from MS Graph API
into the post-processing queue.
Note
----
By default, no config file is expected by the app. All options are taken
either from the command-line interface (CLI) or the execution environment
(ENV).
Config file takes precedence over the CLI, which in turn takes precedence
over ENV.
"""
from ms_graph_exporter.celery.graph_api_base import GraphApiTask
GraphApiTask.config_update(**options)
if options["graph_app_config"]:
GraphApiTask.config_update_from_file()
logger = get_logger(__name__)
map_interval = (GraphApiTask._config["graph_streams"] *
GraphApiTask._config["graph_stream_frame"])
entry = RedBeatSchedulerEntry("periodic_map_streams",
"graph.map_streams",
map_interval,
app=worker.app)
logger.info("adding schedule entry, %s", entry)
entry.save()
def _read_cached_scheduled_tasks():
"""Read in any cached task keys from a previous run if they were left because of a crash
"""
results = []
try:
with open(".task_cache", "r") as f:
for key in f.readlines():
try:
e = Entry.from_key(key.strip(), app=app)
if e is not None:
results.append(e)
except Exception as e:
print("Task not found: ", e)
except FileNotFoundError as e:
return None
except Exception as e:
print("Error: ", e)
return results
def test_next(self):
initial = self.create_entry().save()
now = self.app.now()
n = initial.next(last_run_at=now)
# new entry has updated run info
self.assertNotEqual(initial, n)
self.assertEqual(n.last_run_at, now)
self.assertEqual(initial.total_run_count + 1, n.total_run_count)
# updated meta was stored into redis
loaded = RedBeatSchedulerEntry.from_key(initial.key, app=self.app)
self.assertEqual(loaded.last_run_at, now)
self.assertEqual(loaded.total_run_count, initial.total_run_count + 1)
# new entry updated the schedule
redis = self.app.redbeat_redis
self.assertEqual(redis.zscore(self.app.conf.REDBEAT_SCHEDULE_KEY, n.key), n.score)
def test_next(self):
initial = self.create_entry().save()
now = self.app.now()
n = initial.next(last_run_at=now)
# new entry has updated run info
self.assertNotEqual(initial, n)
self.assertEqual(n.last_run_at, now)
self.assertEqual(initial.total_run_count + 1, n.total_run_count)
# updated meta was stored into redis
loaded = RedBeatSchedulerEntry.from_key(initial.key, app=self.app)
self.assertEqual(loaded.last_run_at, now)
self.assertEqual(loaded.total_run_count, initial.total_run_count + 1)
# new entry updated the schedule
redis = self.app.redbeat_redis
self.assertEqual(redis.zscore(self.app.redbeat_conf.schedule_key, n.key), n.score)
def test_next(self):
initial = self.create_entry().save()
now = self.app.now()
# 3.x is naive but 4.x is aware
now = maybe_make_aware(now)
n = initial.next(last_run_at=now)
self.assertIsNotNone(now.tzinfo)
self.assertEqual(n.last_run_at, now)
self.assertEqual(initial.total_run_count + 1, n.total_run_count)
# updated meta was stored into redis
loaded = RedBeatSchedulerEntry.from_key(initial.key, app=self.app)
self.assertEqual(loaded.last_run_at, now)
self.assertEqual(loaded.total_run_count, initial.total_run_count + 1)
# new entry updated the schedule
redis = self.app.redbeat_redis
self.assertEqual(
redis.zscore(self.app.redbeat_conf.schedule_key, n.key), n.score)
def __init__(self, key='redbeat:watchdog-task'):
self.key = key
# Return KeyError or RedisConnectionError if encounter problem with Redis
self.job = RedBeatSchedulerEntry.from_key(key=self.key, app=celery)
def test_from_key_nonexistent_key(self):
with self.assertRaises(KeyError):
RedBeatSchedulerEntry.from_key('doesntexist', self.app)
from celery.schedules import schedule, crontab, solar
from redbeat import RedBeatSchedulerEntry
import configs
celery = Celery(__name__)
celery.config_from_object(configs)
@celery.task
def my_add(a, b):
result = a + b
print('{} + {} = {}'.format(a, b, result))
return result
@celery.task
def my_subtraction(a, b):
result = a - b
print('{} - {} = {}'.format(a, b, result))
return result
interval = schedule(run_every=5) # seconds
cron = crontab()
entry = RedBeatSchedulerEntry('my_add', '{}.my_add'.format(__name__), interval, args=[5, 2], app=celery)
entry.save()
interval = schedule(run_every=5) # seconds
entry = RedBeatSchedulerEntry('my_subtraction', 'celery_redbeat_demo.my_subtraction', interval, args=[5, 2], app=celery)
entry.save()
def test_from_key_nonexistent_key(self):
with self.assertRaises(KeyError):
RedBeatSchedulerEntry.from_key('doesntexist', self.app)
def _get_entry_from_job(job):
conf = RedBeatConfig(app)
return RedBeatSchedulerEntry.from_key('{}{}'.format(
conf.key_prefix, job.job_id),
app=app)
def test_from_key_missing_meta(self):
initial = self.create_entry().save()
loaded = RedBeatSchedulerEntry.from_key(initial.key, self.app)
self.assertEqual(initial.task, loaded.task)
self.assertIsNone(loaded.last_run_at)
# Demonstrate how to update data in redis beat so worker can pickup updated data without restarting the beat
# Run those command first:
# celery beat -A update_task_redbeat -S redbeat.RedBeatScheduler -l INFO
# celery -A update_task_redbeat worker -l INFO --concurrency=2
# Then in python console, use these to update
from datetime import timedelta
import update_task_redbeat
from redbeat import RedBeatSchedulerEntry as Entry
from celery.schedules import schedule
app = update_task_redbeat.app
e = Entry.from_key('redbeat:task_259', app=app)
# Update schedule to 5 seconds
e.schedule = schedule(5)
e.save()
# Update metadata to print Goodbye instead of Hello
# We simply change the args passed to the task
e.args = ["Goodbye"]
e.save()
# Delete task
e.delete()
"""
"""
import cluster
from redbeat import RedBeatSchedulerEntry as Entry
entry = Entry('thingo', 'cluster.add_task', 10, args=(5, 10), app=cluster.app)
entry.save()
print entry.key, "->", entry.interval
def test_from_key_missing_meta(self):
initial = self.create_entry().save()
loaded = RedBeatSchedulerEntry.from_key(initial.key, self.app)
self.assertEqual(initial.task, loaded.task)
self.assertIsNone(loaded.last_run_at)
def delete_task(mapper, connection, target):
celery_app = celery.get_celery_app()
entry = RedBeatSchedulerEntry.from_key(target.get_redis_key(), celery_app)
entry.delete()
@app.task(bind=True)
def print_task(self, x):
print x
return x
@app.task(bind=True)
def add_task(self, x, y):
return x + y
@app.task(bind=True)
def bad_task(self):
raise RuntimeError("intentional error")
@app.task(bind=True)
def chaining_task(self, add):
return celery.chain(add_task.s(*add), print_task.s()).apply_async()
# you can do this from a separate threads
from redbeat import RedBeatSchedulerEntry as Entry
e = Entry('thingo',
'cluster.chaining_task',
10,
args=([5, 6], ),
options={'schedule_id': 'testid'},
app=app)
e.save()
