def create_periodic_task(cls, name, task, every,
period='seconds', args=[], kwargs={}):
if cls.CELERY_BEAT_SCHEDULER == 'MONGODB':
from celerybeatmongo.models import PeriodicTask
PeriodicTask(
name=name,
task=task,
enabled=True,
args=args,
kwargs=kwargs,
interval=PeriodicTask.Interval(every=every, period=period),
).save()
elif cls.CELERY_BEAT_SCHEDULER == 'REDIS':
from celery.schedules import schedule
from redbeat.schedulers import RedBeatSchedulerEntry
if period != 'seconds':
# do conversion... run_every should be a datetime.timedelta
log.error("Unsupported period {} for redis beat", period)
interval = schedule(run_every=every) # seconds
entry = RedBeatSchedulerEntry(
name,
task,
interval,
args=args,
app=CeleryExt.celery_app
)
entry.save()
else:
log.error(
"Unsupported celery-beat scheduler: {}", cls.CELERY_BEAT_SCHEDULER)
def test_str(self):
periodic = PeriodicTask(task="foo")
periodic.crontab = PeriodicTask.Crontab(minute="0",
hour="*",
day_of_week="*",
day_of_month="10-15",
month_of_year="*")
self.assertEqual("0 * * 10-15 * (m/h/d/dM/MY)", str(periodic.crontab))
def create_crontab_task(
cls,
name: str,
task: str,
minute: str,
hour: str,
day_of_week: str = "*",
day_of_month: str = "*",
month_of_year: str = "*",
args: List[Any] = None,
kwargs: Dict[str, Any] = None,
) -> None:
if args is None:
args = []
if kwargs is None:
kwargs = {}
if cls.CELERYBEAT_SCHEDULER == "MONGODB":
from celerybeatmongo.models import PeriodicTask
PeriodicTask(
name=name,
task=task,
enabled=True,
args=args,
kwargs=kwargs,
crontab=PeriodicTask.Crontab(
minute=minute,
hour=hour,
day_of_week=day_of_week,
day_of_month=day_of_month,
month_of_year=month_of_year,
),
).save()
elif cls.CELERYBEAT_SCHEDULER == "REDIS":
from celery.schedules import crontab
from redbeat.schedulers import RedBeatSchedulerEntry
interval = crontab(
minute=minute,
hour=hour,
day_of_week=day_of_week,
day_of_month=day_of_month,
month_of_year=month_of_year,
)
entry = RedBeatSchedulerEntry(name,
task,
interval,
args=args,
app=CeleryExt.celery_app)
entry.save()
else:
raise AttributeError(
f"Unsupported celery-beat scheduler: {cls.CELERYBEAT_SCHEDULER}"
)
def test_get_from_database(self):
from celerybeatmongo.models import PeriodicTask
PeriodicTask.objects.create(name="a1", task="foo", enabled=True, interval=PeriodicTask.Interval(every=1, period="days"))
PeriodicTask.objects.create(name="b1", task="foo", enabled=True, interval=PeriodicTask.Interval(every=2, period="days"))
PeriodicTask.objects.create(name="c2", task="foo", enabled=False, interval=PeriodicTask.Interval(every=3, period="days"))
scheduler = MongoScheduler(app=self.app)
self.assertEqual(2, len(scheduler.get_from_database())
, "get_from_database should return just enabled tasks")
def task_interval_add_one():
if request.method == "POST":
task = PeriodicTask(
name=str(uuid1()),
task=request.json["task"],
enabled=True,
run_immediately=True,
interval=PeriodicTask.Interval(every=int(request.json["every"]),
period="seconds"),
)
result = task.save()
return result.to_json(), {"Content-Type": "application/json"}
def create_crontab_task(
cls,
name,
task,
minute,
hour,
day_of_week="*",
day_of_month="*",
month_of_year="*",
args=[],
kwargs={},
):
if cls.CELERY_BEAT_SCHEDULER == 'MONGODB':
from celerybeatmongo.models import PeriodicTask
PeriodicTask(
name=name,
task=task,
enabled=True,
args=args,
kwargs=kwargs,
crontab=PeriodicTask.Crontab(
minute=minute,
hour=hour,
day_of_week=day_of_week,
day_of_month=day_of_month,
month_of_year=month_of_year,
),
).save()
elif cls.CELERY_BEAT_SCHEDULER == 'REDIS':
from celery.schedules import crontab
from redbeat.schedulers import RedBeatSchedulerEntry
interval = crontab(
minute=minute,
hour=hour,
day_of_week=day_of_week,
day_of_month=day_of_month,
month_of_year=month_of_year
)
entry = RedBeatSchedulerEntry(
name,
task,
interval,
args=args,
app=CeleryExt.celery_app
)
entry.save()
else:
log.error(
"Unsupported celery-beat scheduler: {}", cls.CELERY_BEAT_SCHEDULER)
def create_periodic_task(
cls,
name: str,
task: str,
every: Union[str, int, timedelta],
period: AllowedTimedeltaPeriods = "seconds",
args: List[Any] = None,
kwargs: Dict[str, Any] = None,
) -> None:
if args is None:
args = []
if kwargs is None:
kwargs = {}
if cls.CELERYBEAT_SCHEDULER == "MONGODB":
from celerybeatmongo.models import PeriodicTask
PeriodicTask(
name=name,
task=task,
enabled=True,
args=args,
kwargs=kwargs,
interval=PeriodicTask.Interval(every=every, period=period),
).save()
elif cls.CELERYBEAT_SCHEDULER == "REDIS":
from celery.schedules import schedule
from redbeat.schedulers import RedBeatSchedulerEntry
# convert strings and integers to timedeltas
if isinstance(every, str) and every.isdigit():
every = get_timedelta(int(every), period)
elif isinstance(every, int):
every = get_timedelta(every, period)
if not isinstance(every, timedelta):
t = type(every).__name__
raise AttributeError(
f"Invalid input parameter every = {every} (type {t})")
interval = schedule(run_every=every)
entry = RedBeatSchedulerEntry(name,
task,
interval,
args=args,
app=CeleryExt.celery_app)
entry.save()
else:
raise AttributeError(
f"Unsupported celery-beat scheduler: {cls.CELERYBEAT_SCHEDULER}"
)
def task_crontab_add_one():
if request.method == "POST":
if "app.mod_task.tasks.send_email_of_logs" in request.json["task"]:
task = PeriodicTask(
name=str(uuid1()),
task=request.json["task"].split(":")[0],
enabled=True,
crontab=PeriodicTask.Crontab(
minute=request.json["minute"],
hour=request.json["hour"],
day_of_month=request.json["day_of_month"],
month_of_year=request.json["month_of_year"],
day_of_week=request.json["day_of_week"],
),
args=[request.json["task"].split(":")[1]],
)
result = task.save()
return result.to_json(), {"Content-Type": "application/json"}
task = PeriodicTask(
name=str(uuid1()),
task=request.json["task"],
enabled=True,
crontab=PeriodicTask.Crontab(
minute=request.json["minute"],
hour=request.json["hour"],
day_of_month=request.json["day_of_month"],
month_of_year=request.json["month_of_year"],
day_of_week=request.json["day_of_week"],
),
)
result = task.save()
return result.to_json(), {"Content-Type": "application/json"}
def test_date_changed(self):
periodic = PeriodicTask(task="foo")
periodic.interval = PeriodicTask.Interval(every=1, period="days")
self.assertIsNone(periodic.date_changed)
periodic.save()
self.assertIsNotNone(periodic.date_changed)
date_changed = periodic.date_changed
periodic.name = "I'm changing now"
periodic.save()
self.assertGreater(periodic.date_changed, date_changed)
def disable_tasks(self, target):
periodic = PeriodicTask.objects(target=target)
for p in periodic:
periodic_document = periodic.get(name=p.name)
if periodic_document.enabled:
periodic_document.enabled = False
periodic_document.save()
def test_creation_date(self):
periodic = PeriodicTask(task="foo")
periodic.interval = PeriodicTask.Interval(every=1, period="days")
self.assertIsNone(
periodic.date_creation,
"date_creation should be none on the first object instantion")
periodic.save()
self.assertIsNotNone(periodic.date_creation)
date_creation = periodic.date_creation
periodic.name = "I'm changing"
periodic.save()
self.assertEqual(
date_creation, periodic.date_creation,
"Update object should not change date_creation value")
def test_cannot_define_both_interval_and_contrab(self):
periodic = PeriodicTask(task="foo")
periodic.interval = PeriodicTask.Interval(every=1, period="days")
periodic.crontab = PeriodicTask.Crontab(minute="0",
hour="*",
day_of_week="*",
day_of_month="10-15",
month_of_year="*")
with self.assertRaises(ValidationError) as err:
periodic.save()
self.assertTrue("Cannot define both interval and crontab schedule." in
err.exception.message)
def delete_all_poll_tasks(self):
periodic = PeriodicTask.objects()
for p in periodic:
if not p.task == "splunk_connect_for_snmp.snmp.tasks.poll":
continue
logger.debug(p)
periodic_document = periodic.get(name=p.name)
logger.debug("Got Schedule")
periodic_document.delete()
logger.debug("Deleting Schedule")
def rerun_all_walks(self):
periodic = PeriodicTask.objects()
for p in periodic:
if not p.task == "splunk_connect_for_snmp.snmp.tasks.walk":
continue
logger.debug(p)
periodic_document = periodic.get(name=p.name)
periodic_document.run_immediately = True
logger.debug("Got Schedule")
periodic_document.save()
def test_str(self):
periodic = PeriodicTask(task="foo")
periodic.interval = PeriodicTask.Interval(every=1, period="days")
self.assertEqual("every day", str(periodic.interval))
periodic.interval = PeriodicTask.Interval(every=2, period="days")
self.assertEqual('every 2 days', str(periodic.interval))
def delete_unused_poll_tasks(self, target: str,
activeschedules: List[str]):
periodic = PeriodicTask.objects(target=target)
for p in periodic:
if not p.task == "splunk_connect_for_snmp.snmp.tasks.poll":
continue
logger.debug(p)
periodic_document = periodic.get(name=p.name)
logger.debug("Got Schedule")
if p.name not in activeschedules:
if periodic_document.enabled:
periodic_document.enabled = False
logger.debug(f"Deleting Schedule: {periodic_document.name}")
def insert(self, task_name, task, args, cron, offset):
''' insert a periodic task into database
params: task_name : string
task: a registered task in celery workers
args: task arguments
cron: crontab object
offset: UTC offset
'''
task = str(task)
task_name = str(task_name)
# the celery scheduler runs on UTC time, add offset so that it
# emulates the user timezone
cron.hour = set(map(lambda x: (x + (offset)) % 24, cron.hour))
# if the integer overflows should a day advance? or will the overflow
# only when UTC time is > 12 therefore the next time a time < 12
# occurs is on the next day?
cronArr = [cron.minute,
cron.hour,
cron.day_of_week, cron.
day_of_month,
cron.month_of_year]
# cron.minute return {number}, change to 'number' for celery schedule
for i in range(len(cronArr)):
cronArr[i] = str(cronArr[i]).replace('{', '')
cronArr[i] = cronArr[i].replace('}', '')
# following mongocelerybeat document schema for schedules
addTask = PeriodicTask()
addTask.name = task_name
addTask.task = task
addTask.crontab = addTask.Crontab(minute=cronArr[0],
hour=cronArr[1],
day_of_week=cronArr[2],
day_of_month=cronArr[3],
month_of_year=cronArr[4])
addTask.enabled = True
addTask.args = args
addTask.save()
def test_cannot_save_interval_schduler_with_a_invalid_period(self):
periodic = PeriodicTask(task="foo")
with self.assertRaises(ValidationError):
periodic.interval = PeriodicTask.Interval(every=1,
period="days111")
periodic.save()
def test_must_define_interval_or_crontab(self):
with self.assertRaises(ValidationError) as err:
periodic = PeriodicTask(task="foo")
periodic.save()
self.assertTrue("Must defined either interval or crontab schedule." in
err.exception.message)
def test_scheduler(self):
periodic = PeriodicTask(task="foo")
periodic.interval = PeriodicTask.Interval(every=1, period="days")
periodic.save()
self.assertIsNotNone(periodic.schedule)
def remove(task_name):
''' remove database entry using task_name as lookup'''
next(PeriodicTask.objects(task_name=task_name)).delete()
def delete_disabled_poll_tasks(self):
periodic = PeriodicTask.objects(enabled=False)
for p in periodic:
periodic_document = periodic.get(name=p.name)
periodic_document.delete()
logger.debug("Deleting Schedule")
def manage_task(self,
run_immediately_if_new: bool = False,
**task_data) -> None:
periodic = PeriodicTask.objects(name=task_data["name"])
if periodic:
logger.debug("Existing Schedule")
isChanged = False
periodic_document = periodic.get(name=task_data["name"])
for key, value in task_data.items():
if key == "interval":
if not periodic_document.interval == PeriodicTask.Interval(
**task_data["interval"]):
periodic_document.interval = PeriodicTask.Interval(
**task_data["interval"])
isChanged = True
elif key == "crontab":
if not periodic_document.crontab == PeriodicTask.Crontab(
**task_data["crontab"]):
periodic_document.crontab = PeriodicTask.Crontab(
**task_data["crontab"])
isChanged = True
elif key == "target":
pass
elif key == "total_run_count":
periodic_document[key] = task_data[key]
else:
if key in periodic_document:
if not periodic_document[key] == task_data[key]:
periodic_document[key] = task_data[key]
isChanged = True
else:
periodic_document[key] = task_data[key]
isChanged = True
else:
logger.debug("New Schedule")
isChanged = True
periodic_document = PeriodicTask(task=task_data["task"])
periodic_document.name = task_data["name"]
periodic_document.args = task_data["args"]
periodic_document.kwargs = task_data["kwargs"]
if "interval" in task_data:
periodic_document.interval = PeriodicTask.Interval(
**task_data["interval"])
else:
periodic_document.crontab = PeriodicTask.Crontab(
**task_data["crontab"])
periodic_document.enabled = task_data["enabled"]
periodic_document.run_immediately = task_data.get(
"run_immediately", run_immediately_if_new)
if "total_run_count" in task_data:
periodic_document["total_run_count"] = task_data[
"total_run_count"]
if "target" in task_data:
periodic_document["target"] = task_data["target"]
if "options" in task_data:
periodic_document["options"] = task_data["options"]
logger.info(
f"Periodic document to save: {periodic_document.to_json()}")
if isChanged:
periodic_document.save()
