def create_repeating_task(user: User, workflow):
now = timezone.now().timestamp()
id = f"{user.username}-repeating-{now}"
eta, seconds = parse_task_eta(workflow['eta'])
schedule, _ = IntervalSchedule.objects.get_or_create(
every=seconds, period=IntervalSchedule.SECONDS)
repo_owner = workflow['repo']['owner']
repo_name = workflow['repo']['name']
repo_branch = workflow['repo']['branch']
if 'logo' in workflow:
logo_path = workflow['logo']
workflow_image_url = f"https://raw.githubusercontent.com/{repo_owner}/{repo_name}/{repo_branch}/{logo_path}"
else:
workflow_image_url = None
task, created = RepeatingTask.objects.get_or_create(
user=user,
interval=schedule,
eta=eta,
workflow_owner=repo_owner,
workflow_name=repo_name,
workflow_branch=repo_branch,
workflow_image_url=workflow_image_url,
name=id,
task='plantit.celery_tasks.create_and_submit_repeating',
args=json.dumps([user.username, workflow, id]))
# manually refresh task schedule
PeriodicTasks.changed(task)
return task, created
def create_delayed_task(user: User, config: dict):
now = timezone.now().timestamp()
id = f"{user.username}-delayed-{now}"
eta, seconds = parse_task_eta(config['eta'])
schedule, _ = IntervalSchedule.objects.get_or_create(
every=seconds, period=IntervalSchedule.SECONDS)
# parse GitHub repo info
repo_owner = config['repo']['owner']
repo_name = config['repo']['name']
repo_branch = config['repo']['branch']
if 'logo' in config:
logo_path = config['logo']
workflow_image_url = f"https://raw.githubusercontent.com/{repo_owner}/{repo_name}/{repo_branch}/{logo_path}"
else:
workflow_image_url = None
task, created = DelayedTask.objects.get_or_create(
user=user,
interval=schedule,
eta=eta,
one_off=True,
workflow_owner=repo_owner,
workflow_name=repo_name,
workflow_branch=repo_branch,
workflow_image_url=workflow_image_url,
name=id,
task='plantit.celery_tasks.create_and_submit_delayed',
args=json.dumps([user.username, config, id]))
# manually refresh task schedule
PeriodicTasks.changed(task)
return task, created
def disable_periodic_task(self, save=True):
if self.periodic_task:
obj = self.periodic_task
obj.delete()
PeriodicTasks.update_changed()
self.periodic_task = None
if save:
self.save()
return self.periodic_task
def save_model(self, request, obj, form, change):
if obj.celery_task_id:
revoke(obj.celery_task_id, terminate=True)
obj.celery_task_id = ''
obj.running = False
obj.status = '-'
obj.task = 'regression.tasks.local_regression_task'
obj.kwargs = json.dumps({'task_name': obj.name})
PeriodicTasks.update_changed()
super().save_model(request, obj, form, change)
def test_track_changes(self):
assert PeriodicTasks.last_change() is None
m1 = self.create_model_interval(schedule(timedelta(seconds=10)))
m1.save()
x = PeriodicTasks.last_change()
assert x
m1.args = '(23, 24)'
m1.save()
y = PeriodicTasks.last_change()
assert y
assert y > x
def test_track_changes(self):
assert PeriodicTasks.last_change() is None
m1 = self.create_model_interval(schedule(timedelta(seconds=10)))
m1.save()
x = PeriodicTasks.last_change()
assert x
m1.args = '(23, 24)'
m1.save()
y = PeriodicTasks.last_change()
assert y
assert y > x
def save(self, commit=True):
"""Save the new password for the user"""
cron = self.cleaned_data.get("cron")
if cron:
cron_list = cron.split(" ")
updated = CrontabScheduleUser.objects.filter(
user=self.user).update(
minute=cron_list[0],
hour=cron_list[1],
day_of_week=cron_list[2],
day_of_month=cron_list[3],
month_of_year=cron_list[4],
timezone=pytz.timezone(self.user.profile.timezone),
user=self.user,
)
if updated == 0 or not updated:
user_crontab = CrontabScheduleUser.objects.create(
minute=cron_list[0],
hour=cron_list[1],
day_of_week=cron_list[2],
day_of_month=cron_list[3],
month_of_year=cron_list[4],
timezone=pytz.timezone(self.user.profile.timezone),
user=self.user,
)
if self.user.profile.notifications_enabled:
ScheduledTasks.objects.create(
crontab=user_crontab,
args=json.dumps([self.user.pk]),
name=f"Notify {self.user.email}",
task="notify",
user=self.user.profile,
)
else:
# f**k too lazy - fix later
user_crontab = CrontabScheduleUser.objects.get(user=self.user)
if self.user.profile.notifications_enabled:
try:
obj = ScheduledTasks.objects.get(user=self.user)
except ObjectDoesNotExist:
obj = ScheduledTasks.objects.create(
crontab=user_crontab,
args=json.dumps([self.user.pk]),
name=f"Notify {self.user.email}",
task="notify",
user=self.user.profile,
)
PeriodicTasks.changed(obj)
return cron
def setup_schedule(self):
"""
Cleanup previous periodic tasks from the database before starting the default scheduler
"""
schedule = self.app.conf.beat_schedule
with transaction.atomic():
num, info = PeriodicTask.objects.\
exclude(task__startswith='celery.').\
exclude(name__in=schedule.keys()).\
delete()
logger.info("Removed %d obsolete periodic tasks.", num)
if num > 0:
PeriodicTasks.update_changed()
super(DatabaseSchedulerWithCleanup, self).setup_schedule()
def save(self, *args, **kwargs):
self.task = "TwilioCaller"
self.exchange = self.exchange or None
self.routing_key = self.routing_key or None
self.queue = self.queue or None
if not self.enabled:
self.last_run_at = None
if not self.pk:
super(CeleryPhoneModel, self).save(*args, **kwargs)
self.args = [self.pk]
super(CeleryPhoneModel, self).save(*args, **kwargs)
else:
self.args = [self.pk]
super(CeleryPhoneModel, self).save(*args, **kwargs)
PeriodicTasks.update_changed()
def enable_periodic_task(self, save=True):
instance_id = self.id
ticker = self.ticker
task_name = f'company-{ticker}-{instance_id}'.lower()
if not self.periodic_task:
schedule = CrontabSchedule.objects.get(id=3)
obj, _ = PeriodicTask.objects.get_or_create(
crontab=schedule,
kwargs=json.dumps({
'instance_id': instance_id,
'service': 'business_insider'
}),
name=task_name,
task='stocks.tasks.company_granular_price_scrape_task')
obj.enabled = True
obj.save()
PeriodicTasks.update_changed()
self.periodic_task = obj
if save:
self.save()
return self.periodic_task
def session_pre_save(sender, instance, **kwargs):
"""
Updates tasks when session time is changed.
"""
try:
old_session = sender.objects.get(pk=instance.pk)
old_schedule, _ = CrontabSchedule.objects.get_or_create(
minute=old_session.end_time.minute, hour=old_session.end_time.hour)
new_schedule, _ = CrontabSchedule.objects.get_or_create(
minute=instance.end_time.minute, hour=instance.end_time.hour)
PeriodicTask.objects \
.filter(
crontab=old_schedule,
task='localhost.core.tasks.cleanup_bids') \
.update(crontab=new_schedule)
for task in PeriodicTask.objects.filter(
crontab=new_schedule,
task='localhost.core.tasks.cleanup_bids'):
PeriodicTasks.changed(task)
old_schedule.delete()
except sender.DoesNotExist:
logger.info(f'{instance} not in db. Skipping pre-save actions.')
def create_periodic_obj(**kwargs):
"""
:param kwargs:
:return:
"""
name = kwargs.get('name', None)
task = kwargs.get('task', 'seecode.services.scan.sched_start')
interval = kwargs.get('interval', None)
crontab = kwargs.get('crontab', None)
args = kwargs.get('args', '')
kkwargs = kwargs.get('kkwargs', '')
queue = kwargs.get('queue', 'sched')
exchange = kwargs.get('exchange', None)
priority = kwargs.get('priority', 1)
if not all((
name,
task,
queue,
)):
raise ParameterIsEmptyException(
u'"name, task, queue" parameters cannot be empty !')
obj = PeriodicTask(
name=name,
task=task,
interval=interval,
crontab=crontab,
args=args,
kwargs=kkwargs,
queue=queue,
exchange=exchange,
priority=priority,
)
obj.save()
PeriodicTasks.changed(obj)
return obj
def save(self, commit=True):
""" Save the new password for the user """
cron = self.cleaned_data.get('cron')
if cron:
cron_list = cron.split(" ")
updated = CrontabScheduleUser.objects.filter(
user=self.user).update(minute=cron_list[0],
hour=cron_list[1],
day_of_week=cron_list[2],
day_of_month=cron_list[3],
month_of_year=cron_list[4],
timezone=pytz.timezone(
self.user.profile.timezone),
user=self.user)
if updated == 0:
user_crontab = CrontabScheduleUser.objects.create(
minute=cron_list[0],
hour=cron_list[1],
day_of_week=cron_list[2],
day_of_month=cron_list[3],
month_of_year=cron_list[4],
timezone=pytz.timezone(self.user.profile.timezone),
user=self.user)
if self.user.profile.notifications_enabled:
obj = ScheduledTasks.objects.create(
crontab=user_crontab,
args=json.dumps([self.user.pk]),
name=f'Notify {self.user.email}',
task='notify',
user=self.user.profile)
else:
obj = ScheduledTasks.objects.get(user=self.user)
PeriodicTasks.changed(obj)
return cron
def create_or_update_celery_periodic_tasks(tasks):
"""
:param tasks: {
'add-every-monday-morning': {
'task': 'tasks.add' # A registered celery task,
'interval': 30,
'crontab': "30 7 * * *",
'args': (16, 16),
'kwargs': {},
'enabled': False,
'description': ''
},
}
:return:
"""
# Todo: check task valid, task and callback must be a celery task
for name, detail in tasks.items():
interval = None
crontab = None
last_run_at = None
try:
IntervalSchedule.objects.all().count()
except (ProgrammingError, OperationalError):
return None
if isinstance(detail.get("interval"), int):
kwargs = dict(
every=detail['interval'],
period=IntervalSchedule.SECONDS,
)
# 不能使用 get_or_create，因为可能会有多个
interval = IntervalSchedule.objects.filter(**kwargs).first()
if interval is None:
interval = IntervalSchedule.objects.create(**kwargs)
last_run_at = local_now()
elif isinstance(detail.get("crontab"), str):
try:
minute, hour, day, month, week = detail["crontab"].split()
except ValueError:
logger.error("crontab is not valid")
return
kwargs = dict(
minute=minute, hour=hour, day_of_week=week,
day_of_month=day, month_of_year=month, timezone=get_current_timezone()
)
crontab = CrontabSchedule.objects.filter(**kwargs).first()
if crontab is None:
crontab = CrontabSchedule.objects.create(**kwargs)
else:
logger.error("Schedule is not valid")
return
defaults = dict(
interval=interval,
crontab=crontab,
name=name,
task=detail['task'],
enabled=detail.get('enabled', True),
args=json.dumps(detail.get('args', [])),
kwargs=json.dumps(detail.get('kwargs', {})),
description=detail.get('description') or '',
last_run_at=last_run_at,
)
task = PeriodicTask.objects.update_or_create(
defaults=defaults, name=name,
)
PeriodicTasks.update_changed()
return task
def toggle_tasks(self, request, queryset):
rows_updated = self._toggle_tasks_activity(queryset)
PeriodicTasks.update_changed()
self._message_user_about_update(request, rows_updated, 'toggled')
def disable_tasks(self, request, queryset):
rows_updated = queryset.update(enabled=False)
PeriodicTasks.update_changed()
self._message_user_about_update(request, rows_updated, 'disabled')
def post(self, request):
periodic_id = request.POST.get('form_periodic_id_u', '')
name = request.POST.get('form_name_u', '')
task = request.POST.get('form_task_u', '')
interval_id = request.POST.get('form_interval_u', '')
if interval_id == "":
interval_id = None
crontab_id = request.POST.get('form_crontab_u', '')
if crontab_id == "":
crontab_id = None
args = request.POST.get('form_args_u', '')
if args == "":
args = "[]"
kwargs = request.POST.get('form_kwargs_u', '')
if kwargs == "":
kwargs = "{}"
queue = request.POST.get('form_queue_u', '')
if queue == "None":
queue = None
exchange = request.POST.get('form_exchange_u', '')
if exchange == "None":
exchange = None
routing_key = request.POST.get('form_routing_key_u', '')
if routing_key == "None":
routing_key = None
headers = request.POST.get('form_headers_u', '')
if headers == "":
headers = "{}"
priority = request.POST.get('form_priority_u', '')
if priority == "None":
priority = None
expires = request.POST.get('form_expires_u', '')
if expires == "None":
expires = None
one_off = request.POST.get('form_one_off_u', '')
if one_off == "":
one_off = 0
elif one_off == "on":
one_off = 1
start_time = request.POST.get('form_start_time_u', '')
if start_time == "None":
start_time = None
enabled = request.POST.get('form_enabled_u', '')
if enabled == "on":
enabled = 1
elif enabled == "":
enabled = 0
description = request.POST.get('form_description_u', '')
if description == "":
description = "空"
PeriodicTask.objects.filter(id=periodic_id).update(
name=name,
task=task,
interval_id=interval_id,
crontab_id=crontab_id,
args=args,
kwargs=kwargs,
queue=queue,
exchange=exchange,
routing_key=routing_key,
headers=headers,
priority=priority,
expires=expires,
one_off=one_off,
start_time=start_time,
enabled=enabled,
description=description,
date_changed=datetime.now(),
)
PeriodicTask.objects.all().update(last_run_at=None)
for task in PeriodicTask.objects.all():
PeriodicTasks.changed(task)
return redirect('/periodic_task/')
def update(self, instance, validated_data):
super(CronScheduleSerializer, self).update(instance, validated_data)
periodic_tasks = instance.periodictask_set.all()
for task in periodic_tasks:
PeriodicTasks.changed(task)
return instance
def disable_celery_periodic_task(task_name):
from django_celery_beat.models import PeriodicTask
PeriodicTask.objects.filter(name=task_name).update(enabled=False)
PeriodicTasks.update_changed()
Websocket.objects.create(visitors=0)
# ping interval
if os.environ.get("PING_INTERVAL"):
if int(os.environ.get("PING_INTERVAL")) < 5:
print(
"Ping interval lower than 5 seconds is not recommended. Please use an interval of 5 seconds or higher. Automatically set to 5 seconds."
)
ping_interval = 5
else:
ping_interval = int(os.environ.get("PING_INTERVAL"))
else:
ping_interval = 5
Settings.objects.update_or_create(id=1, defaults={"interval": ping_interval})
# register device ping task
schedule, created = IntervalSchedule.objects.get_or_create(
every=ping_interval,
period=IntervalSchedule.SECONDS,
)
if created:
PeriodicTask.objects.create(interval=schedule,
name="Ping all devices",
task="wol.tasks.ping_all_devices")
# reset last run to fix time zone changes
# https://django-celery-beat.readthedocs.io/en/latest/#important-warning-about-time-zones
PeriodicTask.objects.update(last_run_at=None)
for task in PeriodicTask.objects.all():
PeriodicTasks.changed(task)
def delete_celery_periodic_task(task_name):
from django_celery_beat.models import PeriodicTask
PeriodicTask.objects.filter(name=task_name).delete()
PeriodicTasks.update_changed()
def update(self, instance, validated_data):
super(PeriodicTaskSerializer, self).update(instance, validated_data)
PeriodicTasks.changed(instance)
return instance