def setup_scheduler(self, app):
self.app = app
self.app.conf.beat_schedule = {}
self.m1 = self.create_model_interval(
schedule(timedelta(seconds=10)))
self.m1.save()
self.m1.refresh_from_db()
self.m2 = self.create_model_interval(
schedule(timedelta(minutes=20)))
self.m2.save()
self.m2.refresh_from_db()
self.m3 = self.create_model_crontab(
crontab(minute='2,4,5'))
self.m3.save()
self.m3.refresh_from_db()
self.m4 = self.create_model_solar(
solar('solar_noon', 48.06, 12.86))
self.m4.save()
self.m4.refresh_from_db()
# disabled, should not be in schedule
m5 = self.create_model_interval(
schedule(timedelta(seconds=1)))
m5.enabled = False
m5.save()
self.s = self.Scheduler(app=self.app)
def test_heap_always_return_the_first_item(self):
interval = 10
s1 = schedule(timedelta(seconds=interval))
m1 = self.create_model_interval(s1, enabled=False)
m1.last_run_at = self.app.now() - timedelta(seconds=interval + 2)
m1.save()
m1.refresh_from_db()
s2 = schedule(timedelta(seconds=interval))
m2 = self.create_model_interval(s2, enabled=True)
m2.last_run_at = self.app.now() - timedelta(seconds=interval + 1)
m2.save()
m2.refresh_from_db()
e1 = EntryTrackSave(m1, self.app)
# because the disabled task e1 runs first, e2 will never be executed
e2 = EntryTrackSave(m2, self.app)
s = self.Scheduler(app=self.app)
s.schedule.clear()
s.schedule[e1.name] = e1
s.schedule[e2.name] = e2
tried = set()
for _ in range(len(s.schedule) * 8):
tick_interval = s.tick()
if tick_interval and tick_interval > 0.0:
tried.add(s._heap[0].entry.name)
time.sleep(tick_interval)
if s.should_sync():
s.sync()
assert len(tried) == 1 and tried == set([e1.name])
def test_entry(self):
m = self.create_model_interval(schedule(timedelta(seconds=10)))
e = self.Entry(m, app=self.app)
assert e.args == [2, 2]
assert e.kwargs == {'callback': 'foo'}
assert e.schedule
assert e.total_run_count == 0
assert isinstance(e.last_run_at, datetime)
assert e.options['queue'] == 'xaz'
assert e.options['exchange'] == 'foo'
assert e.options['routing_key'] == 'cpu'
right_now = self.app.now()
m2 = self.create_model_interval(
schedule(timedelta(seconds=10)),
last_run_at=right_now,
)
assert m2.last_run_at
e2 = self.Entry(m2, app=self.app)
assert e2.last_run_at is right_now
e3 = e2.next()
assert e3.last_run_at > e2.last_run_at
assert e3.total_run_count == 1
def setUp(self):
PeriodicTask.objects.all().delete()
m1 = create_model_interval(schedule(timedelta(seconds=10)))
m2 = create_model_interval(schedule(timedelta(minutes=20)))
m3 = create_model_crontab(crontab(minute="2,4,5"))
for obj in m1, m2, m3:
obj.save()
self.s = self.Scheduler()
self.m1 = PeriodicTask.objects.get(name=m1.name)
self.m2 = PeriodicTask.objects.get(name=m2.name)
self.m3 = PeriodicTask.objects.get(name=m3.name)
def setUp(self):
PeriodicTask.objects.all().delete()
self.prev_schedule = celery.conf.CELERYBEAT_SCHEDULE
celery.conf.CELERYBEAT_SCHEDULE = {}
m1 = create_model_interval(schedule(timedelta(seconds=10)))
m2 = create_model_interval(schedule(timedelta(minutes=20)))
m3 = create_model_crontab(crontab(minute='2,4,5'))
for obj in m1, m2, m3:
obj.save()
self.s = self.Scheduler(app=app)
self.m1 = PeriodicTask.objects.get(name=m1.name)
self.m2 = PeriodicTask.objects.get(name=m2.name)
self.m3 = PeriodicTask.objects.get(name=m3.name)
def test_schedule(self):
d = self.schedule()
result = self.loads(json.dumps(d))
d.pop('__type__')
self.assertEqual(result, schedule(run_every=60))
def test_scheduled_task(self):
tq = TaskQueue('test')
tq.bind_redis(self.conn1)
kw = {
'request': {'method': 'GET',
'url': 'http://httpbin.org'},
'schedule': schedules.crontab('*/10', '1,2-10')
}
self.assertRaises(TaskCNameRequired, tq.add_task, **kw)
kw['cname'] = 'crontest'
task00 = tq.add_task(**kw)
metakey = tq._TaskQueue__metakey(task00['id'])
self.assertEqual(not_bytes(self.conn1.hget(metakey, 'schedule')),
'"*/10 1,2-10 * * *"')
kw['schedule'] = schedules.schedule(30)
kw['cname'] = 'schedtest'
task01 = tq.add_task(**kw)
metakey = tq._TaskQueue__metakey(task01['id'])
self.assertEqual(not_bytes(self.conn1.hget(metakey, 'schedule')),
'"every 30.0 seconds"')
task10 = tq.get_task_by_cname('crontest')
self.assertEqual(task00, task10)
task11 = tq.get_task_by_cname('schedtest')
self.assertEqual(task01, task11)
task = tq._get_task_by_cname('schedtest')
task.dispatch()
task21 = tq.get_task_by_cname('schedtest')
self.assertNotEqual(task11, task21)
now = utcnow()
self.assertTrue(now - timedelta(5) < task21['last_run_at'] < now)
task11.pop('last_run_at')
task11.pop('uuid')
task21.pop('last_run_at')
task21.pop('uuid')
self.assertEqual(task11, task21)
def test_ticks_schedule_change(self):
# initialise schedule and check heap is not initialized
scheduler = mScheduler(app=self.app)
assert scheduler._heap is None
# set initial schedule and check heap is updated
schedule_5 = schedule(5)
scheduler.add(name='test_schedule', schedule=schedule_5)
scheduler.tick()
assert scheduler._heap[0].entry.schedule == schedule_5
# update schedule and check heap is updated
schedule_10 = schedule(10)
scheduler.add(name='test_schedule', schedule=schedule(10))
scheduler.tick()
assert scheduler._heap[0].entry.schedule == schedule_10
def __init__(self, record):
self.name = record.name
self.task = record.task
if record.interval:
i = record.interval
self.schedule = schedules.schedule(timedelta(**{i.period: i.every}))
else:
i = record.crontab
schedules.crontab(minute=i.minute,
hour=i.hour,
day_of_week=i.day_of_week)
try:
self.args = deserialize(record.args or u"[]")
self.kwargs = deserialize(record.kwargs or u"{}")
except ValueError:
record.update_record(no_changes = True, enabled = False)
db(db.celery_periodictasks).update(last_update=datetime.now())
db.commit()
raise
self.options = {"queue": record.queue,
"exchange": record.exchange,
"routing_key": record.routing_key,
"expires": record.expires}
self.total_run_count = record.total_run_count or 0
self.record = record
if not record.last_run_at:
record.update_record(last_run_at = datetime.now())
db(db.celery_periodictasks).update(last_update=datetime.now())
db.commit()
self.last_run_at = record.last_run_at
def create_entry(self, **kwargs):
entry = dict(
name="celery.unittest.add",
schedule=schedule(timedelta(seconds=10)),
args=(2, 2),
options={"routing_key": "cpu"},
)
return self.Entry(**dict(entry, **kwargs))
def setup(self):
self.app.set_current() # @depends_on_current_app
@periodic_task(app=self.app, shared=False,
run_every=schedule(timedelta(hours=1), app=self.app))
def my_periodic():
pass
self.my_periodic = my_periodic
def test_maybe_make_aware(self):
x = schedule(10, app=self.app)
x.utc_enabled = True
d = x.maybe_make_aware(datetime.utcnow())
self.assertTrue(d.tzinfo)
x.utc_enabled = False
d2 = x.maybe_make_aware(datetime.utcnow())
self.assertIsNone(d2.tzinfo)
def test_to_local(self):
x = schedule(10, app=self.app)
x.utc_enabled = True
d = x.to_local(datetime.utcnow())
self.assertIsNone(d.tzinfo)
x.utc_enabled = False
d = x.to_local(datetime.utcnow())
self.assertTrue(d.tzinfo)
def convert_schedule(save_func, call):
"""
Converts one scheduled call from the old schema to the new
:param save_func: a function that takes one parameter, a dictionary that
represents the scheduled call in its new schema. This
function should save the call to the database.
:type save_func: function
:param call: dictionary representing the scheduled call in its old
schema
:type call: dict
"""
call.pop('call_exit_states', None)
call['total_run_count'] = call.pop('call_count')
call['iso_schedule'] = call['schedule']
interval, start_time, occurrences = dateutils.parse_iso8601_interval(call['schedule'])
# this should be a pickled instance of celery.schedules.schedule
call['schedule'] = pickle.dumps(schedule(interval))
call_request = call.pop('serialized_call_request')
# we are no longer storing these pickled.
# these are cast to a string because python 2.6 sometimes fails to
# deserialize json from unicode.
call['args'] = pickle.loads(str(call_request['args']))
call['kwargs'] = pickle.loads(str(call_request['kwargs']))
# keeping this pickled because we don't really know how to use it yet
call['principal'] = call_request['principal']
# this always get calculated on-the-fly now
call.pop('next_run', None)
first_run = call['first_run'].replace(tzinfo=dateutils.utc_tz())
call['first_run'] = dateutils.format_iso8601_datetime(first_run)
last_run = call.pop('last_run')
if last_run:
last_run_at = last_run.replace(tzinfo=dateutils.utc_tz())
call['last_run_at'] = dateutils.format_iso8601_datetime(last_run_at)
else:
call['last_run_at'] = None
call['task'] = NAMES_TO_TASKS[call_request['callable_name']]
# this is a new field that is used to determine when the scheduler needs to
# re-read the collection of schedules.
call['last_updated'] = time.time()
# determine if this is a consumer-related schedule, which we can only identify
# by the consumer resource tag. If it is, save that tag value in the new
# "resource" field, which is the new way that we will identify the
# relationship between a schedule and some other object. This is not
# necessary for repos, because we have a better method above for identifying
# them (move_scheduled_syncs).
tags = call_request.get('tags', [])
for tag in tags:
if tag.startswith('pulp:consumer:'):
call['resource'] = tag
break
save_func(call)
def test_PeriodicTask_schedule_property(self):
p1 = create_model_interval(schedule(timedelta(seconds=10)))
s1 = p1.schedule
self.assertEqual(timedelta_seconds(s1.run_every), 10)
p2 = create_model_crontab(crontab(hour="4, 5", minute="10,20,30"))
s2 = p2.schedule
self.assertSetEqual(s2.hour, set([4, 5]))
self.assertSetEqual(s2.minute, set([10, 20, 30]))
self.assertSetEqual(s2.day_of_week, set([0, 1, 2, 3, 4, 5, 6]))
def decorator(func):
celery_task = self.celery.task(func)
self.celery.conf.CELERYBEAT_SCHEDULE[celery_task.name] = {
"task": celery_task.name,
"schedule": schedule(*args) if args else crontab(**kwargs),
}
self.jobs[celery_task.name] = celery_task
return celery_task
def test_track_changes(self):
self.assertIsNone(PeriodicTasks.last_change())
m1 = create_model_interval(schedule(timedelta(seconds=10)))
m1.save()
x = PeriodicTasks.last_change()
self.assertTrue(x)
m1.args = "(23, 24)"
m1.save()
y = PeriodicTasks.last_change()
self.assertTrue(y)
self.assertGreater(y, x)
def test_task_with_start_time(self):
interval = 10
right_now = self.app.now()
one_interval_ago = right_now - timedelta(seconds=interval)
m = self.create_model_interval(schedule(timedelta(seconds=interval)),
start_time=right_now,
last_run_at=one_interval_ago)
e = self.Entry(m, app=self.app)
isdue, delay = e.is_due()
assert isdue
assert delay == interval
tomorrow = right_now + timedelta(days=1)
m2 = self.create_model_interval(schedule(timedelta(seconds=interval)),
start_time=tomorrow,
last_run_at=one_interval_ago)
e2 = self.Entry(m2, app=self.app)
isdue, delay = e2.is_due()
assert not isdue
assert delay == interval
def test_entry(self):
m = create_model_interval(schedule(timedelta(seconds=10)))
e = self.Entry(m)
self.assertListEqual(e.args, [2, 2])
self.assertDictEqual(e.kwargs, {"callback": "foo"})
self.assertTrue(e.schedule)
self.assertEqual(e.total_run_count, 0)
self.assertIsInstance(e.last_run_at, datetime)
self.assertDictContainsSubset({"queue": "xaz", "exchange": "foo", "routing_key": "cpu"}, e.options)
right_now = celery.now()
m2 = create_model_interval(schedule(timedelta(seconds=10)), last_run_at=right_now)
self.assertTrue(m2.last_run_at)
e2 = self.Entry(m2)
self.assertIs(e2.last_run_at, right_now)
e3 = e2.next()
self.assertGreater(e3.last_run_at, e2.last_run_at)
self.assertEqual(e3.total_run_count, 1)
def test_track_changes(self):
assert PeriodicTasks.last_change() is None
m1 = self.create_model_interval(schedule(timedelta(seconds=10)))
m1.save()
x = PeriodicTasks.last_change()
assert x
m1.args = '(23, 24)'
m1.save()
y = PeriodicTasks.last_change()
assert y
assert y > x
def test_one_off_task(self):
interval = 10
right_now = self.app.now()
one_interval_ago = right_now - timedelta(seconds=interval)
m = self.create_model_interval(schedule(timedelta(seconds=interval)),
one_off=True,
last_run_at=one_interval_ago,
total_run_count=0)
e = self.Entry(m, app=self.app)
isdue, delay = e.is_due()
assert isdue
assert delay == interval
m2 = self.create_model_interval(schedule(timedelta(seconds=interval)),
one_off=True,
last_run_at=one_interval_ago,
total_run_count=1)
e2 = self.Entry(m2, app=self.app)
isdue, delay = e2.is_due()
assert not isdue
assert delay is None
def test_ticks_microseconds(self):
scheduler = mScheduler(app=self.app)
now_ts = 1514797200.2
now = datetime.fromtimestamp(now_ts)
schedule_half = schedule(timedelta(seconds=0.5), nowfun=lambda: now)
scheduler.add(name='half_second_schedule', schedule=schedule_half)
scheduler.tick()
# ensure those 0.2 seconds on now_ts don't get dropped
expected_time = now_ts + 0.5 - 0.010
assert scheduler._heap[0].time == expected_time
def test_ticks_microseconds(self):
scheduler = mScheduler(app=self.app)
now_ts = 1514797200.2
now = datetime.utcfromtimestamp(now_ts)
schedule_half = schedule(timedelta(seconds=0.5), nowfun=lambda: now)
scheduler.add(name='half_second_schedule', schedule=schedule_half)
scheduler.tick()
# ensure those 0.2 seconds on now_ts don't get dropped
expected_time = now_ts + 0.5 - 0.010
assert scheduler._heap[0].time == expected_time
def test_one_off_task(self):
interval = 10
right_now = self.app.now()
one_interval_ago = right_now - timedelta(seconds=interval)
m = self.create_model_interval(schedule(timedelta(seconds=interval)),
one_off=True,
last_run_at=one_interval_ago,
total_run_count=0)
e = self.Entry(m, app=self.app)
isdue, delay = e.is_due()
assert isdue
assert delay == interval
m2 = self.create_model_interval(schedule(timedelta(seconds=interval)),
one_off=True,
last_run_at=one_interval_ago,
total_run_count=1)
e2 = self.Entry(m2, app=self.app)
isdue, delay = e2.is_due()
assert not isdue
assert delay is None
def setup_periodic_tasks(sender, **kwargs): # pylint: disable=unused-argument
"""
Register with celery to run the spot_checker periodically.
"""
logger = logging.getLogger(__name__)
interval = get_polling_interval(logger)
logger.info(
'Setting-up periodic task to run every %s minutes: spot_checker',
interval)
sender.add_periodic_task(
schedule(run_every=interval),
spot_checker.s(),
)
def create_periodic_task(
cls,
name: str,
task: str,
every: Union[str, int, timedelta],
period: AllowedTimedeltaPeriods = "seconds",
args: List[Any] = None,
kwargs: Dict[str, Any] = None,
) -> None:
if args is None:
args = []
if kwargs is None:
kwargs = {}
if cls.CELERYBEAT_SCHEDULER == "MONGODB":
from celerybeatmongo.models import PeriodicTask
PeriodicTask(
name=name,
task=task,
enabled=True,
args=args,
kwargs=kwargs,
interval=PeriodicTask.Interval(every=every, period=period),
).save()
elif cls.CELERYBEAT_SCHEDULER == "REDIS":
from celery.schedules import schedule
from redbeat.schedulers import RedBeatSchedulerEntry
# convert strings and integers to timedeltas
if isinstance(every, str) and every.isdigit():
every = get_timedelta(int(every), period)
elif isinstance(every, int):
every = get_timedelta(every, period)
if not isinstance(every, timedelta):
t = type(every).__name__
raise AttributeError(
f"Invalid input parameter every = {every} (type {t})")
interval = schedule(run_every=every)
entry = RedBeatSchedulerEntry(name,
task,
interval,
args=args,
app=CeleryExt.celery_app)
entry.save()
else:
raise AttributeError(
f"Unsupported celery-beat scheduler: {cls.CELERYBEAT_SCHEDULER}"
)
class Config:
"""
Set base Flask configuration vars.
"""
# General Config
DEBUG = False
TESTING = False
SECRET_KEY = os.environ.get('SECRET_KEY')
# Default number of documents on page (api results pagination)
DEFAULT_PAGINATION_LIMIT = 6
MAX_PAGINATION_LIMIT = 30
# Flasgger Config
SWAGGER = {
'title': 'Monitoring API',
'description': '### The Monitoring API is a simple REST API based on Flask-RESTful library. '
'The main purpose of the application is to monitor the availability of selected services. '
'The application checks at regular intervals the availability of services on the '
'specified ip address (or hostname) and port',
'version': '1.0.0',
'uiversion': 3,
'termsOfService': '',
# 'specs_route': '/apidocs/'
}
# Celery Config
WATCHDOG_CHECK_INTERVAL = 30
BACKGROUND_CHECK_INTERVAL = 20
CELERY = {
'broker_url': os.environ.get("CELERY_BROKER_URL"),
'result_backend': os.environ.get("CELERY_RESULT_BACKEND_URL"),
'redbeat_redis_url': os.environ.get('CELERY_REDBEAT_REDIS_URL'),
'beat_scheduler': 'redbeat.RedBeatScheduler',
'redbeat_key_prefix': 'redbeat:',
'beat_max_loop_interval': 5,
'beat_schedule': {
# Execute every 30sec (by default) after enabled by user
'watchdog-task': {
'task': 'watchdog_task',
'schedule': schedules.schedule(run_every=WATCHDOG_CHECK_INTERVAL),
'relative': True,
'enabled': False
},
# Execute every 20sec (by default) since the application is launched
'background-task': {
'task': 'service_unknown_status_task',
'schedule': BACKGROUND_CHECK_INTERVAL,
'relative': True,
'enabled': True
}
}
}
def create_schedule(self, app):
data = {
'classname': self.classname,
'params': self.kwargs
}
interval = schedules.schedule(run_every=self.interval)
entry = RedBeatSchedulerEntry(
self.taskname,
'kryptobot.workers.harvester.tasks.launch_harvester',
interval,
kwargs=data,
app=app
)
entry.save()
def test_entry(self):
m = create_model_interval(schedule(timedelta(seconds=10)))
e = self.Entry(m)
self.assertListEqual(e.args, [2, 2])
self.assertDictEqual(e.kwargs, {'callback': 'foo'})
self.assertTrue(e.schedule)
self.assertEqual(e.total_run_count, 0)
self.assertIsInstance(e.last_run_at, datetime)
self.assertEqual(e.options.get('queue'), 'xaz')
self.assertEqual(e.options.get('exchange'), 'foo')
self.assertEqual(e.options.get('routing_key'), 'cpu')
right_now = celery.now()
m2 = create_model_interval(schedule(timedelta(seconds=10)),
last_run_at=right_now)
self.assertTrue(m2.last_run_at)
e2 = self.Entry(m2)
self.assertIs(e2.last_run_at, right_now)
e3 = e2.next()
self.assertGreater(e3.last_run_at, e2.last_run_at)
self.assertEqual(e3.total_run_count, 1)
def create_entry(self, name=None, task=None, s=None, run_every=60, **kwargs):
if name is None:
name = 'test'
if task is None:
task = 'tasks.test'
if s is None:
s = schedule(run_every=run_every)
e = RedBeatSchedulerEntry(name, task, s, app=self.app, **kwargs)
return e
def test_entry(self):
m = create_model_interval(schedule(timedelta(seconds=10)))
e = self.Entry(m)
self.assertListEqual(e.args, [2, 2])
self.assertDictEqual(e.kwargs, {'callback': 'foo'})
self.assertTrue(e.schedule)
self.assertEqual(e.total_run_count, 0)
self.assertIsInstance(e.last_run_at, datetime)
self.assertDictContainsSubset({'queue': 'xaz',
'exchange': 'foo',
'routing_key': 'cpu'}, e.options)
right_now = celery.now()
m2 = create_model_interval(schedule(timedelta(seconds=10)),
last_run_at=right_now)
self.assertTrue(m2.last_run_at)
e2 = self.Entry(m2)
self.assertIs(e2.last_run_at, right_now)
e3 = e2.next()
self.assertGreater(e3.last_run_at, e2.last_run_at)
self.assertEqual(e3.total_run_count, 1)
def setup_scheduler(self, app):
self.app = app
self.app.conf.beat_schedule = {}
self.m1 = self.create_model_interval(
schedule(timedelta(seconds=10)))
self.m1.save()
self.m1.refresh_from_db()
self.m2 = self.create_model_interval(
schedule(timedelta(minutes=20)))
self.m2.save()
self.m2.refresh_from_db()
self.m3 = self.create_model_crontab(
crontab(minute='2,4,5'))
self.m3.save()
self.m3.refresh_from_db()
# disabled, should not be in schedule
m4 = self.create_model_interval(
schedule(timedelta(seconds=1)))
m4.enabled = False
m4.save()
self.s = self.Scheduler(app=self.app)
def create(self):
entry = RedBeatSchedulerEntry(
name=self.name,
task=self.task,
schedule=schedule(
run_every=self.run_every,
# setting "now" to the job start datetime
nowfun=lambda: self.run_at,
app=celery,
),
args=(self.spark_job.pk, ),
kwargs={},
app=celery,
)
return entry
def test_PeriodicTask_schedule_property(self):
p1 = create_model_interval(schedule(timedelta(seconds=10)))
s1 = p1.schedule
self.assertEqual(timedelta_seconds(s1.run_every), 10)
p2 = create_model_crontab(crontab(hour='4, 5',
minute='10,20,30',
day_of_month='1-7',
month_of_year='*/3'))
s2 = p2.schedule
self.assertSetEqual(s2.hour, set([4, 5]))
self.assertSetEqual(s2.minute, set([10, 20, 30]))
self.assertSetEqual(s2.day_of_week, set([0, 1, 2, 3, 4, 5, 6]))
self.assertSetEqual(s2.day_of_month, set([1, 2, 3, 4, 5, 6, 7]))
self.assertSetEqual(s2.month_of_year, set([1, 4, 7, 10]))
def create(self):
entry = RedBeatSchedulerEntry(
name=self.name,
task=self.task,
schedule=schedule(
run_every=self.run_every,
# setting "now" to the job start datetime
nowfun=lambda: self.run_at,
app=celery,
),
args=(self.spark_job.pk,),
kwargs={},
app=celery,
)
return entry
def create_interval_schedule(interval_dict):
"""
Creates schedule object from dictionary
:param interval_dict: format:
{
'minutes': '0',
'hours': '0',
'days': '0'
}
:return: celery schedule instance
"""
return schedule(
timedelta(minutes=interval_dict.get('minutes', 0),
hours=interval_dict.get('hours', 0),
days=interval_dict.get('days', 0)))
def test_update(self):
entry = self.create_entry()
assert entry.schedule == timedelta(seconds=10)
assert entry.args == (2, 2)
assert entry.kwargs == {}
assert entry.options == {'routing_key': 'cpu'}
entry2 = self.create_entry(schedule=timedelta(minutes=20),
args=(16, 16),
kwargs={'callback': 'foo.bar.baz'},
options={'routing_key': 'urgent'})
entry.update(entry2)
assert entry.schedule == schedule(timedelta(minutes=20))
assert entry.args == (16, 16)
assert entry.kwargs == {'callback': 'foo.bar.baz'}
assert entry.options == {'routing_key': 'urgent'}
def test_update(self):
entry = self.create_entry()
self.assertEqual(entry.schedule, timedelta(seconds=10))
self.assertTupleEqual(entry.args, (2, 2))
self.assertDictEqual(entry.kwargs, {})
self.assertDictEqual(entry.options, {"routing_key": "cpu"})
entry2 = self.create_entry(schedule=timedelta(minutes=20),
args=(16, 16),
kwargs={"callback": "foo.bar.baz"},
options={"routing_key": "urgent"})
entry.update(entry2)
self.assertEqual(entry.schedule, schedule(timedelta(minutes=20)))
self.assertTupleEqual(entry.args, (16, 16))
self.assertDictEqual(entry.kwargs, {"callback": "foo.bar.baz"})
self.assertDictEqual(entry.options, {"routing_key": "urgent"})
def test_update(self):
entry = self.create_entry()
assert entry.schedule == timedelta(seconds=10)
assert entry.args == (2, 2)
assert entry.kwargs == {}
assert entry.options == {'routing_key': 'cpu'}
entry2 = self.create_entry(schedule=timedelta(minutes=20),
args=(16, 16),
kwargs={'callback': 'foo.bar.baz'},
options={'routing_key': 'urgent'})
entry.update(entry2)
assert entry.schedule == schedule(timedelta(minutes=20))
assert entry.args == (16, 16)
assert entry.kwargs == {'callback': 'foo.bar.baz'}
assert entry.options == {'routing_key': 'urgent'}
def get_interval(minutes, hour, day_of_month):
now = datetime.now()
time_this_month = datetime(now.year,
now.month,
day_of_month,
hour,
minutes,
)
if time_this_month > now:
delta = time_this_month - now
else:
next_time = time_this_month + relativedelta(months=+1)
delta = next_time - now
interval = IntervalSchedule.from_schedule(schedule(delta))
interval.save()
return interval
def delay_task_generator(scan_task_id, run_at):
if run_at is None or run_at < datetime.now():
print 'Time in past'
return
task_name = 'delay_%s' % scan_task_id
task, created = PeriodicTask.objects.get_or_create(
name=task_name,
args=[int(scan_task_id)],
task = 'w3af_webui.tasks.delay_task',
)
delta = run_at - datetime.now()
interval = IntervalSchedule.from_schedule(schedule(delta))
interval.save()
task.interval = interval
task.enabled = True
task.save()
def delay_task_generator(scan_task_id, run_at):
if run_at is None or run_at < datetime.now():
print 'Time in past'
return
task_name = 'delay_%s' % scan_task_id
task, created = PeriodicTask.objects.get_or_create(
name=task_name,
args=[int(scan_task_id)],
task='w3af_webui.tasks.delay_task',
)
delta = run_at - datetime.now()
interval = IntervalSchedule.from_schedule(schedule(delta))
interval.save()
task.interval = interval
task.enabled = True
task.save()
def test_update(self):
entry = self.create_entry()
self.assertEqual(entry.schedule, timedelta(seconds=10))
self.assertTupleEqual(entry.args, (2, 2))
self.assertDictEqual(entry.kwargs, {})
self.assertDictEqual(entry.options, {'routing_key': 'cpu'})
entry2 = self.create_entry(schedule=timedelta(minutes=20),
args=(16, 16),
kwargs={'callback': 'foo.bar.baz'},
options={'routing_key': 'urgent'})
entry.update(entry2)
self.assertEqual(entry.schedule, schedule(timedelta(minutes=20)))
self.assertTupleEqual(entry.args, (16, 16))
self.assertDictEqual(entry.kwargs, {'callback': 'foo.bar.baz'})
self.assertDictEqual(entry.options, {'routing_key': 'urgent'})
def test_PeriodicTask_schedule_property(self):
p1 = self.create_model_interval(schedule(timedelta(seconds=10)))
s1 = p1.schedule
assert s1.run_every.total_seconds() == 10
p2 = self.create_model_crontab(crontab(
hour='4, 5',
minute='10,20,30',
day_of_month='1-7',
month_of_year='*/3',
))
s2 = p2.schedule
assert s2.hour == {4, 5}
assert s2.minute == {10, 20, 30}
assert s2.day_of_week == {0, 1, 2, 3, 4, 5, 6}
assert s2.day_of_month == {1, 2, 3, 4, 5, 6, 7}
assert s2.month_of_year == {1, 4, 7, 10}
def get_interval(minutes, hour, day_of_month):
now = datetime.now()
time_this_month = datetime(
now.year,
now.month,
day_of_month,
hour,
minutes,
)
if time_this_month > now:
delta = time_this_month - now
else:
next_time = time_this_month + relativedelta(months=+1)
delta = next_time - now
interval = IntervalSchedule.from_schedule(schedule(delta))
interval.save()
return interval
def decode_schedule(obj):
if obj is None:
return None
_type = obj['__type__']
value = obj['__value__']
if _type == 'datetime':
return decode_datetime(value)
elif _type == 'crontab':
return crontab(*value.split('\t'))
elif _type == 'solar':
return solar(**value)
elif _type == 'schedule':
return schedule(**value)
else:
raise NotImplementedError('Cannot deserialize schedule %(type)s type' %
{'type': _type})
def dict_to_object(self, d):
if '__type__' not in d:
return d
objtype = d.pop('__type__')
if objtype == 'datetime':
return datetime(**d)
if objtype == 'interval':
return schedule(run_every=d['every'], relative=d['relative'])
if objtype == 'crontab':
return crontab(**d)
d['__type__'] = objtype
return d
def test_eq(self):
assert (self.crontab(day_of_week='1, 2') ==
self.crontab(day_of_week='1-2'))
assert (self.crontab(day_of_month='1, 16, 31') ==
self.crontab(day_of_month='*/15'))
assert (
self.crontab(
minute='1', hour='2', day_of_week='5',
day_of_month='10', month_of_year='5') ==
self.crontab(
minute='1', hour='2', day_of_week='5',
day_of_month='10', month_of_year='5'))
assert crontab(minute='1') != crontab(minute='2')
assert (self.crontab(month_of_year='1') !=
self.crontab(month_of_year='2'))
assert object() != self.crontab(minute='1')
assert self.crontab(minute='1') != object()
assert crontab(month_of_year='1') != schedule(10)
def create_entry(self,
name=None,
task=None,
s=None,
run_every=60,
**kwargs):
if name is None:
name = 'test'
if task is None:
task = 'tasks.test'
if s is None:
s = schedule(run_every=run_every)
e = RedBeatSchedulerEntry(name, task, s, app=self.app, **kwargs)
return e
def monthly_task(*args, **kwargs):
try:
task_id = args[0]
task = PeriodicTask.objects.get(
task='w3af_webui.tasks.monthly_task',
name=task_id,
)
now = datetime.now()
next_time = now + relativedelta(months=+1)
delta = next_time - now
interval = IntervalSchedule.from_schedule(schedule(delta))
interval.save()
logger.info('set interval %s for celery task %s' % (
interval,
task.name,
))
task.interval = interval
task.save()
scan_create_start(task_id)
except Exception, e:
logger.error("monthly task exception %s" % e)
raise Exception, e
def test_eq(self):
self.assertEqual(
self.crontab(day_of_week='1, 2'),
self.crontab(day_of_week='1-2'),
)
self.assertEqual(
self.crontab(day_of_month='1, 16, 31'),
self.crontab(day_of_month='*/15'),
)
self.assertEqual(
self.crontab(minute='1', hour='2', day_of_week='5',
day_of_month='10', month_of_year='5'),
self.crontab(minute='1', hour='2', day_of_week='5',
day_of_month='10', month_of_year='5'),
)
self.assertNotEqual(crontab(minute='1'), crontab(minute='2'))
self.assertNotEqual(
self.crontab(month_of_year='1'),
self.crontab(month_of_year='2'),
)
self.assertNotEqual(object(), self.crontab(minute='1'))
self.assertNotEqual(self.crontab(minute='1'), object())
self.assertNotEqual(crontab(month_of_year='1'), schedule(10))
def test_seralization_1(self):
sch = schedules.schedule(run_every=timedelta(seconds=365))
date = datetime(2017, 5, 3, 23, 3, 47)
entry = ScheduleEntry(name='everytime',
task='celery.runtask',
schedule=sch,
args=['hello', 'world', 5],
kwargs={'time': 4, 'name': 'testers'},
options={'echo': True},
last_run_at=datetime.now(timezone.utc),
total_run_count=99)
dumped = EntrySchema(strict=True).dump(entry).data
loaded = EntrySchema(strict=True).load(dumped).data
assert entry.name == loaded.name
assert entry.task == loaded.task
assert entry.schedule.run_every == loaded.schedule.run_every
assert entry.args == loaded.args
assert entry.kwargs == loaded.kwargs
assert entry.options == loaded.options
assert entry.last_run_at == loaded.last_run_at
assert entry.total_run_count == loaded.total_run_count
def dict_to_object(self, d):
if '__type__' not in d:
return d
objtype = d.pop('__type__')
if objtype == 'datetime':
zone = d.pop('timezone', 'UTC')
try:
tzinfo = FixedOffset(zone / 60)
except TypeError:
tzinfo = timezone.get_timezone(zone)
return datetime(tzinfo=tzinfo, **d)
if objtype == 'interval':
return schedule(run_every=d['every'], relative=d['relative'])
if objtype == 'crontab':
return crontab(**d)
if objtype == 'weekday':
return weekday(**d)
if objtype == 'rrule':
# Decode timestamp values into datetime objects
for key, tz_key in [('dtstart', 'dtstart_tz'),
('until', 'until_tz')]:
timestamp = d.get(key)
tz_minutes = d.pop(tz_key, 0)
if timestamp is not None:
d[key] = from_timestamp(timestamp, tz_minutes)
return rrule(**d)
d['__type__'] = objtype
return d
def test_schedule_equal_different_key_fail(self):
scheduler = beat.Scheduler(app=self.app)
a = {'a': self.create_schedule_entry(schedule=schedule(5))}
b = {'b': self.create_schedule_entry(schedule=schedule(5))}
assert not scheduler.schedules_equal(a, b)
def test_schedule_equal_crontab_vs_schedule_fail(self):
scheduler = beat.Scheduler(app=self.app)
a = {'a': self.create_schedule_entry(schedule=crontab(minute=5))}
b = {'a': self.create_schedule_entry(schedule=schedule(5))}
assert not scheduler.schedules_equal(a, b)
def test_schedule_equal_schedule_vs_schedule_success(self):
scheduler = beat.Scheduler(app=self.app)
a = {'a': self.create_schedule_entry(schedule=schedule(5))}
b = {'a': self.create_schedule_entry(schedule=schedule(5))}
assert scheduler.schedules_equal(a, b)
import datetime
from celery import shared_task
from celery.schedules import schedule
from celery.task import periodic_task
from celery.utils.log import get_task_logger
from tenancy.utils import map_task_per_tenants, tenant_context_or_raise_reject
from .models import FileUpload
logger = get_task_logger(__name__)
@shared_task
def delete_expired_files(tenant_id: int):
with tenant_context_or_raise_reject(tenant_id) as tenant:
deleted, detailed = FileUpload.objects.delete_expired()
logger.info("[%d: %s]: Removed %d expired files" % (
tenant_id,
tenant.schema_name,
deleted,
))
return detailed
@periodic_task(run_every=schedule(run_every=datetime.timedelta(hours=6)))
def delete_expired_files_from_tenants():
return map_task_per_tenants(delete_expired_files)
def create_entry(self, **kwargs):
entry = dict(name="celery.unittest.add",
schedule=schedule(timedelta(seconds=10)),
args=(2, 2),
options={"routing_key": "cpu"})
return self.Entry(**dict(entry, **kwargs))
def test_PeriodicTask_unicode_interval(self):
p = create_model_interval(schedule(timedelta(seconds=10)))
self.assertEqual(unicode(p), '{0}: every 10.0 seconds'.format(p.name))
