def get_periodic_tasks_info():
from celery import current_app
schedule = Service(current_app).get_scheduler().get_schedule()
tasks = []
for key, entry in schedule.items():
# entry.is_due() returns (is_due, time_in_seconds_for_next_execution)
is_due_tpl = entry.is_due()
next_execution = timezone.now() + datetime.timedelta(
seconds=is_due_tpl[1])
# remove delay between the timezone.now and the schedule entry due date
next_execution = next_execution.replace(microsecond=0)
tasks.append({
'name':
key,
'task':
entry.task,
'args':
'(' + ', '.join([json.dumps(arg) for arg in entry.args]) + ')',
'kwargs':
json.dumps(entry.kwargs),
'is_due':
is_due_tpl[0],
'next_execution':
next_execution
})
return tasks
def test_celery_beat_service_connect_function(self):
celery_app = self._scheduler.start()
celery_beat_service = Service(
celery_app,
max_interval=None,
schedule_filename=None,
scheduler_cls=PanoptesCeleryPluginScheduler)
self.assertFalse(
hasattr(celery_beat_service.scheduler, 'panoptes_context'))
self.assertFalse(
hasattr(celery_beat_service.scheduler, 'metadata_kv_store_class'))
self.assertFalse(hasattr(celery_beat_service.scheduler, 'task_prefix'))
with patch(
'yahoo_panoptes.enrichment.enrichment_plugin_scheduler.enrichment_plugin_scheduler'
) as mock_scheduler:
celery_beat_service_started(sender=celery_beat_service)
self.assertTrue(
hasattr(celery_beat_service.scheduler, 'panoptes_context'))
self.assertIsNotNone(
celery_beat_service.scheduler.metadata_kv_store_class)
self.assertIsNotNone(celery_beat_service.scheduler.task_prefix)
mock_scheduler.run.assert_called_with(celery_beat_service, None)
with patch(
'yahoo_panoptes.enrichment.enrichment_plugin_scheduler.enrichment_plugin_scheduler'
) as mock_scheduler:
mock_scheduler.run.side_effect = Exception
with self.assertRaises(SystemExit):
celery_beat_service_started(sender=celery_beat_service)
def test_enrichment_plugin_scheduler_task_config_error(self):
celery_app = self._scheduler.start()
celery_beat_service = Service(celery_app, max_interval=None, schedule_filename=None,
scheduler_cls=PanoptesCeleryPluginScheduler)
with patch('yahoo_panoptes.enrichment.enrichment_plugin_scheduler.const.DEFAULT_CONFIG_FILE_PATH',
self.panoptes_test_conf_file):
mock_getmtime = MagicMock(side_effect=Exception)
with patch('yahoo_panoptes.framework.plugins.panoptes_base_plugin.os.path.getmtime', mock_getmtime):
start_enrichment_plugin_scheduler()
enrichment_plugin_scheduler_task(celery_beat_service)
def test_basic_operations(self):
celery_app = self._scheduler.start()
celery_beat_service = Service(celery_app, max_interval=None, schedule_filename=None,
scheduler_cls=PanoptesCeleryPluginScheduler)
with patch('yahoo_panoptes.enrichment.enrichment_plugin_scheduler.const.DEFAULT_CONFIG_FILE_PATH',
self.panoptes_test_conf_file):
with patch('yahoo_panoptes.enrichment.enrichment_plugin_scheduler.PanoptesResourceCache.get_resources',
mock_get_resources):
start_enrichment_plugin_scheduler()
enrichment_plugin_scheduler_task(celery_beat_service)
def test_basic_operations(self):
celery_app = self._scheduler.start()
celery_beat_service = Service(
celery_app,
max_interval=None,
schedule_filename=None,
scheduler_cls=PanoptesCeleryPluginScheduler)
with patch(
'yahoo_panoptes.discovery.discovery_plugin_scheduler.const.DEFAULT_CONFIG_FILE_PATH',
self.panoptes_test_conf_file):
start_discovery_plugin_scheduler()
discovery_plugin_scheduler_task(celery_beat_service)
def test_error_messages(self):
celery_app = self._scheduler.start()
celery_beat_service = Service(celery_app, max_interval=None, schedule_filename=None,
scheduler_cls=PanoptesCeleryPluginScheduler)
with patch('yahoo_panoptes.enrichment.enrichment_plugin_scheduler.const.DEFAULT_CONFIG_FILE_PATH',
self.panoptes_test_conf_file):
mock_plugin_manager = MagicMock(side_effect=Exception)
with patch('yahoo_panoptes.enrichment.enrichment_plugin_scheduler.PanoptesPluginManager',
mock_plugin_manager):
start_enrichment_plugin_scheduler()
enrichment_plugin_scheduler_task(celery_beat_service)
def test_enrichment_plugin_scheduler_task_bad_plugin(self):
celery_app = self._scheduler.start()
celery_beat_service = Service(celery_app, max_interval=None, schedule_filename=None,
scheduler_cls=PanoptesCeleryPluginScheduler)
with patch('yahoo_panoptes.enrichment.enrichment_plugin_scheduler.const.DEFAULT_CONFIG_FILE_PATH',
self.panoptes_test_conf_file):
with patch('yahoo_panoptes.enrichment.enrichment_plugin_scheduler.PanoptesEnrichmentPluginInfo.'
'execute_frequency', 0):
start_enrichment_plugin_scheduler()
enrichment_plugin_scheduler_task(celery_beat_service)
with patch('yahoo_panoptes.enrichment.enrichment_plugin_scheduler.PanoptesEnrichmentPluginInfo.'
'resource_filter', None):
start_enrichment_plugin_scheduler()
enrichment_plugin_scheduler_task(celery_beat_service)
def test_redundant_shutdown_signal(self):
celery_app = self._scheduler.start()
celery_beat_service = Service(celery_app, max_interval=None, schedule_filename=None,
scheduler_cls=PanoptesCeleryPluginScheduler)
self._scheduler.run(celery_beat_service)
temp_is_set = self._scheduler._shutdown_plugin_scheduler.is_set
self._scheduler._shutdown_plugin_scheduler.is_set = _mock_is_set_true
self._scheduler._signal_handler(signal.SIGTERM, None) # pragma: no cover
self._scheduler._shutdown()
self.assertTrue(self._scheduler._t.isAlive())
with self.assertRaises(SystemExit):
self._scheduler._shutdown_plugin_scheduler.is_set = temp_is_set
self._scheduler._signal_handler(signal.SIGTERM, None) # pragma: no cover
def test_discovery_plugin_scheduler_update_error(self):
celery_app = self._scheduler.start()
celery_beat_service = Service(
celery_app,
max_interval=None,
schedule_filename=None,
scheduler_cls=PanoptesCeleryPluginScheduler)
with patch(
'yahoo_panoptes.discovery.discovery_plugin_scheduler.const.DEFAULT_CONFIG_FILE_PATH',
self.panoptes_test_conf_file):
mock_update = MagicMock(side_effect=Exception)
with patch(
'yahoo_panoptes.framework.celery_manager.PanoptesCeleryPluginScheduler.update',
mock_update):
start_discovery_plugin_scheduler()
discovery_plugin_scheduler_task(celery_beat_service)
def test_resource_cache_error(self):
celery_app = self._scheduler.start()
celery_beat_service = Service(celery_app, max_interval=None, schedule_filename=None,
scheduler_cls=PanoptesCeleryPluginScheduler)
with patch('yahoo_panoptes.enrichment.enrichment_plugin_scheduler.const.DEFAULT_CONFIG_FILE_PATH',
self.panoptes_test_conf_file):
mock_cache = MagicMock(side_effect=Exception)
with patch('yahoo_panoptes.enrichment.enrichment_plugin_scheduler.PanoptesResourceCache', mock_cache):
start_enrichment_plugin_scheduler()
enrichment_plugin_scheduler_task(celery_beat_service)
mock_get_resources_exception = MagicMock(side_effect=Exception)
with patch('yahoo_panoptes.enrichment.enrichment_plugin_scheduler.PanoptesResourceCache.get_resources',
mock_get_resources_exception):
start_enrichment_plugin_scheduler()
enrichment_plugin_scheduler_task(celery_beat_service)
def test_shutdown_after_tour_of_duty(self):
mock_tour_of_duty = create_autospec(PanoptesTourOfDuty)
mock_tour_of_duty.completed.return_value = True
mock_tour_of_duty.tasks_completed.return_value = True
mock_tour_of_duty.time_completed.return_value = True
mock_tour_of_duty.memory_growth_completed.return_value = True
with patch(u'yahoo_panoptes.framework.plugins.scheduler.PanoptesTourOfDuty', mock_tour_of_duty):
self._scheduler = PanoptesPluginScheduler(
panoptes_context=self._panoptes_context,
plugin_type=u"polling",
plugin_type_display_name=u"Polling",
celery_config=self._celery_config,
lock_timeout=1,
plugin_scheduler_task=_callback
)
celery_app = self._scheduler.start()
celery_beat_service = Service(celery_app, max_interval=None, schedule_filename=None,
scheduler_cls=PanoptesCeleryPluginScheduler)
self._scheduler.run(celery_beat_service)
def setup_schedule():
service = Service(app=app)
scheduler = service.get_scheduler()
scheduler.setup_schedule()
from datetime import datetime, timedelta
from celery.beat import Service
from tardis.celery import tardis_app
schedule = Service(tardis_app).get_scheduler().get_schedule()
now = datetime.utcnow()
for task_name, task in schedule.items():
next_run = task.last_run_at.replace(tzinfo=None) + task.schedule.run_every
print("{}: check if {} is less than {}".format(task_name, now, next_run))
assert now < next_run
