def test_run_task_already_started(self, mock_run):
"""Test for CronWorker.run method -
attempt to run already started CronTask.
"""
pid = 1001
cron_task = CronTask.objects.run_now("ClassLockedSleep")[0]
cron_task.mark_as_started(pid)
with patch_ps(active_pids=[pid]):
with patch_kill(active_pids=[pid]):
worker = CronWorker()
worker.slack = mock.MagicMock()
worker.logger = mock.MagicMock()
output = worker.run(cron_task.job_spec())
message = (
"CronTaskInvalidStatus: "
'Unable to start "ClassLockedSleep:task_id={}", '
"because associated CronTask "
'has invalid status "Started".'.format(cron_task.pk)
)
self.assertIn(message, output)
worker.logger.warning.assert_called_once_with(message)
worker.slack.post.assert_called_once_with(
"[{}] {}".format(SYSTEM_NAME, message)
)
mock_run.assert_not_called()
def test_run_task_without_params(self, mock_run):
"""Test for CronWorker.run method - run CronTask without params"""
cron_task = CronTask.objects.run_now("Sleep")[0]
worker = CronWorker()
worker.run(cron_task.job_spec())
mock_run.assert_called_once_with()
cron_task.refresh_from_db()
self.assertEqual(cron_task.status, CronTaskStatus.FINISHED)
def test_run_cron_job_without_slack_notify_done(self, mock_run):
"""Test for CronWorker.run method - run cron job without
`slack_notify_done` option set.
"""
worker = CronWorker()
worker.slack = mock.MagicMock()
output = worker.run("Sleep")
self.assertIn("OK: Processed Sleep", output)
worker.slack.post.assert_not_called()
def create_pid_file(job_spec, pid=None):
"""Utility function to create PID file for given job spec"""
pid = os.getpid() if pid is None else pid
worker = CronWorker()
name, args, kwargs, cron_job_class = worker.parse_job_spec_with_class(
job_spec)
pid_file = worker.get_pid_file(cron_job_class, name, args, kwargs)
with mock.patch("cronman.worker.worker_file.os.getpid", lambda: pid):
pid_file.create()
return pid_file
def create_job_spec_file(job_spec, content=None):
"""Utility function to create JobSpec file for given job spec"""
content = job_spec if content is None else content
worker = CronWorker()
name, args, kwargs, cron_job_class = worker.parse_job_spec_with_class(
job_spec)
pid_file = worker.get_pid_file(cron_job_class, name, args, kwargs)
job_spec_file = pid_file.job_spec_file
job_spec_file.create(content)
return job_spec_file
def test_run_missing_task(self, mock_run):
"""Test for CronWorker.run method - run DELETED CronTask"""
cron_task = CronTask.objects.run_now("Sleep")[0]
job_spec = cron_task.job_spec()
cron_task.delete()
worker = CronWorker()
output = worker.run(job_spec)
mock_run.assert_called_once_with()
self.assertIn("OK:", output) # Cron job passes anyway
def test_run_non_existing_cron_job(self, mock_run):
"""Test for CronWorker.run method - attempt to run non existing
cron job
"""
worker = CronWorker()
with self.assertRaisesMessage(
CronWorkerInvalidParams,
"CronJobNotRegistered: {!r}".format("NoSuchWorker"),
):
worker.run("NoSuchWorker")
mock_run.assert_not_called()
def test_run_task_error(self, mock_run):
"""Test for CronWorker.run method - exception raised while processing
CronTask.
"""
cron_task = CronTask.objects.run_now("Sleep")[0]
worker = CronWorker()
worker.run(cron_task.job_spec())
mock_run.assert_called_once_with()
cron_task.refresh_from_db()
self.assertEqual(cron_task.status, CronTaskStatus.FAILED)
def test_run_invalid_params(self, mock_run):
"""Test for CronWorker.run method - attempt to run cron job with
invalid params
"""
worker = CronWorker()
with self.assertRaisesMessage(
CronWorkerInvalidParams,
"ValueError: In chars 9-10 `5`: "
"Positional argument after named arguments.",
):
worker.run("Sleep:2,test=3,5") # positional arg. after named one
mock_run.assert_not_called()
def test_run_cron_job_with_slack_notify_done(self, mock_run):
"""Test for CronWorker.run method - run cron job with
`slack_notify_done` option set.
"""
worker = CronWorker()
worker.slack = mock.MagicMock()
output = worker.run("SlackNotifyDoneSleep")
self.assertIn("OK: Processed SlackNotifyDoneSleep", output)
mock_run.assert_called_once_with()
worker.slack.post.assert_called_once_with(
'Cron job "SlackNotifyDoneSleep" is done.'
)
def test_run_task_while_class_lock_enabled(self, mock_run):
"""Test for CronWorker.run method -
attempt to run CronTask while class-based lock enabled.
"""
cron_task = CronTask.objects.run_now("ClassLockedSleep")[0]
worker = CronWorker()
worker.slack = mock.MagicMock()
create_pid_file(cron_task.cron_job)
worker.run(cron_task.job_spec())
mock_run.assert_not_called()
cron_task.refresh_from_db()
self.assertEqual(cron_task.status, CronTaskStatus.QUEUED)
def test_run_cron_job_with_lock_ignore_errors(self, mock_run):
"""Test for CronWorker.run method -
attempt to run while params-based lock enabled and `lock_ignore_errors`
option is set.
"""
worker = CronWorker()
worker.slack = mock.MagicMock()
create_pid_file("IgnoreLockErrorsSleep")
output = worker.run("IgnoreLockErrorsSleep")
self.assertIn(
"CronWorkerLocked: "
'Unable to start "IgnoreLockErrorsSleep", '
"because similar process is already running (PID file exists).",
output,
)
mock_run.assert_not_called()
worker.slack.post.assert_not_called()
def test_run_killed_task(self, mock_run):
"""Test for CronWorker.run method - resuming killed CronTask."""
pid = 1001
cron_task = CronTask.objects.run_now("ClassLockedSleep")[0]
cron_task.mark_as_started(pid)
with patch_ps():
with patch_kill():
worker = CronWorker()
worker.slack = mock.MagicMock()
worker.logger = mock.MagicMock()
output = worker.run(cron_task.job_spec())
self.assertIn("OK:", output)
worker.logger.info.assert_has_calls(
[
mock.call(
'Starting "ClassLockedSleep:task_id={}" '
"for killed CronTask.".format(cron_task.pk)
)
]
)
mock_run.assert_called_once_with()
def test_run_cron_job_while_class_lock_enabled(self, mock_run):
"""Test for CronWorker.run method -
attempt to run while class-based lock enabled.
"""
worker = CronWorker()
worker.slack = mock.MagicMock()
create_pid_file("ClassLockedSleep")
output = worker.run("ClassLockedSleep")
self.assertIn(
"CronWorkerLocked: "
'Unable to start "ClassLockedSleep", '
"because similar process is already running (PID file exists).",
output,
)
mock_run.assert_not_called()
worker.slack.post.assert_called_once_with(
(
"[{}] "
"CronWorkerLocked: "
'Unable to start "ClassLockedSleep", '
"because similar process is already running (PID file exists)."
).format(SYSTEM_NAME)
)
def test_run_task_with_invalid_status(self, mock_run):
"""Test for CronWorker.run method -
attempt to run CronTask with invalid status.
"""
cron_task = CronTask.objects.run_now("ClassLockedSleep")[0]
cron_task.mark_as_failed()
worker = CronWorker()
worker.slack = mock.MagicMock()
worker.logger = mock.MagicMock()
output = worker.run(cron_task.job_spec())
message = (
"CronTaskInvalidStatus: "
'Unable to start "ClassLockedSleep:task_id={}", '
"because associated CronTask "
'has invalid status "Failed".'.format(cron_task.pk)
)
self.assertIn(message, output)
worker.logger.warning.assert_called_once_with(message)
worker.slack.post.assert_called_once_with(
"[{}] {}".format(SYSTEM_NAME, message)
)
mock_run.assert_not_called()
def handle(self, **options):
"""Main command logic"""
method_name = options["method"]
arg = options["arg"]
worker = CronWorker()
method = getattr(worker, method_name)
if not arg:
if method_name == "run":
raise CommandError("Job specification is required.")
if arg:
if method_name in ("clean", "suspend"):
raise CommandError(
'Subcommand "{}" does not accept arguments.'.format(
method_name))
return method(arg)
else:
return method()
class CleanCronTasks(BaseCronJob):
"""Changes status of dead CronTasks from STARTED to FAILED."""
cronitor_id = app_settings.CRONMAN_CLEAN_CRON_TASKS_CRONITOR_ID
def __init__(self, logger=None):
super(CleanCronTasks, self).__init__(logger=logger)
self.cron_worker = CronWorker()
self.cron_worker.logger = self.logger
def run(self):
"""Main logic"""
num_failed = 0
cron_tasks = self.get_started_cron_tasks()
if cron_tasks:
active_pids = self.get_active_pids()
for cron_task in cron_tasks:
if cron_task.pid not in active_pids:
cron_task.mark_as_failed()
num_failed += 1
if num_failed:
status_message = "{} CronTask(s) marked as failed.".format(
num_failed)
else:
status_message = "No CronTasks marked as failed."
self.logger.info(status_message)
def get_started_cron_tasks(self):
"""Retrieves started CronTasks"""
allowed_tasks = cron_jobs_module_config("ALLOWED_CRON_TASKS",
default=())
return CronTask.objects.started().filter(cron_job__in=allowed_tasks)
def get_active_pids(self):
"""Retrieves list of all active cron worker PIDs"""
current_cron_jobs = self.cron_worker.get_worker_pid_list().status()[0]
return [
item["pid"] for item in current_cron_jobs
if item["status"] == PIDStatus.ALIVE
]
def test_run_with_params(self, mock_run):
"""Test for CronWorker.run method - run cron job with params"""
worker = CronWorker()
output = worker.run("Sleep:42,path=/tmp/test")
self.assertIn("OK: Processed Sleep:42,path=/tmp/test", output)
mock_run.assert_called_once_with("42", path="/tmp/test")
def cron_worker(self):
"""Cron Worker instance"""
return CronWorker(data_dir=self.data_dir,
debug=self.debug,
logger=self.logger)
def test_run_without_params(self, mock_run):
"""Test for CronWorker.run method - run cron job without params"""
worker = CronWorker()
output = worker.run("Sleep")
self.assertIn("OK: Processed Sleep", output)
mock_run.assert_called_once_with()
def test_run_cron_job_error(self, mock_run):
"""Test for CronWorker.run method - exception raised while processing"""
worker = CronWorker()
output = worker.run("Sleep")
self.assertIn("FAIL: Processed Sleep", output)
mock_run.assert_called_once_with()
def __init__(self, logger=None):
super(CleanCronTasks, self).__init__(logger=logger)
self.cron_worker = CronWorker()
self.cron_worker.logger = self.logger