def test_clean_all_files_active(self):
"""Test for cleaning dead/stalled files - all files active, no deletion"""
pid_1, pid_2, pid_3 = 1001, 1002, 1003
pid_file_1 = create_pid_file("ParamsLockedSleep:seconds=10", pid_1)
pid_file_2 = create_pid_file("ClassLockedSleep:seconds=10", pid_2)
pid_file_3 = create_pid_file("PersistentSleep:seconds=30", pid_3)
job_spec_file_3 = create_job_spec_file("PersistentSleep:seconds=30")
with patch_ps(active_pids=[pid_1, pid_2, pid_3]):
with patch_kill(
active_pids=[pid_1, pid_2, pid_3],
die_on_sigterm_pids=[pid_1, pid_2, pid_3],
):
output = call_command("cron_worker", "clean")
self.assertEqual(
output,
"CLEAN PID FILES:\n"
"No PID file(s) found.\n"
"CLEAN JOBSPEC FILES:\n"
"No JobSpec file(s) found.\n",
)
# No process killed on during cleaning:
self.assertTrue(ProcessManager(pid_1).alive())
self.assertTrue(ProcessManager(pid_2).alive())
self.assertTrue(ProcessManager(pid_3).alive())
# Files associated with active processes are not deleted:
self.assertTrue(os.path.exists(pid_file_1.path))
self.assertTrue(os.path.exists(pid_file_2.path))
self.assertTrue(os.path.exists(pid_file_3.path))
self.assertTrue(os.path.exists(job_spec_file_3.path))
def test_clean_3_dead_pids_1_stalled_jobspec(self):
"""Test for cleaning dead/stalled files - 3 dead PIDfiles,
1 stalled JobSpec file
"""
pid_2, pid_3 = 1002, 1003
hash_1 = get_params_hash([], {"seconds": "10"})
pid_file_1 = create_pid_file("ParamsLockedSleep:seconds=10", "")
pid_file_2 = create_pid_file("ClassLockedSleep:seconds=10", pid_2)
pid_file_3 = create_pid_file("PersistentSleep:seconds=30", pid_3)
job_spec_file_3 = create_job_spec_file("PersistentSleep:seconds=30")
with patch_ps():
with patch_kill():
output = call_command("cron_worker", "clean")
self.assertEqual(
output,
"CLEAN PID FILES:\n"
"ClassLockedSleep\tDELETED\t{pid_2}\n"
"ParamsLockedSleep_{hash_1}\tDELETED\t{pid_1}\n"
"PersistentSleep\tDELETED\t{pid_3}\n"
"TOTAL: 3\n"
"CLEAN JOBSPEC FILES:\n"
"PersistentSleep\tDELETED\tPersistentSleep:seconds=30\n"
"TOTAL: 1\n".format(hash_1=hash_1,
pid_1=None,
pid_2=pid_2,
pid_3=pid_3),
)
# Files associated with already dead processes are deleted:
self.assertFalse(os.path.exists(pid_file_1.path))
self.assertFalse(os.path.exists(pid_file_2.path))
self.assertFalse(os.path.exists(pid_file_3.path))
self.assertFalse(os.path.exists(job_spec_file_3.path))
def test_suspend_cleaning(self):
"""Test for suspending all active workers - clearing dead PIDfiles
before killing.
"""
pid_2, pid_3 = 1002, 1003
hash_1 = get_params_hash([], {"seconds": "10"})
pid_file_1 = create_pid_file("ParamsLockedSleep:seconds=10", "")
pid_file_2 = create_pid_file("ClassLockedSleep:seconds=10", pid_2)
pid_file_3 = create_pid_file("PersistentSleep:seconds=30", pid_3)
job_spec_file_3 = create_job_spec_file("PersistentSleep:seconds=30")
with patch_ps():
with patch_kill():
output = call_command("cron_worker", "suspend")
self.assertEqual(
output,
"CLEAN PID FILES:\n"
"ClassLockedSleep\tDELETED\t{pid_2}\n"
"ParamsLockedSleep_{hash_1}\tDELETED\t{pid_1}\n"
"PersistentSleep\tDELETED\t{pid_3}\n"
"TOTAL: 3\n"
"CLEAN JOBSPEC FILES:\n"
"PersistentSleep\tDELETED\tPersistentSleep:seconds=30\n"
"TOTAL: 1\n"
"KILL:\n"
"No PID file(s) found.\n".format(hash_1=hash_1,
pid_1=None,
pid_2=pid_2,
pid_3=pid_3),
)
# Files associated with already dead processes are deleted (clean):
self.assertFalse(os.path.exists(pid_file_1.path))
self.assertFalse(os.path.exists(pid_file_2.path))
self.assertFalse(os.path.exists(pid_file_3.path))
self.assertFalse(os.path.exists(job_spec_file_3.path))
def test_run_2_started_1_dead(self, mock_get_logger):
"""Test for CleanCronTasks - 2 CronTask objects started, 1 marked as
failed.
"""
now = timezone.now()
# Started Task - running:
pid_s1 = 1001
cron_task_s1 = CronTask.objects.run_now(
"Sleep", now=now - datetime.timedelta(minutes=6))[0]
cron_task_s1.mark_as_started(pid_s1)
# Started Task - dead:
cron_task_s2 = CronTask.objects.run_now(
"ParamsLockedSleep", now=now - datetime.timedelta(minutes=1))[0]
pid_s2 = 1002
cron_task_s2.mark_as_started(pid_s2)
# Queued Task:
cron_task_q1 = CronTask.objects.run_now(
"ClassLockedSleep", now=now - datetime.timedelta(minutes=1))[0]
cron_task_q1.mark_as_queued()
# Waiting Task:
cron_task_w1 = CronTask.objects.run_now(
"ClassLockedSleep",
params="42",
now=now + datetime.timedelta(minutes=6),
)[0]
# Finished Task:
cron_task_f1 = CronTask.objects.run_now("IgnoreLockErrorsSleep")[0]
cron_task_f1.mark_as_finished()
create_pid_file(cron_task_s1.job_spec(), pid_s1)
with patch_ps(active_pids=[pid_s1]):
with patch_kill(active_pids=[pid_s1]):
self.assertTrue(call_worker("CleanCronTasks").ok)
mock_get_logger.return_value.info.assert_has_calls(
[mock.call("1 CronTask(s) marked as failed.")])
# Started Task - running - no status change:
cron_task_s1.refresh_from_db()
self.assertEqual(cron_task_s1.status, CronTaskStatus.STARTED)
# Started Task - dead - marked as failed:
cron_task_s2.refresh_from_db()
self.assertEqual(cron_task_s2.status, CronTaskStatus.FAILED)
# Other statuses - no status change:
cron_task_q1.refresh_from_db()
self.assertEqual(cron_task_q1.status, CronTaskStatus.QUEUED)
cron_task_w1.refresh_from_db()
self.assertEqual(cron_task_w1.status, CronTaskStatus.WAITING)
cron_task_f1.refresh_from_db()
self.assertEqual(cron_task_f1.status, CronTaskStatus.FINISHED)
def test_run_worker_params_lock_enabled_success(self):
"""Test for attempt to run worker while PARAMS-based lock is enabled -
success.
"""
locked_job_spec = "ParamsLockedSleep:seconds=10"
job_spec = "ParamsLockedSleep:seconds=1" # same class, diff. params
pid = 1001
create_pid_file(locked_job_spec, pid)
with patch_ps(active_pids=[pid]):
with patch_kill(active_pids=[pid]):
self.assertTrue(call_worker(job_spec).ok)
def test_run_worker_no_lock_enabled_success(self):
"""Test for running worker while other worker is on, but no lock
is acquired - success.
"""
running_job_spec = "Sleep:seconds=10"
job_spec = "Sleep:seconds=1"
pid = 1001
create_pid_file(running_job_spec, pid)
with patch_ps(active_pids=[pid]):
with patch_kill(active_pids=[pid]):
self.assertTrue(call_worker(job_spec).ok)
def test_resume_truncated_jobspec_file(self, mock_start):
"""Test for resuming workers - truncated JobSpec file case
(no resuming)."""
create_pid_file("PersistentSleep:seconds=20", 1001)
create_job_spec_file("PersistentSleep:seconds=20", "")
with patch_ps():
with patch_kill():
output = call_command("cron_worker", "resume")
self.assertEqual(output, "RESUME:\n"
"No JobSpec file(s) found.\n")
mock_start.assert_not_called()
def test_run_task_while_class_lock_enabled(self, mock_run):
"""Test for CronWorker.run method -
attempt to run CronTask while class-based lock enabled.
"""
cron_task = CronTask.objects.run_now("ClassLockedSleep")[0]
worker = CronWorker()
worker.slack = mock.MagicMock()
create_pid_file(cron_task.cron_job)
worker.run(cron_task.job_spec())
mock_run.assert_not_called()
cron_task.refresh_from_db()
self.assertEqual(cron_task.status, CronTaskStatus.QUEUED)
def test_status_by_pid(self):
"""Test for listing active workers by PID"""
pid_1, pid_2 = 1001, 1002
create_pid_file("ParamsLockedSleep:seconds=10", pid_1)
create_pid_file("ClassLockedSleep:seconds=10", pid_2)
with patch_ps(active_pids=[pid_1, pid_2]):
with patch_kill(active_pids=[pid_1, pid_2]):
output = call_command("cron_worker", "status", str(pid_2))
self.assertEqual(
output,
"STATUS:\n"
"ClassLockedSleep\tALIVE\t{pid_2}\n"
"TOTAL: 1\tALIVE: 1\tDEAD: 0\n".format(pid_2=pid_2),
)
def test_run_worker_params_lock_enabled_failure(self):
"""Test for attempt to run worker while PARAMS-based lock is enabled -
failure.
"""
locked_job_spec = "ParamsLockedSleep:seconds=10"
job_spec = "ParamsLockedSleep:seconds=10" # same class, same params
pid = 1001
create_pid_file(locked_job_spec, pid)
with patch_ps(active_pids=[pid]):
with patch_kill(active_pids=[pid]):
result = call_worker(job_spec)
self.assertEqual(result.exc_class_name, "CronWorkerLocked")
self.assertEqual(
result.exc_message,
'Unable to start "ParamsLockedSleep:seconds=10", '
"because similar process is already running (PID file exists).\n",
)
def test_run_cron_job_with_lock_ignore_errors(self, mock_run):
"""Test for CronWorker.run method -
attempt to run while params-based lock enabled and `lock_ignore_errors`
option is set.
"""
worker = CronWorker()
worker.slack = mock.MagicMock()
create_pid_file("IgnoreLockErrorsSleep")
output = worker.run("IgnoreLockErrorsSleep")
self.assertIn(
"CronWorkerLocked: "
'Unable to start "IgnoreLockErrorsSleep", '
"because similar process is already running (PID file exists).",
output,
)
mock_run.assert_not_called()
worker.slack.post.assert_not_called()
def test_status(self):
"""Test for listing active workers - 2 PID files with active workers"""
pid_1, pid_2 = 1001, 1002
hash_1 = get_params_hash([], {"seconds": "10"})
create_pid_file("ParamsLockedSleep:seconds=10", pid_1)
create_pid_file("ClassLockedSleep:seconds=10", pid_2)
with patch_ps(active_pids=[pid_1, pid_2]):
with patch_kill(active_pids=[pid_1, pid_2]):
output = call_command("cron_worker", "status")
self.assertEqual(
output,
"STATUS:\n"
"ClassLockedSleep\tALIVE\t{pid_2}\n"
"ParamsLockedSleep_{hash_1}\tALIVE\t{pid_1}\n"
"TOTAL: 2\tALIVE: 2\tDEAD: 0\n".format(hash_1=hash_1,
pid_1=pid_1,
pid_2=pid_2),
)
def test_kill_by_name(self):
"""Test for killing all active workers by cron job name"""
pid_1, pid_2 = 1001, 1002
create_pid_file("ParamsLockedSleep:seconds=10", pid_1)
create_pid_file("ClassLockedSleep:seconds=10", pid_2)
with patch_ps(active_pids=[pid_1, pid_2]):
with patch_kill(active_pids=[pid_1, pid_2],
die_on_sigterm_pids=[pid_2]):
output = call_command("cron_worker", "kill",
"ClassLockedSleep")
self.assertEqual(
output,
"KILL:\n"
"ClassLockedSleep\tTERMED\t{pid_2}\n"
"TOTAL: 1\tDEAD: 0\tTERMED: 1\tKILLED: 0\n".format(
pid_2=pid_2),
)
self.assertTrue(ProcessManager(pid_1).alive())
self.assertFalse(ProcessManager(pid_2).alive())
def test_kill_by_pid_existing_worker(self):
"""Test for killing worker by PID - 1 process terminated"""
pid_1, pid_2 = 1001, 1002
hash_1 = get_params_hash([], {"seconds": "10"})
create_pid_file("ParamsLockedSleep:seconds=10", pid_1)
create_pid_file("ClassLockedSleep:seconds=10", pid_2)
with patch_ps(active_pids=[pid_1, pid_2]):
with patch_kill(active_pids=[pid_1, pid_2],
die_on_sigterm_pids=[pid_1]):
output = call_command("cron_worker", "kill", str(pid_1))
self.assertEqual(
output,
"KILL:\n"
"ParamsLockedSleep_{hash_1}\tTERMED\t{pid_1}\n"
"TOTAL: 1\tDEAD: 0\tTERMED: 1\tKILLED: 0\n".format(
hash_1=hash_1, pid_1=pid_1),
)
self.assertFalse(ProcessManager(pid_1).alive())
self.assertTrue(ProcessManager(pid_2).alive())
def test_resume_2_workers(self, mock_start):
"""Test for resuming workers - 2 workers resumed"""
pid_1, pid_2, pid_3 = 1001, 1002, 1003
hash_3 = get_params_hash([], {"seconds": "30"})
create_pid_file("ParamsLockedSleep:seconds=10", pid_1)
create_pid_file("PersistentSleep:seconds=20", pid_2)
job_spec_file_2 = create_job_spec_file("PersistentSleep:seconds=20")
create_pid_file("PersistentSleep2:seconds=30", pid_3)
job_spec_file_3 = create_job_spec_file("PersistentSleep2:seconds=30")
with patch_ps():
with patch_kill():
output = call_command("cron_worker", "resume")
self.assertEqual(
output,
"RESUME:\n"
"PersistentSleep\tRESUMED\tPersistentSleep:seconds=20\n"
"PersistentSleep2_{hash_3}\tRESUMED\t"
"PersistentSleep2:seconds=30\n"
"TOTAL: 2\n".format(hash_3=hash_3),
)
# JobSpec files have been deleted before spawning new processes:
self.assertFalse(os.path.exists(job_spec_file_2.path))
self.assertFalse(os.path.exists(job_spec_file_3.path))
# Processes have been spawned:
mock_start.assert_has_calls([
mock.call("PersistentSleep:seconds=20"),
mock.call("PersistentSleep2:seconds=30"),
])
def test_kill_truncated_pidfile(self):
"""Test for killing all active workers - truncated PIDfile case"""
hash_1 = get_params_hash([], {"seconds": "10"})
create_pid_file("ParamsLockedSleep:seconds=10", "") # empty file
with patch_ps():
with patch_kill():
output = call_command("cron_worker", "kill")
self.assertEqual(
output,
"KILL:\n"
"ParamsLockedSleep_{hash_1}\tDEAD\t{pid_1}\n"
"TOTAL: 1\tDEAD: 1\tTERMED: 0\tKILLED: 0\n".format(
hash_1=hash_1, pid_1=None),
)
# PIDFile should handle IOError (missing file) correctly:
self.assertIsNone(
CronWorkerPIDFile(app_settings.CRONMAN_DATA_DIR, "Fake").pid)
# ProcessManager should handle missing PID
# (due to truncated/missing file) correctly:
process_manager = ProcessManager(None)
self.assertFalse(process_manager.alive())
self.assertEqual(process_manager.status(), "")
def test_run_cron_job_while_class_lock_enabled(self, mock_run):
"""Test for CronWorker.run method -
attempt to run while class-based lock enabled.
"""
worker = CronWorker()
worker.slack = mock.MagicMock()
create_pid_file("ClassLockedSleep")
output = worker.run("ClassLockedSleep")
self.assertIn(
"CronWorkerLocked: "
'Unable to start "ClassLockedSleep", '
"because similar process is already running (PID file exists).",
output,
)
mock_run.assert_not_called()
worker.slack.post.assert_called_once_with(
(
"[{}] "
"CronWorkerLocked: "
'Unable to start "ClassLockedSleep", '
"because similar process is already running (PID file exists)."
).format(SYSTEM_NAME)
)
def test_suspend_3_workers(self):
"""Test for suspending all active workers - 3 processes terminated, 1
can be resumed.
"""
pid_1, pid_2, pid_3 = 1001, 1002, 1003
hash_1 = get_params_hash([], {"seconds": "10"})
pid_file_1 = create_pid_file("ParamsLockedSleep:seconds=10", pid_1)
pid_file_2 = create_pid_file("ClassLockedSleep:seconds=10", pid_2)
pid_file_3 = create_pid_file("PersistentSleep:seconds=30", pid_3)
job_spec_file_3 = create_job_spec_file("PersistentSleep:seconds=30")
with patch_ps(active_pids=[pid_1, pid_2, pid_3]):
with patch_kill(
active_pids=[pid_1, pid_2, pid_3],
die_on_sigterm_pids=[pid_1, pid_2, pid_3],
):
output = call_command("cron_worker", "suspend")
self.assertEqual(
output,
"CLEAN PID FILES:\n"
"No PID file(s) found.\n"
"CLEAN JOBSPEC FILES:\n"
"No JobSpec file(s) found.\n"
"KILL:\n"
"ClassLockedSleep\tTERMED\t{pid_2}\n"
"ParamsLockedSleep_{hash_1}\tTERMED\t{pid_1}\n"
"PersistentSleep\tTERMED\t1003\n"
"TOTAL: 3\tDEAD: 0\tTERMED: 3\tKILLED: 0\n".format(
hash_1=hash_1, pid_1=pid_1, pid_2=pid_2),
)
self.assertFalse(ProcessManager(pid_1).alive())
self.assertFalse(ProcessManager(pid_2).alive())
self.assertFalse(ProcessManager(pid_3).alive())
# Files associated with killed processes are not deleted:
self.assertTrue(os.path.exists(pid_file_1.path))
self.assertTrue(os.path.exists(pid_file_2.path))
self.assertTrue(os.path.exists(pid_file_3.path))
self.assertTrue(os.path.exists(job_spec_file_3.path))
def test_resume_all_files_active(self, mock_start):
"""Test for resuming workers - all workers active, no resuming."""
pid_1, pid_2, pid_3 = 1001, 1002, 1003
create_pid_file("ParamsLockedSleep:seconds=10", pid_1)
create_pid_file("PersistentSleep:seconds=20", pid_2)
create_job_spec_file("PersistentSleep:seconds=20")
create_pid_file("PersistentSleep2:seconds=30", pid_3)
create_job_spec_file("PersistentSleep2:seconds=30")
with patch_ps(active_pids=[pid_1, pid_2, pid_3]):
with patch_kill(
active_pids=[pid_1, pid_2, pid_3],
die_on_sigterm_pids=[pid_1, pid_2, pid_3],
):
output = call_command("cron_worker", "resume")
self.assertEqual(output, "RESUME:\n"
"No JobSpec file(s) found.\n")
mock_start.assert_not_called()
