def test_suspend_cleaning(self): """Test for suspending all active workers - clearing dead PIDfiles before killing. """ pid_2, pid_3 = 1002, 1003 hash_1 = get_params_hash([], {"seconds": "10"}) pid_file_1 = create_pid_file("ParamsLockedSleep:seconds=10", "") pid_file_2 = create_pid_file("ClassLockedSleep:seconds=10", pid_2) pid_file_3 = create_pid_file("PersistentSleep:seconds=30", pid_3) job_spec_file_3 = create_job_spec_file("PersistentSleep:seconds=30") with patch_ps(): with patch_kill(): output = call_command("cron_worker", "suspend") self.assertEqual( output, "CLEAN PID FILES:\n" "ClassLockedSleep\tDELETED\t{pid_2}\n" "ParamsLockedSleep_{hash_1}\tDELETED\t{pid_1}\n" "PersistentSleep\tDELETED\t{pid_3}\n" "TOTAL: 3\n" "CLEAN JOBSPEC FILES:\n" "PersistentSleep\tDELETED\tPersistentSleep:seconds=30\n" "TOTAL: 1\n" "KILL:\n" "No PID file(s) found.\n".format(hash_1=hash_1, pid_1=None, pid_2=pid_2, pid_3=pid_3), ) # Files associated with already dead processes are deleted (clean): self.assertFalse(os.path.exists(pid_file_1.path)) self.assertFalse(os.path.exists(pid_file_2.path)) self.assertFalse(os.path.exists(pid_file_3.path)) self.assertFalse(os.path.exists(job_spec_file_3.path))
def test_clean_3_dead_pids_1_stalled_jobspec(self): """Test for cleaning dead/stalled files - 3 dead PIDfiles, 1 stalled JobSpec file """ pid_2, pid_3 = 1002, 1003 hash_1 = get_params_hash([], {"seconds": "10"}) pid_file_1 = create_pid_file("ParamsLockedSleep:seconds=10", "") pid_file_2 = create_pid_file("ClassLockedSleep:seconds=10", pid_2) pid_file_3 = create_pid_file("PersistentSleep:seconds=30", pid_3) job_spec_file_3 = create_job_spec_file("PersistentSleep:seconds=30") with patch_ps(): with patch_kill(): output = call_command("cron_worker", "clean") self.assertEqual( output, "CLEAN PID FILES:\n" "ClassLockedSleep\tDELETED\t{pid_2}\n" "ParamsLockedSleep_{hash_1}\tDELETED\t{pid_1}\n" "PersistentSleep\tDELETED\t{pid_3}\n" "TOTAL: 3\n" "CLEAN JOBSPEC FILES:\n" "PersistentSleep\tDELETED\tPersistentSleep:seconds=30\n" "TOTAL: 1\n".format(hash_1=hash_1, pid_1=None, pid_2=pid_2, pid_3=pid_3), ) # Files associated with already dead processes are deleted: self.assertFalse(os.path.exists(pid_file_1.path)) self.assertFalse(os.path.exists(pid_file_2.path)) self.assertFalse(os.path.exists(pid_file_3.path)) self.assertFalse(os.path.exists(job_spec_file_3.path))
def test_run_task_already_started(self, mock_run): """Test for CronWorker.run method - attempt to run already started CronTask. """ pid = 1001 cron_task = CronTask.objects.run_now("ClassLockedSleep")[0] cron_task.mark_as_started(pid) with patch_ps(active_pids=[pid]): with patch_kill(active_pids=[pid]): worker = CronWorker() worker.slack = mock.MagicMock() worker.logger = mock.MagicMock() output = worker.run(cron_task.job_spec()) message = ( "CronTaskInvalidStatus: " 'Unable to start "ClassLockedSleep:task_id={}", ' "because associated CronTask " 'has invalid status "Started".'.format(cron_task.pk) ) self.assertIn(message, output) worker.logger.warning.assert_called_once_with(message) worker.slack.post.assert_called_once_with( "[{}] {}".format(SYSTEM_NAME, message) ) mock_run.assert_not_called()
def test_clean_all_files_active(self): """Test for cleaning dead/stalled files - all files active, no deletion""" pid_1, pid_2, pid_3 = 1001, 1002, 1003 pid_file_1 = create_pid_file("ParamsLockedSleep:seconds=10", pid_1) pid_file_2 = create_pid_file("ClassLockedSleep:seconds=10", pid_2) pid_file_3 = create_pid_file("PersistentSleep:seconds=30", pid_3) job_spec_file_3 = create_job_spec_file("PersistentSleep:seconds=30") with patch_ps(active_pids=[pid_1, pid_2, pid_3]): with patch_kill( active_pids=[pid_1, pid_2, pid_3], die_on_sigterm_pids=[pid_1, pid_2, pid_3], ): output = call_command("cron_worker", "clean") self.assertEqual( output, "CLEAN PID FILES:\n" "No PID file(s) found.\n" "CLEAN JOBSPEC FILES:\n" "No JobSpec file(s) found.\n", ) # No process killed on during cleaning: self.assertTrue(ProcessManager(pid_1).alive()) self.assertTrue(ProcessManager(pid_2).alive()) self.assertTrue(ProcessManager(pid_3).alive()) # Files associated with active processes are not deleted: self.assertTrue(os.path.exists(pid_file_1.path)) self.assertTrue(os.path.exists(pid_file_2.path)) self.assertTrue(os.path.exists(pid_file_3.path)) self.assertTrue(os.path.exists(job_spec_file_3.path))
def test_resume_2_workers(self, mock_start): """Test for resuming workers - 2 workers resumed""" pid_1, pid_2, pid_3 = 1001, 1002, 1003 hash_3 = get_params_hash([], {"seconds": "30"}) create_pid_file("ParamsLockedSleep:seconds=10", pid_1) create_pid_file("PersistentSleep:seconds=20", pid_2) job_spec_file_2 = create_job_spec_file("PersistentSleep:seconds=20") create_pid_file("PersistentSleep2:seconds=30", pid_3) job_spec_file_3 = create_job_spec_file("PersistentSleep2:seconds=30") with patch_ps(): with patch_kill(): output = call_command("cron_worker", "resume") self.assertEqual( output, "RESUME:\n" "PersistentSleep\tRESUMED\tPersistentSleep:seconds=20\n" "PersistentSleep2_{hash_3}\tRESUMED\t" "PersistentSleep2:seconds=30\n" "TOTAL: 2\n".format(hash_3=hash_3), ) # JobSpec files have been deleted before spawning new processes: self.assertFalse(os.path.exists(job_spec_file_2.path)) self.assertFalse(os.path.exists(job_spec_file_3.path)) # Processes have been spawned: mock_start.assert_has_calls([ mock.call("PersistentSleep:seconds=20"), mock.call("PersistentSleep2:seconds=30"), ])
def test_run_2_started_1_dead(self, mock_get_logger): """Test for CleanCronTasks - 2 CronTask objects started, 1 marked as failed. """ now = timezone.now() # Started Task - running: pid_s1 = 1001 cron_task_s1 = CronTask.objects.run_now( "Sleep", now=now - datetime.timedelta(minutes=6))[0] cron_task_s1.mark_as_started(pid_s1) # Started Task - dead: cron_task_s2 = CronTask.objects.run_now( "ParamsLockedSleep", now=now - datetime.timedelta(minutes=1))[0] pid_s2 = 1002 cron_task_s2.mark_as_started(pid_s2) # Queued Task: cron_task_q1 = CronTask.objects.run_now( "ClassLockedSleep", now=now - datetime.timedelta(minutes=1))[0] cron_task_q1.mark_as_queued() # Waiting Task: cron_task_w1 = CronTask.objects.run_now( "ClassLockedSleep", params="42", now=now + datetime.timedelta(minutes=6), )[0] # Finished Task: cron_task_f1 = CronTask.objects.run_now("IgnoreLockErrorsSleep")[0] cron_task_f1.mark_as_finished() create_pid_file(cron_task_s1.job_spec(), pid_s1) with patch_ps(active_pids=[pid_s1]): with patch_kill(active_pids=[pid_s1]): self.assertTrue(call_worker("CleanCronTasks").ok) mock_get_logger.return_value.info.assert_has_calls( [mock.call("1 CronTask(s) marked as failed.")]) # Started Task - running - no status change: cron_task_s1.refresh_from_db() self.assertEqual(cron_task_s1.status, CronTaskStatus.STARTED) # Started Task - dead - marked as failed: cron_task_s2.refresh_from_db() self.assertEqual(cron_task_s2.status, CronTaskStatus.FAILED) # Other statuses - no status change: cron_task_q1.refresh_from_db() self.assertEqual(cron_task_q1.status, CronTaskStatus.QUEUED) cron_task_w1.refresh_from_db() self.assertEqual(cron_task_w1.status, CronTaskStatus.WAITING) cron_task_f1.refresh_from_db() self.assertEqual(cron_task_f1.status, CronTaskStatus.FINISHED)
def test_resume_truncated_jobspec_file(self, mock_start): """Test for resuming workers - truncated JobSpec file case (no resuming).""" create_pid_file("PersistentSleep:seconds=20", 1001) create_job_spec_file("PersistentSleep:seconds=20", "") with patch_ps(): with patch_kill(): output = call_command("cron_worker", "resume") self.assertEqual(output, "RESUME:\n" "No JobSpec file(s) found.\n") mock_start.assert_not_called()
def test_run_worker_params_lock_enabled_success(self): """Test for attempt to run worker while PARAMS-based lock is enabled - success. """ locked_job_spec = "ParamsLockedSleep:seconds=10" job_spec = "ParamsLockedSleep:seconds=1" # same class, diff. params pid = 1001 create_pid_file(locked_job_spec, pid) with patch_ps(active_pids=[pid]): with patch_kill(active_pids=[pid]): self.assertTrue(call_worker(job_spec).ok)
def test_run_worker_no_lock_enabled_success(self): """Test for running worker while other worker is on, but no lock is acquired - success. """ running_job_spec = "Sleep:seconds=10" job_spec = "Sleep:seconds=1" pid = 1001 create_pid_file(running_job_spec, pid) with patch_ps(active_pids=[pid]): with patch_kill(active_pids=[pid]): self.assertTrue(call_worker(job_spec).ok)
def test_status_by_pid(self): """Test for listing active workers by PID""" pid_1, pid_2 = 1001, 1002 create_pid_file("ParamsLockedSleep:seconds=10", pid_1) create_pid_file("ClassLockedSleep:seconds=10", pid_2) with patch_ps(active_pids=[pid_1, pid_2]): with patch_kill(active_pids=[pid_1, pid_2]): output = call_command("cron_worker", "status", str(pid_2)) self.assertEqual( output, "STATUS:\n" "ClassLockedSleep\tALIVE\t{pid_2}\n" "TOTAL: 1\tALIVE: 1\tDEAD: 0\n".format(pid_2=pid_2), )
def test_resume_no_pid_file(self, mock_start): """Test for resuming workers - no PID file case (resuming OK).""" job_spec_file_1 = create_job_spec_file("PersistentSleep:seconds=20") with patch_ps(): with patch_kill(): output = call_command("cron_worker", "resume") self.assertEqual( output, "RESUME:\n" "PersistentSleep\tRESUMED\tPersistentSleep:seconds=20\n" "TOTAL: 1\n", ) # JobSpec files have been deleted before spawning new processes: self.assertFalse(os.path.exists(job_spec_file_1.path)) # Processes have been spawned: mock_start.assert_called_once_with("PersistentSleep:seconds=20")
def test_resume_all_files_active(self, mock_start): """Test for resuming workers - all workers active, no resuming.""" pid_1, pid_2, pid_3 = 1001, 1002, 1003 create_pid_file("ParamsLockedSleep:seconds=10", pid_1) create_pid_file("PersistentSleep:seconds=20", pid_2) create_job_spec_file("PersistentSleep:seconds=20") create_pid_file("PersistentSleep2:seconds=30", pid_3) create_job_spec_file("PersistentSleep2:seconds=30") with patch_ps(active_pids=[pid_1, pid_2, pid_3]): with patch_kill( active_pids=[pid_1, pid_2, pid_3], die_on_sigterm_pids=[pid_1, pid_2, pid_3], ): output = call_command("cron_worker", "resume") self.assertEqual(output, "RESUME:\n" "No JobSpec file(s) found.\n") mock_start.assert_not_called()
def test_run_worker_params_lock_enabled_failure(self): """Test for attempt to run worker while PARAMS-based lock is enabled - failure. """ locked_job_spec = "ParamsLockedSleep:seconds=10" job_spec = "ParamsLockedSleep:seconds=10" # same class, same params pid = 1001 create_pid_file(locked_job_spec, pid) with patch_ps(active_pids=[pid]): with patch_kill(active_pids=[pid]): result = call_worker(job_spec) self.assertEqual(result.exc_class_name, "CronWorkerLocked") self.assertEqual( result.exc_message, 'Unable to start "ParamsLockedSleep:seconds=10", ' "because similar process is already running (PID file exists).\n", )
def test_status(self): """Test for listing active workers - 2 PID files with active workers""" pid_1, pid_2 = 1001, 1002 hash_1 = get_params_hash([], {"seconds": "10"}) create_pid_file("ParamsLockedSleep:seconds=10", pid_1) create_pid_file("ClassLockedSleep:seconds=10", pid_2) with patch_ps(active_pids=[pid_1, pid_2]): with patch_kill(active_pids=[pid_1, pid_2]): output = call_command("cron_worker", "status") self.assertEqual( output, "STATUS:\n" "ClassLockedSleep\tALIVE\t{pid_2}\n" "ParamsLockedSleep_{hash_1}\tALIVE\t{pid_1}\n" "TOTAL: 2\tALIVE: 2\tDEAD: 0\n".format(hash_1=hash_1, pid_1=pid_1, pid_2=pid_2), )
def test_kill_by_name(self): """Test for killing all active workers by cron job name""" pid_1, pid_2 = 1001, 1002 create_pid_file("ParamsLockedSleep:seconds=10", pid_1) create_pid_file("ClassLockedSleep:seconds=10", pid_2) with patch_ps(active_pids=[pid_1, pid_2]): with patch_kill(active_pids=[pid_1, pid_2], die_on_sigterm_pids=[pid_2]): output = call_command("cron_worker", "kill", "ClassLockedSleep") self.assertEqual( output, "KILL:\n" "ClassLockedSleep\tTERMED\t{pid_2}\n" "TOTAL: 1\tDEAD: 0\tTERMED: 1\tKILLED: 0\n".format( pid_2=pid_2), ) self.assertTrue(ProcessManager(pid_1).alive()) self.assertFalse(ProcessManager(pid_2).alive())
def test_kill_by_pid_existing_worker(self): """Test for killing worker by PID - 1 process terminated""" pid_1, pid_2 = 1001, 1002 hash_1 = get_params_hash([], {"seconds": "10"}) create_pid_file("ParamsLockedSleep:seconds=10", pid_1) create_pid_file("ClassLockedSleep:seconds=10", pid_2) with patch_ps(active_pids=[pid_1, pid_2]): with patch_kill(active_pids=[pid_1, pid_2], die_on_sigterm_pids=[pid_1]): output = call_command("cron_worker", "kill", str(pid_1)) self.assertEqual( output, "KILL:\n" "ParamsLockedSleep_{hash_1}\tTERMED\t{pid_1}\n" "TOTAL: 1\tDEAD: 0\tTERMED: 1\tKILLED: 0\n".format( hash_1=hash_1, pid_1=pid_1), ) self.assertFalse(ProcessManager(pid_1).alive()) self.assertTrue(ProcessManager(pid_2).alive())
def test_kill_truncated_pidfile(self): """Test for killing all active workers - truncated PIDfile case""" hash_1 = get_params_hash([], {"seconds": "10"}) create_pid_file("ParamsLockedSleep:seconds=10", "") # empty file with patch_ps(): with patch_kill(): output = call_command("cron_worker", "kill") self.assertEqual( output, "KILL:\n" "ParamsLockedSleep_{hash_1}\tDEAD\t{pid_1}\n" "TOTAL: 1\tDEAD: 1\tTERMED: 0\tKILLED: 0\n".format( hash_1=hash_1, pid_1=None), ) # PIDFile should handle IOError (missing file) correctly: self.assertIsNone( CronWorkerPIDFile(app_settings.CRONMAN_DATA_DIR, "Fake").pid) # ProcessManager should handle missing PID # (due to truncated/missing file) correctly: process_manager = ProcessManager(None) self.assertFalse(process_manager.alive()) self.assertEqual(process_manager.status(), "")
def test_run_killed_task(self, mock_run): """Test for CronWorker.run method - resuming killed CronTask.""" pid = 1001 cron_task = CronTask.objects.run_now("ClassLockedSleep")[0] cron_task.mark_as_started(pid) with patch_ps(): with patch_kill(): worker = CronWorker() worker.slack = mock.MagicMock() worker.logger = mock.MagicMock() output = worker.run(cron_task.job_spec()) self.assertIn("OK:", output) worker.logger.info.assert_has_calls( [ mock.call( 'Starting "ClassLockedSleep:task_id={}" ' "for killed CronTask.".format(cron_task.pk) ) ] ) mock_run.assert_called_once_with()
def test_suspend_3_workers(self): """Test for suspending all active workers - 3 processes terminated, 1 can be resumed. """ pid_1, pid_2, pid_3 = 1001, 1002, 1003 hash_1 = get_params_hash([], {"seconds": "10"}) pid_file_1 = create_pid_file("ParamsLockedSleep:seconds=10", pid_1) pid_file_2 = create_pid_file("ClassLockedSleep:seconds=10", pid_2) pid_file_3 = create_pid_file("PersistentSleep:seconds=30", pid_3) job_spec_file_3 = create_job_spec_file("PersistentSleep:seconds=30") with patch_ps(active_pids=[pid_1, pid_2, pid_3]): with patch_kill( active_pids=[pid_1, pid_2, pid_3], die_on_sigterm_pids=[pid_1, pid_2, pid_3], ): output = call_command("cron_worker", "suspend") self.assertEqual( output, "CLEAN PID FILES:\n" "No PID file(s) found.\n" "CLEAN JOBSPEC FILES:\n" "No JobSpec file(s) found.\n" "KILL:\n" "ClassLockedSleep\tTERMED\t{pid_2}\n" "ParamsLockedSleep_{hash_1}\tTERMED\t{pid_1}\n" "PersistentSleep\tTERMED\t1003\n" "TOTAL: 3\tDEAD: 0\tTERMED: 3\tKILLED: 0\n".format( hash_1=hash_1, pid_1=pid_1, pid_2=pid_2), ) self.assertFalse(ProcessManager(pid_1).alive()) self.assertFalse(ProcessManager(pid_2).alive()) self.assertFalse(ProcessManager(pid_3).alive()) # Files associated with killed processes are not deleted: self.assertTrue(os.path.exists(pid_file_1.path)) self.assertTrue(os.path.exists(pid_file_2.path)) self.assertTrue(os.path.exists(pid_file_3.path)) self.assertTrue(os.path.exists(job_spec_file_3.path))