def testCronJobRunExpiry(self): job_id = "job1" self.db.WriteCronJob(rdf_cronjobs.CronJob(cron_job_id=job_id)) fake_time = rdfvalue.RDFDatetime.Now() - rdfvalue.Duration("7d") with test_lib.FakeTime(fake_time): run = rdf_cronjobs.CronJobRun(cron_job_id=job_id, run_id="00000000") self.db.WriteCronJobRun(run) with test_lib.FakeTime(fake_time + rdfvalue.Duration("1d")): run = rdf_cronjobs.CronJobRun(cron_job_id=job_id, run_id="00000001") self.db.WriteCronJobRun(run) with test_lib.FakeTime(fake_time + rdfvalue.Duration("2d")): run = rdf_cronjobs.CronJobRun(cron_job_id=job_id, run_id="00000002") self.db.WriteCronJobRun(run) self.assertLen(self.db.ReadCronJobRuns(job_id), 3) cutoff = fake_time + rdfvalue.Duration("1h") self.db.DeleteOldCronJobRuns(cutoff) jobs = self.db.ReadCronJobRuns(job_id) self.assertLen(jobs, 2) for job in jobs: self.assertGreater(job.timestamp, cutoff) cutoff = fake_time + rdfvalue.Duration("1d") + rdfvalue.Duration("1h") self.db.DeleteOldCronJobRuns(cutoff) jobs = self.db.ReadCronJobRuns(job_id) self.assertLen(jobs, 1) for job in jobs: self.assertGreater(job.timestamp, cutoff)
def testCronJobDeletion(self): job_id = "job0" self.db.WriteCronJob(rdf_cronjobs.CronJob(cron_job_id=job_id)) job_run0 = rdf_cronjobs.CronJobRun(cron_job_id=job_id, run_id="a") job_run1 = rdf_cronjobs.CronJobRun(cron_job_id=job_id, run_id="b") self.db.WriteCronJobRun(job_run0) self.db.WriteCronJobRun(job_run1) self.assertLen(self.db.ReadCronJobRuns(job_id), 2) self.db.DeleteCronJob(job_id) with self.assertRaises(db.UnknownCronJobError): self.db.ReadCronJob(job_id) self.assertEmpty(self.db.ReadCronJobRuns(job_id))
def testCronJobRegistryInstantiation(self): for job_cls in cronjobs.CronJobRegistry.CRON_REGISTRY.values(): job = rdf_cronjobs.CronJob(cron_job_id="foobar") job_run = rdf_cronjobs.CronJobRun(cron_job_id="foobar", status="RUNNING") job_cls(job_run, job) # Should not fail.
def testOSBreakdown(self): """Check that all client stats cron jobs are run.""" run = rdf_cronjobs.CronJobRun() job = rdf_cronjobs.CronJob() system.OSBreakDownCronJob(run, job).Run() self._CheckOSBreakdown()
def testLastAccessStats(self): """Check that all client stats cron jobs are run.""" run = rdf_cronjobs.CronJobRun() job = rdf_cronjobs.CronJob() system.LastAccessStatsCronJob(run, job).Run() self._CheckLastAccessStats()
def testGRRVersionBreakDown(self): """Check that all client stats cron jobs are run.""" cron_run = rdf_cronjobs.CronJobRun() job_data = rdf_cronjobs.CronJob() cron = system.GRRVersionBreakDownCronJob(cron_run, job_data) cron.Run() self._CheckGRRVersionBreakDown()
def testGRRVersionBreakDown(self): """Check that all client stats cron jobs are run.""" cron_run = rdf_cronjobs.CronJobRun() job_data = rdf_cronjobs.CronJob() cron = system.GRRVersionBreakDownCronJob(cron_run, job_data) cron.Run() self._CheckGRRVersionBreakDown() self.assertEqual(cron.run_state.log_message, "Processed 22 clients.")
def testPurgeServerStats(self): if not data_store.RelationalDBReadEnabled(): self.skipTest("Test is only for the relational DB. Skipping...") fake_stats_collector = default_stats_collector.DefaultStatsCollector([ stats_utils.CreateCounterMetadata("fake_counter"), ]) timestamp0 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1) timestamp1 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(2) timestamp2 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3600) timestamp3 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4800) config_overrides = { "Database.useForReads.stats": True, "StatsStore.stats_ttl_hours": 1 } with test_lib.ConfigOverrider(config_overrides), \ stats_test_utils.FakeStatsContext(fake_stats_collector), \ mock.patch.object(system, "_STATS_DELETION_BATCH_SIZE", 1): with test_lib.FakeTime(rdfvalue.RDFDatetime(timestamp0)): stats_store._WriteStats(process_id="fake_process_id") with test_lib.FakeTime(rdfvalue.RDFDatetime(timestamp1)): stats_collector_instance.Get().IncrementCounter("fake_counter") stats_store._WriteStats(process_id="fake_process_id") expected_results = { "fake_process_id": { "fake_counter": [(0, timestamp0), (1, timestamp1)] } } self.assertDictEqual( stats_store.ReadStats("f", "fake_counter"), expected_results) with test_lib.FakeTime(timestamp2): stats_store._WriteStats(process_id="fake_process_id") expected_results = { "fake_process_id": { "fake_counter": [(0, timestamp0), (1, timestamp1), (1, timestamp2)] } } self.assertDictEqual( stats_store.ReadStats("f", "fake_counter"), expected_results) with test_lib.FakeTime(timestamp3): cron = system.PurgeServerStatsCronJob( rdf_cronjobs.CronJobRun(), rdf_cronjobs.CronJob()) cron.Run() # timestamp0 and timestamp1 are older than 1h, so they should get # deleted. expected_results = { "fake_process_id": { "fake_counter": [(1, timestamp2)] } } self.assertDictEqual( stats_store.ReadStats("f", "fake_counter"), expected_results) self.assertIn("Deleted 2 stats entries.", cron.run_state.log_message)
def testCronJobRunExpiry(self): job_id = "job1" self.db.WriteCronJob(rdf_cronjobs.CronJob(cron_job_id=job_id)) fake_time = rdfvalue.RDFDatetime.Now() - rdfvalue.Duration.From( 7, rdfvalue.DAYS) with test_lib.FakeTime(fake_time): run = rdf_cronjobs.CronJobRun(cron_job_id=job_id, run_id="00000000", started_at=fake_time) self.db.WriteCronJobRun(run) fake_time_one_day_later = fake_time + rdfvalue.Duration.From( 1, rdfvalue.DAYS) with test_lib.FakeTime(fake_time_one_day_later): run = rdf_cronjobs.CronJobRun(cron_job_id=job_id, run_id="00000001", started_at=fake_time_one_day_later) self.db.WriteCronJobRun(run) fake_time_two_days_later = fake_time + rdfvalue.Duration.From( 2, rdfvalue.DAYS) with test_lib.FakeTime(fake_time_two_days_later): run = rdf_cronjobs.CronJobRun(cron_job_id=job_id, run_id="00000002", started_at=fake_time_two_days_later) self.db.WriteCronJobRun(run) self.assertLen(self.db.ReadCronJobRuns(job_id), 3) cutoff = fake_time + rdfvalue.Duration.From(1, rdfvalue.HOURS) self.db.DeleteOldCronJobRuns(cutoff) jobs = self.db.ReadCronJobRuns(job_id) self.assertLen(jobs, 2) for job in jobs: self.assertGreater(job.timestamp, cutoff) cutoff = fake_time + rdfvalue.Duration.From(25, rdfvalue.HOURS) self.db.DeleteOldCronJobRuns(cutoff) jobs = self.db.ReadCronJobRuns(job_id) self.assertLen(jobs, 1) for job in jobs: self.assertGreater(job.timestamp, cutoff)
def RunJob(self, job): """Does the actual work of the Cron, if the job is due to run. Args: job: The cronjob rdfvalue that should be run. Must be leased. Returns: A boolean indicating if this cron job was due to run. Raises: LockError: if the object is not locked. ValueError: If the job argument is invalid. """ if not job.leased_until: raise LockError("CronJob must be leased for Run() to be called.") if job.leased_until < rdfvalue.RDFDatetime.Now(): raise LockError("CronJob lease expired for %s." % job.cron_job_id) if not self.JobDueToRun(job): return False logging.info("Running cron job: %s", job.cron_job_id) run_state = rdf_cronjobs.CronJobRun(cron_job_id=job.cron_job_id, status="RUNNING") run_state.GenerateRunId() if job.args.action_type == job.args.ActionType.SYSTEM_CRON_ACTION: cls_name = job.args.system_cron_action.job_class_name job_cls = registry.SystemCronJobRegistry.CronJobClassByName( cls_name) name = "%s runner" % cls_name elif job.args.action_type == job.args.ActionType.HUNT_CRON_ACTION: job_cls = registry.CronJobRegistry.CronJobClassByName("RunHunt") name = "Hunt runner" else: raise ValueError("CronJob %s doesn't have a valid args type set." % job.cron_job_id) data_store.REL_DB.WriteCronJobRun(run_state) data_store.REL_DB.UpdateCronJob( job.cron_job_id, last_run_time=rdfvalue.RDFDatetime.Now(), current_run_id=run_state.run_id, forced_run_requested=False) run_obj = job_cls(run_state, job) if self.max_threads == 1: run_obj.StartRun() else: self._GetThreadPool().AddTask(target=run_obj.StartRun, name=name) return True
def testCronJobRegistryInstantiation(self): # We import the `server_startup` module to ensure that all cron jobs classes # that are really used on the server are imported and populate the registry. # pylint: disable=unused-variable, g-import-not-at-top from grr_response_server import server_startup # pylint: enable=unused-variable, g-import-not-at-top for job_cls in cronjobs.CronJobRegistry.CRON_REGISTRY.values(): job = rdf_cronjobs.CronJob(cron_job_id="foobar") job_run = rdf_cronjobs.CronJobRun(cron_job_id="foobar", status="RUNNING") job_cls(job_run, job) # Should not fail.
def testCronJobRuns(self): with self.assertRaises(db.UnknownCronJobError): self.db.WriteCronJobRun( rdf_cronjobs.CronJobRun(cron_job_id="job1", run_id="00000000")) now = rdfvalue.RDFDatetime.Now() with test_lib.FakeTime(now): for j in range(1, 3): self.db.WriteCronJob( rdf_cronjobs.CronJob(cron_job_id="job%d" % j)) for r in range(1, 3): run = rdf_cronjobs.CronJobRun(cron_job_id="job%d" % j, run_id="abcd123%d" % r) self.db.WriteCronJobRun(run) for j in range(1, 3): job_id = "job%d" % j jobs = self.db.ReadCronJobRuns(job_id) self.assertLen(jobs, 2) for job in jobs: self.assertEqual(job.cron_job_id, job_id) self.assertEqual(job.timestamp, now) job = self.db.ReadCronJobRun("job1", "abcd1231") self.assertEqual(job.cron_job_id, "job1") self.assertEqual(job.run_id, "abcd1231") self.assertEqual(job.timestamp, now) with self.assertRaises(ValueError): self.db.ReadCronJobRun(job_id, "invalid_id") with self.assertRaises(db.UnknownCronJobRunError): self.db.ReadCronJobRun(job_id, "abcd1234") with self.assertRaises(db.UnknownCronJobRunError): self.db.ReadCronJobRun("doesntexist", "abcd1231") self.assertEqual(self.db.ReadCronJobRuns("doesntexist"), [])
def testPurgeServerStats(self): if not data_store.RelationalDBReadEnabled(): self.skipTest("Test is only for the relational DB. Skipping...") fake_stats_collector = default_stats_collector.DefaultStatsCollector([ stats_utils.CreateCounterMetadata("fake_counter"), ]) timestamp1 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1) timestamp2 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3600) timestamp3 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4800) config_overrides = { "Database.useForReads.stats": True, "StatsStore.stats_ttl_hours": 1 } with test_lib.ConfigOverrider(config_overrides): with stats_test_utils.FakeStatsContext(fake_stats_collector): with test_lib.FakeTime(rdfvalue.RDFDatetime(timestamp1)): stats_collector_instance.Get().IncrementCounter( "fake_counter") stats_store._WriteStats(process_id="fake_process_id") expected_results = { "fake_process_id": { "fake_counter": [(1, timestamp1)] } } self.assertDictEqual( stats_store.ReadStats("f", "fake_counter"), expected_results) with test_lib.FakeTime(timestamp2): stats_store._WriteStats(process_id="fake_process_id") expected_results = { "fake_process_id": { "fake_counter": [(1, timestamp1), (1, timestamp2)] } } self.assertDictEqual( stats_store.ReadStats("f", "fake_counter"), expected_results) with test_lib.FakeTime(timestamp3): system.PurgeServerStatsCronJob( rdf_cronjobs.CronJobRun(), rdf_cronjobs.CronJob()).Run() # timestamp1 is older than 1h, so it should get deleted. expected_results = { "fake_process_id": { "fake_counter": [(1, timestamp2)] } } self.assertDictEqual( stats_store.ReadStats("f", "fake_counter"), expected_results)
def ProcessHuntOutputPlugins(self): if data_store.RelationalDBFlowsEnabled(): job = rdf_cronjobs.CronJob( cron_job_id="some/id", lifetime=rdfvalue.Duration("1h")) run_state = rdf_cronjobs.CronJobRun( cron_job_id="some/id", status="RUNNING", started_at=rdfvalue.RDFDatetime.Now()) process_results.ProcessHuntResultCollectionsCronJob(run_state, job).Run() else: flow_urn = flow.StartAFF4Flow( flow_name=process_results.ProcessHuntResultCollectionsCronFlow .__name__, token=self.token) flow_test_lib.TestFlowHelper(flow_urn, token=self.token) return flow_urn
def testCronJobUpdates(self): job = self._CreateCronJob() job.last_run_status = rdf_cronjobs.CronJobRun.CronJobRunStatus.FINISHED self.db.WriteCronJob(job) err = rdf_cronjobs.CronJobRun.CronJobRunStatus.ERROR self.db.UpdateCronJob(job.cron_job_id, last_run_status=err) new_job = self.db.ReadCronJob(job.cron_job_id) self.assertEqual(job.last_run_status, "FINISHED") self.assertEqual(new_job.last_run_status, "ERROR") t = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1000000) self.db.UpdateCronJob(job.cron_job_id, last_run_time=t) new_job = self.db.ReadCronJob(job.cron_job_id) self.assertEqual(new_job.last_run_time, t) # To test `current_run_id` we first need to create a CronJobRun with it. job_run0 = rdf_cronjobs.CronJobRun(cron_job_id=job.cron_job_id, run_id="ABCD1234") self.db.WriteCronJobRun(job_run0) self.db.UpdateCronJob(job.cron_job_id, current_run_id="ABCD1234") new_job = self.db.ReadCronJob(job.cron_job_id) self.assertEqual(new_job.current_run_id, "ABCD1234") # None is accepted to clear the current_run_id. self.db.UpdateCronJob(job.cron_job_id, current_run_id=None) new_job = self.db.ReadCronJob(job.cron_job_id) self.assertFalse(new_job.current_run_id) state = job.state self.assertNotIn("test", state) state["test"] = 12345 self.db.UpdateCronJob(job.cron_job_id, state=state) new_job = self.db.ReadCronJob(job.cron_job_id) self.assertEqual(new_job.state.get("test"), 12345) self.db.UpdateCronJob(job.cron_job_id, forced_run_requested=True) new_job = self.db.ReadCronJob(job.cron_job_id) self.assertEqual(new_job.forced_run_requested, True) with self.assertRaises(db.UnknownCronJobError): self.db.UpdateCronJob("Does not exist", current_run_id="12345678")
def testCronJobRunsOverwrite(self): self.db.WriteCronJob(rdf_cronjobs.CronJob(cron_job_id="job")) run = rdf_cronjobs.CronJobRun(cron_job_id="job", run_id="abcd1234") self.db.WriteCronJobRun(run) original_ts = self.db.ReadCronJobRun("job", "abcd1234").timestamp now = rdfvalue.RDFDatetime.Now() run.backtrace = "error" run.log_message = "log" run.started_at = now - rdfvalue.Duration("5s") run.finished_at = now self.db.WriteCronJobRun(run) read = self.db.ReadCronJobRun("job", "abcd1234") self.assertEqual(read.backtrace, run.backtrace) self.assertEqual(read.log_message, run.log_message) self.assertEqual(read.started_at, run.started_at) self.assertEqual(read.finished_at, run.finished_at) self.assertNotEqual(read.timestamp, original_ts)
def _RunPurgeClientStats(self): run = rdf_cronjobs.CronJobRun() job = rdf_cronjobs.CronJob() system.PurgeClientStatsCronJob(run, job).Run()
def testPurgeServerStats(self): if not data_store.RelationalDBReadEnabled(): self.skipTest("Test is only for the relational DB. Skipping...") fake_stats_collector = prometheus_stats_collector.PrometheusStatsCollector( [ stats_utils.CreateCounterMetadata("fake_counter"), ]) timestamp0 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1) timestamp1 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(2) timestamp2 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3600) timestamp3 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4800) config_overrides = { "Database.useForReads.stats": True, "StatsStore.stats_ttl_hours": 1 } zero_duration = rdfvalue.Duration(0) # Backslash continuation is explicitly allowed by Google's style guide for # nested context manager expressions spanning 3 or more lines. # pylint: disable=g-backslash-continuation with test_lib.ConfigOverrider(config_overrides), \ stats_test_utils.FakeStatsContext(fake_stats_collector), \ mock.patch.object(system, "_STATS_DELETION_BATCH_SIZE", 1), \ mock.patch.object(system, "_stats_checkpoint_period", zero_duration): with test_lib.FakeTime(rdfvalue.RDFDatetime(timestamp0)): stats_store._WriteStats(process_id="fake_process_id") with test_lib.FakeTime(rdfvalue.RDFDatetime(timestamp1)): stats_collector_instance.Get().IncrementCounter("fake_counter") stats_store._WriteStats(process_id="fake_process_id") expected_results = { "fake_process_id": { "fake_counter": [(0, timestamp0), (1, timestamp1)] } } self.assertDictEqual( stats_store.ReadStats("f", "fake_counter"), expected_results) with test_lib.FakeTime(timestamp2): stats_store._WriteStats(process_id="fake_process_id") expected_results = { "fake_process_id": { "fake_counter": [(0, timestamp0), (1, timestamp1), (1, timestamp2)] } } self.assertDictEqual( stats_store.ReadStats("f", "fake_counter"), expected_results) with test_lib.FakeTime(timestamp3): cron_name = compatibility.GetName( system.PurgeServerStatsCronJob) cronjobs.ScheduleSystemCronJobs(names=[cron_name]) job_data = data_store.REL_DB.ReadCronJobs([cron_name])[0] cron_run = rdf_cronjobs.CronJobRun(cron_job_id=cron_name) cron_run.GenerateRunId() cron_run.started_at = rdfvalue.RDFDatetime.Now() cron = system.PurgeServerStatsCronJob(cron_run, job_data) cron.Run() # timestamp0 and timestamp1 are older than 1h, so they should get # deleted. expected_results = { "fake_process_id": { "fake_counter": [(1, timestamp2)] } } self.assertDictEqual( stats_store.ReadStats("f", "fake_counter"), expected_results) self.assertEqual( "Deleted 2 stats entries.\nDeleted 1 stats entries.", cron.run_state.log_message)
def testCronJob(self, fs_conn_mock): if not data_store.RelationalDBReadEnabled(): self.skipTest("Test is only for the relational DB. Skipping...") client_id1 = "C.0000000000000001" client_id2 = "C.0000000000000002" client_id3 = "C.0000000000000003" client_id4 = "C.0000000000000004" client_id5 = "C.0000000000000005" client_id6 = "C.0000000000000006" client_id7 = "C.0000000000000007" data_store.REL_DB.WriteClientMetadata(client_id1, fleetspeak_enabled=False) data_store.REL_DB.WriteClientMetadata(client_id2, fleetspeak_enabled=True) data_store.REL_DB.WriteClientMetadata( client_id3, last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3), fleetspeak_enabled=True) data_store.REL_DB.WriteClientMetadata( client_id4, last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(41), fleetspeak_enabled=True) data_store.REL_DB.WriteClientMetadata( client_id5, last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(5), fleetspeak_enabled=True) data_store.REL_DB.WriteClientMetadata( client_id6, last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(61), fleetspeak_enabled=True) data_store.REL_DB.WriteClientMetadata( client_id7, last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(68), fleetspeak_enabled=True) fs_enabled_ids = [ client_id2, client_id3, client_id4, client_id5, client_id6, client_id7 ] fs_clients = {} for i, client_id in enumerate(fs_enabled_ids): client_number = i + 2 fs_client_id = fleetspeak_utils.GRRIDToFleetspeakID(client_id) fs_client = admin_pb2.Client(client_id=fs_client_id) fs_client.last_contact_time.FromSeconds(client_number * 10) fs_clients[fs_client_id] = fs_client def FakeListClients(list_request): clients = [] for fs_client_id in list_request.client_ids: clients.append(fs_clients[fs_client_id]) return admin_pb2.ListClientsResponse(clients=clients) fs_conn_mock.outgoing.ListClients = FakeListClients cron_run = rdf_cronjobs.CronJobRun() job_data = rdf_cronjobs.CronJob() cron = system.UpdateFSLastPingTimestamps(cron_run, job_data) with test_lib.FakeTime( rdfvalue.RDFDatetime.FromSecondsSinceEpoch(100)): with test_lib.ConfigOverrider({ "Server.fleetspeak_last_ping_threshold": "35s", "Server.fleetspeak_list_clients_batch_size": 2, }): cron.Run() actual_timestamps = data_store.REL_DB.ReadClientLastPings() expected_timestamps = { # Skipped because not a Fleetspeak client. client_id1: None, client_id2: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(20), client_id3: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(30), # Skipped because FS timestamp is old. client_id4: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(41), client_id5: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(50), # Skipped because FS timestamp is old. client_id6: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(61), # Skipped because existing GRR timestamp is too recent. client_id7: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(68), } self.assertEqual(actual_timestamps, expected_timestamps) self.assertMultiLineEqual(cron._log_messages.popleft(), "Updated timestamps for 3 clients.")
def _RunCleanup(self): run = rdf_cronjobs.CronJobRun() job = rdf_cronjobs.CronJob() self.cleaner_job = data_retention.CleanInactiveClientsCronJob(run, job) self.cleaner_job.Run()
def RunJob(self, job): """Does the actual work of the Cron, if the job is due to run. Args: job: The cronjob rdfvalue that should be run. Must be leased. Returns: A boolean indicating if this cron job was started or not. False may be returned when the threadpool is already full. Raises: LockError: if the object is not locked. ValueError: If the job argument is invalid. """ if not job.leased_until: raise LockError("CronJob must be leased for Run() to be called.") if job.leased_until < rdfvalue.RDFDatetime.Now(): raise LockError("CronJob lease expired for %s." % job.cron_job_id) logging.info("Starting cron job: %s", job.cron_job_id) if job.args.action_type == job.args.ActionType.SYSTEM_CRON_ACTION: cls_name = job.args.system_cron_action.job_class_name job_cls = SystemCronJobRegistry.CronJobClassByName(cls_name) name = "%s runner" % cls_name elif job.args.action_type == job.args.ActionType.HUNT_CRON_ACTION: job_cls = CronJobRegistry.CronJobClassByName("RunHunt") name = "Hunt runner" else: raise ValueError("CronJob %s doesn't have a valid args type set." % job.cron_job_id) run_state = rdf_cronjobs.CronJobRun( cron_job_id=job.cron_job_id, status="RUNNING") run_state.GenerateRunId() run_obj = job_cls(run_state, job) wait_for_start_event, signal_event, wait_for_write_event = ( threading.Event(), threading.Event(), threading.Event()) try: self._GetThreadPool().AddTask( target=run_obj.StartRun, args=(wait_for_start_event, signal_event, wait_for_write_event), name=name, blocking=False, inline=False) if not wait_for_start_event.wait(TASK_STARTUP_WAIT): logging.error("Cron job run task for %s is too slow to start.", job.cron_job_id) # Most likely the thread pool is full and the task is sitting on the # queue. Make sure we don't put more things on the queue by returning # False. return False # We know that the cron job task has started, unblock it by setting # the signal event. If signal_event is not set (this happens if the # task sits on a ThreadPool's queue doing nothing, see the # if-statement above) the task will just be a no-op when ThreadPool # finally gets to it. This way we can ensure that we can safely return # the lease and let another worker schedule the same job. signal_event.set() wait_for_write_event.wait(TASK_STARTUP_WAIT) return True except threadpool.Full: return False