def testCronJobRunExpiry(self):
job_id = "job1"
self.db.WriteCronJob(rdf_cronjobs.CronJob(cron_job_id=job_id))
fake_time = rdfvalue.RDFDatetime.Now() - rdfvalue.Duration("7d")
with test_lib.FakeTime(fake_time):
run = rdf_cronjobs.CronJobRun(cron_job_id=job_id,
run_id="00000000")
self.db.WriteCronJobRun(run)
with test_lib.FakeTime(fake_time + rdfvalue.Duration("1d")):
run = rdf_cronjobs.CronJobRun(cron_job_id=job_id,
run_id="00000001")
self.db.WriteCronJobRun(run)
with test_lib.FakeTime(fake_time + rdfvalue.Duration("2d")):
run = rdf_cronjobs.CronJobRun(cron_job_id=job_id,
run_id="00000002")
self.db.WriteCronJobRun(run)
self.assertLen(self.db.ReadCronJobRuns(job_id), 3)
cutoff = fake_time + rdfvalue.Duration("1h")
self.db.DeleteOldCronJobRuns(cutoff)
jobs = self.db.ReadCronJobRuns(job_id)
self.assertLen(jobs, 2)
for job in jobs:
self.assertGreater(job.timestamp, cutoff)
cutoff = fake_time + rdfvalue.Duration("1d") + rdfvalue.Duration("1h")
self.db.DeleteOldCronJobRuns(cutoff)
jobs = self.db.ReadCronJobRuns(job_id)
self.assertLen(jobs, 1)
for job in jobs:
self.assertGreater(job.timestamp, cutoff)
def testCronJobDeletion(self):
job_id = "job0"
self.db.WriteCronJob(rdf_cronjobs.CronJob(cron_job_id=job_id))
job_run0 = rdf_cronjobs.CronJobRun(cron_job_id=job_id, run_id="a")
job_run1 = rdf_cronjobs.CronJobRun(cron_job_id=job_id, run_id="b")
self.db.WriteCronJobRun(job_run0)
self.db.WriteCronJobRun(job_run1)
self.assertLen(self.db.ReadCronJobRuns(job_id), 2)
self.db.DeleteCronJob(job_id)
with self.assertRaises(db.UnknownCronJobError):
self.db.ReadCronJob(job_id)
self.assertEmpty(self.db.ReadCronJobRuns(job_id))
def testCronJobRegistryInstantiation(self):
for job_cls in cronjobs.CronJobRegistry.CRON_REGISTRY.values():
job = rdf_cronjobs.CronJob(cron_job_id="foobar")
job_run = rdf_cronjobs.CronJobRun(cron_job_id="foobar",
status="RUNNING")
job_cls(job_run, job) # Should not fail.
def testOSBreakdown(self):
"""Check that all client stats cron jobs are run."""
run = rdf_cronjobs.CronJobRun()
job = rdf_cronjobs.CronJob()
system.OSBreakDownCronJob(run, job).Run()
self._CheckOSBreakdown()
def testLastAccessStats(self):
"""Check that all client stats cron jobs are run."""
run = rdf_cronjobs.CronJobRun()
job = rdf_cronjobs.CronJob()
system.LastAccessStatsCronJob(run, job).Run()
self._CheckLastAccessStats()
def testGRRVersionBreakDown(self):
"""Check that all client stats cron jobs are run."""
cron_run = rdf_cronjobs.CronJobRun()
job_data = rdf_cronjobs.CronJob()
cron = system.GRRVersionBreakDownCronJob(cron_run, job_data)
cron.Run()
self._CheckGRRVersionBreakDown()
def testGRRVersionBreakDown(self):
"""Check that all client stats cron jobs are run."""
cron_run = rdf_cronjobs.CronJobRun()
job_data = rdf_cronjobs.CronJob()
cron = system.GRRVersionBreakDownCronJob(cron_run, job_data)
cron.Run()
self._CheckGRRVersionBreakDown()
self.assertEqual(cron.run_state.log_message, "Processed 22 clients.")
def testPurgeServerStats(self):
if not data_store.RelationalDBReadEnabled():
self.skipTest("Test is only for the relational DB. Skipping...")
fake_stats_collector = default_stats_collector.DefaultStatsCollector([
stats_utils.CreateCounterMetadata("fake_counter"),
])
timestamp0 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1)
timestamp1 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(2)
timestamp2 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3600)
timestamp3 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4800)
config_overrides = {
"Database.useForReads.stats": True,
"StatsStore.stats_ttl_hours": 1
}
with test_lib.ConfigOverrider(config_overrides), \
stats_test_utils.FakeStatsContext(fake_stats_collector), \
mock.patch.object(system, "_STATS_DELETION_BATCH_SIZE", 1):
with test_lib.FakeTime(rdfvalue.RDFDatetime(timestamp0)):
stats_store._WriteStats(process_id="fake_process_id")
with test_lib.FakeTime(rdfvalue.RDFDatetime(timestamp1)):
stats_collector_instance.Get().IncrementCounter("fake_counter")
stats_store._WriteStats(process_id="fake_process_id")
expected_results = {
"fake_process_id": {
"fake_counter": [(0, timestamp0), (1, timestamp1)]
}
}
self.assertDictEqual(
stats_store.ReadStats("f", "fake_counter"),
expected_results)
with test_lib.FakeTime(timestamp2):
stats_store._WriteStats(process_id="fake_process_id")
expected_results = {
"fake_process_id": {
"fake_counter": [(0, timestamp0), (1, timestamp1),
(1, timestamp2)]
}
}
self.assertDictEqual(
stats_store.ReadStats("f", "fake_counter"),
expected_results)
with test_lib.FakeTime(timestamp3):
cron = system.PurgeServerStatsCronJob(
rdf_cronjobs.CronJobRun(), rdf_cronjobs.CronJob())
cron.Run()
# timestamp0 and timestamp1 are older than 1h, so they should get
# deleted.
expected_results = {
"fake_process_id": {
"fake_counter": [(1, timestamp2)]
}
}
self.assertDictEqual(
stats_store.ReadStats("f", "fake_counter"),
expected_results)
self.assertIn("Deleted 2 stats entries.",
cron.run_state.log_message)
def testCronJobRunExpiry(self):
job_id = "job1"
self.db.WriteCronJob(rdf_cronjobs.CronJob(cron_job_id=job_id))
fake_time = rdfvalue.RDFDatetime.Now() - rdfvalue.Duration.From(
7, rdfvalue.DAYS)
with test_lib.FakeTime(fake_time):
run = rdf_cronjobs.CronJobRun(cron_job_id=job_id,
run_id="00000000",
started_at=fake_time)
self.db.WriteCronJobRun(run)
fake_time_one_day_later = fake_time + rdfvalue.Duration.From(
1, rdfvalue.DAYS)
with test_lib.FakeTime(fake_time_one_day_later):
run = rdf_cronjobs.CronJobRun(cron_job_id=job_id,
run_id="00000001",
started_at=fake_time_one_day_later)
self.db.WriteCronJobRun(run)
fake_time_two_days_later = fake_time + rdfvalue.Duration.From(
2, rdfvalue.DAYS)
with test_lib.FakeTime(fake_time_two_days_later):
run = rdf_cronjobs.CronJobRun(cron_job_id=job_id,
run_id="00000002",
started_at=fake_time_two_days_later)
self.db.WriteCronJobRun(run)
self.assertLen(self.db.ReadCronJobRuns(job_id), 3)
cutoff = fake_time + rdfvalue.Duration.From(1, rdfvalue.HOURS)
self.db.DeleteOldCronJobRuns(cutoff)
jobs = self.db.ReadCronJobRuns(job_id)
self.assertLen(jobs, 2)
for job in jobs:
self.assertGreater(job.timestamp, cutoff)
cutoff = fake_time + rdfvalue.Duration.From(25, rdfvalue.HOURS)
self.db.DeleteOldCronJobRuns(cutoff)
jobs = self.db.ReadCronJobRuns(job_id)
self.assertLen(jobs, 1)
for job in jobs:
self.assertGreater(job.timestamp, cutoff)
def RunJob(self, job):
"""Does the actual work of the Cron, if the job is due to run.
Args:
job: The cronjob rdfvalue that should be run. Must be leased.
Returns:
A boolean indicating if this cron job was due to run.
Raises:
LockError: if the object is not locked.
ValueError: If the job argument is invalid.
"""
if not job.leased_until:
raise LockError("CronJob must be leased for Run() to be called.")
if job.leased_until < rdfvalue.RDFDatetime.Now():
raise LockError("CronJob lease expired for %s." % job.cron_job_id)
if not self.JobDueToRun(job):
return False
logging.info("Running cron job: %s", job.cron_job_id)
run_state = rdf_cronjobs.CronJobRun(cron_job_id=job.cron_job_id,
status="RUNNING")
run_state.GenerateRunId()
if job.args.action_type == job.args.ActionType.SYSTEM_CRON_ACTION:
cls_name = job.args.system_cron_action.job_class_name
job_cls = registry.SystemCronJobRegistry.CronJobClassByName(
cls_name)
name = "%s runner" % cls_name
elif job.args.action_type == job.args.ActionType.HUNT_CRON_ACTION:
job_cls = registry.CronJobRegistry.CronJobClassByName("RunHunt")
name = "Hunt runner"
else:
raise ValueError("CronJob %s doesn't have a valid args type set." %
job.cron_job_id)
data_store.REL_DB.WriteCronJobRun(run_state)
data_store.REL_DB.UpdateCronJob(
job.cron_job_id,
last_run_time=rdfvalue.RDFDatetime.Now(),
current_run_id=run_state.run_id,
forced_run_requested=False)
run_obj = job_cls(run_state, job)
if self.max_threads == 1:
run_obj.StartRun()
else:
self._GetThreadPool().AddTask(target=run_obj.StartRun, name=name)
return True
def testCronJobRegistryInstantiation(self):
# We import the `server_startup` module to ensure that all cron jobs classes
# that are really used on the server are imported and populate the registry.
# pylint: disable=unused-variable, g-import-not-at-top
from grr_response_server import server_startup
# pylint: enable=unused-variable, g-import-not-at-top
for job_cls in cronjobs.CronJobRegistry.CRON_REGISTRY.values():
job = rdf_cronjobs.CronJob(cron_job_id="foobar")
job_run = rdf_cronjobs.CronJobRun(cron_job_id="foobar",
status="RUNNING")
job_cls(job_run, job) # Should not fail.
def testCronJobRuns(self):
with self.assertRaises(db.UnknownCronJobError):
self.db.WriteCronJobRun(
rdf_cronjobs.CronJobRun(cron_job_id="job1", run_id="00000000"))
now = rdfvalue.RDFDatetime.Now()
with test_lib.FakeTime(now):
for j in range(1, 3):
self.db.WriteCronJob(
rdf_cronjobs.CronJob(cron_job_id="job%d" % j))
for r in range(1, 3):
run = rdf_cronjobs.CronJobRun(cron_job_id="job%d" % j,
run_id="abcd123%d" % r)
self.db.WriteCronJobRun(run)
for j in range(1, 3):
job_id = "job%d" % j
jobs = self.db.ReadCronJobRuns(job_id)
self.assertLen(jobs, 2)
for job in jobs:
self.assertEqual(job.cron_job_id, job_id)
self.assertEqual(job.timestamp, now)
job = self.db.ReadCronJobRun("job1", "abcd1231")
self.assertEqual(job.cron_job_id, "job1")
self.assertEqual(job.run_id, "abcd1231")
self.assertEqual(job.timestamp, now)
with self.assertRaises(ValueError):
self.db.ReadCronJobRun(job_id, "invalid_id")
with self.assertRaises(db.UnknownCronJobRunError):
self.db.ReadCronJobRun(job_id, "abcd1234")
with self.assertRaises(db.UnknownCronJobRunError):
self.db.ReadCronJobRun("doesntexist", "abcd1231")
self.assertEqual(self.db.ReadCronJobRuns("doesntexist"), [])
def testPurgeServerStats(self):
if not data_store.RelationalDBReadEnabled():
self.skipTest("Test is only for the relational DB. Skipping...")
fake_stats_collector = default_stats_collector.DefaultStatsCollector([
stats_utils.CreateCounterMetadata("fake_counter"),
])
timestamp1 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1)
timestamp2 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3600)
timestamp3 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4800)
config_overrides = {
"Database.useForReads.stats": True,
"StatsStore.stats_ttl_hours": 1
}
with test_lib.ConfigOverrider(config_overrides):
with stats_test_utils.FakeStatsContext(fake_stats_collector):
with test_lib.FakeTime(rdfvalue.RDFDatetime(timestamp1)):
stats_collector_instance.Get().IncrementCounter(
"fake_counter")
stats_store._WriteStats(process_id="fake_process_id")
expected_results = {
"fake_process_id": {
"fake_counter": [(1, timestamp1)]
}
}
self.assertDictEqual(
stats_store.ReadStats("f", "fake_counter"),
expected_results)
with test_lib.FakeTime(timestamp2):
stats_store._WriteStats(process_id="fake_process_id")
expected_results = {
"fake_process_id": {
"fake_counter": [(1, timestamp1), (1, timestamp2)]
}
}
self.assertDictEqual(
stats_store.ReadStats("f", "fake_counter"),
expected_results)
with test_lib.FakeTime(timestamp3):
system.PurgeServerStatsCronJob(
rdf_cronjobs.CronJobRun(),
rdf_cronjobs.CronJob()).Run()
# timestamp1 is older than 1h, so it should get deleted.
expected_results = {
"fake_process_id": {
"fake_counter": [(1, timestamp2)]
}
}
self.assertDictEqual(
stats_store.ReadStats("f", "fake_counter"),
expected_results)
def ProcessHuntOutputPlugins(self):
if data_store.RelationalDBFlowsEnabled():
job = rdf_cronjobs.CronJob(
cron_job_id="some/id", lifetime=rdfvalue.Duration("1h"))
run_state = rdf_cronjobs.CronJobRun(
cron_job_id="some/id",
status="RUNNING",
started_at=rdfvalue.RDFDatetime.Now())
process_results.ProcessHuntResultCollectionsCronJob(run_state, job).Run()
else:
flow_urn = flow.StartAFF4Flow(
flow_name=process_results.ProcessHuntResultCollectionsCronFlow
.__name__,
token=self.token)
flow_test_lib.TestFlowHelper(flow_urn, token=self.token)
return flow_urn
def testCronJobUpdates(self):
job = self._CreateCronJob()
job.last_run_status = rdf_cronjobs.CronJobRun.CronJobRunStatus.FINISHED
self.db.WriteCronJob(job)
err = rdf_cronjobs.CronJobRun.CronJobRunStatus.ERROR
self.db.UpdateCronJob(job.cron_job_id, last_run_status=err)
new_job = self.db.ReadCronJob(job.cron_job_id)
self.assertEqual(job.last_run_status, "FINISHED")
self.assertEqual(new_job.last_run_status, "ERROR")
t = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1000000)
self.db.UpdateCronJob(job.cron_job_id, last_run_time=t)
new_job = self.db.ReadCronJob(job.cron_job_id)
self.assertEqual(new_job.last_run_time, t)
# To test `current_run_id` we first need to create a CronJobRun with it.
job_run0 = rdf_cronjobs.CronJobRun(cron_job_id=job.cron_job_id,
run_id="ABCD1234")
self.db.WriteCronJobRun(job_run0)
self.db.UpdateCronJob(job.cron_job_id, current_run_id="ABCD1234")
new_job = self.db.ReadCronJob(job.cron_job_id)
self.assertEqual(new_job.current_run_id, "ABCD1234")
# None is accepted to clear the current_run_id.
self.db.UpdateCronJob(job.cron_job_id, current_run_id=None)
new_job = self.db.ReadCronJob(job.cron_job_id)
self.assertFalse(new_job.current_run_id)
state = job.state
self.assertNotIn("test", state)
state["test"] = 12345
self.db.UpdateCronJob(job.cron_job_id, state=state)
new_job = self.db.ReadCronJob(job.cron_job_id)
self.assertEqual(new_job.state.get("test"), 12345)
self.db.UpdateCronJob(job.cron_job_id, forced_run_requested=True)
new_job = self.db.ReadCronJob(job.cron_job_id)
self.assertEqual(new_job.forced_run_requested, True)
with self.assertRaises(db.UnknownCronJobError):
self.db.UpdateCronJob("Does not exist", current_run_id="12345678")
def testCronJobRunsOverwrite(self):
self.db.WriteCronJob(rdf_cronjobs.CronJob(cron_job_id="job"))
run = rdf_cronjobs.CronJobRun(cron_job_id="job", run_id="abcd1234")
self.db.WriteCronJobRun(run)
original_ts = self.db.ReadCronJobRun("job", "abcd1234").timestamp
now = rdfvalue.RDFDatetime.Now()
run.backtrace = "error"
run.log_message = "log"
run.started_at = now - rdfvalue.Duration("5s")
run.finished_at = now
self.db.WriteCronJobRun(run)
read = self.db.ReadCronJobRun("job", "abcd1234")
self.assertEqual(read.backtrace, run.backtrace)
self.assertEqual(read.log_message, run.log_message)
self.assertEqual(read.started_at, run.started_at)
self.assertEqual(read.finished_at, run.finished_at)
self.assertNotEqual(read.timestamp, original_ts)
def _RunPurgeClientStats(self):
run = rdf_cronjobs.CronJobRun()
job = rdf_cronjobs.CronJob()
system.PurgeClientStatsCronJob(run, job).Run()
def testPurgeServerStats(self):
if not data_store.RelationalDBReadEnabled():
self.skipTest("Test is only for the relational DB. Skipping...")
fake_stats_collector = prometheus_stats_collector.PrometheusStatsCollector(
[
stats_utils.CreateCounterMetadata("fake_counter"),
])
timestamp0 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1)
timestamp1 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(2)
timestamp2 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3600)
timestamp3 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4800)
config_overrides = {
"Database.useForReads.stats": True,
"StatsStore.stats_ttl_hours": 1
}
zero_duration = rdfvalue.Duration(0)
# Backslash continuation is explicitly allowed by Google's style guide for
# nested context manager expressions spanning 3 or more lines.
# pylint: disable=g-backslash-continuation
with test_lib.ConfigOverrider(config_overrides), \
stats_test_utils.FakeStatsContext(fake_stats_collector), \
mock.patch.object(system, "_STATS_DELETION_BATCH_SIZE", 1), \
mock.patch.object(system, "_stats_checkpoint_period", zero_duration):
with test_lib.FakeTime(rdfvalue.RDFDatetime(timestamp0)):
stats_store._WriteStats(process_id="fake_process_id")
with test_lib.FakeTime(rdfvalue.RDFDatetime(timestamp1)):
stats_collector_instance.Get().IncrementCounter("fake_counter")
stats_store._WriteStats(process_id="fake_process_id")
expected_results = {
"fake_process_id": {
"fake_counter": [(0, timestamp0), (1, timestamp1)]
}
}
self.assertDictEqual(
stats_store.ReadStats("f", "fake_counter"),
expected_results)
with test_lib.FakeTime(timestamp2):
stats_store._WriteStats(process_id="fake_process_id")
expected_results = {
"fake_process_id": {
"fake_counter": [(0, timestamp0), (1, timestamp1),
(1, timestamp2)]
}
}
self.assertDictEqual(
stats_store.ReadStats("f", "fake_counter"),
expected_results)
with test_lib.FakeTime(timestamp3):
cron_name = compatibility.GetName(
system.PurgeServerStatsCronJob)
cronjobs.ScheduleSystemCronJobs(names=[cron_name])
job_data = data_store.REL_DB.ReadCronJobs([cron_name])[0]
cron_run = rdf_cronjobs.CronJobRun(cron_job_id=cron_name)
cron_run.GenerateRunId()
cron_run.started_at = rdfvalue.RDFDatetime.Now()
cron = system.PurgeServerStatsCronJob(cron_run, job_data)
cron.Run()
# timestamp0 and timestamp1 are older than 1h, so they should get
# deleted.
expected_results = {
"fake_process_id": {
"fake_counter": [(1, timestamp2)]
}
}
self.assertDictEqual(
stats_store.ReadStats("f", "fake_counter"),
expected_results)
self.assertEqual(
"Deleted 2 stats entries.\nDeleted 1 stats entries.",
cron.run_state.log_message)
def testCronJob(self, fs_conn_mock):
if not data_store.RelationalDBReadEnabled():
self.skipTest("Test is only for the relational DB. Skipping...")
client_id1 = "C.0000000000000001"
client_id2 = "C.0000000000000002"
client_id3 = "C.0000000000000003"
client_id4 = "C.0000000000000004"
client_id5 = "C.0000000000000005"
client_id6 = "C.0000000000000006"
client_id7 = "C.0000000000000007"
data_store.REL_DB.WriteClientMetadata(client_id1,
fleetspeak_enabled=False)
data_store.REL_DB.WriteClientMetadata(client_id2,
fleetspeak_enabled=True)
data_store.REL_DB.WriteClientMetadata(
client_id3,
last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3),
fleetspeak_enabled=True)
data_store.REL_DB.WriteClientMetadata(
client_id4,
last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(41),
fleetspeak_enabled=True)
data_store.REL_DB.WriteClientMetadata(
client_id5,
last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(5),
fleetspeak_enabled=True)
data_store.REL_DB.WriteClientMetadata(
client_id6,
last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(61),
fleetspeak_enabled=True)
data_store.REL_DB.WriteClientMetadata(
client_id7,
last_ping=rdfvalue.RDFDatetime.FromSecondsSinceEpoch(68),
fleetspeak_enabled=True)
fs_enabled_ids = [
client_id2, client_id3, client_id4, client_id5, client_id6,
client_id7
]
fs_clients = {}
for i, client_id in enumerate(fs_enabled_ids):
client_number = i + 2
fs_client_id = fleetspeak_utils.GRRIDToFleetspeakID(client_id)
fs_client = admin_pb2.Client(client_id=fs_client_id)
fs_client.last_contact_time.FromSeconds(client_number * 10)
fs_clients[fs_client_id] = fs_client
def FakeListClients(list_request):
clients = []
for fs_client_id in list_request.client_ids:
clients.append(fs_clients[fs_client_id])
return admin_pb2.ListClientsResponse(clients=clients)
fs_conn_mock.outgoing.ListClients = FakeListClients
cron_run = rdf_cronjobs.CronJobRun()
job_data = rdf_cronjobs.CronJob()
cron = system.UpdateFSLastPingTimestamps(cron_run, job_data)
with test_lib.FakeTime(
rdfvalue.RDFDatetime.FromSecondsSinceEpoch(100)):
with test_lib.ConfigOverrider({
"Server.fleetspeak_last_ping_threshold":
"35s",
"Server.fleetspeak_list_clients_batch_size":
2,
}):
cron.Run()
actual_timestamps = data_store.REL_DB.ReadClientLastPings()
expected_timestamps = {
# Skipped because not a Fleetspeak client.
client_id1: None,
client_id2: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(20),
client_id3: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(30),
# Skipped because FS timestamp is old.
client_id4: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(41),
client_id5: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(50),
# Skipped because FS timestamp is old.
client_id6: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(61),
# Skipped because existing GRR timestamp is too recent.
client_id7: rdfvalue.RDFDatetime.FromSecondsSinceEpoch(68),
}
self.assertEqual(actual_timestamps, expected_timestamps)
self.assertMultiLineEqual(cron._log_messages.popleft(),
"Updated timestamps for 3 clients.")
def _RunCleanup(self): run = rdf_cronjobs.CronJobRun() job = rdf_cronjobs.CronJob() self.cleaner_job = data_retention.CleanInactiveClientsCronJob(run, job) self.cleaner_job.Run()
def RunJob(self, job):
"""Does the actual work of the Cron, if the job is due to run.
Args:
job: The cronjob rdfvalue that should be run. Must be leased.
Returns:
A boolean indicating if this cron job was started or not. False may
be returned when the threadpool is already full.
Raises:
LockError: if the object is not locked.
ValueError: If the job argument is invalid.
"""
if not job.leased_until:
raise LockError("CronJob must be leased for Run() to be called.")
if job.leased_until < rdfvalue.RDFDatetime.Now():
raise LockError("CronJob lease expired for %s." % job.cron_job_id)
logging.info("Starting cron job: %s", job.cron_job_id)
if job.args.action_type == job.args.ActionType.SYSTEM_CRON_ACTION:
cls_name = job.args.system_cron_action.job_class_name
job_cls = SystemCronJobRegistry.CronJobClassByName(cls_name)
name = "%s runner" % cls_name
elif job.args.action_type == job.args.ActionType.HUNT_CRON_ACTION:
job_cls = CronJobRegistry.CronJobClassByName("RunHunt")
name = "Hunt runner"
else:
raise ValueError("CronJob %s doesn't have a valid args type set." %
job.cron_job_id)
run_state = rdf_cronjobs.CronJobRun(
cron_job_id=job.cron_job_id, status="RUNNING")
run_state.GenerateRunId()
run_obj = job_cls(run_state, job)
wait_for_start_event, signal_event, wait_for_write_event = (
threading.Event(), threading.Event(), threading.Event())
try:
self._GetThreadPool().AddTask(
target=run_obj.StartRun,
args=(wait_for_start_event, signal_event, wait_for_write_event),
name=name,
blocking=False,
inline=False)
if not wait_for_start_event.wait(TASK_STARTUP_WAIT):
logging.error("Cron job run task for %s is too slow to start.",
job.cron_job_id)
# Most likely the thread pool is full and the task is sitting on the
# queue. Make sure we don't put more things on the queue by returning
# False.
return False
# We know that the cron job task has started, unblock it by setting
# the signal event. If signal_event is not set (this happens if the
# task sits on a ThreadPool's queue doing nothing, see the
# if-statement above) the task will just be a no-op when ThreadPool
# finally gets to it. This way we can ensure that we can safely return
# the lease and let another worker schedule the same job.
signal_event.set()
wait_for_write_event.wait(TASK_STARTUP_WAIT)
return True
except threadpool.Full:
return False