def testLogging(self):
class LoggingCronJob(cronjobs.SystemCronJobBase):
lifetime = rdfvalue.Duration("1h")
frequency = rdfvalue.Duration("2h")
def Run(self):
for i in range(7):
self.Log("Log message %d." % i)
cron_name = compatibility.GetName(LoggingCronJob)
cronjobs.ScheduleSystemCronJobs(names=[cron_name])
cron_manager = cronjobs.CronManager()
try:
cron_manager.RunOnce()
finally:
cron_manager._GetThreadPool().Join()
runs = cron_manager.ReadJobRuns(cron_name)
self.assertLen(runs, 1)
self.assertEmpty(runs[0].backtrace)
self.assertEqual(runs[0].status,
rdf_cronjobs.CronJobRun.CronJobRunStatus.FINISHED)
# The first two log messages should be discarded since
# _MAX_LOG_MESSAGES is 5.
self.assertMultiLineEqual(
runs[0].log_message,
"Log message 6.\nLog message 5.\nLog message 4.\nLog message 3.\n"
"Log message 2.")
def testSystemCronJobsGetScheduledWhenDisabledListInvalid(self):
with test_lib.ConfigOverrider({"Cron.disabled_cron_jobs": ["NonExistent"]}):
with self.assertRaises(ValueError):
cronjobs.ScheduleSystemCronJobs(names=[DummySystemCronJobRel.__name__])
jobs = cronjobs.CronManager().ListJobs()
self.assertIn("DummySystemCronJobRel", jobs)
def _TestHeartBeat(self, cron_class, cron_started_event, heartbeat_event):
"""Helper for heartbeat tests."""
cron_name = compatibility.GetName(cron_class)
cronjobs.ScheduleSystemCronJobs(names=[cron_name])
cron_manager = cronjobs.CronManager()
jobs = cronjobs.CronManager().ListJobs()
self.assertIn(cron_name, jobs)
try:
cron_manager.RunOnce()
cron_started_event.wait()
runs = cron_manager.ReadJobRuns(cron_name)
self.assertLen(runs, 1)
self.assertEqual(runs[0].status,
rdf_cronjobs.CronJobRun.CronJobRunStatus.RUNNING)
finally:
heartbeat_event.set()
cron_manager._GetThreadPool().Join()
runs = cron_manager.ReadJobRuns(cron_name)
self.assertLen(runs, 1)
if cron_class.allow_overruns:
expected_status = rdf_cronjobs.CronJobRun.CronJobRunStatus.FINISHED
else:
expected_status = (
rdf_cronjobs.CronJobRun.CronJobRunStatus.LIFETIME_EXCEEDED)
self.assertEqual(runs[0].status, expected_status)
def ScheduleSystemCronFlows(names=None, token=None):
"""Schedule all the SystemCronFlows found."""
if data_store.RelationalDBReadEnabled(category="cronjobs"):
return cronjobs.ScheduleSystemCronJobs(names=names)
errors = []
for name in config.CONFIG["Cron.disabled_system_jobs"]:
try:
cls = registry.FlowRegistry.FlowClassByName(name)
except ValueError:
errors.append("No such flow: %s." % name)
continue
if not aff4.issubclass(cls, SystemCronFlow):
errors.append("Disabled system cron job name doesn't correspond to "
"a flow inherited from SystemCronFlow: %s" % name)
if names is None:
names = iterkeys(registry.FlowRegistry.FLOW_REGISTRY)
for name in names:
cls = registry.FlowRegistry.FlowClassByName(name)
if not aff4.issubclass(cls, SystemCronFlow):
continue
cron_args = rdf_cronjobs.CreateCronJobFlowArgs(
periodicity=cls.frequency,
lifetime=cls.lifetime,
allow_overruns=cls.allow_overruns)
cron_args.flow_runner_args.flow_name = name
if cls.enabled:
enabled = name not in config.CONFIG["Cron.disabled_system_jobs"]
else:
enabled = False
job_urn = CronManager.CRON_JOBS_PATH.Add(name)
with aff4.FACTORY.Create(
job_urn,
aff4_type=CronJob,
mode="rw",
token=token,
force_new_version=False) as cron_job:
# If the cronjob was already present we don't want to overwrite the
# original start_time.
existing_cron_args = cron_job.Get(cron_job.Schema.CRON_ARGS)
if cron_args != existing_cron_args:
cron_job.Set(cron_job.Schema.CRON_ARGS(cron_args))
cron_job.Set(cron_job.Schema.DISABLED(not enabled))
if errors:
raise ValueError(
"Error(s) while parsing Cron.disabled_system_jobs: %s" % errors)
def testSystemCronJobsGetScheduledAutomatically(self):
cronjobs.ScheduleSystemCronJobs(names=[DummySystemCronJobRel.__name__])
jobs = cronjobs.CronManager().ListJobs()
self.assertIn("DummySystemCronJobRel", jobs)
# System cron job should be enabled by default.
job = cronjobs.CronManager().ReadJob("DummySystemCronJobRel")
self.assertTrue(job.enabled)
def setUp(self):
super().setUp()
cron_job_names = [
cron_system.GRRVersionBreakDownCronJob.__name__,
cron_system.OSBreakDownCronJob.__name__,
cron_system.LastAccessStatsCronJob.__name__
]
cronjobs.ScheduleSystemCronJobs(cron_job_names)
def _ScheduleCronJob(self):
if data_store.RelationalDBReadEnabled(category="cronjobs"):
cron_job_id = unicode(cron_system.OSBreakDownCronJob.__name__)
cronjobs.ScheduleSystemCronJobs(names=[cron_job_id])
else:
cron_job_id = unicode(cron_system.OSBreakDown.__name__)
aff4_cronjobs.ScheduleSystemCronFlows(names=[cron_job_id],
token=self.token)
aff4_cronjobs.GetCronManager().DisableJob(job_id=cron_job_id)
return cron_job_id
def _ScheduleCronJob(self):
if data_store.RelationalDBReadEnabled():
cron_job_id = compatibility.GetName(cron_system.OSBreakDownCronJob)
cronjobs.ScheduleSystemCronJobs(names=[cron_job_id])
else:
cron_job_id = compatibility.GetName(cron_system.OSBreakDown)
aff4_cronjobs.ScheduleSystemCronFlows(names=[cron_job_id],
token=self.token)
aff4_cronjobs.GetCronManager().DisableJob(job_id=cron_job_id)
return cron_job_id
def testSystemCronJobsWithDisabledAttributeDoNotGetScheduled(self):
cronjobs.ScheduleSystemCronJobs(
names=[DummyDisabledSystemCronJobRel.__name__])
jobs = cronjobs.CronManager().ListJobs()
self.assertIn("DummyDisabledSystemCronJobRel", jobs)
# System cron job should be enabled by default.
job = cronjobs.CronManager().ReadJob("DummyDisabledSystemCronJobRel")
self.assertFalse(job.enabled)
def _SetupAndRunVersionBreakDownCronjob():
with test_lib.FakeTime(44):
manager = cronjobs.CronManager()
cron_job_name = compatibility.GetName(
cron_system.GRRVersionBreakDownCronJob)
cronjobs.ScheduleSystemCronJobs(names=[cron_job_name])
manager.RunOnce()
manager._GetThreadPool().Stop()
return cron_job_name
def _CreateOSBreakDownCronJobApproval(self):
if data_store.RelationalDBReadEnabled("cronjobs"):
job_name = compatibility.GetName(cron_system.OSBreakDownCronJob)
cronjobs.ScheduleSystemCronJobs(names=[job_name])
else:
job_name = compatibility.GetName(cron_system.OSBreakDown)
aff4_cronjobs.ScheduleSystemCronFlows(names=[job_name], token=self.token)
aff4_cronjobs.GetCronManager().DisableJob(job_id=job_name)
return job_name
def testSystemCronJobsMayBeDisabledViaConfig(self):
with test_lib.ConfigOverrider(
{"Cron.disabled_cron_jobs": ["DummySystemCronJobRel"]}):
cronjobs.ScheduleSystemCronJobs()
cron_manager = cronjobs.CronManager()
jobs = cron_manager.ListJobs()
self.assertIn("DummySystemCronJobRel", jobs)
# This cron job should be disabled, because it's listed in
# Cron.disabled_cron_jobs config variable.
job = cron_manager.ReadJob("DummySystemCronJobRel")
self.assertFalse(job.enabled)
# Now remove the cron job from the list and check that it gets disabled
# after next ScheduleSystemCronJobs() call.
with test_lib.ConfigOverrider({"Cron.disabled_cron_jobs": []}):
cronjobs.ScheduleSystemCronJobs()
# System cron job should be enabled.
job = cron_manager.ReadJob("DummySystemCronJobRel")
self.assertTrue(job.enabled)
def _SetupAndRunVersionBreakDownCronjob(token=None):
with test_lib.FakeTime(44):
manager = aff4_cronjobs.GetCronManager()
if data_store.RelationalDBReadEnabled():
cron_job_name = cron_system.GRRVersionBreakDownCronJob.__name__
cronjobs.ScheduleSystemCronJobs(names=[cron_job_name])
manager.RunOnce()
manager._GetThreadPool().Join()
else:
cron_job_name = cron_system.GRRVersionBreakDown.__name__
aff4_cronjobs.ScheduleSystemCronFlows(names=[cron_job_name], token=token)
manager.RunOnce(token=token)
run_id = _GetRunId(cron_job_name, token=token)
flow_test_lib.TestFlowHelper(
rdfvalue.RDFURN("aff4:/cron/%s/%s" % (cron_job_name, run_id)),
token=token)
manager.RunOnce(token=token)
return cron_job_name
def _CreateOSBreakDownCronJobApproval(self):
job_name = compatibility.GetName(cron_system.OSBreakDownCronJob)
cronjobs.ScheduleSystemCronJobs(names=[job_name])
cronjobs.CronManager().DisableJob(job_name)
return job_name
def _ScheduleCronJob(self):
cron_job_id = compatibility.GetName(cron_system.OSBreakDownCronJob)
cronjobs.ScheduleSystemCronJobs(names=[cron_job_id])
cronjobs.CronManager().DisableJob(cron_job_id)
return cron_job_id
def testPurgeServerStats(self):
if not data_store.RelationalDBReadEnabled():
self.skipTest("Test is only for the relational DB. Skipping...")
fake_stats_collector = prometheus_stats_collector.PrometheusStatsCollector(
[
stats_utils.CreateCounterMetadata("fake_counter"),
])
timestamp0 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1)
timestamp1 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(2)
timestamp2 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3600)
timestamp3 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4800)
config_overrides = {
"Database.useForReads.stats": True,
"StatsStore.stats_ttl_hours": 1
}
zero_duration = rdfvalue.Duration(0)
# Backslash continuation is explicitly allowed by Google's style guide for
# nested context manager expressions spanning 3 or more lines.
# pylint: disable=g-backslash-continuation
with test_lib.ConfigOverrider(config_overrides), \
stats_test_utils.FakeStatsContext(fake_stats_collector), \
mock.patch.object(system, "_STATS_DELETION_BATCH_SIZE", 1), \
mock.patch.object(system, "_stats_checkpoint_period", zero_duration):
with test_lib.FakeTime(rdfvalue.RDFDatetime(timestamp0)):
stats_store._WriteStats(process_id="fake_process_id")
with test_lib.FakeTime(rdfvalue.RDFDatetime(timestamp1)):
stats_collector_instance.Get().IncrementCounter("fake_counter")
stats_store._WriteStats(process_id="fake_process_id")
expected_results = {
"fake_process_id": {
"fake_counter": [(0, timestamp0), (1, timestamp1)]
}
}
self.assertDictEqual(
stats_store.ReadStats("f", "fake_counter"),
expected_results)
with test_lib.FakeTime(timestamp2):
stats_store._WriteStats(process_id="fake_process_id")
expected_results = {
"fake_process_id": {
"fake_counter": [(0, timestamp0), (1, timestamp1),
(1, timestamp2)]
}
}
self.assertDictEqual(
stats_store.ReadStats("f", "fake_counter"),
expected_results)
with test_lib.FakeTime(timestamp3):
cron_name = compatibility.GetName(
system.PurgeServerStatsCronJob)
cronjobs.ScheduleSystemCronJobs(names=[cron_name])
job_data = data_store.REL_DB.ReadCronJobs([cron_name])[0]
cron_run = rdf_cronjobs.CronJobRun(cron_job_id=cron_name)
cron_run.GenerateRunId()
cron_run.started_at = rdfvalue.RDFDatetime.Now()
cron = system.PurgeServerStatsCronJob(cron_run, job_data)
cron.Run()
# timestamp0 and timestamp1 are older than 1h, so they should get
# deleted.
expected_results = {
"fake_process_id": {
"fake_counter": [(1, timestamp2)]
}
}
self.assertDictEqual(
stats_store.ReadStats("f", "fake_counter"),
expected_results)
self.assertEqual(
"Deleted 2 stats entries.\nDeleted 1 stats entries.",
cron.run_state.log_message)
