def testLogging(self): class LoggingCronJob(cronjobs.SystemCronJobBase): lifetime = rdfvalue.Duration("1h") frequency = rdfvalue.Duration("2h") def Run(self): for i in range(7): self.Log("Log message %d." % i) cron_name = compatibility.GetName(LoggingCronJob) cronjobs.ScheduleSystemCronJobs(names=[cron_name]) cron_manager = cronjobs.CronManager() try: cron_manager.RunOnce() finally: cron_manager._GetThreadPool().Join() runs = cron_manager.ReadJobRuns(cron_name) self.assertLen(runs, 1) self.assertEmpty(runs[0].backtrace) self.assertEqual(runs[0].status, rdf_cronjobs.CronJobRun.CronJobRunStatus.FINISHED) # The first two log messages should be discarded since # _MAX_LOG_MESSAGES is 5. self.assertMultiLineEqual( runs[0].log_message, "Log message 6.\nLog message 5.\nLog message 4.\nLog message 3.\n" "Log message 2.")
def testSystemCronJobsGetScheduledWhenDisabledListInvalid(self): with test_lib.ConfigOverrider({"Cron.disabled_cron_jobs": ["NonExistent"]}): with self.assertRaises(ValueError): cronjobs.ScheduleSystemCronJobs(names=[DummySystemCronJobRel.__name__]) jobs = cronjobs.CronManager().ListJobs() self.assertIn("DummySystemCronJobRel", jobs)
def _TestHeartBeat(self, cron_class, cron_started_event, heartbeat_event): """Helper for heartbeat tests.""" cron_name = compatibility.GetName(cron_class) cronjobs.ScheduleSystemCronJobs(names=[cron_name]) cron_manager = cronjobs.CronManager() jobs = cronjobs.CronManager().ListJobs() self.assertIn(cron_name, jobs) try: cron_manager.RunOnce() cron_started_event.wait() runs = cron_manager.ReadJobRuns(cron_name) self.assertLen(runs, 1) self.assertEqual(runs[0].status, rdf_cronjobs.CronJobRun.CronJobRunStatus.RUNNING) finally: heartbeat_event.set() cron_manager._GetThreadPool().Join() runs = cron_manager.ReadJobRuns(cron_name) self.assertLen(runs, 1) if cron_class.allow_overruns: expected_status = rdf_cronjobs.CronJobRun.CronJobRunStatus.FINISHED else: expected_status = ( rdf_cronjobs.CronJobRun.CronJobRunStatus.LIFETIME_EXCEEDED) self.assertEqual(runs[0].status, expected_status)
def ScheduleSystemCronFlows(names=None, token=None): """Schedule all the SystemCronFlows found.""" if data_store.RelationalDBReadEnabled(category="cronjobs"): return cronjobs.ScheduleSystemCronJobs(names=names) errors = [] for name in config.CONFIG["Cron.disabled_system_jobs"]: try: cls = registry.FlowRegistry.FlowClassByName(name) except ValueError: errors.append("No such flow: %s." % name) continue if not aff4.issubclass(cls, SystemCronFlow): errors.append("Disabled system cron job name doesn't correspond to " "a flow inherited from SystemCronFlow: %s" % name) if names is None: names = iterkeys(registry.FlowRegistry.FLOW_REGISTRY) for name in names: cls = registry.FlowRegistry.FlowClassByName(name) if not aff4.issubclass(cls, SystemCronFlow): continue cron_args = rdf_cronjobs.CreateCronJobFlowArgs( periodicity=cls.frequency, lifetime=cls.lifetime, allow_overruns=cls.allow_overruns) cron_args.flow_runner_args.flow_name = name if cls.enabled: enabled = name not in config.CONFIG["Cron.disabled_system_jobs"] else: enabled = False job_urn = CronManager.CRON_JOBS_PATH.Add(name) with aff4.FACTORY.Create( job_urn, aff4_type=CronJob, mode="rw", token=token, force_new_version=False) as cron_job: # If the cronjob was already present we don't want to overwrite the # original start_time. existing_cron_args = cron_job.Get(cron_job.Schema.CRON_ARGS) if cron_args != existing_cron_args: cron_job.Set(cron_job.Schema.CRON_ARGS(cron_args)) cron_job.Set(cron_job.Schema.DISABLED(not enabled)) if errors: raise ValueError( "Error(s) while parsing Cron.disabled_system_jobs: %s" % errors)
def testSystemCronJobsGetScheduledAutomatically(self): cronjobs.ScheduleSystemCronJobs(names=[DummySystemCronJobRel.__name__]) jobs = cronjobs.CronManager().ListJobs() self.assertIn("DummySystemCronJobRel", jobs) # System cron job should be enabled by default. job = cronjobs.CronManager().ReadJob("DummySystemCronJobRel") self.assertTrue(job.enabled)
def setUp(self): super().setUp() cron_job_names = [ cron_system.GRRVersionBreakDownCronJob.__name__, cron_system.OSBreakDownCronJob.__name__, cron_system.LastAccessStatsCronJob.__name__ ] cronjobs.ScheduleSystemCronJobs(cron_job_names)
def _ScheduleCronJob(self): if data_store.RelationalDBReadEnabled(category="cronjobs"): cron_job_id = unicode(cron_system.OSBreakDownCronJob.__name__) cronjobs.ScheduleSystemCronJobs(names=[cron_job_id]) else: cron_job_id = unicode(cron_system.OSBreakDown.__name__) aff4_cronjobs.ScheduleSystemCronFlows(names=[cron_job_id], token=self.token) aff4_cronjobs.GetCronManager().DisableJob(job_id=cron_job_id) return cron_job_id
def _ScheduleCronJob(self): if data_store.RelationalDBReadEnabled(): cron_job_id = compatibility.GetName(cron_system.OSBreakDownCronJob) cronjobs.ScheduleSystemCronJobs(names=[cron_job_id]) else: cron_job_id = compatibility.GetName(cron_system.OSBreakDown) aff4_cronjobs.ScheduleSystemCronFlows(names=[cron_job_id], token=self.token) aff4_cronjobs.GetCronManager().DisableJob(job_id=cron_job_id) return cron_job_id
def testSystemCronJobsWithDisabledAttributeDoNotGetScheduled(self): cronjobs.ScheduleSystemCronJobs( names=[DummyDisabledSystemCronJobRel.__name__]) jobs = cronjobs.CronManager().ListJobs() self.assertIn("DummyDisabledSystemCronJobRel", jobs) # System cron job should be enabled by default. job = cronjobs.CronManager().ReadJob("DummyDisabledSystemCronJobRel") self.assertFalse(job.enabled)
def _SetupAndRunVersionBreakDownCronjob(): with test_lib.FakeTime(44): manager = cronjobs.CronManager() cron_job_name = compatibility.GetName( cron_system.GRRVersionBreakDownCronJob) cronjobs.ScheduleSystemCronJobs(names=[cron_job_name]) manager.RunOnce() manager._GetThreadPool().Stop() return cron_job_name
def _CreateOSBreakDownCronJobApproval(self): if data_store.RelationalDBReadEnabled("cronjobs"): job_name = compatibility.GetName(cron_system.OSBreakDownCronJob) cronjobs.ScheduleSystemCronJobs(names=[job_name]) else: job_name = compatibility.GetName(cron_system.OSBreakDown) aff4_cronjobs.ScheduleSystemCronFlows(names=[job_name], token=self.token) aff4_cronjobs.GetCronManager().DisableJob(job_id=job_name) return job_name
def testSystemCronJobsMayBeDisabledViaConfig(self): with test_lib.ConfigOverrider( {"Cron.disabled_cron_jobs": ["DummySystemCronJobRel"]}): cronjobs.ScheduleSystemCronJobs() cron_manager = cronjobs.CronManager() jobs = cron_manager.ListJobs() self.assertIn("DummySystemCronJobRel", jobs) # This cron job should be disabled, because it's listed in # Cron.disabled_cron_jobs config variable. job = cron_manager.ReadJob("DummySystemCronJobRel") self.assertFalse(job.enabled) # Now remove the cron job from the list and check that it gets disabled # after next ScheduleSystemCronJobs() call. with test_lib.ConfigOverrider({"Cron.disabled_cron_jobs": []}): cronjobs.ScheduleSystemCronJobs() # System cron job should be enabled. job = cron_manager.ReadJob("DummySystemCronJobRel") self.assertTrue(job.enabled)
def _SetupAndRunVersionBreakDownCronjob(token=None): with test_lib.FakeTime(44): manager = aff4_cronjobs.GetCronManager() if data_store.RelationalDBReadEnabled(): cron_job_name = cron_system.GRRVersionBreakDownCronJob.__name__ cronjobs.ScheduleSystemCronJobs(names=[cron_job_name]) manager.RunOnce() manager._GetThreadPool().Join() else: cron_job_name = cron_system.GRRVersionBreakDown.__name__ aff4_cronjobs.ScheduleSystemCronFlows(names=[cron_job_name], token=token) manager.RunOnce(token=token) run_id = _GetRunId(cron_job_name, token=token) flow_test_lib.TestFlowHelper( rdfvalue.RDFURN("aff4:/cron/%s/%s" % (cron_job_name, run_id)), token=token) manager.RunOnce(token=token) return cron_job_name
def _CreateOSBreakDownCronJobApproval(self): job_name = compatibility.GetName(cron_system.OSBreakDownCronJob) cronjobs.ScheduleSystemCronJobs(names=[job_name]) cronjobs.CronManager().DisableJob(job_name) return job_name
def _ScheduleCronJob(self): cron_job_id = compatibility.GetName(cron_system.OSBreakDownCronJob) cronjobs.ScheduleSystemCronJobs(names=[cron_job_id]) cronjobs.CronManager().DisableJob(cron_job_id) return cron_job_id
def testPurgeServerStats(self): if not data_store.RelationalDBReadEnabled(): self.skipTest("Test is only for the relational DB. Skipping...") fake_stats_collector = prometheus_stats_collector.PrometheusStatsCollector( [ stats_utils.CreateCounterMetadata("fake_counter"), ]) timestamp0 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(1) timestamp1 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(2) timestamp2 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(3600) timestamp3 = rdfvalue.RDFDatetime.FromSecondsSinceEpoch(4800) config_overrides = { "Database.useForReads.stats": True, "StatsStore.stats_ttl_hours": 1 } zero_duration = rdfvalue.Duration(0) # Backslash continuation is explicitly allowed by Google's style guide for # nested context manager expressions spanning 3 or more lines. # pylint: disable=g-backslash-continuation with test_lib.ConfigOverrider(config_overrides), \ stats_test_utils.FakeStatsContext(fake_stats_collector), \ mock.patch.object(system, "_STATS_DELETION_BATCH_SIZE", 1), \ mock.patch.object(system, "_stats_checkpoint_period", zero_duration): with test_lib.FakeTime(rdfvalue.RDFDatetime(timestamp0)): stats_store._WriteStats(process_id="fake_process_id") with test_lib.FakeTime(rdfvalue.RDFDatetime(timestamp1)): stats_collector_instance.Get().IncrementCounter("fake_counter") stats_store._WriteStats(process_id="fake_process_id") expected_results = { "fake_process_id": { "fake_counter": [(0, timestamp0), (1, timestamp1)] } } self.assertDictEqual( stats_store.ReadStats("f", "fake_counter"), expected_results) with test_lib.FakeTime(timestamp2): stats_store._WriteStats(process_id="fake_process_id") expected_results = { "fake_process_id": { "fake_counter": [(0, timestamp0), (1, timestamp1), (1, timestamp2)] } } self.assertDictEqual( stats_store.ReadStats("f", "fake_counter"), expected_results) with test_lib.FakeTime(timestamp3): cron_name = compatibility.GetName( system.PurgeServerStatsCronJob) cronjobs.ScheduleSystemCronJobs(names=[cron_name]) job_data = data_store.REL_DB.ReadCronJobs([cron_name])[0] cron_run = rdf_cronjobs.CronJobRun(cron_job_id=cron_name) cron_run.GenerateRunId() cron_run.started_at = rdfvalue.RDFDatetime.Now() cron = system.PurgeServerStatsCronJob(cron_run, job_data) cron.Run() # timestamp0 and timestamp1 are older than 1h, so they should get # deleted. expected_results = { "fake_process_id": { "fake_counter": [(1, timestamp2)] } } self.assertDictEqual( stats_store.ReadStats("f", "fake_counter"), expected_results) self.assertEqual( "Deleted 2 stats entries.\nDeleted 1 stats entries.", cron.run_state.log_message)