def testTimeoutOfLongRunningJobIsHandledCorrectly(self):
wait_event = threading.Event()
signal_event = threading.Event()
waiting_func = functools.partial(WaitAndSignal, wait_event, signal_event)
fake_time = rdfvalue.RDFDatetime.Now()
with mock.patch.object(cronjobs.RunHunt, "Run", wraps=waiting_func):
with test_lib.FakeTime(fake_time):
cron_manager = cronjobs.CronManager()
create_flow_args = rdf_cronjobs.CreateCronJobArgs()
create_flow_args.lifetime = "1h"
job_id = cron_manager.CreateJob(cron_args=create_flow_args)
cron_manager.RunOnce(token=self.token)
# Make sure the cron job has actually been started.
signal_event.wait(10)
cron_job = cron_manager.ReadJob(job_id, token=self.token)
self.assertTrue(cron_manager.JobIsRunning(cron_job))
runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(runs, 1)
run = runs[0]
self.assertEqual(cron_job.current_run_id, run.run_id)
self.assertEqual(run.status, "RUNNING")
prev_timeout_value = cronjobs.CRON_JOB_TIMEOUT.GetValue([job_id])
prev_latency_value = cronjobs.CRON_JOB_LATENCY.GetValue([job_id])
fake_time += rdfvalue.Duration.From(2, rdfvalue.HOURS)
with test_lib.FakeTime(fake_time):
wait_event.set()
cron_manager._GetThreadPool().Join()
cron_job = cron_manager.ReadJob(job_id, token=self.token)
runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(runs, 1)
run = runs[0]
self.assertEqual(cron_job.last_run_status, "LIFETIME_EXCEEDED")
self.assertEqual(run.status, "LIFETIME_EXCEEDED")
# Check that timeout counter got updated.
current_timeout_value = cronjobs.CRON_JOB_TIMEOUT.GetValue([job_id])
self.assertEqual(current_timeout_value - prev_timeout_value, 1)
# Check that latency stat got updated.
current_latency_value = cronjobs.CRON_JOB_LATENCY.GetValue([job_id])
self.assertEqual(current_latency_value.count - prev_latency_value.count,
1)
self.assertEqual(
current_latency_value.sum - prev_latency_value.sum,
rdfvalue.Duration.From(2, rdfvalue.HOURS).ToInt(rdfvalue.SECONDS))
def testCronManagerListJobsDoesNotListDeletedJobs(self):
cron_manager = cronjobs.CronManager()
create_flow_args = rdf_cronjobs.CreateCronJobArgs()
cron_job_id = cron_manager.CreateJob(cron_args=create_flow_args)
self.assertLen(cron_manager.ListJobs(), 1)
cron_manager.DeleteJob(cron_job_id)
self.assertEmpty(cron_manager.ListJobs())
def testToolbarStateForDisabledCronJob(self):
cronjobs.CronManager().DisableJob(job_id=u"OSBreakDownCronJob")
self.Open("/")
self.Click("css=a[grrtarget=crons]")
self.Click("css=td:contains('OSBreakDownCronJob')")
self.assertTrue(
self.IsElementPresent("css=button[name=EnableCronJob]:not([disabled])"))
self.assertTrue(
self.IsElementPresent("css=button[name=DisableCronJob][disabled]"))
self.assertTrue(
self.IsElementPresent("css=button[name=DeleteCronJob]:not([disabled])"))
def testEnableCronJob(self):
cronjobs.CronManager().DisableJob(job_id=u"OSBreakDownCronJob")
self.Open("/")
self.Click("css=a[grrtarget=crons]")
self.Click("css=td:contains('OSBreakDownCronJob')")
# Click on Enable button and check that dialog appears.
self.Click("css=button[name=EnableCronJob]")
self.WaitUntil(self.IsTextPresent,
"Are you sure you want to ENABLE this cron job?")
# Click on "Proceed" and wait for authorization dialog to appear.
self.Click("css=button[name=Proceed]")
# This should be rejected now and a form request is made.
self.WaitUntil(self.IsTextPresent, "Create a new approval")
self.Click("css=grr-request-approval-dialog button[name=Cancel]")
# Wait for dialog to disappear.
self.WaitUntilNot(self.IsVisible, "css=.modal-open")
self.RequestAndGrantCronJobApproval(u"OSBreakDownCronJob")
# Click on Enable button and check that dialog appears.
self.Click("css=button[name=EnableCronJob]")
self.WaitUntil(self.IsTextPresent,
"Are you sure you want to ENABLE this cron job?")
# Click on "Proceed" and wait for success label to appear.
# Also check that "Proceed" button gets disabled.
self.Click("css=button[name=Proceed]")
self.WaitUntil(self.IsTextPresent,
"Cron job was ENABLED successfully!")
self.assertFalse(self.IsElementPresent("css=button[name=Proceed]"))
# Click on "Close" and check that dialog disappears.
self.Click("css=button[name=Close]")
self.WaitUntilNot(self.IsVisible, "css=.modal-open")
# TODO(amoser): The lower pane does not refresh automatically so we need to
# workaround. Remove when we have implemented this auto refresh.
self.Open("/")
self.Click("css=a[grrtarget=crons]")
self.Click("css=td:contains('OSBreakDownCronJob')")
self.WaitUntil(self.IsTextPresent,
cron_system.OSBreakDownCronJob.__name__)
self.WaitUntil(self.IsElementPresent,
"css=div:contains('Enabled') dd:contains('true')")
def testForceRun(self):
event = threading.Event()
waiting_func = functools.partial(WaitForEvent, event)
cron_manager = cronjobs.CronManager()
with mock.patch.object(cronjobs.RunHunt, "Run", wraps=waiting_func):
try:
fake_time = rdfvalue.RDFDatetime.Now()
with test_lib.FakeTime(fake_time):
create_flow_args = rdf_cronjobs.CreateCronJobArgs(
allow_overruns=False,
frequency="1h",
flow_name=transfer.GetFile.__name__)
job_id = cron_manager.CreateJob(cron_args=create_flow_args)
cron_manager.RunOnce()
cron_job_runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(cron_job_runs, 1)
job = cron_manager.ReadJob(job_id)
self.assertTrue(cron_manager.JobIsRunning(job))
# At this point, there is a run currently executing and also the job
# is not due to run for another hour. We can still force execute the
# job.
cron_manager.RunOnce()
cron_job_runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(cron_job_runs, 1)
cron_manager.RequestForcedRun(job_id)
cron_manager.RunOnce()
cron_job_runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(cron_job_runs, 2)
# The only way to prevent a forced run is to disable the job.
cron_manager.DisableJob(job_id)
cron_manager.RequestForcedRun(job_id)
cron_manager.RunOnce()
cron_job_runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(cron_job_runs, 2)
# And enable again.
cron_manager.EnableJob(job_id)
cron_manager.RequestForcedRun(job_id)
cron_manager.RunOnce()
cron_job_runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(cron_job_runs, 3)
finally:
event.set()
cron_manager._GetThreadPool().Join()
def CreateCronJob(self,
flow_name,
periodicity="1d",
lifetime="7d",
description="",
enabled=True,
token=None):
args = rdf_cronjobs.CreateCronJobArgs(flow_name=flow_name,
description=description,
frequency=periodicity,
lifetime=lifetime)
return cronjobs.CronManager().CreateJob(args,
enabled=enabled,
token=token)
def testRunningJobs(self):
event = threading.Event()
waiting_func = functools.partial(WaitForEvent, event)
with mock.patch.object(cronjobs.RunHunt, "Run", wraps=waiting_func):
cron_manager = cronjobs.CronManager()
create_flow_args = rdf_cronjobs.CreateCronJobArgs(
frequency="1w",
lifetime="1d",
flow_name=transfer.GetFile.__name__)
job_id = cron_manager.CreateJob(cron_args=create_flow_args)
prev_timeout_value = cronjobs.CRON_JOB_TIMEOUT.GetValue(
fields=[job_id])
prev_latency_value = cronjobs.CRON_JOB_LATENCY.GetValue([job_id])
cron_manager.RunOnce()
cron_job = cron_manager.ReadJob(job_id)
self.assertTrue(cron_manager.JobIsRunning(cron_job))
runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(runs, 1)
run = runs[0]
self.assertEqual(cron_job.current_run_id, run.run_id)
self.assertEqual(run.status, "RUNNING")
event.set()
cron_manager._GetThreadPool().Join()
cron_job = cron_manager.ReadJob(job_id)
self.assertFalse(cron_manager.JobIsRunning(cron_job))
runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(runs, 1)
run = runs[0]
self.assertFalse(cron_job.current_run_id)
self.assertEqual(run.status, "FINISHED")
# Check that timeout counter got updated.
current_timeout_value = cronjobs.CRON_JOB_TIMEOUT.GetValue(
[job_id])
self.assertEqual(current_timeout_value, prev_timeout_value)
# Check that latency stat got updated.
current_latency_value = cronjobs.CRON_JOB_LATENCY.GetValue(
[job_id])
self.assertEqual(
current_latency_value.count - prev_latency_value.count, 1)
def testRunningJobs(self):
event = threading.Event()
waiting_func = functools.partial(WaitForEvent, event)
with mock.patch.object(standard.RunHunt, "Run", wraps=waiting_func):
cron_manager = cronjobs.CronManager()
create_flow_args = rdf_cronjobs.CreateCronJobArgs(
frequency="1w", lifetime="1d")
job_id = cron_manager.CreateJob(cron_args=create_flow_args)
prev_timeout_value = stats_collector_instance.Get().GetMetricValue(
"cron_job_timeout", fields=[job_id])
prev_latency_value = stats_collector_instance.Get().GetMetricValue(
"cron_job_latency", fields=[job_id])
cron_manager.RunOnce(token=self.token)
cron_job = cron_manager.ReadJob(job_id, token=self.token)
self.assertTrue(cron_manager.JobIsRunning(cron_job))
runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(runs, 1)
run = runs[0]
self.assertEqual(cron_job.current_run_id, run.run_id)
self.assertEqual(run.status, "RUNNING")
event.set()
cron_manager._GetThreadPool().Join()
cron_job = cron_manager.ReadJob(job_id, token=self.token)
self.assertFalse(cron_manager.JobIsRunning(cron_job))
runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(runs, 1)
run = runs[0]
self.assertFalse(cron_job.current_run_id)
self.assertEqual(run.status, "FINISHED")
# Check that timeout counter got updated.
current_timeout_value = stats_collector_instance.Get().GetMetricValue(
"cron_job_timeout", fields=[job_id])
self.assertEqual(current_timeout_value, prev_timeout_value)
# Check that latency stat got updated.
current_latency_value = stats_collector_instance.Get().GetMetricValue(
"cron_job_latency", fields=[job_id])
self.assertEqual(current_latency_value.count - prev_latency_value.count,
1)
def Handle(self, args, context=None):
if not args.count:
stop = None
else:
stop = args.offset + args.count
cron_manager = cronjobs.CronManager()
all_jobs = list(cron_manager.ReadJobs())
all_jobs.sort(
key=lambda job: (getattr(job, "cron_job_id", None) or job.urn))
cron_jobs = all_jobs[args.offset:stop]
items = [ApiCronJob.InitFromObject(cron_job) for cron_job in cron_jobs]
return ApiListCronJobsResult(items=items, total_count=len(all_jobs))
def testRendersRequestedCronJobApproval(self):
cron_manager = cronjobs.CronManager()
cron_args = rdf_cronjobs.CreateCronJobArgs(frequency="1d",
allow_overruns=False)
cron_job_id = cron_manager.CreateJob(cron_args=cron_args)
self.RequestCronJobApproval(cron_job_id,
reason=self.token.reason,
approver=u"approver",
requestor=self.context.username)
args = user_plugin.ApiListCronJobApprovalsArgs()
result = self.handler.Handle(args, context=self.context)
self.assertLen(result.items, 1)
def testCronMaxThreadsLimitIsRespectedAndCorrectlyHandled(self):
cron_manager = cronjobs.CronManager()
event = threading.Event()
waiting_func = functools.partial(WaitForEvent, event)
try:
create_flow_args = rdf_cronjobs.CreateCronJobArgs(
frequency="1h",
lifetime="1h",
flow_name=transfer.GetFile.__name__)
with mock.patch.object(cronjobs.RunHunt, "Run",
wraps=waiting_func):
job_ids = []
for i in range(cron_manager.max_threads * 2):
# TODO: The CronJob ID space is small. Using 20 random
# IDs already causes flaky tests. Use hardcoded IDs instead.
job_ids.append(
cron_manager.CreateJob(cron_args=create_flow_args,
job_id=f"{i}"))
cron_manager.RunOnce()
count_scheduled = 0
for job_id in job_ids:
count_scheduled += len(cron_manager.ReadJobRuns(job_id))
self.assertEqual(count_scheduled, cron_manager.max_threads)
finally:
event.set()
cron_manager._GetThreadPool().Join()
count_scheduled = 0
for job_id in job_ids:
count_scheduled += len(cron_manager.ReadJobRuns(job_id))
# Check that tasks that were not scheduled due to max_threads limit
# run later.
self.assertEqual(count_scheduled, cron_manager.max_threads)
# Now all the cron jobs that weren't scheduled in previous RunOnce call
# due to max_threads limit should get scheduled.
cron_manager.RunOnce()
count_scheduled = 0
for job_id in job_ids:
count_scheduled += len(cron_manager.ReadJobRuns(job_id))
self.assertEqual(count_scheduled, cron_manager.max_threads * 2)
def testCronJobStartsRun(self):
cron_manager = cronjobs.CronManager()
create_flow_args = rdf_cronjobs.CreateCronJobArgs()
job_id = cron_manager.CreateJob(cron_args=create_flow_args)
cron_job = cron_manager.ReadJob(job_id, token=self.token)
self.assertFalse(cron_manager.JobIsRunning(cron_job, token=self.token))
# The job never ran, so JobDueToRun() should return true.
self.assertTrue(cron_manager.JobDueToRun(cron_job))
cron_manager.RunOnce(token=self.token)
runs = cron_manager.ReadJobRuns(job_id, token=self.token)
self.assertEqual(len(runs), 1)
self.assertTrue(runs[0].run_id)
def setUp(self):
super(ApiCreateCronJobApprovalHandlerTest, self).setUp()
self.SetUpApprovalTest()
cron_manager = cronjobs.CronManager()
cron_args = rdf_cronjobs.CreateCronJobArgs(
frequency="1d", allow_overruns=False)
cron_id = cron_manager.CreateJob(cron_args=cron_args)
self.handler = user_plugin.ApiCreateCronJobApprovalHandler()
self.args = user_plugin.ApiCreateCronJobApprovalArgs(cron_job_id=cron_id)
self.args.approval.reason = self.token.reason
self.args.approval.notified_users = [u"approver"]
self.args.approval.email_cc_addresses = ["*****@*****.**"]
def Run(self):
with test_lib.FakeTime(42):
self.CreateAdminUser(u"approver")
cron_manager = cronjobs.CronManager()
cron_args = rdf_cronjobs.CreateCronJobArgs(frequency="1d",
allow_overruns=False)
cron1_id = cron_manager.CreateJob(cron_args=cron_args,
token=self.token)
cron2_id = cron_manager.CreateJob(cron_args=cron_args,
token=self.token)
with test_lib.FakeTime(44):
approval1_id = self.RequestCronJobApproval(cron1_id,
reason="foo",
approver=u"approver")
with test_lib.FakeTime(45):
approval2_id = self.RequestCronJobApproval(cron2_id,
reason="bar",
approver=u"approver")
with test_lib.FakeTime(84):
self.GrantCronJobApproval(cron2_id,
approval_id=approval2_id,
approver=u"approver")
with test_lib.FakeTime(126):
self.Check("GetCronJobApproval",
args=user_plugin.ApiGetCronJobApprovalArgs(
username=self.token.username,
cron_job_id=cron1_id,
approval_id=approval1_id),
replace={
cron1_id: "CronJob_123456",
approval1_id: "approval:111111"
})
self.Check("GetCronJobApproval",
args=user_plugin.ApiGetCronJobApprovalArgs(
username=self.token.username,
cron_job_id=cron2_id,
approval_id=approval2_id),
replace={
cron2_id: "CronJob_567890",
approval2_id: "approval:222222"
})
def testDeleteCronJob(self):
cronjobs.CronManager().EnableJob(job_id=u"OSBreakDown")
self.Open("/")
self.Click("css=a[grrtarget=crons]")
self.Click("css=td:contains('OSBreakDown')")
# Click on Delete button and check that dialog appears.
self.Click("css=button[name=DeleteCronJob]:not([disabled])")
self.WaitUntil(self.IsTextPresent,
"Are you sure you want to DELETE this cron job?")
# Click on "Proceed" and wait for authorization dialog to appear.
self.Click("css=button[name=Proceed]")
self.WaitUntil(self.IsTextPresent, "Create a new approval")
self.Click("css=grr-request-approval-dialog button[name=Cancel]")
# Wait for dialog to disappear.
self.WaitUntilNot(self.IsVisible, "css=.modal-open")
self.RequestAndGrantCronJobApproval(u"OSBreakDown")
# Click on Delete button and check that dialog appears.
self.Click("css=button[name=DeleteCronJob]:not([disabled])")
self.WaitUntil(self.IsTextPresent,
"Are you sure you want to DELETE this cron job?")
# Click on "Proceed" and wait for success label to appear.
# Also check that "Proceed" button gets disabled.
self.Click("css=button[name=Proceed]")
self.WaitUntil(self.IsTextPresent,
"Cron job was deleted successfully!")
self.assertFalse(self.IsElementPresent("css=button[name=Proceed]"))
# Click on "Close" and check that dialog disappears.
self.Click("css=button[name=Close]")
self.WaitUntilNot(self.IsVisible, "css=.modal-open")
# View should be refreshed automatically.
self.WaitUntil(
self.IsElementPresent, "css=grr-cron-jobs-list "
"td:contains('GRRVersionBreakDown')")
self.WaitUntilNot(
self.IsElementPresent, "css=grr-cron-jobs-list "
"td:contains('OSBreakDown')")
def testRendersRequestedCronJobApproval(self):
cron_manager = cronjobs.CronManager()
cron_args = rdf_cronjobs.CreateCronJobArgs(
frequency="1d",
allow_overruns=False,
flow_name=file.CollectSingleFile.__name__)
cron_job_id = cron_manager.CreateJob(cron_args=cron_args)
self.RequestCronJobApproval(cron_job_id,
reason="Running tests",
approver="approver",
requestor=self.context.username)
args = user_plugin.ApiListCronJobApprovalsArgs()
result = self.handler.Handle(args, context=self.context)
self.assertLen(result.items, 1)
def Handle(self, args, token=None):
if data_store.RelationalDBEnabled():
run = cronjobs.CronManager().ReadJobRun(
str(args.cron_job_id), str(args.run_id))
if not run:
raise CronJobRunNotFoundError(
"Cron job run with id %s could not be found" % args.run_id)
return ApiCronJobRun().InitFromRunObject(run)
else:
# Note: this is a legacy AFF4 implementation.
flow_urn = args.run_id.ToURN(args.cron_job_id)
flow_obj = aff4.FACTORY.Open(
flow_urn, aff4_type=flow.GRRFlow, mode="r", token=token)
f = api_plugins_flow.ApiFlow().InitFromAff4Object(
flow_obj, with_state_and_context=True)
return ApiCronJobRun().InitFromApiFlow(f, cron_job_id=args.cron_job_id)
def setUp(self):
super().setUp()
self.SetUpApprovalTest()
cron_manager = cronjobs.CronManager()
cron_args = rdf_cronjobs.CreateCronJobArgs(
frequency="1d",
allow_overruns=False,
flow_name=file.CollectSingleFile.__name__)
cron_id = cron_manager.CreateJob(cron_args=cron_args)
self.handler = user_plugin.ApiCreateCronJobApprovalHandler()
self.args = user_plugin.ApiCreateCronJobApprovalArgs(cron_job_id=cron_id)
self.args.approval.reason = "Running tests"
self.args.approval.notified_users = [u"approver"]
self.args.approval.email_cc_addresses = ["*****@*****.**"]
def testCronMaxThreadsLimitIsRespectedAndCorrectlyHandled(self):
cron_manager = cronjobs.CronManager()
event = threading.Event()
waiting_func = functools.partial(WaitForEvent, event)
try:
create_flow_args = rdf_cronjobs.CreateCronJobArgs(frequency="1h",
lifetime="1h")
with mock.patch.object(cronjobs.RunHunt, "Run",
wraps=waiting_func):
job_ids = []
for _ in range(cron_manager.max_threads * 2):
job_ids.append(
cron_manager.CreateJob(cron_args=create_flow_args))
cron_manager.RunOnce()
count_scheduled = 0
for job_id in job_ids:
count_scheduled += len(cron_manager.ReadJobRuns(job_id))
self.assertEqual(count_scheduled, cron_manager.max_threads)
finally:
event.set()
cron_manager._GetThreadPool().Join()
count_scheduled = 0
for job_id in job_ids:
count_scheduled += len(cron_manager.ReadJobRuns(job_id))
# Check that tasks that were not scheduled due to max_threads limit
# run later.
self.assertEqual(count_scheduled, cron_manager.max_threads)
# Now all the cron jobs that weren't scheduled in previous RunOnce call
# due to max_threads limit should get scheduled.
cron_manager.RunOnce()
count_scheduled = 0
for job_id in job_ids:
count_scheduled += len(cron_manager.ReadJobRuns(job_id))
self.assertEqual(count_scheduled, cron_manager.max_threads * 2)
def testDisabledCronJobDoesNotCreateJobs(self):
cron_manager = cronjobs.CronManager()
create_flow_args = rdf_cronjobs.CreateCronJobArgs()
job_id1 = cron_manager.CreateJob(cron_args=create_flow_args)
job_id2 = cron_manager.CreateJob(cron_args=create_flow_args)
cron_manager.DisableJob(job_id1, token=self.token)
cron_manager.RunOnce(token=self.token)
cron_job1 = cron_manager.ReadJob(job_id1, token=self.token)
cron_job2 = cron_manager.ReadJob(job_id2, token=self.token)
# Disabled flow shouldn't be running, while not-disabled flow should run
# as usual.
self.assertFalse(cron_manager.JobIsRunning(cron_job1,
token=self.token))
self.assertTrue(cron_manager.JobIsRunning(cron_job2, token=self.token))
def testNonExistingSystemCronJobDoesNotPreventOtherCronJobsFromRunning(
self):
# Have a fake non-existing cron job. We assume that cron jobs are going
# to be processed in alphabetical order, according to their cron job ids.
args = rdf_cronjobs.CronJobAction(
action_type=rdf_cronjobs.CronJobAction.ActionType.
SYSTEM_CRON_ACTION,
system_cron_action=rdf_cronjobs.SystemCronAction(
job_class_name="__AbstractFakeCronJob__"))
job = rdf_cronjobs.CronJob(
cron_job_id="cron_1",
args=args,
enabled=True,
frequency=rdfvalue.Duration.From(2, rdfvalue.HOURS),
lifetime=rdfvalue.Duration.From(1, rdfvalue.HOURS),
allow_overruns=False)
data_store.REL_DB.WriteCronJob(job)
# Have a proper cron job.
cron_manager = cronjobs.CronManager()
args = rdf_cronjobs.CronJobAction(
action_type=rdf_cronjobs.CronJobAction.ActionType.
SYSTEM_CRON_ACTION,
system_cron_action=rdf_cronjobs.SystemCronAction(
job_class_name="DummyStatefulSystemCronJobRel"))
job = rdf_cronjobs.CronJob(
cron_job_id="cron_2",
args=args,
enabled=True,
frequency=rdfvalue.Duration.From(2, rdfvalue.HOURS),
lifetime=rdfvalue.Duration.From(1, rdfvalue.HOURS),
allow_overruns=False)
data_store.REL_DB.WriteCronJob(job)
with self.assertRaises(cronjobs.OneOrMoreCronJobsFailedError):
cron_manager.RunOnce()
cron_manager._GetThreadPool().Join()
self.assertEmpty(cron_manager.ReadJobRuns("cron_1"))
self.assertLen(cron_manager.ReadJobRuns("cron_2"), 1)
def testFailingCronJobIsHighlighted(self):
# Run all cron jobs once to put them into the OK state.
manager = cronjobs.CronManager()
manager.RunOnce()
manager._GetThreadPool().Stop()
self.AddJobStatus(u"OSBreakDownCronJob",
rdf_cronjobs.CronJobRunStatus.Status.ERROR)
self.Open("/")
self.WaitUntil(self.IsElementPresent, "client_query")
self.Click("css=a[grrtarget=crons]")
# OSBreakDownCronJob's row should have an 'error' class
self.WaitUntil(self.IsElementPresent,
"css=tr.danger td:contains('OSBreakDownCronJob')")
# Check that only OSBreakDownCronJob is highlighted
self.WaitUntilNot(
self.IsElementPresent,
"css=tr.danger td:contains('GRRVersionBreakDownCronJob')")
def testCronJobStartsRun(self):
cron_manager = cronjobs.CronManager()
create_flow_args = rdf_cronjobs.CreateCronJobArgs()
job_id = cron_manager.CreateJob(cron_args=create_flow_args)
cron_job = cron_manager.ReadJob(job_id, token=self.token)
self.assertFalse(cron_manager.JobIsRunning(cron_job, token=self.token))
# The job never ran, so JobDueToRun() should return true.
self.assertTrue(cron_manager.JobDueToRun(cron_job))
cron_manager.RunOnce(token=self.token)
cron_manager._GetThreadPool().Join()
runs = cron_manager.ReadJobRuns(job_id, token=self.token)
self.assertLen(runs, 1)
run = runs[0]
self.assertTrue(run.run_id)
self.assertTrue(run.started_at)
self.assertTrue(run.finished_at)
self.assertEqual(run.status, "FINISHED")
def Handle(self, args, token=None):
if data_store.RelationalDBEnabled():
runs = cronjobs.CronManager().ReadJobRuns(str(args.cron_job_id))
start = args.offset
if args.count:
end = args.offset + args.count
else:
end = db.MAX_COUNT
return ApiListCronJobRunsResult(items=[
ApiCronJobRun().InitFromRunObject(run) for run in runs[start:end]
])
else:
# Note: this is a legacy AFF4 implementation.
flows_result = api_plugins_flow.ApiListFlowsHandler.BuildFlowList(
args.cron_job_id.ToURN(),
args.count,
args.offset,
with_state_and_context=True,
token=token)
return ApiListCronJobRunsResult(items=[
ApiCronJobRun().InitFromApiFlow(f, cron_job_id=args.cron_job_id)
for f in flows_result.items
])
def testCronJobRunDoesNothingIfDueTimeHasNotComeYet(self):
fake_time = rdfvalue.RDFDatetime.Now()
with test_lib.FakeTime(fake_time):
cron_manager = cronjobs.CronManager()
create_flow_args = rdf_cronjobs.CreateCronJobArgs(
allow_overruns=False, frequency="1h")
job_id = cron_manager.CreateJob(cron_args=create_flow_args)
cron_manager.RunOnce(token=self.token)
cron_job_runs = cron_manager.ReadJobRuns(job_id, token=self.token)
self.assertLen(cron_job_runs, 1)
# Let 59 minutes pass. Frequency is 1 hour, so new flow is not
# supposed to start.
fake_time += rdfvalue.Duration("59m")
with test_lib.FakeTime(fake_time):
cron_manager.RunOnce(token=self.token)
cron_job_runs = cron_manager.ReadJobRuns(job_id, token=self.token)
self.assertLen(cron_job_runs, 1)
def testError(self):
with mock.patch.object(
standard.RunHunt,
"Run",
side_effect=ValueError("Random cron job error.")):
cron_manager = cronjobs.CronManager()
create_flow_args = rdf_cronjobs.CreateCronJobArgs()
job_id = cron_manager.CreateJob(cron_args=create_flow_args)
prev_failure_value = stats_collector_instance.Get().GetMetricValue(
"cron_job_failure", fields=[job_id])
prev_latency_value = stats_collector_instance.Get().GetMetricValue(
"cron_job_latency", fields=[job_id])
cron_manager.RunOnce(token=self.token)
cron_manager._GetThreadPool().Join()
cron_job = cron_manager.ReadJob(job_id, token=self.token)
self.assertFalse(cron_manager.JobIsRunning(cron_job))
runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(runs, 1)
run = runs[0]
self.assertEqual(cron_job.last_run_status, "ERROR")
self.assertEqual(run.status, "ERROR")
self.assertTrue(run.backtrace)
self.assertIn("cron job error", run.backtrace)
current_failure_value = stats_collector_instance.Get().GetMetricValue(
"cron_job_failure", fields=[job_id])
current_latency_value = stats_collector_instance.Get().GetMetricValue(
"cron_job_latency", fields=[job_id])
self.assertEqual(current_failure_value, prev_failure_value + 1)
self.assertEqual(current_latency_value.count,
prev_latency_value.count + 1)
def testError(self):
with mock.patch.object(
cronjobs.RunHunt,
"Run",
side_effect=ValueError("Random cron job error.")):
cron_manager = cronjobs.CronManager()
create_flow_args = rdf_cronjobs.CreateCronJobArgs()
create_flow_args.flow_name = transfer.GetFile.__name__
job_id = cron_manager.CreateJob(cron_args=create_flow_args)
prev_failure_value = cronjobs.CRON_JOB_FAILURE.GetValue([job_id])
prev_latency_value = cronjobs.CRON_JOB_LATENCY.GetValue([job_id])
cron_manager.RunOnce()
cron_manager._GetThreadPool().Join()
cron_job = cron_manager.ReadJob(job_id)
self.assertFalse(cron_manager.JobIsRunning(cron_job))
runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(runs, 1)
run = runs[0]
self.assertEqual(cron_job.last_run_status, "ERROR")
self.assertEqual(run.status, "ERROR")
self.assertTrue(run.backtrace)
self.assertIn("cron job error", run.backtrace)
current_failure_value = cronjobs.CRON_JOB_FAILURE.GetValue(
[job_id])
current_latency_value = cronjobs.CRON_JOB_LATENCY.GetValue(
[job_id])
self.assertEqual(current_failure_value, prev_failure_value + 1)
self.assertEqual(current_latency_value.count,
prev_latency_value.count + 1)
def testSystemCronJobsMayBeDisabledViaConfig(self):
with test_lib.ConfigOverrider(
{"Cron.disabled_cron_jobs": ["DummySystemCronJobRel"]}):
cronjobs.ScheduleSystemCronJobs()
cron_manager = cronjobs.CronManager()
jobs = cron_manager.ListJobs()
self.assertIn("DummySystemCronJobRel", jobs)
# This cron job should be disabled, because it's listed in
# Cron.disabled_cron_jobs config variable.
job = cron_manager.ReadJob("DummySystemCronJobRel")
self.assertFalse(job.enabled)
# Now remove the cron job from the list and check that it gets disabled
# after next ScheduleSystemCronJobs() call.
with test_lib.ConfigOverrider({"Cron.disabled_cron_jobs": []}):
cronjobs.ScheduleSystemCronJobs()
# System cron job should be enabled.
job = cron_manager.ReadJob("DummySystemCronJobRel")
self.assertTrue(job.enabled)
def testDisabledCronJobDoesNotCreateJobs(self):
cron_manager = cronjobs.CronManager()
create_flow_args = rdf_cronjobs.CreateCronJobArgs()
job_id1 = cron_manager.CreateJob(cron_args=create_flow_args)
job_id2 = cron_manager.CreateJob(cron_args=create_flow_args)
cron_manager.DisableJob(job_id1, token=self.token)
event = threading.Event()
waiting_func = functools.partial(WaitForEvent, event)
try:
with mock.patch.object(standard.RunHunt, "Run", wraps=waiting_func):
cron_manager.RunOnce(token=self.token)
cron_job1 = cron_manager.ReadJob(job_id1, token=self.token)
cron_job2 = cron_manager.ReadJob(job_id2, token=self.token)
# Disabled flow shouldn't be running, while not-disabled flow should run
# as usual.
self.assertFalse(cron_manager.JobIsRunning(cron_job1, token=self.token))
self.assertTrue(cron_manager.JobIsRunning(cron_job2, token=self.token))
finally:
event.set()
def testCronJobRunPreventsOverrunsWhenAllowOverrunsIsTrue(self):
event = threading.Event()
waiting_func = functools.partial(WaitForEvent, event)
cron_manager = cronjobs.CronManager()
try:
with mock.patch.object(cronjobs.RunHunt, "Run",
wraps=waiting_func):
fake_time = rdfvalue.RDFDatetime.Now()
with test_lib.FakeTime(fake_time):
create_flow_args = rdf_cronjobs.CreateCronJobArgs(
allow_overruns=True,
frequency="1h",
flow_name=transfer.GetFile.__name__)
job_id = cron_manager.CreateJob(cron_args=create_flow_args)
cron_manager.RunOnce()
cron_job_runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(cron_job_runs, 1)
# Let two hours pass. Frequency is 1h (i.e. cron job iterations are
# supposed to be started every hour), so the new flow should be started
# by RunOnce(). However, as allow_overruns is False, and previous
# iteration flow hasn't finished yet, no flow will be started.
fake_time += rdfvalue.Duration.From(2, rdfvalue.HOURS)
with test_lib.FakeTime(fake_time):
cron_manager.RunOnce()
cron_job_runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(cron_job_runs, 2)
finally:
event.set()
cron_manager._GetThreadPool().Join()
