def testCronJobRespectsStartTime(self):
with test_lib.FakeTime(0):
cron_manager = aff4_cronjobs.GetCronManager()
start_time1 = rdfvalue.RDFDatetime(100 * 1000 * 1000)
cron_args1 = rdf_cronjobs.CreateCronJobFlowArgs(start_time=start_time1)
cron_args1.flow_runner_args.flow_name = "FakeCronJob"
cron_args2 = rdf_cronjobs.CreateCronJobFlowArgs()
cron_args2.flow_runner_args.flow_name = "FakeCronJob"
cron_job_id1 = cron_manager.CreateJob(cron_args1, token=self.token)
cron_job_id2 = cron_manager.CreateJob(cron_args2, token=self.token)
cron_manager.RunOnce(token=self.token)
cron_job1 = cron_manager.ReadJob(cron_job_id1, token=self.token)
cron_job2 = cron_manager.ReadJob(cron_job_id2, token=self.token)
self.assertEqual(
cron_job1.Get(cron_job1.Schema.CRON_ARGS).start_time, start_time1)
# Flow without a start time should now be running
self.assertFalse(cron_job1.IsRunning())
self.assertTrue(cron_job2.IsRunning())
# Move the clock past the start time
with test_lib.FakeTime(500):
cron_manager.RunOnce(token=self.token)
cron_job1 = cron_manager.ReadJob(cron_job_id1, token=self.token)
cron_job2 = cron_manager.ReadJob(cron_job_id2, token=self.token)
# Start time should be the same
self.assertEqual(
cron_job1.Get(cron_job1.Schema.CRON_ARGS).start_time, start_time1)
# Now both should be running
self.assertTrue(cron_job1.IsRunning())
self.assertTrue(cron_job2.IsRunning())
# Check setting a bad flow urn is handled and removed
with aff4.FACTORY.OpenWithLock(
cron_job2.urn, aff4_type=aff4_cronjobs.CronJob,
token=self.token) as cron_job2:
cron_job2.Set(cron_job2.Schema.CURRENT_FLOW_URN("aff4:/does/not/exist"))
self.assertFalse(cron_job2.IsRunning())
cron_job2 = cron_manager.ReadJob(cron_job_id2, token=self.token)
self.assertFalse(cron_job2.Get(cron_job2.Schema.CURRENT_FLOW_URN))
def testCronJobRespectsStartTime(self):
with test_lib.FakeTime(0):
cron_manager = aff4_cronjobs.GetCronManager()
start_time1 = rdfvalue.RDFDatetime(100 * 1000 * 1000)
cron_args1 = rdf_cronjobs.CreateCronJobFlowArgs(start_time=start_time1)
cron_args1.flow_runner_args.flow_name = "FakeCronJob"
cron_args2 = rdf_cronjobs.CreateCronJobFlowArgs()
cron_args2.flow_runner_args.flow_name = "FakeCronJob"
cron_job_id1 = cron_manager.CreateJob(cron_args1)
cron_job_id2 = cron_manager.CreateJob(cron_args2)
cron_manager.RunOnce(token=self.token)
cron_job1 = cron_manager.ReadJob(cron_job_id1, token=self.token)
cron_job2 = cron_manager.ReadJob(cron_job_id2, token=self.token)
self.assertEqual(cron_job1.cron_args.start_time, start_time1)
# Flow without a start time should now be running
self.assertFalse(cron_manager.JobIsRunning(cron_job1, token=self.token))
self.assertTrue(cron_manager.JobIsRunning(cron_job2, token=self.token))
# Move the clock past the start time
with test_lib.FakeTime(500):
cron_manager.RunOnce(token=self.token)
cron_job1 = cron_manager.ReadJob(cron_job_id1, token=self.token)
cron_job2 = cron_manager.ReadJob(cron_job_id2, token=self.token)
# Start time should be the same
self.assertEqual(cron_job1.cron_args.start_time, start_time1)
# Now both should be running
self.assertTrue(cron_manager.JobIsRunning(cron_job1, token=self.token))
self.assertTrue(cron_manager.JobIsRunning(cron_job2, token=self.token))
# Check setting a bad run id is handled.
data_store.REL_DB.UpdateCronJob(cron_job2.job_id, current_run_id=12345)
cron_job2 = cron_manager.ReadJob(cron_job_id2, token=self.token)
self.assertFalse(cron_manager.JobIsRunning(cron_job2, token=self.token))
# Job got updated right away.
self.assertFalse(cron_job2.current_run_id)
# DB also reflects the removed run id.
cron_job2 = cron_manager.ReadJob(cron_job_id2, token=self.token)
self.assertFalse(cron_job2.current_run_id)
def testLastRunStatusGetsUpdated(self):
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobFlowArgs()
cron_args.flow_runner_args.flow_name = "OccasionallyFailingFakeCronJob"
cron_args.periodicity = "30s"
job_id = cron_manager.CreateJob(cron_args=cron_args)
statuses = []
for fake_time in [0, 60]:
with test_lib.FakeTime(fake_time):
# This call should start a new cron job flow
cron_manager.RunOnce(token=self.token)
run_urn = cron_manager.ReadJobRuns(job_id)[-1].urn
flow_test_lib.TestFlowHelper(
run_urn, check_flow_errors=False, token=self.token)
# This RunOnce call should determine that the flow has finished
cron_manager.RunOnce(token=self.token)
cron_job = cron_manager.ReadJob(job_id, token=self.token)
statuses.append(cron_job.last_run_status)
statuses = sorted(statuses, key=lambda x: x.age)
self.assertEqual(len(statuses), 2)
self.assertEqual(statuses[0], rdf_cronjobs.CronJobRunStatus.Status.OK)
self.assertEqual(statuses[1], rdf_cronjobs.CronJobRunStatus.Status.ERROR)
def Run(self):
with test_lib.FakeTime(42):
self.CreateUser("approver")
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobFlowArgs(periodicity="1d",
allow_overruns=False)
cron_id = cron_manager.CreateJob(cron_args=cron_args, token=self.token)
def ReplaceCronAndApprovalIds():
approvals = self.ListCronJobApprovals()
return {
approvals[0].id: "approval:112233",
cron_id: "CronJob_123456"
}
with test_lib.FakeTime(126):
self.Check("CreateCronJobApproval",
args=user_plugin.ApiCreateCronJobApprovalArgs(
cron_job_id=cron_id,
approval=user_plugin.ApiCronJobApproval(
reason="really important reason!",
notified_users=["approver1", "approver2"],
email_cc_addresses=["*****@*****.**"])),
replace=ReplaceCronAndApprovalIds)
def testCronJobRunAllowsOverrunsWhenAllowOverrunsIsTrue(self):
with test_lib.FakeTime(0):
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobFlowArgs(
allow_overruns=True, periodicity="1h")
cron_args.flow_runner_args.flow_name = "FakeCronJob"
job_id = cron_manager.CreateJob(cron_args=cron_args)
cron_manager.RunOnce(token=self.token)
cron_job_runs = cron_manager.ReadJobRuns(job_id, token=self.token)
self.assertEqual(len(cron_job_runs), 1)
# Let an hour pass. Frequency is 1h (i.e. cron job iterations are
# supposed to be started every hour), so the new flow should be started
# by RunOnce(). Previous iteration flow hasn't finished yet, but
# allow_overruns is True, so it's ok to start new iteration.
time.time = lambda: 60 * 60 + 1
cron_manager.RunOnce(token=self.token)
cron_job_runs = cron_manager.ReadJobRuns(job_id, token=self.token)
self.assertEqual(len(cron_job_runs), 2)
def testCronJobPreservesFlowNameAndArguments(self):
"""Testing initialization of a ConfigManager."""
pathspec = rdf_paths.PathSpec(
path="/foo", pathtype=rdf_paths.PathSpec.PathType.TSK)
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobFlowArgs(
periodicity="1d", allow_overruns=False)
cron_args.flow_runner_args.flow_name = transfer.GetFile.__name__
cron_args.flow_args.pathspec = pathspec
job_id = cron_manager.CreateJob(cron_args=cron_args)
# Check that CronJob definition is saved properly
jobs = cron_manager.ListJobs(token=self.token)
self.assertEqual(len(jobs), 1)
self.assertEqual(jobs[0], job_id)
cron_job = cron_manager.ReadJob(job_id, token=self.token)
self.assertEqual(cron_job.cron_args.flow_runner_args.flow_name,
transfer.GetFile.__name__)
self.assertEqual(cron_job.cron_args.flow_args.pathspec, pathspec)
self.assertEqual(cron_job.cron_args.periodicity, rdfvalue.Duration("1d"))
self.assertEqual(cron_job.cron_args.allow_overruns, False)
def testLatencyStatsAreCorrectlyRecorded(self):
with test_lib.FakeTime(0):
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobFlowArgs()
cron_args.flow_runner_args.flow_name = "FakeCronJob"
cron_args.periodicity = "1w"
cron_job_id = cron_manager.CreateJob(
cron_args=cron_args, token=self.token)
cron_manager.RunOnce(token=self.token)
prev_metric_value = stats.STATS.GetMetricValue(
"cron_job_latency", fields=[cron_job_id])
# Fast forward one minute
with test_lib.FakeTime(60):
cron_manager.RunOnce(token=self.token)
cron_job = cron_manager.ReadJob(cron_job_id, token=self.token)
cron_flow_urn = cron_job.Get(cron_job.Schema.CURRENT_FLOW_URN)
flow_test_lib.TestFlowHelper(
cron_flow_urn, check_flow_errors=False, token=self.token)
# This RunOnce call should determine that the flow has finished
cron_manager.RunOnce(token=self.token)
# Check that stats got updated
current_metric_value = stats.STATS.GetMetricValue(
"cron_job_latency", fields=[cron_job_id])
self.assertEqual(current_metric_value.count - prev_metric_value.count, 1)
self.assertEqual(current_metric_value.sum - prev_metric_value.sum, 60)
def testLastRunStatusGetsUpdatedOnEveryRun(self):
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobFlowArgs()
cron_args.flow_runner_args.flow_name = "OccasionallyFailingFakeCronJob"
cron_args.periodicity = "30s"
job_id = cron_manager.CreateJob(cron_args=cron_args, token=self.token)
for fake_time in [0, 60]:
with test_lib.FakeTime(fake_time):
# This call should start a new cron job flow
cron_manager.RunOnce(token=self.token)
cron_job = cron_manager.ReadJob(job_id, token=self.token)
cron_flow_urn = cron_job.Get(cron_job.Schema.CURRENT_FLOW_URN)
flow_test_lib.TestFlowHelper(
cron_flow_urn, check_flow_errors=False, token=self.token)
# This RunOnce call should determine that the flow has finished
cron_manager.RunOnce(token=self.token)
cron_job = cron_manager.ReadJob(job_id, token=self.token)
statuses = list(
cron_job.GetValuesForAttribute(cron_job.Schema.LAST_RUN_STATUS))
statuses = sorted(statuses, key=lambda x: x.age)
self.assertEqual(len(statuses), 2)
self.assertEqual(statuses[0].age,
rdfvalue.RDFDatetime.FromSecondsSinceEpoch(0))
self.assertEqual(statuses[1].age,
rdfvalue.RDFDatetime.FromSecondsSinceEpoch(60))
self.assertEqual(statuses[0].status,
rdf_cronjobs.CronJobRunStatus.Status.OK)
self.assertEqual(statuses[1].status,
rdf_cronjobs.CronJobRunStatus.Status.ERROR)
def Run(self):
with test_lib.FakeTime(42):
self.CreateAdminUser("requestor")
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobFlowArgs(
periodicity="1d", allow_overruns=False)
cron_id = cron_manager.CreateJob(cron_args=cron_args, token=self.token)
with test_lib.FakeTime(44):
approval_id = self.RequestCronJobApproval(
cron_id,
approver=self.token.username,
requestor="requestor",
reason="foo")
with test_lib.FakeTime(126):
self.Check(
"GrantCronJobApproval",
args=user_plugin.ApiGrantCronJobApprovalArgs(
cron_job_id=cron_id,
approval_id=approval_id,
username="******"),
replace={
cron_id: "CronJob_123456",
approval_id: "approval:111111"
})
def testSchedulingJobWithFixedNamePreservesTheName(self):
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobFlowArgs(
allow_overruns=True, periodicity="1d")
cron_args.flow_runner_args.flow_name = "FakeCronJob"
job_id = cron_manager.CreateJob(cron_args=cron_args, job_id="TheJob")
self.assertEqual("TheJob", job_id)
def setUp(self):
super(ApiListCronJobFlowsHandlerRegressionTest, self).setUp()
with test_lib.FakeTime(44):
cron_args = rdf_cronjobs.CreateCronJobFlowArgs(periodicity="7d",
lifetime="1d")
cron_args.flow_runner_args.flow_name = self.flow_name
cronjobs.GetCronManager().CreateJob(cron_args,
job_id=self.flow_name,
token=self.token)
cronjobs.GetCronManager().RunOnce(token=self.token)
def setUp(self):
super(TestCronView, self).setUp()
for flow_name in [
cron_system.GRRVersionBreakDown.__name__,
cron_system.OSBreakDown.__name__, cron_system.LastAccessStats.__name__
]:
cron_args = rdf_cronjobs.CreateCronJobFlowArgs(
periodicity="7d", lifetime="1d")
cron_args.flow_runner_args.flow_name = flow_name
cronjobs.GetCronManager().CreateJob(
cron_args, job_id=flow_name, token=self.token)
cronjobs.GetCronManager().RunOnce(token=self.token)
def ScheduleSystemCronFlows(names=None, token=None):
"""Schedule all the SystemCronFlows found."""
errors = []
for name in config.CONFIG["Cron.disabled_system_jobs"]:
try:
cls = registry.FlowRegistry.FlowClassByName(name)
except ValueError:
errors.append("No such flow: %s." % name)
continue
if not aff4.issubclass(cls, SystemCronFlow):
errors.append(
"Disabled system cron job name doesn't correspond to "
"a flow inherited from SystemCronFlow: %s" % name)
if names is None:
names = registry.FlowRegistry.FLOW_REGISTRY.keys()
for name in names:
cls = registry.FlowRegistry.FlowClassByName(name)
if aff4.issubclass(cls, SystemCronFlow):
cron_args = rdf_cronjobs.CreateCronJobFlowArgs(
periodicity=cls.frequency)
cron_args.flow_runner_args.flow_name = name
cron_args.lifetime = cls.lifetime
cron_args.allow_overruns = cls.allow_overruns
cron_args.start_time = GetStartTime(cls)
if cls.disabled:
disabled = True
else:
disabled = name in config.CONFIG["Cron.disabled_system_jobs"]
manager = GetCronManager()
if data_store.RelationalDBReadEnabled(category="cronjobs"):
manager.CreateJob(cron_args=cron_args,
job_id=name,
disabled=disabled)
else:
manager.CreateJob(cron_args=cron_args,
job_id=name,
token=token,
disabled=disabled)
if errors:
raise ValueError(
"Error(s) while parsing Cron.disabled_system_jobs: %s" % errors)
def testKillOldFlows(self):
with test_lib.FakeTime(0):
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobFlowArgs()
cron_args.flow_runner_args.flow_name = "FakeCronJob"
cron_args.periodicity = "1w"
cron_args.lifetime = FakeCronJob.lifetime
job_id = cron_manager.CreateJob(cron_args=cron_args, token=self.token)
cron_manager.RunOnce(token=self.token)
cron_job = cron_manager.ReadJob(job_id, token=self.token)
self.assertTrue(cron_job.IsRunning())
self.assertFalse(cron_job.KillOldFlows())
prev_timeout_value = stats.STATS.GetMetricValue(
"cron_job_timeout", fields=[job_id])
prev_latency_value = stats.STATS.GetMetricValue(
"cron_job_latency", fields=[job_id])
# Fast forward one day
with test_lib.FakeTime(24 * 60 * 60 + 1):
flow_urn = cron_job.Get(cron_job.Schema.CURRENT_FLOW_URN)
cron_manager.RunOnce(token=self.token)
cron_job = cron_manager.ReadJob(job_id, token=self.token)
self.assertFalse(cron_job.IsRunning())
# Check the termination log
log_collection = flow.GRRFlow.LogCollectionForFID(flow_urn)
for line in log_collection:
if line.urn == flow_urn:
self.assertTrue("lifetime exceeded" in str(line.log_message))
# Check that timeout counter got updated.
current_timeout_value = stats.STATS.GetMetricValue(
"cron_job_timeout", fields=[job_id])
self.assertEqual(current_timeout_value - prev_timeout_value, 1)
# Check that latency stat got updated.
current_latency_value = stats.STATS.GetMetricValue(
"cron_job_latency", fields=[job_id])
self.assertEqual(current_latency_value.count - prev_latency_value.count,
1)
self.assertEqual(current_latency_value.sum - prev_latency_value.sum,
24 * 60 * 60 + 1)
def testStatefulSystemCronFlowMaintainsState(self):
DummyStatefulSystemCronJob.VALUES = []
# We need to have a cron job started to have a place to maintain
# state.
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobFlowArgs()
cron_args.flow_runner_args.flow_name = "DummyStatefulSystemCronJob"
cron_manager.CreateJob(job_id="RelationalStateCronJob", cron_args=cron_args)
for i in range(3):
cron_manager.RunOnce(force=True, token=self.token)
runs = cron_manager.ReadJobRuns("RelationalStateCronJob")
self.assertEqual(len(runs), i + 1)
flow_test_lib.TestFlowHelper(runs[-1].urn, token=self.token)
self.assertListEqual(DummyStatefulSystemCronJob.VALUES, [0, 1, 2])
def CreateCronJob(self,
flow_name,
periodicity="1d",
lifetime="7d",
description="",
disabled=False,
token=None):
cron_args = rdf_cronjobs.CreateCronJobFlowArgs(periodicity=periodicity,
lifetime=lifetime,
description=description)
cron_args.flow_runner_args.flow_name = flow_name
return cronjobs.GetCronManager().CreateJob(cron_args,
job_id=flow_name,
disabled=disabled,
token=token)
def testRendersRequestedCronJobApproval(self):
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobFlowArgs(periodicity="1d",
allow_overruns=False)
cron_job_id = cron_manager.CreateJob(cron_args=cron_args,
token=self.token)
self.RequestCronJobApproval(cron_job_id,
reason=self.token.reason,
approver="approver",
requestor=self.token.username)
args = user_plugin.ApiListCronJobApprovalsArgs()
result = self.handler.Handle(args, token=self.token)
self.assertEqual(len(result.items), 1)
def testCronJobRunDoesNothingIfCurrentFlowIsRunning(self):
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobFlowArgs(
allow_overruns=False, periodicity="1d")
cron_args.flow_runner_args.flow_name = "FakeCronJob"
job_id = cron_manager.CreateJob(cron_args=cron_args)
cron_manager.RunOnce(token=self.token)
cron_job_runs = cron_manager.ReadJobRuns(job_id, token=self.token)
self.assertEqual(len(cron_job_runs), 1)
cron_manager.RunOnce(token=self.token)
cron_job_runs = cron_manager.ReadJobRuns(job_id, token=self.token)
self.assertEqual(len(cron_job_runs), 1)
def testCronManagerListJobsDoesNotListDeletedJobs(self):
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobFlowArgs(
allow_overruns=True, periodicity="1d")
cron_args.flow_runner_args.flow_name = "FakeCronJob"
cron_job_urn = cron_manager.CreateJob(cron_args=cron_args, token=self.token)
cron_jobs = list(cron_manager.ListJobs(token=self.token))
self.assertEqual(len(cron_jobs), 1)
cron_manager.DeleteJob(cron_job_urn, token=self.token)
cron_jobs = list(cron_manager.ListJobs(token=self.token))
self.assertEqual(len(cron_jobs), 0)
def setUp(self):
super(ApiCreateCronJobApprovalHandlerTest, self).setUp()
self.SetUpApprovalTest()
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobFlowArgs(periodicity="1d",
allow_overruns=False)
cron_id = cron_manager.CreateJob(cron_args=cron_args, token=self.token)
self.handler = user_plugin.ApiCreateCronJobApprovalHandler()
self.args = user_plugin.ApiCreateCronJobApprovalArgs(
cron_job_id=cron_id)
self.args.approval.reason = self.token.reason
self.args.approval.notified_users = ["approver"]
self.args.approval.email_cc_addresses = ["*****@*****.**"]
def setUp(self):
super(CleanCronJobsTest, self).setUp()
with test_lib.FakeTime(40):
cron_args = rdf_cronjobs.CreateCronJobFlowArgs(
periodicity=RetentionTestSystemCronJob.frequency)
cron_args.flow_runner_args.flow_name = RetentionTestSystemCronJob.__name__
cron_args.lifetime = RetentionTestSystemCronJob.lifetime
self.cron_jobs_names = []
self.cron_jobs_names.append(cronjobs.GetCronManager().CreateJob(
cron_args=cron_args, job_id="Foo", token=self.token, disabled=False))
self.cron_jobs_names.append(cronjobs.GetCronManager().CreateJob(
cron_args=cron_args, job_id="Bar", token=self.token, disabled=False))
for i in range(self.NUM_CRON_RUNS):
with test_lib.FakeTime(40 + 60 * i):
cronjobs.GetCronManager().RunOnce(token=self.token, force=True)
def testDisabledCronJobDoesNotCreateJobs(self):
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobFlowArgs()
cron_args.flow_runner_args.flow_name = "FakeCronJob"
job_id1 = cron_manager.CreateJob(cron_args)
job_id2 = cron_manager.CreateJob(cron_args)
cron_manager.DisableJob(job_id1, token=self.token)
cron_manager.RunOnce(token=self.token)
cron_job1 = cron_manager.ReadJob(job_id1, token=self.token)
cron_job2 = cron_manager.ReadJob(job_id2, token=self.token)
# Disabled flow shouldn't be running, while not-disabled flow should run
# as usual.
self.assertFalse(cron_manager.JobIsRunning(cron_job1, token=self.token))
self.assertTrue(cron_manager.JobIsRunning(cron_job2, token=self.token))
def testReschedulingJobWithFixedNameDoesNotCreateNewObjectVersion(self):
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobFlowArgs(
allow_overruns=True, periodicity="1d")
cron_args.flow_runner_args.flow_name = "FakeCronJob"
# Schedule cron job with a fixed name. Check that we have 1 version
# of "TYPE" attribute.
cron_manager.CreateJob(
cron_args=cron_args, token=self.token, job_id="TheJob")
cron_job = cron_manager.ReadJob("TheJob", token=self.token)
attr_values = list(cron_job.GetValuesForAttribute(cron_job.Schema.TYPE))
self.assertTrue(len(attr_values) == 1)
# Reschedule the job. Check that we still have only one "TYPE" version.
cron_manager.CreateJob(
cron_args=cron_args, token=self.token, job_id="TheJob")
cron_job = cron_manager.ReadJob("TheJob", token=self.token)
attr_values = list(cron_job.GetValuesForAttribute(cron_job.Schema.TYPE))
self.assertTrue(len(attr_values) == 1)
def testCronJobRunMonitorsRunningFlowState(self):
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobFlowArgs(
allow_overruns=False, periodicity="1d")
cron_args.flow_runner_args.flow_name = "FakeCronJob"
job_id = cron_manager.CreateJob(cron_args)
# Run() wasn't called, so nothing is supposed to be running
cron_job = cron_manager.ReadJob(job_id, token=self.token)
self.assertFalse(cron_manager.JobIsRunning(cron_job, token=self.token))
cron_manager.RunOnce(token=self.token)
# Run() was called and flow was started, so the job should be
# considered running.
cron_job = cron_manager.ReadJob(job_id, token=self.token)
self.assertTrue(cron_manager.JobIsRunning(cron_job, token=self.token))
# Find the flow that is currently running for the job and terminate it.
cron_job = cron_manager.ReadJob(job_id, token=self.token)
self.assertTrue(cron_manager.JobIsRunning(cron_job, token=self.token))
self.assertTrue(cron_job.current_run_id)
runs = cron_manager.ReadJobRuns(job_id)
self.assertTrue(runs)
for run in runs:
flow.GRRFlow.TerminateFlow(run.urn, token=self.token)
# Check we're dead
cron_job = cron_manager.ReadJob(job_id, token=self.token)
self.assertFalse(cron_manager.JobIsRunning(cron_job, token=self.token))
# This will understand that current flow has terminated. New flow won't be
# started, because iterations are supposed to be started once per day
# (frequency=1d).
cron_manager.RunOnce(token=self.token)
# Still dead
cron_job = cron_manager.ReadJob(job_id, token=self.token)
self.assertFalse(cron_manager.JobIsRunning(cron_job, token=self.token))
def testFailedFlowUpdatesStats(self):
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobFlowArgs(
allow_overruns=False, periodicity="1d")
cron_args.flow_runner_args.flow_name = "FailingFakeCronJob"
job_id = cron_manager.CreateJob(cron_args=cron_args)
prev_metric_value = stats.STATS.GetMetricValue(
"cron_job_failure", fields=[job_id])
cron_manager.RunOnce(token=self.token)
run_urn = cron_manager.ReadJobRuns(job_id)[-1].urn
flow_test_lib.TestFlowHelper(
run_urn, check_flow_errors=False, token=self.token)
# This RunOnce call should determine that the flow has failed
cron_manager.RunOnce(token=self.token)
# Check that stats got updated
current_metric_value = stats.STATS.GetMetricValue(
"cron_job_failure", fields=[job_id])
self.assertEqual(current_metric_value - prev_metric_value, 1)
def Handle(self, args, token=None):
args.flow_args.hunt_runner_args.hunt_name = "GenericHunt"
# TODO(user): The following should be asserted in a more elegant way.
# Also, it's not clear whether cron job scheduling UI is used often enough
# to justify its existence. We should check with opensource users whether
# they find this feature useful and if not, deprecate it altogether.
if args.flow_name != standard.CreateAndRunGenericHuntFlow.__name__:
raise ValueError(
"Only CreateAndRunGenericHuntFlow flows are supported "
"here (got: %s)." % args.flow_name)
# Clear all fields marked with HIDDEN, except for output_plugins - they are
# marked HIDDEN, because we have a separate UI for them, not because they
# shouldn't be shown to the user at all.
#
# TODO(user): Refactor the code to remove the HIDDEN label from
# FlowRunnerArgs.output_plugins.
args.flow_runner_args.ClearFieldsWithLabel(
rdf_structs.SemanticDescriptor.Labels.HIDDEN,
exceptions="output_plugins")
if not args.flow_runner_args.flow_name:
args.flow_runner_args.flow_name = args.flow_name
cron_args = rdf_cronjobs.CreateCronJobFlowArgs(
description=args.description,
periodicity=args.periodicity,
flow_runner_args=args.flow_runner_args,
flow_args=args.flow_args,
allow_overruns=args.allow_overruns,
lifetime=args.lifetime)
name = aff4_cronjobs.GetCronManager().CreateJob(cron_args=cron_args,
disabled=True,
token=token)
fd = aff4_cronjobs.GetCronManager().ReadJob(name)
return ApiCronJob().InitFromObject(fd)
def testCronJobStartsFlowAndCreatesSymlinkOnRun(self):
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobFlowArgs()
cron_args.flow_runner_args.flow_name = "FakeCronJob"
job_id = cron_manager.CreateJob(cron_args=cron_args)
cron_job = cron_manager.ReadJob(job_id, token=self.token)
self.assertFalse(cron_manager.JobIsRunning(cron_job, token=self.token))
# The job never ran, so JobDueToRun() should return true.
self.assertTrue(cron_manager.JobDueToRun(cron_job))
cron_manager.RunOnce(token=self.token)
cron_job = cron_manager.ReadJob(job_id, token=self.token)
self.assertTrue(cron_manager.JobIsRunning(cron_job, token=self.token))
# Check that a link to the flow is created under job object.
runs = cron_manager.ReadJobRuns(job_id, token=self.token)
self.assertEqual(len(runs), 1)
# Check that the link points to the correct flow.
self.assertEqual(runs[0].runner_args.flow_name, "FakeCronJob")
def testCronJobRunDoesNothingIfDueTimeHasNotComeYet(self):
with test_lib.FakeTime(0):
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobFlowArgs(
allow_overruns=False, periodicity="1h")
cron_args.flow_runner_args.flow_name = "FakeCronJob"
job_id = cron_manager.CreateJob(cron_args=cron_args)
cron_manager.RunOnce(token=self.token)
cron_job_runs = cron_manager.ReadJobRuns(job_id, token=self.token)
self.assertEqual(len(cron_job_runs), 1)
# Let 59 minutes pass. Frequency is 1 hour, so new flow is not
# supposed to start.
time.time = lambda: 59 * 60
cron_manager.RunOnce(token=self.token)
cron_job_runs = cron_manager.ReadJobRuns(job_id, token=self.token)
self.assertEqual(len(cron_job_runs), 1)
