def testCronJobRunAllowsOverrunsWhenAllowOverrunsIsTrue(self):
with test_lib.FakeTime(0):
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobArgs(allow_overruns=True,
frequency="1h",
flow_name="FakeCronJob")
job_id = cron_manager.CreateJob(cron_args=cron_args,
token=self.token)
cron_manager.RunOnce(token=self.token)
cron_job_runs = cron_manager.ReadJobRuns(job_id, token=self.token)
self.assertEqual(len(cron_job_runs), 1)
# Let an hour pass. Frequency is 1h (i.e. cron job iterations are
# supposed to be started every hour), so the new flow should be started
# by RunOnce(). Previous iteration flow hasn't finished yet, but
# allow_overruns is True, so it's ok to start new iteration.
time.time = lambda: 60 * 60 + 1
cron_manager.RunOnce(token=self.token)
cron_job_runs = cron_manager.ReadJobRuns(job_id, token=self.token)
self.assertEqual(len(cron_job_runs), 2)
def testError(self):
with utils.Stubber(standard.RunHunt, "Run", Error):
cron_manager = cronjobs.CronManager()
create_flow_args = rdf_cronjobs.CreateCronJobArgs()
job_id = cron_manager.CreateJob(cron_args=create_flow_args)
prev_failure_value = stats_collector_instance.Get().GetMetricValue(
"cron_job_failure", fields=[job_id])
prev_latency_value = stats_collector_instance.Get().GetMetricValue(
"cron_job_latency", fields=[job_id])
cron_manager.RunOnce(token=self.token)
cron_manager._GetThreadPool().Join()
cron_job = cron_manager.ReadJob(job_id, token=self.token)
self.assertFalse(cron_manager.JobIsRunning(cron_job))
runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(runs, 1)
run = runs[0]
self.assertEqual(cron_job.last_run_status, "ERROR")
self.assertEqual(run.status, "ERROR")
self.assertTrue(run.backtrace)
self.assertIn("cron job error", run.backtrace)
current_failure_value = stats_collector_instance.Get(
).GetMetricValue("cron_job_failure", fields=[job_id])
current_latency_value = stats_collector_instance.Get(
).GetMetricValue("cron_job_latency", fields=[job_id])
self.assertEqual(current_failure_value, prev_failure_value + 1)
self.assertEqual(current_latency_value.count,
prev_latency_value.count + 1)
def testCronJobRunPreventsOverrunsWhenAllowOverrunsIsTrue(self):
event = threading.Event()
waiting_func = functools.partial(WaitForEvent, event)
cron_manager = cronjobs.CronManager()
try:
with utils.Stubber(standard.RunHunt, "Run", waiting_func):
fake_time = rdfvalue.RDFDatetime.Now()
with test_lib.FakeTime(fake_time):
create_flow_args = rdf_cronjobs.CreateCronJobArgs(
allow_overruns=True, frequency="1h")
job_id = cron_manager.CreateJob(cron_args=create_flow_args)
cron_manager.RunOnce(token=self.token)
cron_job_runs = cron_manager.ReadJobRuns(job_id,
token=self.token)
self.assertLen(cron_job_runs, 1)
# Let two hours pass. Frequency is 1h (i.e. cron job iterations are
# supposed to be started every hour), so the new flow should be started
# by RunOnce(). However, as allow_overruns is False, and previous
# iteration flow hasn't finished yet, no flow will be started.
fake_time += rdfvalue.Duration("2h")
with test_lib.FakeTime(fake_time):
cron_manager.RunOnce(token=self.token)
cron_job_runs = cron_manager.ReadJobRuns(job_id,
token=self.token)
self.assertLen(cron_job_runs, 2)
finally:
event.set()
cron_manager._GetThreadPool().Join()
def testCronJobRunDoesNothingIfCurrentFlowIsRunning(self):
event = threading.Event()
waiting_func = functools.partial(WaitForEvent, event)
cron_manager = cronjobs.CronManager()
with utils.Stubber(standard.RunHunt, "Run", waiting_func):
try:
fake_time = rdfvalue.RDFDatetime.Now()
with test_lib.FakeTime(fake_time):
create_flow_args = rdf_cronjobs.CreateCronJobArgs(
allow_overruns=False, frequency="1h")
job_id = cron_manager.CreateJob(cron_args=create_flow_args)
cron_manager.RunOnce(token=self.token)
cron_job_runs = cron_manager.ReadJobRuns(job_id,
token=self.token)
self.assertLen(cron_job_runs, 1)
job = cron_manager.ReadJob(job_id)
self.assertTrue(cron_manager.JobIsRunning(job))
fake_time += rdfvalue.Duration("2h")
with test_lib.FakeTime(fake_time):
cron_manager.RunOnce(token=self.token)
cron_job_runs = cron_manager.ReadJobRuns(job_id,
token=self.token)
self.assertLen(cron_job_runs, 1)
finally:
event.set()
cron_manager._GetThreadPool().Join()
def testReschedulingJobWithFixedNameDoesNotCreateNewObjectVersion(self):
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobArgs(allow_overruns=True,
frequency="1d",
flow_name="FakeCronJob")
# Schedule cron job with a fixed name. Check that we have 1 version
# of "TYPE" attribute.
cron_manager.CreateJob(cron_args=cron_args,
token=self.token,
job_id="TheJob")
cron_job = cron_manager.ReadJob("TheJob", token=self.token)
attr_values = list(cron_job.GetValuesForAttribute(
cron_job.Schema.TYPE))
self.assertTrue(len(attr_values) == 1)
# Reschedule the job. Check that we still have only one "TYPE" version.
cron_manager.CreateJob(cron_args=cron_args,
token=self.token,
job_id="TheJob")
cron_job = cron_manager.ReadJob("TheJob", token=self.token)
attr_values = list(cron_job.GetValuesForAttribute(
cron_job.Schema.TYPE))
self.assertTrue(len(attr_values) == 1)
def Run(self):
with test_lib.FakeTime(42):
self.CreateAdminUser(u"requestor")
cron_manager = cronjobs.CronManager()
cron_args = rdf_cronjobs.CreateCronJobArgs(
frequency="1d",
allow_overruns=False,
flow_name=discovery.Interrogate.__name__)
cron_id = cron_manager.CreateJob(cron_args=cron_args)
with test_lib.FakeTime(44):
approval_id = self.RequestCronJobApproval(
cron_id,
approver=self.test_username,
requestor=u"requestor",
reason="foo")
with test_lib.FakeTime(126):
self.Check("GrantCronJobApproval",
args=user_plugin.ApiGrantCronJobApprovalArgs(
cron_job_id=cron_id,
approval_id=approval_id,
username=u"requestor"),
replace={
cron_id: "CronJob_123456",
approval_id: "approval:111111"
})
def testError(self):
with mock.patch.object(
cronjobs.RunHunt,
"Run",
side_effect=ValueError("Random cron job error.")):
cron_manager = cronjobs.CronManager()
create_flow_args = rdf_cronjobs.CreateCronJobArgs()
job_id = cron_manager.CreateJob(cron_args=create_flow_args)
prev_failure_value = cronjobs.CRON_JOB_FAILURE.GetValue([job_id])
prev_latency_value = cronjobs.CRON_JOB_LATENCY.GetValue([job_id])
cron_manager.RunOnce(token=self.token)
cron_manager._GetThreadPool().Join()
cron_job = cron_manager.ReadJob(job_id, token=self.token)
self.assertFalse(cron_manager.JobIsRunning(cron_job))
runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(runs, 1)
run = runs[0]
self.assertEqual(cron_job.last_run_status, "ERROR")
self.assertEqual(run.status, "ERROR")
self.assertTrue(run.backtrace)
self.assertIn("cron job error", run.backtrace)
current_failure_value = cronjobs.CRON_JOB_FAILURE.GetValue(
[job_id])
current_latency_value = cronjobs.CRON_JOB_LATENCY.GetValue(
[job_id])
self.assertEqual(current_failure_value, prev_failure_value + 1)
self.assertEqual(current_latency_value.count,
prev_latency_value.count + 1)
def testCronJobRunPreventsOverrunsWhenAllowOverrunsIsFalse(self):
event = threading.Event()
waiting_func = functools.partial(WaitForEvent, event)
cron_manager = cronjobs.CronManager()
try:
with mock.patch.object(cronjobs.RunHunt, "Run", wraps=waiting_func):
fake_time = rdfvalue.RDFDatetime.Now()
with test_lib.FakeTime(fake_time):
create_flow_args = rdf_cronjobs.CreateCronJobArgs(
allow_overruns=False,
frequency="1h",
flow_name=transfer.GetFile.__name__)
job_id = cron_manager.CreateJob(cron_args=create_flow_args)
cron_manager.RunOnce()
cron_job_runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(cron_job_runs, 1)
# Let two hours pass. Frequency is 1h (i.e. cron job iterations are
# supposed to be started every hour), so the new flow should be started
# by RunOnce(). However, as allow_overruns is False, and previous
# iteration flow hasn't finished yet, no flow will be started.
fake_time += rdfvalue.Duration.From(2, rdfvalue.HOURS)
with test_lib.FakeTime(fake_time):
cron_manager.RunOnce()
cron_job_runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(cron_job_runs, 1)
finally:
event.set()
cron_manager._GetThreadPool().Join()
def testLatencyStatsAreCorrectlyRecorded(self):
with test_lib.FakeTime(0):
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobArgs(
flow_name="FakeCronJob", frequency="1w")
cron_job_id = cron_manager.CreateJob(
cron_args=cron_args, token=self.token)
cron_manager.RunOnce(token=self.token)
prev_metric_value = stats.STATS.GetMetricValue(
"cron_job_latency", fields=[cron_job_id])
# Fast forward one minute
with test_lib.FakeTime(60):
cron_manager.RunOnce(token=self.token)
cron_job = cron_manager.ReadJob(cron_job_id, token=self.token)
cron_flow_urn = cron_job.Get(cron_job.Schema.CURRENT_FLOW_URN)
flow_test_lib.TestFlowHelper(
cron_flow_urn, check_flow_errors=False, token=self.token)
# This RunOnce call should determine that the flow has finished
cron_manager.RunOnce(token=self.token)
# Check that stats got updated
current_metric_value = stats.STATS.GetMetricValue(
"cron_job_latency", fields=[cron_job_id])
self.assertEqual(current_metric_value.count - prev_metric_value.count, 1)
self.assertEqual(current_metric_value.sum - prev_metric_value.sum, 60)
def testCronJobRunDoesNothingIfDueTimeHasNotComeYet(self):
fake_time = rdfvalue.RDFDatetime.Now()
with test_lib.FakeTime(fake_time):
cron_manager = cronjobs.CronManager()
create_flow_args = rdf_cronjobs.CreateCronJobArgs(
allow_overruns=False,
frequency="1h",
flow_name=transfer.GetFile.__name__)
job_id = cron_manager.CreateJob(cron_args=create_flow_args)
cron_manager.RunOnce()
cron_job_runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(cron_job_runs, 1)
# Let 59 minutes pass. Frequency is 1 hour, so new flow is not
# supposed to start.
fake_time += rdfvalue.Duration.From(59, rdfvalue.MINUTES)
with test_lib.FakeTime(fake_time):
cron_manager.RunOnce()
cron_job_runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(cron_job_runs, 1)
def Run(self):
with test_lib.FakeTime(42):
self.CreateUser(u"approver1")
self.CreateUser(u"approver2")
cron_manager = cronjobs.CronManager()
cron_args = rdf_cronjobs.CreateCronJobArgs(
frequency="1d",
allow_overruns=False,
flow_name=discovery.Interrogate.__name__)
cron_id = cron_manager.CreateJob(cron_args=cron_args)
def ReplaceCronAndApprovalIds():
approvals = self.ListCronJobApprovals()
return {
approvals[0].id: "approval:112233",
cron_id: "CronJob_123456"
}
with test_lib.FakeTime(126):
self.Check("CreateCronJobApproval",
args=user_plugin.ApiCreateCronJobApprovalArgs(
cron_job_id=cron_id,
approval=user_plugin.ApiCronJobApproval(
reason="really important reason!",
notified_users=["approver1", "approver2"],
email_cc_addresses=["*****@*****.**"])),
replace=ReplaceCronAndApprovalIds)
def testDisabledCronJobDoesNotCreateJobs(self):
cron_manager = cronjobs.CronManager()
create_flow_args = rdf_cronjobs.CreateCronJobArgs()
create_flow_args.flow_name = transfer.GetFile.__name__
job_id1 = cron_manager.CreateJob(cron_args=create_flow_args)
job_id2 = cron_manager.CreateJob(cron_args=create_flow_args)
cron_manager.DisableJob(job_id1)
event = threading.Event()
waiting_func = functools.partial(WaitForEvent, event)
try:
with mock.patch.object(cronjobs.RunHunt, "Run", wraps=waiting_func):
cron_manager.RunOnce()
cron_job1 = cron_manager.ReadJob(job_id1)
cron_job2 = cron_manager.ReadJob(job_id2)
# Disabled flow shouldn't be running, while not-disabled flow should run
# as usual.
self.assertFalse(cron_manager.JobIsRunning(cron_job1))
self.assertTrue(cron_manager.JobIsRunning(cron_job2))
finally:
event.set()
def setUp(self):
super(CleanCronJobsFlowTest, self).setUp()
with test_lib.FakeTime(40):
cron_args = rdf_cronjobs.CreateCronJobArgs(
frequency=RetentionTestSystemCronJob.frequency,
flow_name=RetentionTestSystemCronJob.__name__,
lifetime=RetentionTestSystemCronJob.lifetime)
self.cron_jobs_names = []
self.cron_jobs_names.append(cronjobs.GetCronManager().CreateJob(
cron_args=cron_args,
job_id="Foo",
token=self.token,
enabled=True))
self.cron_jobs_names.append(cronjobs.GetCronManager().CreateJob(
cron_args=cron_args,
job_id="Bar",
token=self.token,
enabled=True))
manager = cronjobs.GetCronManager()
for i in range(self.NUM_CRON_RUNS):
with test_lib.FakeTime(40 + 60 * i):
manager.RunOnce(token=self.token)
if data_store.RelationalDBReadEnabled(category="cronjobs"):
manager._GetThreadPool().Join()
if data_store.RelationalDBReadEnabled(category="cronjobs"):
manager._GetThreadPool().Stop()
def testCronJobRunPreventsOverrunsWhenAllowOverrunsIsFalse(self):
with test_lib.FakeTime(0):
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobArgs(allow_overruns=False,
frequency="1h",
flow_name="FakeCronJob")
job_id = cron_manager.CreateJob(cron_args=cron_args,
token=self.token)
cron_manager.RunOnce(token=self.token)
cron_job_runs = cron_manager.ReadJobRuns(job_id, token=self.token)
self.assertLen(cron_job_runs, 1)
# Let an hour pass. Frequency is 1h (i.e. cron job iterations are
# supposed to be started every hour), so the new flow should be started
# by RunOnce(). However, as allow_overruns is False, and previous
# iteration flow hasn't finished yet, no flow will be started.
time.time = lambda: 60 * 60 + 1
cron_manager.RunOnce(token=self.token)
cron_job_runs = cron_manager.ReadJobRuns(job_id, token=self.token)
self.assertLen(cron_job_runs, 1)
def testSchedulingJobWithFixedNamePreservesTheName(self):
cron_manager = cronjobs.CronManager()
create_flow_args = rdf_cronjobs.CreateCronJobArgs()
job_id = cron_manager.CreateJob(cron_args=create_flow_args,
job_id="TheJob")
self.assertEqual("TheJob", job_id)
def testCronJobPreservesFlowNameAndArguments(self):
pathspec = rdf_paths.PathSpec(path="/foo",
pathtype=rdf_paths.PathSpec.PathType.TSK)
cron_manager = cronjobs.CronManager()
flow_name = transfer.GetFile.__name__
cron_args = rdf_cronjobs.CreateCronJobArgs(frequency="1d",
allow_overruns=False,
flow_name=flow_name)
cron_args.flow_args.pathspec = pathspec
job_id = cron_manager.CreateJob(cron_args=cron_args)
# Check that CronJob definition is saved properly
jobs = cron_manager.ListJobs(token=self.token)
self.assertLen(jobs, 1)
self.assertEqual(jobs[0], job_id)
cron_job = cron_manager.ReadJob(job_id, token=self.token)
hunt_args = cron_job.args.hunt_cron_action
self.assertEqual(hunt_args.flow_name, flow_name)
self.assertEqual(hunt_args.flow_args.pathspec, pathspec)
self.assertEqual(cron_job.frequency, rdfvalue.Duration("1d"))
self.assertEqual(cron_job.allow_overruns, False)
def Run(self):
with test_lib.FakeTime(42):
self.CreateAdminUser("requestor")
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobArgs(frequency="1d",
allow_overruns=False)
cron_id = cron_manager.CreateJob(cron_args=cron_args,
token=self.token)
with test_lib.FakeTime(44):
approval_id = self.RequestCronJobApproval(
cron_id,
approver=self.token.username,
requestor="requestor",
reason="foo")
with test_lib.FakeTime(126):
self.Check("GrantCronJobApproval",
args=user_plugin.ApiGrantCronJobApprovalArgs(
cron_job_id=cron_id,
approval_id=approval_id,
username="******"),
replace={
cron_id: "CronJob_123456",
approval_id: "approval:111111"
})
def testCronJobPreservesFlowNameAndArguments(self):
"""Testing initialization of a ConfigManager."""
pathspec = rdf_paths.PathSpec(
path="/foo", pathtype=rdf_paths.PathSpec.PathType.TSK)
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobArgs(
frequency="1d",
allow_overruns=False,
flow_name=transfer.GetFile.__name__)
cron_args.flow_args.pathspec = pathspec
job_id = cron_manager.CreateJob(cron_args=cron_args, token=self.token)
# Check that CronJob definition is saved properly
jobs = cron_manager.ListJobs(token=self.token)
self.assertEqual(len(jobs), 1)
self.assertEqual(jobs[0], job_id)
cron_job = cron_manager.ReadJob(job_id, token=self.token)
cron_args = cron_job.Get(cron_job.Schema.CRON_ARGS)
# CreateJob creates a cronjob that runs the CreateAndRunGenericHuntFlow
# which in turn starts a hunt with the flow that we passed in.
self.assertEqual(cron_args.flow_runner_args.flow_name,
standard.CreateAndRunGenericHuntFlow.__name__)
self.assertEqual(cron_args.flow_args.hunt_args.flow_runner_args.flow_name,
transfer.GetFile.__name__)
self.assertEqual(cron_args.flow_args.hunt_args.flow_args.pathspec, pathspec)
self.assertEqual(cron_args.periodicity, rdfvalue.Duration("1d"))
self.assertEqual(cron_args.allow_overruns, False)
def testCronJobRunDoesNothingIfCurrentFlowIsRunning(self):
event = threading.Event()
waiting_func = functools.partial(WaitForEvent, event)
cron_manager = cronjobs.CronManager()
with mock.patch.object(cronjobs.RunHunt, "Run", wraps=waiting_func):
try:
fake_time = rdfvalue.RDFDatetime.Now()
with test_lib.FakeTime(fake_time):
create_flow_args = rdf_cronjobs.CreateCronJobArgs(
allow_overruns=False,
frequency="1h",
flow_name=transfer.GetFile.__name__)
job_id = cron_manager.CreateJob(cron_args=create_flow_args)
cron_manager.RunOnce()
cron_job_runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(cron_job_runs, 1)
job = cron_manager.ReadJob(job_id)
self.assertTrue(cron_manager.JobIsRunning(job))
fake_time += rdfvalue.Duration.From(2, rdfvalue.HOURS)
with test_lib.FakeTime(fake_time):
cron_manager.RunOnce()
cron_job_runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(cron_job_runs, 1)
finally:
event.set()
cron_manager._GetThreadPool().Join()
def testSchedulingJobWithFixedNamePreservesTheName(self):
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobArgs(
allow_overruns=True, frequency="1d", flow_name="FakeCronJob")
job_id = cron_manager.CreateJob(
cron_args=cron_args, token=self.token, job_id="TheJob")
self.assertEqual("TheJob", job_id)
def testTimeout(self):
wait_event = threading.Event()
signal_event = threading.Event()
waiting_func = functools.partial(WaitAndSignal, wait_event,
signal_event)
fake_time = rdfvalue.RDFDatetime.Now()
with utils.Stubber(standard.RunHunt, "Run", waiting_func):
with test_lib.FakeTime(fake_time):
cron_manager = cronjobs.CronManager()
create_flow_args = rdf_cronjobs.CreateCronJobArgs()
create_flow_args.lifetime = "1h"
job_id = cron_manager.CreateJob(cron_args=create_flow_args)
cron_manager.RunOnce(token=self.token)
# Make sure the cron job has actually been started.
signal_event.wait(10)
cron_job = cron_manager.ReadJob(job_id, token=self.token)
self.assertTrue(cron_manager.JobIsRunning(cron_job))
runs = cron_manager.ReadJobRuns(job_id)
self.assertEqual(len(runs), 1)
run = runs[0]
self.assertEqual(cron_job.current_run_id, run.run_id)
self.assertEqual(run.status, "RUNNING")
prev_timeout_value = stats_collector_instance.Get().GetMetricValue(
"cron_job_timeout", fields=[job_id])
prev_latency_value = stats_collector_instance.Get().GetMetricValue(
"cron_job_latency", fields=[job_id])
fake_time += rdfvalue.Duration("2h")
with test_lib.FakeTime(fake_time):
wait_event.set()
cron_manager._GetThreadPool().Join()
cron_job = cron_manager.ReadJob(job_id, token=self.token)
runs = cron_manager.ReadJobRuns(job_id)
self.assertEqual(len(runs), 1)
run = runs[0]
self.assertEqual(cron_job.last_run_status, "LIFETIME_EXCEEDED")
self.assertEqual(run.status, "LIFETIME_EXCEEDED")
# Check that timeout counter got updated.
current_timeout_value = stats_collector_instance.Get(
).GetMetricValue("cron_job_timeout", fields=[job_id])
self.assertEqual(current_timeout_value - prev_timeout_value, 1)
# Check that latency stat got updated.
current_latency_value = stats_collector_instance.Get(
).GetMetricValue("cron_job_latency", fields=[job_id])
self.assertEqual(
current_latency_value.count - prev_latency_value.count, 1)
self.assertEqual(
current_latency_value.sum - prev_latency_value.sum,
rdfvalue.Duration("2h").seconds)
def testTimeoutOfLongRunningJobIsHandledCorrectly(self):
wait_event = threading.Event()
signal_event = threading.Event()
waiting_func = functools.partial(WaitAndSignal, wait_event,
signal_event)
fake_time = rdfvalue.RDFDatetime.Now()
with mock.patch.object(cronjobs.RunHunt, "Run", wraps=waiting_func):
with test_lib.FakeTime(fake_time):
cron_manager = cronjobs.CronManager()
create_flow_args = rdf_cronjobs.CreateCronJobArgs()
create_flow_args.lifetime = "1h"
create_flow_args.flow_name = transfer.GetFile.__name__
job_id = cron_manager.CreateJob(cron_args=create_flow_args)
cron_manager.RunOnce()
# Make sure the cron job has actually been started.
signal_event.wait(10)
cron_job = cron_manager.ReadJob(job_id)
self.assertTrue(cron_manager.JobIsRunning(cron_job))
runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(runs, 1)
run = runs[0]
self.assertEqual(cron_job.current_run_id, run.run_id)
self.assertEqual(run.status, "RUNNING")
prev_timeout_value = cronjobs.CRON_JOB_TIMEOUT.GetValue([job_id])
prev_latency_value = cronjobs.CRON_JOB_LATENCY.GetValue([job_id])
fake_time += rdfvalue.Duration.From(2, rdfvalue.HOURS)
with test_lib.FakeTime(fake_time):
wait_event.set()
cron_manager._GetThreadPool().Join()
cron_job = cron_manager.ReadJob(job_id)
runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(runs, 1)
run = runs[0]
self.assertEqual(cron_job.last_run_status, "LIFETIME_EXCEEDED")
self.assertEqual(run.status, "LIFETIME_EXCEEDED")
# Check that timeout counter got updated.
current_timeout_value = cronjobs.CRON_JOB_TIMEOUT.GetValue(
[job_id])
self.assertEqual(current_timeout_value - prev_timeout_value, 1)
# Check that latency stat got updated.
current_latency_value = cronjobs.CRON_JOB_LATENCY.GetValue(
[job_id])
self.assertEqual(
current_latency_value.count - prev_latency_value.count, 1)
self.assertEqual(
current_latency_value.sum - prev_latency_value.sum,
rdfvalue.Duration.From(2, rdfvalue.HOURS).ToInt(
rdfvalue.SECONDS))
def CreateCronJob(self,
flow_name,
periodicity="1d",
lifetime="7d",
description="",
enabled=True):
args = rdf_cronjobs.CreateCronJobArgs(flow_name=flow_name,
description=description,
frequency=periodicity,
lifetime=lifetime)
return cronjobs.CronManager().CreateJob(args, enabled=enabled)
def testForceRun(self):
event = threading.Event()
waiting_func = functools.partial(WaitForEvent, event)
cron_manager = cronjobs.CronManager()
with utils.Stubber(standard.RunHunt, "Run", waiting_func):
try:
fake_time = rdfvalue.RDFDatetime.Now()
with test_lib.FakeTime(fake_time):
create_flow_args = rdf_cronjobs.CreateCronJobArgs(
allow_overruns=False, frequency="1h")
job_id = cron_manager.CreateJob(cron_args=create_flow_args)
cron_manager.RunOnce(token=self.token)
cron_job_runs = cron_manager.ReadJobRuns(job_id,
token=self.token)
self.assertEqual(len(cron_job_runs), 1)
job = cron_manager.ReadJob(job_id)
self.assertTrue(cron_manager.JobIsRunning(job))
# At this point, there is a run currently executing and also the job
# is not due to run for another hour. We can still force execute the
# job.
cron_manager.RunOnce(token=self.token)
cron_job_runs = cron_manager.ReadJobRuns(job_id,
token=self.token)
self.assertEqual(len(cron_job_runs), 1)
cron_manager.RequestForcedRun(job_id)
cron_manager.RunOnce(token=self.token)
cron_job_runs = cron_manager.ReadJobRuns(job_id,
token=self.token)
self.assertEqual(len(cron_job_runs), 2)
# The only way to prevent a forced run is to disable the job.
cron_manager.DisableJob(job_id)
cron_manager.RequestForcedRun(job_id)
cron_manager.RunOnce(token=self.token)
cron_job_runs = cron_manager.ReadJobRuns(job_id,
token=self.token)
self.assertEqual(len(cron_job_runs), 2)
# And enable again.
cron_manager.EnableJob(job_id)
cron_manager.RequestForcedRun(job_id)
cron_manager.RunOnce(token=self.token)
cron_job_runs = cron_manager.ReadJobRuns(job_id,
token=self.token)
self.assertEqual(len(cron_job_runs), 3)
finally:
event.set()
cron_manager._GetThreadPool().Join()
def testCronManagerListJobsDoesNotListDeletedJobs(self):
cron_manager = cronjobs.CronManager()
create_flow_args = rdf_cronjobs.CreateCronJobArgs()
cron_job_id = cron_manager.CreateJob(cron_args=create_flow_args)
self.assertLen(cron_manager.ListJobs(), 1)
cron_manager.DeleteJob(cron_job_id)
self.assertEmpty(cron_manager.ListJobs())
def testRunningJobs(self):
event = threading.Event()
waiting_func = functools.partial(WaitForEvent, event)
with mock.patch.object(cronjobs.RunHunt, "Run", wraps=waiting_func):
cron_manager = cronjobs.CronManager()
create_flow_args = rdf_cronjobs.CreateCronJobArgs(
frequency="1w",
lifetime="1d",
flow_name=transfer.GetFile.__name__)
job_id = cron_manager.CreateJob(cron_args=create_flow_args)
prev_timeout_value = cronjobs.CRON_JOB_TIMEOUT.GetValue(
fields=[job_id])
prev_latency_value = cronjobs.CRON_JOB_LATENCY.GetValue([job_id])
cron_manager.RunOnce()
cron_job = cron_manager.ReadJob(job_id)
self.assertTrue(cron_manager.JobIsRunning(cron_job))
runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(runs, 1)
run = runs[0]
self.assertEqual(cron_job.current_run_id, run.run_id)
self.assertEqual(run.status, "RUNNING")
event.set()
cron_manager._GetThreadPool().Join()
cron_job = cron_manager.ReadJob(job_id)
self.assertFalse(cron_manager.JobIsRunning(cron_job))
runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(runs, 1)
run = runs[0]
self.assertFalse(cron_job.current_run_id)
self.assertEqual(run.status, "FINISHED")
# Check that timeout counter got updated.
current_timeout_value = cronjobs.CRON_JOB_TIMEOUT.GetValue(
[job_id])
self.assertEqual(current_timeout_value, prev_timeout_value)
# Check that latency stat got updated.
current_latency_value = cronjobs.CRON_JOB_LATENCY.GetValue(
[job_id])
self.assertEqual(
current_latency_value.count - prev_latency_value.count, 1)
def testKillOldFlows(self):
with test_lib.FakeTime(0):
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobArgs(
flow_name="FakeCronJob",
frequency="1w",
lifetime=FakeCronJob.lifetime)
job_id = cron_manager.CreateJob(cron_args=cron_args,
token=self.token)
cron_manager.RunOnce(token=self.token)
cron_job = cron_manager.ReadJob(job_id, token=self.token)
self.assertTrue(cron_job.IsRunning())
self.assertFalse(cron_job.KillOldFlows())
prev_timeout_value = stats.STATS.GetMetricValue("cron_job_timeout",
fields=[job_id])
prev_latency_value = stats.STATS.GetMetricValue("cron_job_latency",
fields=[job_id])
# Fast forward one day
with test_lib.FakeTime(24 * 60 * 60 + 1):
flow_urn = cron_job.Get(cron_job.Schema.CURRENT_FLOW_URN)
cron_manager.RunOnce(token=self.token)
cron_job = cron_manager.ReadJob(job_id, token=self.token)
self.assertFalse(cron_job.IsRunning())
# Check the termination log
log_collection = flow.GRRFlow.LogCollectionForFID(flow_urn)
for line in log_collection:
if line.urn == flow_urn:
self.assertTrue(
"lifetime exceeded" in str(line.log_message))
# Check that timeout counter got updated.
current_timeout_value = stats.STATS.GetMetricValue(
"cron_job_timeout", fields=[job_id])
self.assertEqual(current_timeout_value - prev_timeout_value, 1)
# Check that latency stat got updated.
current_latency_value = stats.STATS.GetMetricValue(
"cron_job_latency", fields=[job_id])
self.assertEqual(
current_latency_value.count - prev_latency_value.count, 1)
self.assertEqual(
current_latency_value.sum - prev_latency_value.sum,
24 * 60 * 60 + 1)
def testRunningJobs(self):
event = threading.Event()
waiting_func = functools.partial(WaitForEvent, event)
with mock.patch.object(standard.RunHunt, "Run", wraps=waiting_func):
cron_manager = cronjobs.CronManager()
create_flow_args = rdf_cronjobs.CreateCronJobArgs(
frequency="1w", lifetime="1d")
job_id = cron_manager.CreateJob(cron_args=create_flow_args)
prev_timeout_value = stats_collector_instance.Get().GetMetricValue(
"cron_job_timeout", fields=[job_id])
prev_latency_value = stats_collector_instance.Get().GetMetricValue(
"cron_job_latency", fields=[job_id])
cron_manager.RunOnce(token=self.token)
cron_job = cron_manager.ReadJob(job_id, token=self.token)
self.assertTrue(cron_manager.JobIsRunning(cron_job))
runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(runs, 1)
run = runs[0]
self.assertEqual(cron_job.current_run_id, run.run_id)
self.assertEqual(run.status, "RUNNING")
event.set()
cron_manager._GetThreadPool().Join()
cron_job = cron_manager.ReadJob(job_id, token=self.token)
self.assertFalse(cron_manager.JobIsRunning(cron_job))
runs = cron_manager.ReadJobRuns(job_id)
self.assertLen(runs, 1)
run = runs[0]
self.assertFalse(cron_job.current_run_id)
self.assertEqual(run.status, "FINISHED")
# Check that timeout counter got updated.
current_timeout_value = stats_collector_instance.Get().GetMetricValue(
"cron_job_timeout", fields=[job_id])
self.assertEqual(current_timeout_value, prev_timeout_value)
# Check that latency stat got updated.
current_latency_value = stats_collector_instance.Get().GetMetricValue(
"cron_job_latency", fields=[job_id])
self.assertEqual(current_latency_value.count - prev_latency_value.count,
1)
def testCronManagerListJobsDoesNotListDeletedJobs(self):
cron_manager = aff4_cronjobs.GetCronManager()
cron_args = rdf_cronjobs.CreateCronJobArgs(
allow_overruns=True, frequency="1d", flow_name="FakeCronJob")
cron_job_urn = cron_manager.CreateJob(cron_args=cron_args, token=self.token)
cron_jobs = list(cron_manager.ListJobs(token=self.token))
self.assertEqual(len(cron_jobs), 1)
cron_manager.DeleteJob(cron_job_urn, token=self.token)
cron_jobs = list(cron_manager.ListJobs(token=self.token))
self.assertEqual(len(cron_jobs), 0)
def testRendersRequestedCronJobApproval(self):
cron_manager = cronjobs.CronManager()
cron_args = rdf_cronjobs.CreateCronJobArgs(frequency="1d",
allow_overruns=False)
cron_job_id = cron_manager.CreateJob(cron_args=cron_args)
self.RequestCronJobApproval(cron_job_id,
reason=self.token.reason,
approver=u"approver",
requestor=self.context.username)
args = user_plugin.ApiListCronJobApprovalsArgs()
result = self.handler.Handle(args, context=self.context)
self.assertLen(result.items, 1)
