def testStatefulSystemCronJobMaintainsState(self):
DummyStatefulSystemCronJobRel.VALUES = []
# We need to have a cron job started to have a place to maintain
# state.
cron_manager = cronjobs.CronManager()
args = rdf_cronjobs.CronJobAction(
action_type=rdf_cronjobs.CronJobAction.ActionType.SYSTEM_CRON_ACTION,
system_cron_action=rdf_cronjobs.SystemCronAction(
job_class_name="DummyStatefulSystemCronJobRel"))
job = rdf_cronjobs.CronJob(
cron_job_id="test_cron",
args=args,
enabled=True,
frequency=rdfvalue.Duration("2h"),
lifetime=rdfvalue.Duration("1h"),
allow_overruns=False)
data_store.REL_DB.WriteCronJob(job)
fake_time = rdfvalue.RDFDatetime.Now()
for i in range(3):
with test_lib.FakeTime(fake_time + rdfvalue.Duration("%dh" % (3 * i))):
cron_manager.RunOnce()
cron_manager._GetThreadPool().Join()
runs = cron_manager.ReadJobRuns("test_cron")
self.assertLen(runs, i + 1)
for run in runs:
self.assertEqual(run.status, "FINISHED")
self.assertListEqual(DummyStatefulSystemCronJobRel.VALUES, [0, 1, 2])
def InitFromAff4Object(self, cron_job):
cron_args = cron_job.Get(cron_job.Schema.CRON_ARGS)
flow_name = cron_args.flow_runner_args.flow_name
if flow_name == "CreateAndRunGenericHuntFlow":
action_type = rdf_cronjobs.CronJobAction.ActionType.HUNT_CRON_ACTION
hunt_args = cron_args.flow_args.hunt_args
# Hunt name is always GenericHunt, no need to keep it around.
cron_args.flow_args.hunt_runner_args.hunt_name = None
args = rdf_cronjobs.CronJobAction(
action_type=action_type,
hunt_cron_action=rdf_cronjobs.HuntCronAction(
hunt_runner_args=cron_args.flow_args.hunt_runner_args,
flow_args=hunt_args.flow_args,
flow_name=hunt_args.flow_runner_args.flow_name,
))
else:
action_type = rdf_cronjobs.CronJobAction.ActionType.SYSTEM_CRON_ACTION
args = rdf_cronjobs.CronJobAction(
action_type=action_type,
system_cron_action=rdf_cronjobs.SystemCronAction(
job_class_name=cron_args.flow_runner_args.flow_name))
api_cron_job = ApiCronJob(
cron_job_id=cron_job.urn.Basename(),
args=args,
enabled=not cron_job.Get(cron_job.Schema.DISABLED),
last_run_status=self._StatusFromCronJobRunStatus(
cron_job.Get(cron_job.Schema.LAST_RUN_STATUS)),
last_run_time=cron_job.Get(cron_job.Schema.LAST_RUN_TIME),
frequency=cron_args.periodicity,
lifetime=cron_args.lifetime or None,
allow_overruns=cron_args.allow_overruns,
description=cron_args.description,
is_failing=self._IsCronJobFailing(cron_job))
state_dict = cron_job.Get(cron_job.Schema.STATE_DICT)
if state_dict:
state = api_call_handler_utils.ApiDataObject()
state.InitFromDataObject(state_dict)
api_cron_job.state = state
current_flow_urn = cron_job.Get(cron_job.Schema.CURRENT_FLOW_URN)
if current_flow_urn:
api_cron_job.current_run_id = current_flow_urn.Basename()
return api_cron_job
def testNonExistingSystemCronJobDoesNotPreventOtherCronJobsFromRunning(
self):
# Have a fake non-existing cron job. We assume that cron jobs are going
# to be processed in alphabetical order, according to their cron job ids.
args = rdf_cronjobs.CronJobAction(
action_type=rdf_cronjobs.CronJobAction.ActionType.
SYSTEM_CRON_ACTION,
system_cron_action=rdf_cronjobs.SystemCronAction(
job_class_name="__AbstractFakeCronJob__"))
job = rdf_cronjobs.CronJob(
cron_job_id="cron_1",
args=args,
enabled=True,
frequency=rdfvalue.Duration.From(2, rdfvalue.HOURS),
lifetime=rdfvalue.Duration.From(1, rdfvalue.HOURS),
allow_overruns=False)
data_store.REL_DB.WriteCronJob(job)
# Have a proper cron job.
cron_manager = cronjobs.CronManager()
args = rdf_cronjobs.CronJobAction(
action_type=rdf_cronjobs.CronJobAction.ActionType.
SYSTEM_CRON_ACTION,
system_cron_action=rdf_cronjobs.SystemCronAction(
job_class_name="DummyStatefulSystemCronJobRel"))
job = rdf_cronjobs.CronJob(
cron_job_id="cron_2",
args=args,
enabled=True,
frequency=rdfvalue.Duration.From(2, rdfvalue.HOURS),
lifetime=rdfvalue.Duration.From(1, rdfvalue.HOURS),
allow_overruns=False)
data_store.REL_DB.WriteCronJob(job)
with self.assertRaises(cronjobs.OneOrMoreCronJobsFailedError):
cron_manager.RunOnce()
cron_manager._GetThreadPool().Join()
self.assertEmpty(cron_manager.ReadJobRuns("cron_1"))
self.assertLen(cron_manager.ReadJobRuns("cron_2"), 1)
def CreateJob(self, cron_args=None, job_id=None, enabled=True, token=None):
"""Creates a cron job that runs given flow with a given frequency.
Args:
cron_args: A protobuf of type rdf_cronjobs.CreateCronJobArgs.
job_id: Use this job_id instead of an autogenerated unique name (used
for system cron jobs - we want them to have well-defined
persistent name).
enabled: If False, the job object will be created, but will be disabled.
token: Security token used for data store access. Unused.
Returns:
URN of the cron job created.
Raises:
ValueError: This function expects an arg protobuf that starts a
CreateAndRunGenericHuntFlow flow. If the args specify
something else, ValueError is raised.
"""
# TODO(amoser): Remove the token from this method once the aff4
# cronjobs are gone.
del token
if not job_id:
uid = utils.PRNG.GetUInt16()
job_id = "%s_%s" % (cron_args.flow_name, uid)
args = rdf_cronjobs.CronJobAction(
action_type=rdf_cronjobs.CronJobAction.ActionType.HUNT_CRON_ACTION,
hunt_cron_action=rdf_cronjobs.HuntCronAction(
flow_name=cron_args.flow_name,
flow_args=cron_args.flow_args,
hunt_runner_args=cron_args.hunt_runner_args))
job = rdf_cronjobs.CronJob(
cron_job_id=job_id,
frequency=cron_args.frequency,
lifetime=cron_args.lifetime,
allow_overruns=cron_args.allow_overruns,
args=args,
enabled=enabled)
data_store.REL_DB.WriteCronJob(job)
return job_id
def CreateJob(self, cron_args=None, job_id=None, enabled=True):
"""Creates a cron job that runs given flow with a given frequency.
Args:
cron_args: A protobuf of type rdf_cronjobs.CreateCronJobArgs.
job_id: Use this job_id instead of an autogenerated unique name (used for
system cron jobs - we want them to have well-defined persistent name).
enabled: If False, the job object will be created, but will be disabled.
Returns:
URN of the cron job created.
Raises:
ValueError: This function expects an arg protobuf that starts a
CreateAndRunGenericHuntFlow flow. If the args specify
something else, ValueError is raised.
"""
if not cron_args.flow_name:
raise ValueError("Unspecified flow name")
if not job_id:
# TODO: UInt16 is too small for randomly generated IDs.
uid = random.UInt16()
job_id = "%s_%s" % (cron_args.flow_name, uid)
args = rdf_cronjobs.CronJobAction(
action_type=rdf_cronjobs.CronJobAction.ActionType.HUNT_CRON_ACTION,
hunt_cron_action=rdf_cronjobs.HuntCronAction(
flow_name=cron_args.flow_name,
flow_args=cron_args.flow_args,
hunt_runner_args=cron_args.hunt_runner_args))
job = rdf_cronjobs.CronJob(
cron_job_id=job_id,
description=cron_args.description,
frequency=cron_args.frequency,
lifetime=cron_args.lifetime,
allow_overruns=cron_args.allow_overruns,
args=args,
enabled=enabled)
data_store.REL_DB.WriteCronJob(job)
return job_id
def ScheduleSystemCronJobs(names=None):
"""Schedules all system cron jobs."""
errors = []
disabled_classes = config.CONFIG["Cron.disabled_cron_jobs"]
for name in disabled_classes:
try:
cls = registry.SystemCronJobRegistry.CronJobClassByName(name)
except ValueError:
errors.append("Cron job not found: %s." % name)
continue
if names is None:
names = iterkeys(registry.SystemCronJobRegistry.SYSTEM_CRON_REGISTRY)
for name in names:
cls = registry.SystemCronJobRegistry.CronJobClassByName(name)
enabled = cls.enabled and name not in disabled_classes
system = rdf_cronjobs.CronJobAction.ActionType.SYSTEM_CRON_ACTION
args = rdf_cronjobs.CronJobAction(
action_type=system,
system_cron_action=rdf_cronjobs.SystemCronAction(
job_class_name=name))
job = rdf_cronjobs.CronJob(cron_job_id=name,
args=args,
enabled=enabled,
frequency=cls.frequency,
lifetime=cls.lifetime,
allow_overruns=cls.allow_overruns)
data_store.REL_DB.WriteCronJob(job)
if errors:
raise ValueError("Error(s) while parsing Cron.disabled_cron_jobs: %s" %
errors)