def test_sleep_time(self):
assert_equal(mcp.sleep_time(timeutils.current_time()), 0)
assert_equal(mcp.sleep_time(timeutils.current_time() - datetime.timedelta(seconds=5)), 0)
seconds = 5
time = mcp.sleep_time(timeutils.current_time() + datetime.timedelta(seconds=seconds))
assert equals_with_delta(time, seconds, .01)
def seconds_until_run_time(self):
run_time = self.run_time
if run_time.tzinfo:
now = timeutils.current_time(tz=run_time.tzinfo)
else:
now = timeutils.current_time()
return max(0, timeutils.delta_total_seconds(run_time - now))
def seconds_until_run_time(self):
run_time = self.run_time
if run_time.tzinfo:
now = timeutils.current_time(tz=run_time.tzinfo)
else:
now = timeutils.current_time()
return max(0, timeutils.delta_total_seconds(run_time - now))
def test_sleep_time(self):
assert_equal(mcp.sleep_time(timeutils.current_time()), 0)
assert_equal(
mcp.sleep_time(timeutils.current_time() -
datetime.timedelta(seconds=5)), 0)
seconds = 5
time = mcp.sleep_time(timeutils.current_time() +
datetime.timedelta(seconds=seconds))
assert equals_with_delta(time, seconds, .01)
def test_success_but_recent_failure(self):
job_run = turtle.Turtle()
job_run.is_success = True
job_run.end_time = timeutils.current_time() - datetime.timedelta(seconds=30)
self.job.runs.append(job_run)
bad_job_run = turtle.Turtle()
bad_job_run.is_success = False
bad_job_run.end_time = timeutils.current_time()
self.job.runs.append(bad_job_run)
assert self.resource.is_ready
def test_success_but_recent_failure(self):
job_run = turtle.Turtle()
job_run.is_success = True
job_run.end_time = timeutils.current_time() - datetime.timedelta(
seconds=30)
self.job.runs.append(job_run)
bad_job_run = turtle.Turtle()
bad_job_run.is_success = False
bad_job_run.end_time = timeutils.current_time()
self.job.runs.append(bad_job_run)
assert self.resource.is_ready
def start(self):
if not self.machine.check('start'):
raise InvalidStartStateError(self.state)
log.info("Starting action run %s", self.id)
self.start_time = timeutils.current_time()
self.end_time = None
self.machine.transition('start')
assert self.state == self.STATE_STARTING, self.state
self._open_output_file()
if not self.is_valid_command:
log.error("Command for action run %s is invalid: %r",
self.id, self.action.command)
self.fail(-1)
return
# And now we try to actually start some work....
self.action_command = ActionCommand(self.id,
self.command,
stdout=self.stdout_file,
stderr=self.stderr_file)
self.action_command.machine.listen(True, self._handle_action_command)
try:
df = self.node.run(self.action_command)
df.addErrback(self._handle_errback)
except node.Error, e:
log.warning("Failed to start %s: %r", self.id, e)
def render_GET(self, request):
run_output = []
state = job_run_state(self._run)
for action_run in self._run.runs:
action_state = job_run_state(action_run)
last_time = action_run.end_time if action_run.end_time else timeutils.current_time()
duration = str(last_time - action_run.start_time) if action_run.start_time else ""
run_output.append(
{
"id": action_run.id,
"name": action_run.action.name,
"run_time": action_run.run_time and str(action_run.run_time),
"start_time": action_run.start_time and str(action_run.start_time),
"end_time": action_run.end_time and str(action_run.end_time),
"exit_status": action_run.exit_status,
"duration": duration,
"state": action_state,
}
)
output = {"runs": run_output, "id": self._run.id, "state": state, "node": self._run.node.hostname}
return respond(request, output)
def next_run_time(self, start_time):
"""Find the next time to run."""
if not start_time:
start_time = timeutils.current_time(tz=self.time_zone)
elif self.time_zone:
if start_time.tzinfo is None or start_time.tzinfo.utcoffset(
start_time,
) is None:
# tz-naive start times need to be localized first to the requested
# time zone.
try:
start_time = self.time_zone.localize(
start_time,
is_dst=None,
)
except AmbiguousTimeError:
# We are in the infamous 1 AM block which happens twice on
# fall-back. Pretend like it's the first time, every time.
start_time = self.time_zone.localize(
start_time,
is_dst=True,
)
except NonExistentTimeError:
# We are in the infamous 2:xx AM block which does not
# exist. Pretend like it's the later time, every time.
start_time = self.time_zone.localize(
start_time,
is_dst=True,
)
return self.time_spec.get_match(start_time) + get_jitter(self.jitter)
def _done(self, target, exit_status=0):
log.info("Action run %s completed with %s and exit status %r", self.id,
target, exit_status)
if self.machine.check(target):
self.exit_status = exit_status
self.end_time = timeutils.current_time()
return self.machine.transition(target)
def test_success_too_old(self):
job_run = turtle.Turtle()
job_run.is_success = True
job_run.end_time = timeutils.current_time() - datetime.timedelta(days=1)
self.job.runs.append(job_run)
assert not self.resource.is_ready
def test_success(self):
job_run = turtle.Turtle()
job_run.is_success = True
job_run.end_time = timeutils.current_time() - datetime.timedelta(seconds=30)
self.job.runs.append(job_run)
assert self.resource.is_ready
def start(self):
log.info("Starting action job %s", self.job.name)
self.start_time = timeutils.current_time()
self.end_time = None
for r in self.runs:
r.attempt_start()
def start(self):
log.info("Starting action job %s", self.job.name)
self.start_time = timeutils.current_time()
self.end_time = None
for r in self.runs:
r.attempt_start()
def start(self):
"""Start this ActionRun."""
if not self.machine.check('start'):
return False
log.info("Starting action run %s", self.id)
self.start_time = timeutils.current_time()
self.machine.transition('start')
if not self.is_valid_command:
log.error(
"Command for action run %s is invalid: %r",
self.id, self.bare_command,
)
self.fail(-1)
return
action_command = self.build_action_command()
try:
self.node.submit_command(action_command)
except node.Error as e:
log.warning("Failed to start %s: %r", self.id, e)
self.fail(-2)
return
return True
def next_runs(self, job):
# Find the next time to run
if job.runs:
start_time = job.runs[0].run_time
else:
start_time = timeutils.current_time()
if self.time_zone:
try:
start_time = self.time_zone.localize(start_time,
is_dst=None)
except AmbiguousTimeError:
# We are in the infamous 1 AM block which happens twice on
# fall-back. Pretend like it's the first time, every time.
start_time = self.time_zone.localize(start_time,
is_dst=True)
except NonExistentTimeError:
# We are in the infamous 2:xx AM block which does not
# exist. Pretend like it's the later time, every time.
start_time = self.time_zone.localize(start_time,
is_dst=True)
run_time = self.time_spec.GetMatch(start_time)
job_runs = job.build_runs()
for job_run in job_runs:
job_run.set_run_time(run_time)
return job_runs
def manual_start(self, run_time=None):
"""Trigger a job run manually (instead of from the scheduler)."""
run_time = run_time or timeutils.current_time(tz=self.job.time_zone)
manual_runs = list(self.job.build_new_runs(run_time, manual=True))
for r in manual_runs:
r.start()
return manual_runs
def manual_start(self, run_time=None):
"""Trigger a job run manually (instead of from the scheduler)."""
run_time = run_time or timeutils.current_time()
manual_runs = list(self.job.build_new_runs(run_time, manual=True))
for r in manual_runs:
r.start()
return manual_runs
def _check_job_runs(self):
if timeutils.current_time() < self.next_check_time:
return
min_success_time = None
if self.last_succeed_interval:
min_success_time = timeutils.current_time() - self.last_succeed_interval
for run in reversed(self.job.runs):
if run.is_success and (min_success_time is None or run.end_time >= min_success_time):
self._is_ready = True
break
else:
self._is_ready = False
self.last_check = timeutils.current_time()
def render_POST(self, request):
cmd = request.args['command'][0]
log.info("Handling '%s' request for job run %s", cmd, self._job.name)
if cmd == 'enable':
self._master_control.enable_job(self._job)
return respond(request, {'result': "Job %s is enabled" % self._job.name})
if cmd == 'disable':
self._master_control.disable_job(self._job)
return respond(request, {'result': "Job %s is disabled" % self._job.name})
if cmd == 'start':
if 'run_time' in request.args:
run_time_str = request.args['run_time'][0]
run_time = datetime.datetime.strptime(run_time_str, "%Y-%m-%d %H:%M:%S")
else:
run_time = timeutils.current_time()
runs = self._job.manual_start(run_time=run_time)
return respond(request, {'result': "New Job Runs %s created" % [r.id for r in runs]})
log.warning("Unknown request job command %s", request.args['command'])
return respond(request, None, code=http.NOT_IMPLEMENTED)
def start(self):
log.info("Starting action job %s", self.id)
self.start_time = timeutils.current_time()
self.end_time = None
for action in self.action_runs:
action.attempt_start()
def _done(self, target, exit_status=0):
log.info("Action run %s completed with %s and exit status %r",
self.id, target, exit_status)
if self.machine.check(target):
self.exit_status = exit_status
self.end_time = timeutils.current_time()
return self.machine.transition(target)
def render_GET(self, request):
run_output = []
state = job_run_state(self._run)
for action_run in self._run.action_runs:
action_state = job_run_state(action_run)
last_time = action_run.end_time if action_run.end_time else timeutils.current_time()
duration = str(last_time - action_run.start_time) if action_run.start_time else ""
run_output.append({
'id': action_run.id,
'name': action_run.action.name,
'run_time': action_run.run_time and str(action_run.run_time),
'start_time': action_run.start_time and str(action_run.start_time),
'end_time': action_run.end_time and str(action_run.end_time),
'exit_status': action_run.exit_status,
'duration': duration,
'state': action_state,
'command': action_run.command,
})
output = {
'runs': run_output,
'id': self._run.id,
'state': state,
'node': self._run.node.hostname,
}
return respond(request, output)
def test_success_too_old(self):
job_run = turtle.Turtle()
job_run.is_success = True
job_run.end_time = timeutils.current_time() - datetime.timedelta(
days=1)
self.job.runs.append(job_run)
assert not self.resource.is_ready
def test_success(self):
job_run = turtle.Turtle()
job_run.is_success = True
job_run.end_time = timeutils.current_time() - datetime.timedelta(
seconds=30)
self.job.runs.append(job_run)
assert self.resource.is_ready
def fail(self, exit_status=0):
"""Mark the run as having failed, providing an exit status"""
log.info("Action run %s failed with exit status %r",
self.id, exit_status)
if self.machine.transition('fail'):
self.exit_status = exit_status
self.end_time = timeutils.current_time()
return True
def test_set_run_time(self):
jr = self.job.next_runs()[0]
time = timeutils.current_time()
jr.set_run_time(time)
assert_equal(jr.run_time, time)
assert_equal(jr.action_runs[0].run_time, time)
assert_equal(jr.action_runs[1].run_time, time)
def next_runs(self, job):
run_time = timeutils.current_time() + self.interval
job_runs = job.build_runs()
for job_run in job_runs:
job_run.set_run_time(run_time)
return job_runs
def test_set_run_time(self):
jr = self.job.next_runs()[0]
time = timeutils.current_time()
jr.set_run_time(time)
assert_equal(jr.run_time, time)
assert_equal(jr.runs[0].run_time, time)
assert not jr.runs[1].run_time
def next_runs(self, job):
# Find the next time to run
if job.runs:
next_day = job.runs[0].run_time + datetime.timedelta(days=self.wait_days[job.runs[0].run_time.weekday()])
else:
next_day = timeutils.current_time() + datetime.timedelta(self.wait_days[timeutils.current_time().weekday()])
run_time = next_day.replace(
hour=self.start_time.hour,
minute=self.start_time.minute,
second=self.start_time.second)
job_runs = job.build_runs()
for job_run in job_runs:
job_run.set_run_time(run_time)
return job_runs
def next_runs(self, job):
run_time = timeutils.current_time() + self.interval
job_runs = job.build_runs()
for job_run in job_runs:
job_run.set_run_time(run_time)
return job_runs
def fake_action_runs(self):
mock_unknown_machine = Mock(autospec=True)
mock_ok_machine = Mock(autospec=True)
mock_unknown_machine.state = ActionRun.UNKNOWN
mock_ok_machine.state = ActionRun.SUCCEEDED
self.action_runs = [
SSHActionRun(
job_run_id="test.unknown",
name="test.unknown",
node=Mock(),
command_config=Mock(),
machine=mock_unknown_machine,
end_time=timeutils.current_time(),
),
SSHActionRun(
job_run_id="test.succeeded",
name="test.succeeded",
node=Mock(),
command_config=Mock(),
machine=mock_ok_machine,
),
MesosActionRun(
job_run_id="test.succeeded",
name="test.succeeded",
node=Mock(),
command_config=Mock(),
machine=mock_ok_machine,
),
MesosActionRun(
job_run_id="test.unknown-mesos",
name="test.unknown-mesos",
node=Mock(),
command_config=Mock(),
machine=mock_unknown_machine,
),
MesosActionRun(
job_run_id="test.unknown-mesos-done",
name="test.unknown-mesos-done",
node=Mock(),
command_config=Mock(),
machine=mock_unknown_machine,
end_time=timeutils.current_time(),
),
]
def next_runs(self, job):
if job.next_to_finish():
return []
job_runs = job.build_runs()
for job_run in job_runs:
job_run.set_run_time(timeutils.current_time())
return job_runs
def next_runs(self, job):
if job.next_to_finish():
return []
job_runs = job.build_runs()
for job_run in job_runs:
job_run.set_run_time(timeutils.current_time())
return job_runs
def fail(self, exit_status):
"""Mark the run as having failed, providing an exit status"""
log.info("Action run %s failed with exit status %r", self.id, exit_status)
self.state = ACTION_RUN_FAILED
self.exit_status = exit_status
self.end_time = timeutils.current_time()
self.complete_callback()
self.state_callback()
def next_runs(self, job):
# Find the next time to run
if job.runs:
next_day = job.runs[0].run_time + datetime.timedelta(
days=self.wait_days[job.runs[0].run_time.weekday()])
else:
next_day = timeutils.current_time() + datetime.timedelta(
self.wait_days[timeutils.current_time().weekday()])
run_time = next_day.replace(hour=self.start_time.hour,
minute=self.start_time.minute,
second=self.start_time.second)
job_runs = job.build_runs()
for job_run in job_runs:
job_run.set_run_time(run_time)
return job_runs
def _check_job_runs(self):
if timeutils.current_time() < self.next_check_time:
return
min_success_time = None
if self.last_succeed_interval:
min_success_time = timeutils.current_time(
) - self.last_succeed_interval
for run in reversed(self.job.runs):
if run.is_success and (min_success_time is None
or run.end_time >= min_success_time):
self._is_ready = True
break
else:
self._is_ready = False
self.last_check = timeutils.current_time()
def fail(self, exit_status):
"""Mark the run as having failed, providing an exit status"""
log.info("Action run %s failed with exit status %r", self.id,
exit_status)
self.state = ACTION_RUN_FAILED
self.exit_status = exit_status
self.end_time = timeutils.current_time()
self.complete_callback()
self.state_callback()
def seconds_until_run_time(self):
run_time = self.run_time
tz = self.job.scheduler.time_zone
now = timeutils.current_time()
if tz is not None:
now = tz.localize(now)
sleep = run_time - now
seconds = (sleep.days * SECS_PER_DAY + sleep.seconds +
sleep.microseconds * MICRO_SEC)
return max(0, seconds)
def test_next_run_time(self):
one_day = datetime.timedelta(days=1)
today = self.now.date()
yesterday = self.now - one_day
tomorrow = today + one_day
next_run = self.scheduler.next_run_time(timeutils.current_time())
assert_equal(tomorrow, next_run.date())
next_run = self.scheduler.next_run_time(yesterday)
assert_equal(today, next_run.date())
def cleanup_completed(self):
self.end_time = timeutils.current_time()
if self.is_failure:
self.event_recorder.emit_critical("failed")
else:
self.event_recorder.emit_ok("succeeded")
next = self.job.next_to_finish()
if next and next.is_queued:
next.attempt_start()
def _monitor_complete_callback(self):
"""Callback when our monitor has completed"""
assert self.monitor_action
self.last_check = timeutils.current_time()
log.debug("Monitor callback with exit %r", self.monitor_action.exit_status)
if self.monitor_action.exit_status != 0:
self.machine.transition("down")
else:
self.machine.transition("up")
self._queue_monitor()
self.monitor_action = None
def setup_action_run(self):
anode = turtle.Turtle()
self.output_path = filehandler.OutputPath(tempfile.mkdtemp())
self.command = "do command %(actionname)s"
self.rendered_command = "do command action_name"
self.action_run = ActionRun(
"id",
"action_name",
anode,
timeutils.current_time(),
self.command,
output_path=self.output_path)
def _done(self, target, exit_status=0):
if self.machine.check(target):
if self.triggered_by:
EventBus.clear_subscriptions(self.__hash__())
self.exit_status = exit_status
self.end_time = timeutils.current_time()
log.info(f"{self} completed with {target}, transitioned to "
f"{self.state}, exit status: {exit_status}")
return self.transition_and_notify(target)
else:
log.debug(
f"{self} cannot transition from {self.state} via {target}")
def fake_action_runs(self):
mock_unknown_machine = Mock(autospec=True)
mock_ok_machine = Mock(autospec=True)
mock_unknown_machine.state = ActionRun.UNKNOWN
mock_ok_machine.state = ActionRun.SUCCEEDED
self.action_runs = [
SSHActionRun(
job_run_id="test.unknown",
name="test.unknown",
node=Mock(),
machine=mock_unknown_machine,
end_time=timeutils.current_time(),
),
SSHActionRun(
job_run_id="test.succeeded",
name="test.succeeded",
node=Mock(),
machine=mock_ok_machine,
),
MesosActionRun(
job_run_id="test.succeeded",
name="test.succeeded",
node=Mock(),
machine=mock_ok_machine,
),
MesosActionRun(
job_run_id="test.unknown-mesos",
name="test.unknown-mesos",
node=Mock(),
machine=mock_unknown_machine,
),
MesosActionRun(
job_run_id="test.unknown-mesos-done",
name="test.unknown-mesos-done",
node=Mock(),
machine=mock_unknown_machine,
end_time=timeutils.current_time(),
),
]
def run_completed(self):
if self.is_success:
self.last_success_check()
if self.job.constant and self.job.enabled:
self.job.build_run().start()
if self.is_done:
self.end_time = timeutils.current_time()
next = self.job.next_to_finish()
if next and next.is_queued:
next.attempt_start()
def create_attempt(self, original_command=True):
current_time = timeutils.current_time()
command_config = self.command_config.copy()
if original_command:
command_config.command = self.original_command
rendered_command = self.render_command(command_config.command)
new_attempt = ActionRunAttempt(
command_config=command_config,
start_time=current_time,
rendered_command=rendered_command,
)
self.attempts.append(new_attempt)
return new_attempt
def next_run_time(self, start_time):
"""Find the next time to run."""
if not start_time:
start_time = timeutils.current_time(tz=self.time_zone)
elif self.time_zone:
if start_time.tzinfo is None or start_time.tzinfo.utcoffset(
start_time,
) is None:
# tz-naive start times need to be localized first to the requested
# time zone.
start_time = trontimespec.naive_as_timezone(start_time, self.time_zone)
return self.time_spec.get_match(start_time) + get_jitter(self.jitter)
def next_runs(self, job):
# Find the next time to run
if job.runs:
start_time = job.runs[0].run_time
else:
start_time = timeutils.current_time()
run_time = self.time_spec.GetMatch(start_time)
job_runs = job.build_runs()
for job_run in job_runs:
job_run.set_run_time(run_time)
return job_runs
def manual_start(self, run_time=None):
scheduled = deque()
while self.runs and self.runs[0].is_scheduled:
scheduled.appendleft(self.runs.popleft())
man_runs = self.build_runs()
self.runs.extendleft(scheduled)
for r in man_runs:
r.set_run_time(run_time or timeutils.current_time())
r.manual_start()
return man_runs
def next_run_time(self, start_time):
"""Find the next time to run."""
if not start_time:
start_time = timeutils.current_time(tz=self.time_zone)
elif self.time_zone:
if start_time.tzinfo is None or start_time.tzinfo.utcoffset(
start_time, ) is None:
# tz-naive start times need to be localized first to the requested
# time zone.
start_time = trontimespec.naive_as_timezone(
start_time, self.time_zone)
return self.time_spec.get_match(start_time) + get_jitter(self.jitter)
def start(self):
log.info("Starting action run %s", self.id)
self.start_time = timeutils.current_time()
self.end_time = None
self.state = ACTION_RUN_RUNNING
self._open_output_file()
# And now we try to actually start some work....
ret = self.node.run(self)
if isinstance(ret, defer.Deferred):
self._setup_callbacks(ret)
self.state_callback()
def setup_subscriptions(self):
remaining_triggers = self.remaining_triggers
if not remaining_triggers:
return
if self.trigger_timeout_timestamp:
now = timeutils.current_time().timestamp()
delay = max(self.trigger_timeout_timestamp - now, 1)
self.trigger_timeout_call = reactor.callLater(
delay, self.trigger_timeout_reached)
else:
log.error(f"{self} has no trigger_timeout_timestamp")
for trigger in remaining_triggers:
EventBus.subscribe(trigger, self.__hash__(), self.trigger_notify)
def get_run_data(self, request, run):
state = job_run_state(run)
last_time = run.end_time if run.end_time else timeutils.current_time()
duration = str(last_time - run.start_time) if run.start_time else ""
return {
'id': run.id,
'href': request.childLink(run.id),
'node': run.node.hostname if run.node else None,
'run_time': run.run_time and str(run.run_time),
'start_time': run.start_time and str(run.start_time),
'end_time': run.end_time and str(run.end_time),
'duration': duration,
'run_num': run.run_num,
'state': state,
}
def next_run_time(self, start_time):
"""Find the next time to run."""
if not start_time:
start_time = timeutils.current_time()
elif self.time_zone:
try:
start_time = self.time_zone.localize(start_time, is_dst=None)
except AmbiguousTimeError:
# We are in the infamous 1 AM block which happens twice on
# fall-back. Pretend like it's the first time, every time.
start_time = self.time_zone.localize(start_time, is_dst=True)
except NonExistentTimeError:
# We are in the infamous 2:xx AM block which does not
# exist. Pretend like it's the later time, every time.
start_time = self.time_zone.localize(start_time, is_dst=True)
return self.time_spec.get_match(start_time) + get_jitter(self.jitter)
def test_recover_action_run_action_runner(self):
action_runner = SubprocessActionRunnerFactory(
status_path='/tmp/foo',
exec_path='/bin/foo',
)
mock_node = mock.Mock()
action_run = SSHActionRun(job_run_id="test.succeeded",
name="test.succeeded",
node=mock_node,
action_runner=action_runner,
end_time=timeutils.current_time(),
exit_status=0)
action_run.machine.state = ActionRun.UNKNOWN
recover_action_run(action_run, action_runner)
mock_node.submit_command.assert_called_once()
assert action_run.machine.state == ActionRun.RUNNING
assert action_run.end_time is None
assert action_run.exit_status is None
def render_GET(self, request):
run_output = []
state = job_run_state(self._run)
for action_run in self._run.runs:
action_state = job_run_state(action_run)
last_time = action_run.end_time if action_run.end_time else timeutils.current_time(
)
duration = str(
last_time -
action_run.start_time) if action_run.start_time else ""
run_output.append({
'id':
action_run.id,
'name':
action_run.action.name,
'run_time':
action_run.run_time and str(action_run.run_time),
'start_time':
action_run.start_time and str(action_run.start_time),
'end_time':
action_run.end_time and str(action_run.end_time),
'exit_status':
action_run.exit_status,
'duration':
duration,
'state':
action_state,
})
output = {
'runs': run_output,
'id': self._run.id,
'state': state,
'node': self._run.node.hostname,
}
return respond(request, output)
def start(self):
"""Start this ActionRun."""
if self.in_delay is not None:
log.warning(f"{self} cancelling suspend timer")
self.in_delay.cancel()
self.in_delay = None
if not self.machine.check('start'):
return False
if len(self.exit_statuses) == 0:
log.info(f"{self} starting")
else:
log.info(f"{self} restarting, retry {len(self.exit_statuses)}")
self.start_time = timeutils.current_time()
self.transition_and_notify('start')
if not self.is_valid_command:
log.error(f"{self} invalid command: {self.bare_command}")
self.fail(-1)
return
return self.submit_command()
def next_run_time(self, last_run_time, time_zone=None):
last_run_time = last_run_time or timeutils.current_time(tz=time_zone)
return last_run_time + self.interval + get_jitter(self.jitter)
def next_run_time(self, _):
return timeutils.current_time()
