def test_sleep_time(self): assert_equal(mcp.sleep_time(timeutils.current_time()), 0) assert_equal(mcp.sleep_time(timeutils.current_time() - datetime.timedelta(seconds=5)), 0) seconds = 5 time = mcp.sleep_time(timeutils.current_time() + datetime.timedelta(seconds=seconds)) assert equals_with_delta(time, seconds, .01)
def seconds_until_run_time(self): run_time = self.run_time if run_time.tzinfo: now = timeutils.current_time(tz=run_time.tzinfo) else: now = timeutils.current_time() return max(0, timeutils.delta_total_seconds(run_time - now))
def test_sleep_time(self): assert_equal(mcp.sleep_time(timeutils.current_time()), 0) assert_equal( mcp.sleep_time(timeutils.current_time() - datetime.timedelta(seconds=5)), 0) seconds = 5 time = mcp.sleep_time(timeutils.current_time() + datetime.timedelta(seconds=seconds)) assert equals_with_delta(time, seconds, .01)
def test_success_but_recent_failure(self): job_run = turtle.Turtle() job_run.is_success = True job_run.end_time = timeutils.current_time() - datetime.timedelta(seconds=30) self.job.runs.append(job_run) bad_job_run = turtle.Turtle() bad_job_run.is_success = False bad_job_run.end_time = timeutils.current_time() self.job.runs.append(bad_job_run) assert self.resource.is_ready
def test_success_but_recent_failure(self): job_run = turtle.Turtle() job_run.is_success = True job_run.end_time = timeutils.current_time() - datetime.timedelta( seconds=30) self.job.runs.append(job_run) bad_job_run = turtle.Turtle() bad_job_run.is_success = False bad_job_run.end_time = timeutils.current_time() self.job.runs.append(bad_job_run) assert self.resource.is_ready
def start(self): if not self.machine.check('start'): raise InvalidStartStateError(self.state) log.info("Starting action run %s", self.id) self.start_time = timeutils.current_time() self.end_time = None self.machine.transition('start') assert self.state == self.STATE_STARTING, self.state self._open_output_file() if not self.is_valid_command: log.error("Command for action run %s is invalid: %r", self.id, self.action.command) self.fail(-1) return # And now we try to actually start some work.... self.action_command = ActionCommand(self.id, self.command, stdout=self.stdout_file, stderr=self.stderr_file) self.action_command.machine.listen(True, self._handle_action_command) try: df = self.node.run(self.action_command) df.addErrback(self._handle_errback) except node.Error, e: log.warning("Failed to start %s: %r", self.id, e)
def render_GET(self, request): run_output = [] state = job_run_state(self._run) for action_run in self._run.runs: action_state = job_run_state(action_run) last_time = action_run.end_time if action_run.end_time else timeutils.current_time() duration = str(last_time - action_run.start_time) if action_run.start_time else "" run_output.append( { "id": action_run.id, "name": action_run.action.name, "run_time": action_run.run_time and str(action_run.run_time), "start_time": action_run.start_time and str(action_run.start_time), "end_time": action_run.end_time and str(action_run.end_time), "exit_status": action_run.exit_status, "duration": duration, "state": action_state, } ) output = {"runs": run_output, "id": self._run.id, "state": state, "node": self._run.node.hostname} return respond(request, output)
def next_run_time(self, start_time): """Find the next time to run.""" if not start_time: start_time = timeutils.current_time(tz=self.time_zone) elif self.time_zone: if start_time.tzinfo is None or start_time.tzinfo.utcoffset( start_time, ) is None: # tz-naive start times need to be localized first to the requested # time zone. try: start_time = self.time_zone.localize( start_time, is_dst=None, ) except AmbiguousTimeError: # We are in the infamous 1 AM block which happens twice on # fall-back. Pretend like it's the first time, every time. start_time = self.time_zone.localize( start_time, is_dst=True, ) except NonExistentTimeError: # We are in the infamous 2:xx AM block which does not # exist. Pretend like it's the later time, every time. start_time = self.time_zone.localize( start_time, is_dst=True, ) return self.time_spec.get_match(start_time) + get_jitter(self.jitter)
def _done(self, target, exit_status=0): log.info("Action run %s completed with %s and exit status %r", self.id, target, exit_status) if self.machine.check(target): self.exit_status = exit_status self.end_time = timeutils.current_time() return self.machine.transition(target)
def test_success_too_old(self): job_run = turtle.Turtle() job_run.is_success = True job_run.end_time = timeutils.current_time() - datetime.timedelta(days=1) self.job.runs.append(job_run) assert not self.resource.is_ready
def test_success(self): job_run = turtle.Turtle() job_run.is_success = True job_run.end_time = timeutils.current_time() - datetime.timedelta(seconds=30) self.job.runs.append(job_run) assert self.resource.is_ready
def start(self): log.info("Starting action job %s", self.job.name) self.start_time = timeutils.current_time() self.end_time = None for r in self.runs: r.attempt_start()
def start(self): """Start this ActionRun.""" if not self.machine.check('start'): return False log.info("Starting action run %s", self.id) self.start_time = timeutils.current_time() self.machine.transition('start') if not self.is_valid_command: log.error( "Command for action run %s is invalid: %r", self.id, self.bare_command, ) self.fail(-1) return action_command = self.build_action_command() try: self.node.submit_command(action_command) except node.Error as e: log.warning("Failed to start %s: %r", self.id, e) self.fail(-2) return return True
def next_runs(self, job): # Find the next time to run if job.runs: start_time = job.runs[0].run_time else: start_time = timeutils.current_time() if self.time_zone: try: start_time = self.time_zone.localize(start_time, is_dst=None) except AmbiguousTimeError: # We are in the infamous 1 AM block which happens twice on # fall-back. Pretend like it's the first time, every time. start_time = self.time_zone.localize(start_time, is_dst=True) except NonExistentTimeError: # We are in the infamous 2:xx AM block which does not # exist. Pretend like it's the later time, every time. start_time = self.time_zone.localize(start_time, is_dst=True) run_time = self.time_spec.GetMatch(start_time) job_runs = job.build_runs() for job_run in job_runs: job_run.set_run_time(run_time) return job_runs
def manual_start(self, run_time=None): """Trigger a job run manually (instead of from the scheduler).""" run_time = run_time or timeutils.current_time(tz=self.job.time_zone) manual_runs = list(self.job.build_new_runs(run_time, manual=True)) for r in manual_runs: r.start() return manual_runs
def manual_start(self, run_time=None): """Trigger a job run manually (instead of from the scheduler).""" run_time = run_time or timeutils.current_time() manual_runs = list(self.job.build_new_runs(run_time, manual=True)) for r in manual_runs: r.start() return manual_runs
def _check_job_runs(self): if timeutils.current_time() < self.next_check_time: return min_success_time = None if self.last_succeed_interval: min_success_time = timeutils.current_time() - self.last_succeed_interval for run in reversed(self.job.runs): if run.is_success and (min_success_time is None or run.end_time >= min_success_time): self._is_ready = True break else: self._is_ready = False self.last_check = timeutils.current_time()
def render_POST(self, request): cmd = request.args['command'][0] log.info("Handling '%s' request for job run %s", cmd, self._job.name) if cmd == 'enable': self._master_control.enable_job(self._job) return respond(request, {'result': "Job %s is enabled" % self._job.name}) if cmd == 'disable': self._master_control.disable_job(self._job) return respond(request, {'result': "Job %s is disabled" % self._job.name}) if cmd == 'start': if 'run_time' in request.args: run_time_str = request.args['run_time'][0] run_time = datetime.datetime.strptime(run_time_str, "%Y-%m-%d %H:%M:%S") else: run_time = timeutils.current_time() runs = self._job.manual_start(run_time=run_time) return respond(request, {'result': "New Job Runs %s created" % [r.id for r in runs]}) log.warning("Unknown request job command %s", request.args['command']) return respond(request, None, code=http.NOT_IMPLEMENTED)
def start(self): log.info("Starting action job %s", self.id) self.start_time = timeutils.current_time() self.end_time = None for action in self.action_runs: action.attempt_start()
def render_GET(self, request): run_output = [] state = job_run_state(self._run) for action_run in self._run.action_runs: action_state = job_run_state(action_run) last_time = action_run.end_time if action_run.end_time else timeutils.current_time() duration = str(last_time - action_run.start_time) if action_run.start_time else "" run_output.append({ 'id': action_run.id, 'name': action_run.action.name, 'run_time': action_run.run_time and str(action_run.run_time), 'start_time': action_run.start_time and str(action_run.start_time), 'end_time': action_run.end_time and str(action_run.end_time), 'exit_status': action_run.exit_status, 'duration': duration, 'state': action_state, 'command': action_run.command, }) output = { 'runs': run_output, 'id': self._run.id, 'state': state, 'node': self._run.node.hostname, } return respond(request, output)
def test_success_too_old(self): job_run = turtle.Turtle() job_run.is_success = True job_run.end_time = timeutils.current_time() - datetime.timedelta( days=1) self.job.runs.append(job_run) assert not self.resource.is_ready
def test_success(self): job_run = turtle.Turtle() job_run.is_success = True job_run.end_time = timeutils.current_time() - datetime.timedelta( seconds=30) self.job.runs.append(job_run) assert self.resource.is_ready
def fail(self, exit_status=0): """Mark the run as having failed, providing an exit status""" log.info("Action run %s failed with exit status %r", self.id, exit_status) if self.machine.transition('fail'): self.exit_status = exit_status self.end_time = timeutils.current_time() return True
def test_set_run_time(self): jr = self.job.next_runs()[0] time = timeutils.current_time() jr.set_run_time(time) assert_equal(jr.run_time, time) assert_equal(jr.action_runs[0].run_time, time) assert_equal(jr.action_runs[1].run_time, time)
def next_runs(self, job): run_time = timeutils.current_time() + self.interval job_runs = job.build_runs() for job_run in job_runs: job_run.set_run_time(run_time) return job_runs
def test_set_run_time(self): jr = self.job.next_runs()[0] time = timeutils.current_time() jr.set_run_time(time) assert_equal(jr.run_time, time) assert_equal(jr.runs[0].run_time, time) assert not jr.runs[1].run_time
def next_runs(self, job): # Find the next time to run if job.runs: next_day = job.runs[0].run_time + datetime.timedelta(days=self.wait_days[job.runs[0].run_time.weekday()]) else: next_day = timeutils.current_time() + datetime.timedelta(self.wait_days[timeutils.current_time().weekday()]) run_time = next_day.replace( hour=self.start_time.hour, minute=self.start_time.minute, second=self.start_time.second) job_runs = job.build_runs() for job_run in job_runs: job_run.set_run_time(run_time) return job_runs
def fake_action_runs(self): mock_unknown_machine = Mock(autospec=True) mock_ok_machine = Mock(autospec=True) mock_unknown_machine.state = ActionRun.UNKNOWN mock_ok_machine.state = ActionRun.SUCCEEDED self.action_runs = [ SSHActionRun( job_run_id="test.unknown", name="test.unknown", node=Mock(), command_config=Mock(), machine=mock_unknown_machine, end_time=timeutils.current_time(), ), SSHActionRun( job_run_id="test.succeeded", name="test.succeeded", node=Mock(), command_config=Mock(), machine=mock_ok_machine, ), MesosActionRun( job_run_id="test.succeeded", name="test.succeeded", node=Mock(), command_config=Mock(), machine=mock_ok_machine, ), MesosActionRun( job_run_id="test.unknown-mesos", name="test.unknown-mesos", node=Mock(), command_config=Mock(), machine=mock_unknown_machine, ), MesosActionRun( job_run_id="test.unknown-mesos-done", name="test.unknown-mesos-done", node=Mock(), command_config=Mock(), machine=mock_unknown_machine, end_time=timeutils.current_time(), ), ]
def next_runs(self, job): if job.next_to_finish(): return [] job_runs = job.build_runs() for job_run in job_runs: job_run.set_run_time(timeutils.current_time()) return job_runs
def fail(self, exit_status): """Mark the run as having failed, providing an exit status""" log.info("Action run %s failed with exit status %r", self.id, exit_status) self.state = ACTION_RUN_FAILED self.exit_status = exit_status self.end_time = timeutils.current_time() self.complete_callback() self.state_callback()
def next_runs(self, job): # Find the next time to run if job.runs: next_day = job.runs[0].run_time + datetime.timedelta( days=self.wait_days[job.runs[0].run_time.weekday()]) else: next_day = timeutils.current_time() + datetime.timedelta( self.wait_days[timeutils.current_time().weekday()]) run_time = next_day.replace(hour=self.start_time.hour, minute=self.start_time.minute, second=self.start_time.second) job_runs = job.build_runs() for job_run in job_runs: job_run.set_run_time(run_time) return job_runs
def _check_job_runs(self): if timeutils.current_time() < self.next_check_time: return min_success_time = None if self.last_succeed_interval: min_success_time = timeutils.current_time( ) - self.last_succeed_interval for run in reversed(self.job.runs): if run.is_success and (min_success_time is None or run.end_time >= min_success_time): self._is_ready = True break else: self._is_ready = False self.last_check = timeutils.current_time()
def seconds_until_run_time(self): run_time = self.run_time tz = self.job.scheduler.time_zone now = timeutils.current_time() if tz is not None: now = tz.localize(now) sleep = run_time - now seconds = (sleep.days * SECS_PER_DAY + sleep.seconds + sleep.microseconds * MICRO_SEC) return max(0, seconds)
def test_next_run_time(self): one_day = datetime.timedelta(days=1) today = self.now.date() yesterday = self.now - one_day tomorrow = today + one_day next_run = self.scheduler.next_run_time(timeutils.current_time()) assert_equal(tomorrow, next_run.date()) next_run = self.scheduler.next_run_time(yesterday) assert_equal(today, next_run.date())
def cleanup_completed(self): self.end_time = timeutils.current_time() if self.is_failure: self.event_recorder.emit_critical("failed") else: self.event_recorder.emit_ok("succeeded") next = self.job.next_to_finish() if next and next.is_queued: next.attempt_start()
def _monitor_complete_callback(self): """Callback when our monitor has completed""" assert self.monitor_action self.last_check = timeutils.current_time() log.debug("Monitor callback with exit %r", self.monitor_action.exit_status) if self.monitor_action.exit_status != 0: self.machine.transition("down") else: self.machine.transition("up") self._queue_monitor() self.monitor_action = None
def setup_action_run(self): anode = turtle.Turtle() self.output_path = filehandler.OutputPath(tempfile.mkdtemp()) self.command = "do command %(actionname)s" self.rendered_command = "do command action_name" self.action_run = ActionRun( "id", "action_name", anode, timeutils.current_time(), self.command, output_path=self.output_path)
def _done(self, target, exit_status=0): if self.machine.check(target): if self.triggered_by: EventBus.clear_subscriptions(self.__hash__()) self.exit_status = exit_status self.end_time = timeutils.current_time() log.info(f"{self} completed with {target}, transitioned to " f"{self.state}, exit status: {exit_status}") return self.transition_and_notify(target) else: log.debug( f"{self} cannot transition from {self.state} via {target}")
def fake_action_runs(self): mock_unknown_machine = Mock(autospec=True) mock_ok_machine = Mock(autospec=True) mock_unknown_machine.state = ActionRun.UNKNOWN mock_ok_machine.state = ActionRun.SUCCEEDED self.action_runs = [ SSHActionRun( job_run_id="test.unknown", name="test.unknown", node=Mock(), machine=mock_unknown_machine, end_time=timeutils.current_time(), ), SSHActionRun( job_run_id="test.succeeded", name="test.succeeded", node=Mock(), machine=mock_ok_machine, ), MesosActionRun( job_run_id="test.succeeded", name="test.succeeded", node=Mock(), machine=mock_ok_machine, ), MesosActionRun( job_run_id="test.unknown-mesos", name="test.unknown-mesos", node=Mock(), machine=mock_unknown_machine, ), MesosActionRun( job_run_id="test.unknown-mesos-done", name="test.unknown-mesos-done", node=Mock(), machine=mock_unknown_machine, end_time=timeutils.current_time(), ), ]
def run_completed(self): if self.is_success: self.last_success_check() if self.job.constant and self.job.enabled: self.job.build_run().start() if self.is_done: self.end_time = timeutils.current_time() next = self.job.next_to_finish() if next and next.is_queued: next.attempt_start()
def create_attempt(self, original_command=True): current_time = timeutils.current_time() command_config = self.command_config.copy() if original_command: command_config.command = self.original_command rendered_command = self.render_command(command_config.command) new_attempt = ActionRunAttempt( command_config=command_config, start_time=current_time, rendered_command=rendered_command, ) self.attempts.append(new_attempt) return new_attempt
def next_run_time(self, start_time): """Find the next time to run.""" if not start_time: start_time = timeutils.current_time(tz=self.time_zone) elif self.time_zone: if start_time.tzinfo is None or start_time.tzinfo.utcoffset( start_time, ) is None: # tz-naive start times need to be localized first to the requested # time zone. start_time = trontimespec.naive_as_timezone(start_time, self.time_zone) return self.time_spec.get_match(start_time) + get_jitter(self.jitter)
def next_runs(self, job): # Find the next time to run if job.runs: start_time = job.runs[0].run_time else: start_time = timeutils.current_time() run_time = self.time_spec.GetMatch(start_time) job_runs = job.build_runs() for job_run in job_runs: job_run.set_run_time(run_time) return job_runs
def manual_start(self, run_time=None): scheduled = deque() while self.runs and self.runs[0].is_scheduled: scheduled.appendleft(self.runs.popleft()) man_runs = self.build_runs() self.runs.extendleft(scheduled) for r in man_runs: r.set_run_time(run_time or timeutils.current_time()) r.manual_start() return man_runs
def next_run_time(self, start_time): """Find the next time to run.""" if not start_time: start_time = timeutils.current_time(tz=self.time_zone) elif self.time_zone: if start_time.tzinfo is None or start_time.tzinfo.utcoffset( start_time, ) is None: # tz-naive start times need to be localized first to the requested # time zone. start_time = trontimespec.naive_as_timezone( start_time, self.time_zone) return self.time_spec.get_match(start_time) + get_jitter(self.jitter)
def start(self): log.info("Starting action run %s", self.id) self.start_time = timeutils.current_time() self.end_time = None self.state = ACTION_RUN_RUNNING self._open_output_file() # And now we try to actually start some work.... ret = self.node.run(self) if isinstance(ret, defer.Deferred): self._setup_callbacks(ret) self.state_callback()
def setup_subscriptions(self): remaining_triggers = self.remaining_triggers if not remaining_triggers: return if self.trigger_timeout_timestamp: now = timeutils.current_time().timestamp() delay = max(self.trigger_timeout_timestamp - now, 1) self.trigger_timeout_call = reactor.callLater( delay, self.trigger_timeout_reached) else: log.error(f"{self} has no trigger_timeout_timestamp") for trigger in remaining_triggers: EventBus.subscribe(trigger, self.__hash__(), self.trigger_notify)
def get_run_data(self, request, run): state = job_run_state(run) last_time = run.end_time if run.end_time else timeutils.current_time() duration = str(last_time - run.start_time) if run.start_time else "" return { 'id': run.id, 'href': request.childLink(run.id), 'node': run.node.hostname if run.node else None, 'run_time': run.run_time and str(run.run_time), 'start_time': run.start_time and str(run.start_time), 'end_time': run.end_time and str(run.end_time), 'duration': duration, 'run_num': run.run_num, 'state': state, }
def next_run_time(self, start_time): """Find the next time to run.""" if not start_time: start_time = timeutils.current_time() elif self.time_zone: try: start_time = self.time_zone.localize(start_time, is_dst=None) except AmbiguousTimeError: # We are in the infamous 1 AM block which happens twice on # fall-back. Pretend like it's the first time, every time. start_time = self.time_zone.localize(start_time, is_dst=True) except NonExistentTimeError: # We are in the infamous 2:xx AM block which does not # exist. Pretend like it's the later time, every time. start_time = self.time_zone.localize(start_time, is_dst=True) return self.time_spec.get_match(start_time) + get_jitter(self.jitter)
def test_recover_action_run_action_runner(self): action_runner = SubprocessActionRunnerFactory( status_path='/tmp/foo', exec_path='/bin/foo', ) mock_node = mock.Mock() action_run = SSHActionRun(job_run_id="test.succeeded", name="test.succeeded", node=mock_node, action_runner=action_runner, end_time=timeutils.current_time(), exit_status=0) action_run.machine.state = ActionRun.UNKNOWN recover_action_run(action_run, action_runner) mock_node.submit_command.assert_called_once() assert action_run.machine.state == ActionRun.RUNNING assert action_run.end_time is None assert action_run.exit_status is None
def render_GET(self, request): run_output = [] state = job_run_state(self._run) for action_run in self._run.runs: action_state = job_run_state(action_run) last_time = action_run.end_time if action_run.end_time else timeutils.current_time( ) duration = str( last_time - action_run.start_time) if action_run.start_time else "" run_output.append({ 'id': action_run.id, 'name': action_run.action.name, 'run_time': action_run.run_time and str(action_run.run_time), 'start_time': action_run.start_time and str(action_run.start_time), 'end_time': action_run.end_time and str(action_run.end_time), 'exit_status': action_run.exit_status, 'duration': duration, 'state': action_state, }) output = { 'runs': run_output, 'id': self._run.id, 'state': state, 'node': self._run.node.hostname, } return respond(request, output)
def start(self): """Start this ActionRun.""" if self.in_delay is not None: log.warning(f"{self} cancelling suspend timer") self.in_delay.cancel() self.in_delay = None if not self.machine.check('start'): return False if len(self.exit_statuses) == 0: log.info(f"{self} starting") else: log.info(f"{self} restarting, retry {len(self.exit_statuses)}") self.start_time = timeutils.current_time() self.transition_and_notify('start') if not self.is_valid_command: log.error(f"{self} invalid command: {self.bare_command}") self.fail(-1) return return self.submit_command()
def next_run_time(self, last_run_time, time_zone=None): last_run_time = last_run_time or timeutils.current_time(tz=time_zone) return last_run_time + self.interval + get_jitter(self.jitter)
def next_run_time(self, _): return timeutils.current_time()