def main(mytimer: func.TimerRequest, dashboard: func.Out[str]) -> None: # noqa: F841 expired_tasks = Task.search_expired() for task in expired_tasks: logging.info("stopping expired task: %s", task.job_id) task.stopping() expired_jobs = Job.search_expired() for job in expired_jobs: logging.info("stopping expired job: %s", job.job_id) job.stopping() jobs = Job.search_states(states=JobState.needs_work()) for job in jobs: logging.info("update job: %s", job.job_id) process_state_updates(job) tasks = Task.search_states(states=TaskState.needs_work()) for task in tasks: logging.info("update task: %s", task.task_id) process_state_updates(task) schedule_tasks() Job.stop_never_started_jobs() events = get_events() if events: dashboard.set(events)
def main(mytimer: func.TimerRequest, dashboard: func.Out[str]) -> None: # noqa: F841 proxies = Proxy.search_states(states=VmState.needs_work()) for proxy in proxies: logging.info("requeueing update proxy vm: %s", proxy.region) proxy.queue() vms = Repro.search_states(states=VmState.needs_work()) for vm in vms: logging.info("requeueing update vm: %s", vm.vm_id) vm.queue() tasks = Task.search_states(states=TaskState.needs_work()) for task in tasks: logging.info("requeueing update task: %s", task.task_id) task.queue() jobs = Job.search_states(states=JobState.needs_work()) for job in jobs: logging.info("requeueing update job: %s", job.job_id) job.queue() pools = Pool.search_states(states=PoolState.needs_work()) for pool in pools: logging.info("queuing update pool: %s (%s)", pool.pool_id, pool.name) pool.queue() nodes = Node.search_states(states=NodeState.needs_work()) for node in nodes: logging.info("queuing update node: %s", node.machine_id) node.queue() expired_tasks = Task.search_expired() for task in expired_tasks: logging.info("queuing stop for task: %s", task.job_id) task.queue_stop() expired_jobs = Job.search_expired() for job in expired_jobs: logging.info("queuing stop for job: %s", job.job_id) job.queue_stop() # Reminder, proxies are created on-demand. If something is "wrong" with # a proxy, the plan is: delete and recreate it. for proxy in Proxy.search(): if not proxy.is_alive(): logging.error("proxy alive check failed, stopping: %s", proxy.region) proxy.state = VmState.stopping proxy.save() else: proxy.save_proxy_config() event = get_event() if event: dashboard.set(event)
def check_current_job(self) -> Job: job = self.onefuzz.jobs.get(self.job.job_id) if job.state in JobState.shutting_down(): raise StoppedEarly("job unexpectedly stopped early") errors = [] for task in self.onefuzz.tasks.list(job_id=self.job.job_id, state=TaskState.shutting_down()): if task.error: errors.append("%s: %s" % (task.config.task.type, task.error)) else: errors.append("%s" % task.config.task.type) if errors: raise StoppedEarly("tasks stopped unexpectedly.\n%s" % "\n".join(errors)) return job
def get_running_tasks_checked(self) -> List[Task]: self.job = self.onefuzz.jobs.get(self.job.job_id) if self.job.state in JobState.shutting_down(): raise StoppedEarly("job unexpectedly stopped early") errors = [] tasks = [] for task in self.onefuzz.tasks.list(job_id=self.job.job_id): if task.state in TaskState.shutting_down(): if task.error: errors.append("%s: %s" % (task.config.task.type, task.error)) else: errors.append("%s" % task.config.task.type) tasks.append(task) if errors: raise StoppedEarly("tasks stopped unexpectedly.\n%s" % "\n".join(errors)) return tasks
def project( self, *, project: str, name: str, build: Optional[str] = None, all_jobs: Optional[bool] = False, ) -> None: job_state = JobState.available() if all_jobs: job_state = [x for x in JobState] for job in self.onefuzz.jobs.list(job_state=job_state): if job.config.project != project: continue if job.config.name != name: continue if build is not None and job.config.build != build: continue self.job(job.job_id)
def search_expired(cls) -> List["Job"]: time_filter = "end_time lt datetime'%s'" % datetime.utcnow().isoformat( ) return cls.search(query={"state": JobState.available()}, raw_unchecked_filter=time_filter)