def main(mytimer: func.TimerRequest,
dashboard: func.Out[str]) -> None: # noqa: F841
expired_tasks = Task.search_expired()
for task in expired_tasks:
logging.info("stopping expired task: %s", task.job_id)
task.stopping()
expired_jobs = Job.search_expired()
for job in expired_jobs:
logging.info("stopping expired job: %s", job.job_id)
job.stopping()
jobs = Job.search_states(states=JobState.needs_work())
for job in jobs:
logging.info("update job: %s", job.job_id)
process_state_updates(job)
tasks = Task.search_states(states=TaskState.needs_work())
for task in tasks:
logging.info("update task: %s", task.task_id)
process_state_updates(task)
schedule_tasks()
Job.stop_never_started_jobs()
events = get_events()
if events:
dashboard.set(events)
def main(mytimer: func.TimerRequest,
dashboard: func.Out[str]) -> None: # noqa: F841
proxies = Proxy.search_states(states=VmState.needs_work())
for proxy in proxies:
logging.info("requeueing update proxy vm: %s", proxy.region)
proxy.queue()
vms = Repro.search_states(states=VmState.needs_work())
for vm in vms:
logging.info("requeueing update vm: %s", vm.vm_id)
vm.queue()
tasks = Task.search_states(states=TaskState.needs_work())
for task in tasks:
logging.info("requeueing update task: %s", task.task_id)
task.queue()
jobs = Job.search_states(states=JobState.needs_work())
for job in jobs:
logging.info("requeueing update job: %s", job.job_id)
job.queue()
pools = Pool.search_states(states=PoolState.needs_work())
for pool in pools:
logging.info("queuing update pool: %s (%s)", pool.pool_id, pool.name)
pool.queue()
nodes = Node.search_states(states=NodeState.needs_work())
for node in nodes:
logging.info("queuing update node: %s", node.machine_id)
node.queue()
expired_tasks = Task.search_expired()
for task in expired_tasks:
logging.info("queuing stop for task: %s", task.job_id)
task.queue_stop()
expired_jobs = Job.search_expired()
for job in expired_jobs:
logging.info("queuing stop for job: %s", job.job_id)
job.queue_stop()
# Reminder, proxies are created on-demand. If something is "wrong" with
# a proxy, the plan is: delete and recreate it.
for proxy in Proxy.search():
if not proxy.is_alive():
logging.error("proxy alive check failed, stopping: %s",
proxy.region)
proxy.state = VmState.stopping
proxy.save()
else:
proxy.save_proxy_config()
event = get_event()
if event:
dashboard.set(event)
def check_current_job(self) -> Job:
job = self.onefuzz.jobs.get(self.job.job_id)
if job.state in JobState.shutting_down():
raise StoppedEarly("job unexpectedly stopped early")
errors = []
for task in self.onefuzz.tasks.list(job_id=self.job.job_id,
state=TaskState.shutting_down()):
if task.error:
errors.append("%s: %s" % (task.config.task.type, task.error))
else:
errors.append("%s" % task.config.task.type)
if errors:
raise StoppedEarly("tasks stopped unexpectedly.\n%s" %
"\n".join(errors))
return job
def get_running_tasks_checked(self) -> List[Task]:
self.job = self.onefuzz.jobs.get(self.job.job_id)
if self.job.state in JobState.shutting_down():
raise StoppedEarly("job unexpectedly stopped early")
errors = []
tasks = []
for task in self.onefuzz.tasks.list(job_id=self.job.job_id):
if task.state in TaskState.shutting_down():
if task.error:
errors.append("%s: %s" %
(task.config.task.type, task.error))
else:
errors.append("%s" % task.config.task.type)
tasks.append(task)
if errors:
raise StoppedEarly("tasks stopped unexpectedly.\n%s" %
"\n".join(errors))
return tasks
def project(
self,
*,
project: str,
name: str,
build: Optional[str] = None,
all_jobs: Optional[bool] = False,
) -> None:
job_state = JobState.available()
if all_jobs:
job_state = [x for x in JobState]
for job in self.onefuzz.jobs.list(job_state=job_state):
if job.config.project != project:
continue
if job.config.name != name:
continue
if build is not None and job.config.build != build:
continue
self.job(job.job_id)
def search_expired(cls) -> List["Job"]:
time_filter = "end_time lt datetime'%s'" % datetime.utcnow().isoformat(
)
return cls.search(query={"state": JobState.available()},
raw_unchecked_filter=time_filter)