def main(mytimer: func.TimerRequest) -> None: # noqa: F841
# NOTE: Update pools first, such that scalesets impacted by pool updates
# (such as shutdown or resize) happen during this iteration `timer_worker`
# rather than the following iteration.
pools = Pool.search()
for pool in pools:
if pool.state in PoolState.needs_work():
logging.info("update pool: %s (%s)", pool.pool_id, pool.name)
process_state_updates(pool)
if pool.state in PoolState.available() and pool.autoscale:
autoscale_pool(pool)
# NOTE: Nodes, and Scalesets should be processed in a consistent order such
# during 'pool scale down' operations. This means that pools that are
# scaling down will more likely remove from the same scalesets over time.
# By more likely removing from the same scalesets, we are more likely to
# get to empty scalesets, which can safely be deleted.
Node.mark_outdated_nodes()
Node.cleanup_busy_nodes_without_work()
nodes = Node.search_states(states=NodeState.needs_work())
for node in sorted(nodes, key=lambda x: x.machine_id):
logging.info("update node: %s", node.machine_id)
process_state_updates(node)
scalesets = Scaleset.search()
for scaleset in sorted(scalesets, key=lambda x: x.scaleset_id):
process_scaleset(scaleset)
def main(mytimer: func.TimerRequest,
dashboard: func.Out[str]) -> None: # noqa: F841
proxies = Proxy.search_states(states=VmState.needs_work())
for proxy in proxies:
logging.info("requeueing update proxy vm: %s", proxy.region)
proxy.queue()
vms = Repro.search_states(states=VmState.needs_work())
for vm in vms:
logging.info("requeueing update vm: %s", vm.vm_id)
vm.queue()
tasks = Task.search_states(states=TaskState.needs_work())
for task in tasks:
logging.info("requeueing update task: %s", task.task_id)
task.queue()
jobs = Job.search_states(states=JobState.needs_work())
for job in jobs:
logging.info("requeueing update job: %s", job.job_id)
job.queue()
pools = Pool.search_states(states=PoolState.needs_work())
for pool in pools:
logging.info("queuing update pool: %s (%s)", pool.pool_id, pool.name)
pool.queue()
nodes = Node.search_states(states=NodeState.needs_work())
for node in nodes:
logging.info("queuing update node: %s", node.machine_id)
node.queue()
expired_tasks = Task.search_expired()
for task in expired_tasks:
logging.info("queuing stop for task: %s", task.job_id)
task.queue_stop()
expired_jobs = Job.search_expired()
for job in expired_jobs:
logging.info("queuing stop for job: %s", job.job_id)
job.queue_stop()
# Reminder, proxies are created on-demand. If something is "wrong" with
# a proxy, the plan is: delete and recreate it.
for proxy in Proxy.search():
if not proxy.is_alive():
logging.error("proxy alive check failed, stopping: %s",
proxy.region)
proxy.state = VmState.stopping
proxy.save()
else:
proxy.save_proxy_config()
event = get_event()
if event:
dashboard.set(event)
def main(mytimer: func.TimerRequest,
dashboard: func.Out[str]) -> None: # noqa: F841
Node.mark_outdated_nodes()
nodes = Node.search_states(states=NodeState.needs_work())
for node in nodes:
logging.info("update node: %s", node.machine_id)
process_state_updates(node)
scalesets = Scaleset.search()
for scaleset in scalesets:
process_scaleset(scaleset)
pools = Pool.search()
for pool in pools:
if pool.state in PoolState.needs_work():
logging.info("update pool: %s (%s)", pool.pool_id, pool.name)
process_state_updates(pool)
elif pool.state in PoolState.available() and pool.autoscale:
autoscale_pool(pool)
event = get_event()
if event:
dashboard.set(event)
