def main(mytimer: func.TimerRequest,
dashboard: func.Out[str]) -> None: # noqa: F841
proxies = Proxy.search_states(states=VmState.needs_work())
for proxy in proxies:
logging.info("requeueing update proxy vm: %s", proxy.region)
proxy.queue()
vms = Repro.search_states(states=VmState.needs_work())
for vm in vms:
logging.info("requeueing update vm: %s", vm.vm_id)
vm.queue()
tasks = Task.search_states(states=TaskState.needs_work())
for task in tasks:
logging.info("requeueing update task: %s", task.task_id)
task.queue()
jobs = Job.search_states(states=JobState.needs_work())
for job in jobs:
logging.info("requeueing update job: %s", job.job_id)
job.queue()
pools = Pool.search_states(states=PoolState.needs_work())
for pool in pools:
logging.info("queuing update pool: %s (%s)", pool.pool_id, pool.name)
pool.queue()
nodes = Node.search_states(states=NodeState.needs_work())
for node in nodes:
logging.info("queuing update node: %s", node.machine_id)
node.queue()
expired_tasks = Task.search_expired()
for task in expired_tasks:
logging.info("queuing stop for task: %s", task.job_id)
task.queue_stop()
expired_jobs = Job.search_expired()
for job in expired_jobs:
logging.info("queuing stop for job: %s", job.job_id)
job.queue_stop()
# Reminder, proxies are created on-demand. If something is "wrong" with
# a proxy, the plan is: delete and recreate it.
for proxy in Proxy.search():
if not proxy.is_alive():
logging.error("proxy alive check failed, stopping: %s",
proxy.region)
proxy.state = VmState.stopping
proxy.save()
else:
proxy.save_proxy_config()
event = get_event()
if event:
dashboard.set(event)
def is_outdated(self) -> bool:
if self.state not in VmState.available():
return True
if self.version != __version__:
logging.info(
PROXY_LOG_PREFIX +
"mismatch version: proxy:%s service:%s state:%s",
self.version,
__version__,
self.state,
)
return True
if self.created_timestamp is not None:
proxy_timestamp = self.created_timestamp
if proxy_timestamp < (datetime.datetime.now(
tz=datetime.timezone.utc) - PROXY_LIFESPAN):
logging.info(
PROXY_LOG_PREFIX +
"proxy older than 7 days:proxy-created:%s state:%s",
self.created_timestamp,
self.state,
)
return True
return False
def main(mytimer: func.TimerRequest,
dashboard: func.Out[str]) -> None: # noqa: F841
# Reminder, proxies are created on-demand. If something is "wrong" with
# a proxy, the plan is: delete and recreate it.
for proxy in Proxy.search():
if not proxy.is_alive():
logging.error(
PROXY_LOG_PREFIX + "alive check failed, stopping: %s",
proxy.region)
proxy.state = VmState.stopping
proxy.save()
else:
proxy.save_proxy_config()
if proxy.state in VmState.needs_work():
logging.info(
PROXY_LOG_PREFIX + "update state. proxy:%s state:%s",
proxy.region,
proxy.state,
)
process_state_updates(proxy)
events = get_events()
if events:
dashboard.set(events)
def main(mytimer: func.TimerRequest) -> None: # noqa: F841
expired = Repro.search_expired()
for repro in expired:
logging.info("stopping repro: %s", repro.vm_id)
repro.stopping()
expired_vm_ids = [x.vm_id for x in expired]
for repro in Repro.search_states(states=VmState.needs_work()):
if repro.vm_id in expired_vm_ids:
# this VM already got processed during the expired phase
continue
logging.info("update repro: %s", repro.vm_id)
process_state_updates(repro)
def get(req: func.HttpRequest) -> func.HttpResponse:
request = parse_request(ReproGet, req)
if isinstance(request, Error):
return not_ok(request, context="repro_vm get")
if request.vm_id:
vm = Repro.get(request.vm_id)
if not vm:
return not_ok(
Error(code=ErrorCode.INVALID_REQUEST, errors=["no such VM"]),
context=request.vm_id,
)
return ok(vm)
else:
vms = Repro.search_states(states=VmState.available())
for vm in vms:
vm.auth = None
return ok(vms)
def get_or_create(cls, region: Region) -> Optional["Proxy"]:
proxy_list = Proxy.search(query={
"region": [region],
"outdated": [False]
})
for proxy in proxy_list:
if proxy.is_outdated():
proxy.outdated = True
proxy.save()
continue
if proxy.state not in VmState.available():
continue
return proxy
logging.info(PROXY_LOG_PREFIX + "creating proxy: region:%s", region)
proxy = Proxy(region=region)
proxy.save()
send_event(EventProxyCreated(region=region, proxy_id=proxy.proxy_id))
return proxy
def main(mytimer: func.TimerRequest) -> None: # noqa: F841
proxies = Proxy.search()
for proxy in proxies:
if proxy.state in VmState.available():
# Note, outdated checked at the start, but set at the end of this loop.
# As this function is called via a timer, this works around a user
# requesting to use the proxy while this function is checking if it's
# out of date
if proxy.outdated and not proxy.is_used():
proxy.set_state(VmState.stopping)
# If something is "wrong" with a proxy, delete & recreate it
elif not proxy.is_alive():
logging.error(
PROXY_LOG_PREFIX + "alive check failed, stopping: %s", proxy.region
)
proxy.set_state(VmState.stopping)
else:
proxy.save_proxy_config()
if proxy.state in VmState.needs_work():
logging.info(
PROXY_LOG_PREFIX + "update state. proxy:%s state:%s",
proxy.region,
proxy.state,
)
process_state_updates(proxy)
if proxy.state != VmState.stopped and proxy.is_outdated():
proxy.outdated = True
proxy.save()
# make sure there is a proxy for every currently active region
scalesets = Scaleset.search()
regions = set(x.region for x in scalesets)
for region in regions:
if all(x.outdated for x in proxies if x.region == region):
Proxy.get_or_create(region)
logging.info("Creating new proxy in region %s" % region)
# this is required in order to support upgrade from non-nsg to
# nsg enabled OneFuzz this will overwrite existing NSG
# assignment though. This behavior is acceptable at this point
# since we do not support bring your own NSG
if get_nsg(region):
network = Network(region)
subnet = network.get_subnet()
if subnet:
result = associate_subnet(region, network.get_vnet(), subnet)
if isinstance(result, Error):
logging.error(
"Failed to associate NSG and subnet due to %s in region %s"
% (result, region)
)
# if there are NSGs with name same as the region that they are allocated
# and have no NIC associated with it then delete the NSG
for nsg in list_nsgs():
if ok_to_delete(regions, nsg.location, nsg.name):
if nsg.network_interfaces is None and nsg.subnets is None:
start_delete_nsg(nsg.name)