def build(self, batch, code, scope):
assert scope in ('deploy', 'test', 'dev')
for step in self.steps:
if step.scopes is None or scope in step.scopes:
step.build(batch, code, scope)
if scope == 'dev':
return
step_to_parent_steps = defaultdict(set)
for step in self.steps:
for dep in step.all_deps():
step_to_parent_steps[dep].add(step)
for step in self.steps:
parent_jobs = flatten([
parent_step.wrapped_job()
for parent_step in step_to_parent_steps[step]
])
log.info(
f"Cleanup {step.name} after running {[parent_step.name for parent_step in step_to_parent_steps[step]]}"
)
if step.scopes is None or scope in step.scopes:
step.cleanup(batch, scope, parent_jobs)
async def fetch_prices(pricing_client: aioazure.AzurePricingClient, regions: List[str]) -> List[AzurePrice]:
# Azure seems to have a limit on how long the OData filter request can be so we split the query into smaller groups
vm_coros = [
vm_prices_by_region(pricing_client, region, machine_types)
for region in regions
for machine_types in grouped(8, azure_valid_machine_types)
]
disk_coros = [managed_disk_prices_by_region(pricing_client, region) for region in regions]
prices = await asyncio.gather(*vm_coros, *disk_coros)
return flatten(prices)
def deps_parents(self):
if not self.deps:
return None
return flatten([d.wrapped_job() for d in self.deps])
