def build(self, batch, code, scope): assert scope in ('deploy', 'test', 'dev') for step in self.steps: if step.scopes is None or scope in step.scopes: step.build(batch, code, scope) if scope == 'dev': return step_to_parent_steps = defaultdict(set) for step in self.steps: for dep in step.all_deps(): step_to_parent_steps[dep].add(step) for step in self.steps: parent_jobs = flatten([ parent_step.wrapped_job() for parent_step in step_to_parent_steps[step] ]) log.info( f"Cleanup {step.name} after running {[parent_step.name for parent_step in step_to_parent_steps[step]]}" ) if step.scopes is None or scope in step.scopes: step.cleanup(batch, scope, parent_jobs)
async def fetch_prices(pricing_client: aioazure.AzurePricingClient, regions: List[str]) -> List[AzurePrice]: # Azure seems to have a limit on how long the OData filter request can be so we split the query into smaller groups vm_coros = [ vm_prices_by_region(pricing_client, region, machine_types) for region in regions for machine_types in grouped(8, azure_valid_machine_types) ] disk_coros = [managed_disk_prices_by_region(pricing_client, region) for region in regions] prices = await asyncio.gather(*vm_coros, *disk_coros) return flatten(prices)
def deps_parents(self): if not self.deps: return None return flatten([d.wrapped_job() for d in self.deps])