def _run_parallel(args):
jobs = []
if args.regions:
enabled_regions = args.regions
else:
enabled_regions = get_all_regions(args.tests_config)
for region in enabled_regions:
p = multiprocessing.Process(target=_run_test_in_region, args=(region, args, OUT_DIR, LOGS_DIR))
jobs.append(p)
p.start()
for job in jobs:
job.join()
def _log_collected_tests(session):
from xdist import get_xdist_worker_id
# Write collected tests in a single worker
# get_xdist_worker_id returns the id of the current worker ('gw0', 'gw1', etc) or 'master'
if get_xdist_worker_id(session) in ["master", "gw0"]:
collected_tests = list(map(lambda item: item.nodeid, session.items))
logging.info(
"Collected tests in regions %s (total=%d):\n%s",
session.config.getoption("regions") or get_all_regions(session.config.getoption("tests_config")),
len(session.items),
json.dumps(collected_tests, indent=2),
)
out_dir = session.config.getoption("output_dir")
with open(f"{out_dir}/collected_tests.txt", "a") as out_f:
out_f.write("\n".join(collected_tests))
out_f.write("\n")
def vpc_stacks(cfn_stacks_factory, request):
"""Create VPC used by integ tests in all configured regions."""
regions = request.config.getoption("regions") or get_all_regions(request.config.getoption("tests_config"))
vpc_stacks = {}
for region in regions:
# Creating private_subnet_different_cidr in a different AZ for test_efs
# To-do: isolate this logic and create a compute subnet in different AZ than head node in test_efs
# if region has a non-empty list in AVAILABILITY_ZONE_OVERRIDES, select a subset of those AZs
credential = request.config.getoption("credential")
az_ids_for_region = AVAILABILITY_ZONE_OVERRIDES.get(region, [])
if az_ids_for_region:
az_id_to_az_name = get_az_id_to_az_name_map(region, credential)
az_names = [az_id_to_az_name.get(az_id) for az_id in az_ids_for_region]
# if only one AZ can be used for the given region, use it multiple times
if len(az_names) == 1:
az_names *= 2
availability_zones = random.sample(az_names, k=2)
# otherwise, select a subset of all AZs in the region
else:
az_list = get_availability_zones(region, credential)
# if number of available zones is smaller than 2, available zones should be [None, None]
if len(az_list) < 2:
availability_zones = [None, None]
else:
availability_zones = random.sample(az_list, k=2)
# Subnets visual representation:
# http://www.davidc.net/sites/default/subnets/subnets.html?network=192.168.0.0&mask=16&division=7.70
public_subnet = SubnetConfig(
name="Public",
cidr="192.168.32.0/19", # 8190 IPs
map_public_ip_on_launch=True,
has_nat_gateway=True,
availability_zone=availability_zones[0],
default_gateway=Gateways.INTERNET_GATEWAY,
)
private_subnet = SubnetConfig(
name="Private",
cidr="192.168.64.0/18", # 16382 IPs
map_public_ip_on_launch=False,
has_nat_gateway=False,
availability_zone=availability_zones[0],
default_gateway=Gateways.NAT_GATEWAY,
)
private_subnet_different_cidr = SubnetConfig(
name="PrivateAdditionalCidr",
cidr="192.168.128.0/17", # 32766 IPs
map_public_ip_on_launch=False,
has_nat_gateway=False,
availability_zone=availability_zones[1],
default_gateway=Gateways.NAT_GATEWAY,
)
vpc_config = VPCConfig(
cidr="192.168.0.0/17",
additional_cidr_blocks=["192.168.128.0/17"],
subnets=[public_subnet, private_subnet, private_subnet_different_cidr],
)
template = NetworkTemplateBuilder(vpc_configuration=vpc_config, availability_zone=availability_zones[0]).build()
vpc_stacks[region] = _create_vpc_stack(request, template, region, cfn_stacks_factory)
return vpc_stacks