def make(n, q, args): image = args.image ci_subnet_index = 123 # TODO fix inital setup stuff scenario = args.scenario data_path = args.data_path fragment_path = args.fragment_path if args.debug: debug.debug = True test_id = uuid.uuid4().hex print test_id networks = {} subnets = {} ports = {} # Ci network with external route # There can be only one of these per tenant # because overlapping subnets + router doesn't work networks['ci'] = make_network(q, 'ci') subnets['ci'] = make_subnet(q, 'ci', networks['ci'], ci_subnet_index, gateway=True) set_external_routing(q, get_ci_subnet(q)) ci_subnet_index = ci_subnet_index + 1 with open(data_path + '/nodes/' + scenario + '.yaml') as scenario_yaml_file: scenario_yaml = yaml.load(scenario_yaml_file.read()) # Find needed internal networks for node, props in scenario_yaml['nodes'].items(): for network in props['networks']: if network != 'ci': # build network with NAT services networks[network] = False # Create internal networks for network, gate in networks.items(): if network != 'ci': networks[network] = make_network(q, 'ci-' + network + '-' + test_id) subnets[network] = make_subnet(q, 'ci-' + network + '-' + test_id, networks[network], index=ci_subnet_index, gateway=gate) ci_subnet_index = ci_subnet_index + 1 # Allocate ports for node, props in scenario_yaml['nodes'].items(): for network in props['networks']: if node not in ports: ports[node] = {} ports[node][network] = allocate_ports(q, networks[network]['id'], test_id) else: ports[node][network] = allocate_ports(q, networks[network]['id'], test_id) dprint("networks") for net, value in networks.items(): dprint (net + str(value)) dprint ("subnets") for snet, value in subnets.items(): dprint (snet + str(value)) dprint ("ports") dprint (json.dumps(ports,sort_keys=True, indent=4)) # Quantum won't schedule dhcp agents to networks when using # port preallocation. This requires the admin role agent = q.list_dhcp_agent_hosting_networks(networks['ci']['id']) for name, network in networks.items(): if name != 'ci': q.add_network_to_dhcp_agent(agent['agents'][0]['id'], {"network_id" : network['id']}) # config is a dictionary updated from env vars and user supplied # yaml files to serve as input to hiera and build scripts initial_config_meta = build_metadata(data_path, scenario, 'config') hiera_config_meta = build_metadata(data_path, scenario, 'user') # IP addresses of particular nodes mapped to specified config # values to go into hiera + build scripts. See data/nodes/2_role.yaml meta_update = {} for node, props in scenario_yaml['nodes'].items(): for network, mappings in props['networks'].items(): if mappings != None: for mapping in mappings: meta_update[mapping] = str(ports[node][network][0]['fixed_ips'][0]['ip_address']) hiera_config_meta.update(meta_update) initial_config_meta.update(meta_update) # fragment composition deploy_files = {} for node, props in scenario_yaml['nodes'].items(): deploy_files[node] = fragment.compose(node, data_path, fragment_path, scenario, initial_config_meta) dprint(node + 'deploy:\n' + deploy_files[node]) user_config_yaml = yaml.dump(hiera_config_meta, default_flow_style=False) initial_config_yaml = yaml.dump(initial_config_meta, default_flow_style=False) dprint('Config Yaml: \n' + str(initial_config_yaml)) dprint('User Yaml: \n' + str(user_config_yaml)) port_list = {} for node, props in scenario_yaml['nodes'].items(): nics = [] for network in props['networks']: nics.append(ports[node][network][0]['id']) port_list[node] = build_nic_port_list(nics) for node, props in scenario_yaml['nodes'].items(): boot_puppetised_instance(n, node, image, port_list[node], deploy=cloud_init, files={ u'/root/deploy' : deploy_files[node], u'/root/user.yaml' : user_config_yaml, u'/root/config.yaml' : initial_config_yaml}, meta={'ci_test_id' : test_id} )
def make(n, q, k, args): image = args.image ci_subnet_index = 123 # TODO fix inital setup stuff scenario = args.scenario data_path = args.data_path fragment_path = args.fragment_path public_network = args.public_network nameserver = args.nameserver if args.debug: debug.debug = True test_id = uuid.uuid4().hex print test_id networks = {} subnets = {} ports = {} # Ci network with external route # There can be only one of these per tenant # because overlapping subnets + router doesn't work networks['ci'] = make_network(q, 'ci') subnets['ci'] = make_subnet(q, 'ci', networks['ci'], ci_subnet_index, gateway=True, dns_nameserver=nameserver) set_external_routing(q, get_ci_subnet(q, networks['ci']), public_network) ci_subnet_index = ci_subnet_index + 1 with open(data_path + '/nodes/' + scenario + '.yaml') as scenario_yaml_file: scenario_yaml = yaml.load(scenario_yaml_file.read()) # Find needed internal networks for node, props in scenario_yaml['nodes'].items(): for network in props['networks']: if network != 'ci': # build network with NAT services networks[network] = False # Create internal networks for network, gate in networks.items(): if network != 'ci': networks[network] = make_network(q, 'ci-' + network + '-' + test_id) subnets[network] = make_subnet(q, 'ci-' + network + '-' + test_id, networks[network], index=ci_subnet_index, gateway=gate) ci_subnet_index = ci_subnet_index + 1 # There seems to be a bug in quantum where networks are not scheduled a dhcp agent unless a VM # boots on that network without a pre-made port. So we boot an instance that will do this # on all our networks dummynets = [network for network in networks.values()] dummy = boot_puppetised_instance(n, 'dummy', image, build_nic_net_list(dummynets), deploy=cloud_init, meta={'ci_test_id' : test_id}, os_flavor=u'm1.small' ) while dummy.status != u'ACTIVE': dummy = n.servers.get(dummy) dprint('dummy status: ' + str(dummy.status)) dummy.delete() # Allocate ports for node, props in scenario_yaml['nodes'].items(): for network in props['networks']: if node not in ports: ports[node] = {} ports[node][network] = allocate_ports(q, networks[network]['id'], test_id) else: ports[node][network] = allocate_ports(q, networks[network]['id'], test_id) dprint("networks") for net, value in networks.items(): dprint (net + str(value)) dprint ("subnets") for snet, value in subnets.items(): dprint (snet + str(value)) dprint ("ports") dprint (json.dumps(ports,sort_keys=True, indent=4)) # config is a dictionary updated from env vars and user supplied # yaml files to serve as input to hiera and build scripts initial_config_meta = build_metadata(data_path, scenario, 'config') hiera_config_meta = build_metadata(data_path, scenario, 'user') global_config_meta = build_metadata(data_path, scenario, 'global') meta_update = metadata_update(scenario_yaml, ports) hiera_config_meta.update(meta_update) initial_config_meta.update(meta_update) # fragment composition deploy_files = {} for node, props in scenario_yaml['nodes'].items(): deploy_files[node] = fragment.compose(node, data_path, fragment_path, scenario, initial_config_meta) dprint(node + 'deploy:\n' + deploy_files[node]) user_config_yaml = yaml.dump(hiera_config_meta, default_flow_style=False) initial_config_yaml = yaml.dump(initial_config_meta, default_flow_style=False) global_config_yaml = yaml.dump(global_config_meta, default_flow_style=False) dprint('Config Yaml: \n' + str(initial_config_yaml)) dprint('User Yaml: \n' + str(user_config_yaml)) dprint('Global Yaml: \n' + str(global_config_yaml)) port_list = {} for node, props in scenario_yaml['nodes'].items(): nics = [] for network in props['networks']: nics.append(ports[node][network][0]['id']) port_list[node] = build_nic_port_list(nics) for node, props in scenario_yaml['nodes'].items(): boot_puppetised_instance(n, node, image, port_list[node], deploy=cloud_init, files={ u'/root/deploy' : deploy_files[node], u'/root/user.yaml' : user_config_yaml, u'/root/config.yaml' : initial_config_yaml, u'/root/global.yaml' : global_config_yaml}, meta={'ci_test_id' : test_id} )
def rebuild(n,q,k,args): image = args.image ci_subnet_index = 123 # TODO fix inital setup stuff scenario = args.scenario data_path = args.data_path fragment_path = args.fragment_path test_id = args.test_id if args.debug: debug.debug = True with open(data_path + '/nodes/' + scenario + '.yaml') as scenario_yaml_file: scenario_yaml = yaml.load(scenario_yaml_file.read()) current_instances = build.get(n,q,k,args)[unicode(test_id)] print current_instances # determine which instances are missing current_instance_names = [instance.name for instance in current_instances] missing_instance_names = [] for instance in scenario_yaml['nodes'].keys(): if instance not in current_instance_names: missing_instance_names.append(instance) dprint("present instances: " + str(current_instance_names)) dprint("missing instances: " + str(missing_instance_names)) networks = {} subnets = {} ports = {} networks['ci'] = build.get_ci_network(q,k) all_nets = q.list_networks() # build networks list, assume they're all there for node, props in scenario_yaml['nodes'].items(): for network in props['networks']: if network != 'ci': networks[network] = get_network(all_nets, network, test_id) dprint('networks ' + str(networks)) # New ports for the missing instances for node, props in scenario_yaml['nodes'].items(): if node in missing_instance_names: for network in props['networks']: if node not in ports: ports[node] = {} dprint('creating port for node' + str(node) + ' on network ' + network) ports[node][network] = build.allocate_ports(q, networks[network]['id'], test_id) else: dprint('creating port for node' + str(node) + ' on network ' + network) ports[node][network] = build.allocate_ports(q, networks[network]['id'], test_id) # Recreate port dictionary for existing instances for node in current_instances: nodename = str(node.name) for network, ips in node.networks.items(): if nodename not in ports: ports[nodename] = {} ports[nodename][network] = [{'fixed_ips' : [{'ip_address' : ips[0]}]}] else: ports[nodename][network] = [{'fixed_ips' : [{'ip_address' : ips[0]}]}] dprint("Ports" + str(ports)) # Re-create metadata and deploy initial_config_meta = build_metadata(data_path, scenario, 'config') hiera_config_meta = build_metadata(data_path, scenario, 'user') meta_update = build.metadata_update(scenario_yaml, ports) dprint('runtime metadata: ' + str(meta_update)) hiera_config_meta.update(meta_update) initial_config_meta.update(meta_update) # fragment composition deploy_files = {} for node, props in scenario_yaml['nodes'].items(): deploy_files[node] = fragment.compose(node, data_path, fragment_path, scenario, initial_config_meta) dprint(node + 'deploy:\n' + deploy_files[node]) user_config_yaml = yaml.dump(hiera_config_meta, default_flow_style=False) initial_config_yaml = yaml.dump(initial_config_meta, default_flow_style=False) # Create missing instances port_list = {} for node, props in scenario_yaml['nodes'].items(): if node in missing_instance_names: nics = [] for network in props['networks']: nics.append(ports[node][network][0]['id']) port_list[node] = build.build_nic_port_list(nics) for node, props in scenario_yaml['nodes'].items(): if node in missing_instance_names: dprint("booting " + str(node)) build.boot_puppetised_instance(n, node, image, port_list[node], deploy=build.cloud_init, files={ u'/root/deploy' : deploy_files[node], u'/root/user.yaml' : user_config_yaml, u'/root/config.yaml' : initial_config_yaml}, meta={'ci_test_id' : test_id} )
def make(n, q, k, args): image = args.image ci_subnet_index = 123 # TODO fix inital setup stuff scenario = args.scenario data_path = args.data_path fragment_path = args.fragment_path public_network = args.public_network nameserver = args.nameserver if args.debug: debug.debug = True test_id = uuid.uuid4().hex print test_id networks = {} subnets = {} ports = {} # Ci network with external route # There can be only one of these per tenant # because overlapping subnets + router doesn't work networks['ci'] = make_network(q, 'ci') subnets['ci'] = make_subnet(q, 'ci', networks['ci'], ci_subnet_index, gateway=True, dns_nameserver=nameserver) set_external_routing(q, get_ci_subnet(q, networks['ci']), public_network) ci_subnet_index = ci_subnet_index + 1 with open(data_path + '/nodes/' + scenario + '.yaml') as scenario_yaml_file: scenario_yaml = yaml.load(scenario_yaml_file.read()) # Find needed internal networks for node, props in scenario_yaml['nodes'].items(): for network in props['networks']: if network != 'ci': # build network with NAT services networks[network] = False # Create internal networks for network, gate in networks.items(): if network != 'ci': networks[network] = make_network(q, 'ci-' + network + '-' + test_id) subnets[network] = make_subnet(q, 'ci-' + network + '-' + test_id, networks[network], index=ci_subnet_index, gateway=gate) ci_subnet_index = ci_subnet_index + 1 # There seems to be a bug in quantum where networks are not scheduled a dhcp agent unless a VM # boots on that network without a pre-made port. So we boot an instance that will do this # on all our networks dummynets = [network for network in networks.values()] dummy = boot_puppetised_instance(n, 'dummy', image, build_nic_net_list(dummynets), deploy=cloud_init, meta={'ci_test_id': test_id}, os_flavor=u'm1.small') while dummy.status != u'ACTIVE': dummy = n.servers.get(dummy) dprint('dummy status: ' + str(dummy.status)) dummy.delete() # Allocate ports for node, props in scenario_yaml['nodes'].items(): for network in props['networks']: if node not in ports: ports[node] = {} ports[node][network] = allocate_ports(q, networks[network]['id'], test_id) else: ports[node][network] = allocate_ports(q, networks[network]['id'], test_id) dprint("networks") for net, value in networks.items(): dprint(net + str(value)) dprint("subnets") for snet, value in subnets.items(): dprint(snet + str(value)) dprint("ports") dprint(json.dumps(ports, sort_keys=True, indent=4)) # config is a dictionary updated from env vars and user supplied # yaml files to serve as input to hiera and build scripts initial_config_meta = build_metadata(data_path, scenario, 'config') hiera_config_meta = build_metadata(data_path, scenario, 'user') global_config_meta = build_metadata(data_path, scenario, 'global') meta_update = metadata_update(scenario_yaml, ports) hiera_config_meta.update(meta_update) initial_config_meta.update(meta_update) # fragment composition deploy_files = {} for node, props in scenario_yaml['nodes'].items(): deploy_files[node] = fragment.compose(node, data_path, fragment_path, scenario, initial_config_meta) dprint(node + 'deploy:\n' + deploy_files[node]) user_config_yaml = yaml.dump(hiera_config_meta, default_flow_style=False) initial_config_yaml = yaml.dump(initial_config_meta, default_flow_style=False) global_config_yaml = yaml.dump(global_config_meta, default_flow_style=False) dprint('Config Yaml: \n' + str(initial_config_yaml)) dprint('User Yaml: \n' + str(user_config_yaml)) dprint('Global Yaml: \n' + str(global_config_yaml)) port_list = {} for node, props in scenario_yaml['nodes'].items(): nics = [] for network in props['networks']: nics.append(ports[node][network][0]['id']) port_list[node] = build_nic_port_list(nics) for node, props in scenario_yaml['nodes'].items(): boot_puppetised_instance(n, node, image, port_list[node], deploy=cloud_init, files={ u'/root/deploy': deploy_files[node], u'/root/user.yaml': user_config_yaml, u'/root/config.yaml': initial_config_yaml, u'/root/global.yaml': global_config_yaml }, meta={'ci_test_id': test_id})