def show(n, q, k, args): hostname = args.node yaml_dir = args.yaml_dir fragment_dir = args.fragment_dir scenario = args.scenario print compose(hostname, yaml_dir, fragment_dir, scenario, build_metadata("./data", "2_role", "config"))
def show(n, q, args): hostname = args.node yaml_dir = args.yaml_dir fragment_dir = args.fragment_dir scenario = args.scenario print compose(hostname, yaml_dir, fragment_dir, scenario, build_metadata('./data', '2_role', 'config'))
def show(n, q, k, args): hostname = args.node yaml_dir = args.yaml_dir fragment_dir = args.fragment_dir scenario = args.scenario print compose(hostname, yaml_dir, fragment_dir, scenario, build_metadata('./data', '2_role', 'config'))
def heat(args): data_path = args.data_path scenario = args.scenario t = datetime.now() test_id = ''.join(map(str, [t.year, t.month, t.day, t.hour, t.minute, t.second, t.microsecond])) with open(data_path + '/nodes/heat/bootstrap.sh') as user_data_raw_file: user_data_raw = user_data_raw_file.read() with open(data_path + '/nodes/heat/' + scenario + '.yaml') as heat_template: heat_yaml = yaml.load(heat_template.read()) for resource_set,details_set in heat_yaml['resources'].items(): if details_set['type'] == 'OS::Nova::Server': if 'wait_nodes' in details_set['properties']['metadata']: for wait_node in details_set['properties']['metadata']['wait_nodes'].split(' '): details_set['properties']['metadata']['NODE_' + wait_node] = { 'get_attr': [ wait_node, 'first_address' ] } # config is a dictionary updated from env vars and user supplied # yaml files to serve as input to hiera and build scripts initial_config_meta = build_metadata(data_path, scenario, 'config') hiera_config_meta = build_metadata(data_path, scenario, 'user') global_config_meta = build_metadata(data_path, scenario, 'global') for node,details in heat_yaml['resources'].items(): if details['type'] == 'OS::Nova::Server': role = bootstrap.load_role(data_path, node, scenario) class_groups = bootstrap.load_class_groups(data_path, role, node, scenario) details['properties']['metadata']['role'] = role details['properties']['user_data'] = user_data_raw details['properties']['metadata']['class_groups'] = ' '.join(class_groups) for config, value in initial_config_meta.items(): details['properties']['metadata'][config] = value for user, value in hiera_config_meta.items(): details['properties']['metadata']['OS_USER_'+user] = value for glob, value in global_config_meta.items(): details['properties']['metadata']['OS_GLOB_'+glob] = value with open('heat.yaml', 'w' ) as temp_file: temp_file.write(str(yaml.dump(heat_yaml)))
def make(n, q, args): image = args.image ci_subnet_index = 123 # TODO fix inital setup stuff scenario = args.scenario data_path = args.data_path fragment_path = args.fragment_path if args.debug: debug.debug = True test_id = uuid.uuid4().hex print test_id networks = {} subnets = {} ports = {} # Ci network with external route # There can be only one of these per tenant # because overlapping subnets + router doesn't work networks['ci'] = make_network(q, 'ci') subnets['ci'] = make_subnet(q, 'ci', networks['ci'], ci_subnet_index, gateway=True) set_external_routing(q, get_ci_subnet(q)) ci_subnet_index = ci_subnet_index + 1 with open(data_path + '/nodes/' + scenario + '.yaml') as scenario_yaml_file: scenario_yaml = yaml.load(scenario_yaml_file.read()) # Find needed internal networks for node, props in scenario_yaml['nodes'].items(): for network in props['networks']: if network != 'ci': # build network with NAT services networks[network] = False # Create internal networks for network, gate in networks.items(): if network != 'ci': networks[network] = make_network(q, 'ci-' + network + '-' + test_id) subnets[network] = make_subnet(q, 'ci-' + network + '-' + test_id, networks[network], index=ci_subnet_index, gateway=gate) ci_subnet_index = ci_subnet_index + 1 # Allocate ports for node, props in scenario_yaml['nodes'].items(): for network in props['networks']: if node not in ports: ports[node] = {} ports[node][network] = allocate_ports(q, networks[network]['id'], test_id) else: ports[node][network] = allocate_ports(q, networks[network]['id'], test_id) dprint("networks") for net, value in networks.items(): dprint (net + str(value)) dprint ("subnets") for snet, value in subnets.items(): dprint (snet + str(value)) dprint ("ports") dprint (json.dumps(ports,sort_keys=True, indent=4)) # Quantum won't schedule dhcp agents to networks when using # port preallocation. This requires the admin role agent = q.list_dhcp_agent_hosting_networks(networks['ci']['id']) for name, network in networks.items(): if name != 'ci': q.add_network_to_dhcp_agent(agent['agents'][0]['id'], {"network_id" : network['id']}) # config is a dictionary updated from env vars and user supplied # yaml files to serve as input to hiera and build scripts initial_config_meta = build_metadata(data_path, scenario, 'config') hiera_config_meta = build_metadata(data_path, scenario, 'user') # IP addresses of particular nodes mapped to specified config # values to go into hiera + build scripts. See data/nodes/2_role.yaml meta_update = {} for node, props in scenario_yaml['nodes'].items(): for network, mappings in props['networks'].items(): if mappings != None: for mapping in mappings: meta_update[mapping] = str(ports[node][network][0]['fixed_ips'][0]['ip_address']) hiera_config_meta.update(meta_update) initial_config_meta.update(meta_update) # fragment composition deploy_files = {} for node, props in scenario_yaml['nodes'].items(): deploy_files[node] = fragment.compose(node, data_path, fragment_path, scenario, initial_config_meta) dprint(node + 'deploy:\n' + deploy_files[node]) user_config_yaml = yaml.dump(hiera_config_meta, default_flow_style=False) initial_config_yaml = yaml.dump(initial_config_meta, default_flow_style=False) dprint('Config Yaml: \n' + str(initial_config_yaml)) dprint('User Yaml: \n' + str(user_config_yaml)) port_list = {} for node, props in scenario_yaml['nodes'].items(): nics = [] for network in props['networks']: nics.append(ports[node][network][0]['id']) port_list[node] = build_nic_port_list(nics) for node, props in scenario_yaml['nodes'].items(): boot_puppetised_instance(n, node, image, port_list[node], deploy=cloud_init, files={ u'/root/deploy' : deploy_files[node], u'/root/user.yaml' : user_config_yaml, u'/root/config.yaml' : initial_config_yaml}, meta={'ci_test_id' : test_id} )
def rebuild(n,q,k,args): image = args.image ci_subnet_index = 123 # TODO fix inital setup stuff scenario = args.scenario data_path = args.data_path fragment_path = args.fragment_path test_id = args.test_id if args.debug: debug.debug = True with open(data_path + '/nodes/' + scenario + '.yaml') as scenario_yaml_file: scenario_yaml = yaml.load(scenario_yaml_file.read()) current_instances = build.get(n,q,k,args)[unicode(test_id)] print current_instances # determine which instances are missing current_instance_names = [instance.name for instance in current_instances] missing_instance_names = [] for instance in scenario_yaml['nodes'].keys(): if instance not in current_instance_names: missing_instance_names.append(instance) dprint("present instances: " + str(current_instance_names)) dprint("missing instances: " + str(missing_instance_names)) networks = {} subnets = {} ports = {} networks['ci'] = build.get_ci_network(q,k) all_nets = q.list_networks() # build networks list, assume they're all there for node, props in scenario_yaml['nodes'].items(): for network in props['networks']: if network != 'ci': networks[network] = get_network(all_nets, network, test_id) dprint('networks ' + str(networks)) # New ports for the missing instances for node, props in scenario_yaml['nodes'].items(): if node in missing_instance_names: for network in props['networks']: if node not in ports: ports[node] = {} dprint('creating port for node' + str(node) + ' on network ' + network) ports[node][network] = build.allocate_ports(q, networks[network]['id'], test_id) else: dprint('creating port for node' + str(node) + ' on network ' + network) ports[node][network] = build.allocate_ports(q, networks[network]['id'], test_id) # Recreate port dictionary for existing instances for node in current_instances: nodename = str(node.name) for network, ips in node.networks.items(): if nodename not in ports: ports[nodename] = {} ports[nodename][network] = [{'fixed_ips' : [{'ip_address' : ips[0]}]}] else: ports[nodename][network] = [{'fixed_ips' : [{'ip_address' : ips[0]}]}] dprint("Ports" + str(ports)) # Re-create metadata and deploy initial_config_meta = build_metadata(data_path, scenario, 'config') hiera_config_meta = build_metadata(data_path, scenario, 'user') meta_update = build.metadata_update(scenario_yaml, ports) dprint('runtime metadata: ' + str(meta_update)) hiera_config_meta.update(meta_update) initial_config_meta.update(meta_update) # fragment composition deploy_files = {} for node, props in scenario_yaml['nodes'].items(): deploy_files[node] = fragment.compose(node, data_path, fragment_path, scenario, initial_config_meta) dprint(node + 'deploy:\n' + deploy_files[node]) user_config_yaml = yaml.dump(hiera_config_meta, default_flow_style=False) initial_config_yaml = yaml.dump(initial_config_meta, default_flow_style=False) # Create missing instances port_list = {} for node, props in scenario_yaml['nodes'].items(): if node in missing_instance_names: nics = [] for network in props['networks']: nics.append(ports[node][network][0]['id']) port_list[node] = build.build_nic_port_list(nics) for node, props in scenario_yaml['nodes'].items(): if node in missing_instance_names: dprint("booting " + str(node)) build.boot_puppetised_instance(n, node, image, port_list[node], deploy=build.cloud_init, files={ u'/root/deploy' : deploy_files[node], u'/root/user.yaml' : user_config_yaml, u'/root/config.yaml' : initial_config_yaml}, meta={'ci_test_id' : test_id} )
def make(n, q, k, args): image = args.image ci_subnet_index = 123 # TODO fix inital setup stuff scenario = args.scenario data_path = args.data_path public_network = args.public_network nameserver = args.nameserver if args.debug: debug.debug = True test_id = uuid.uuid4().hex print test_id networks = {} subnets = {} ports = {} # Ci network with external route # There can be only one of these per tenant # because overlapping subnets + router doesn't work networks['ci'] = make_network(q, 'ci') subnets['ci'] = make_subnet(q, 'ci', networks['ci'], ci_subnet_index, gateway=True, dns_nameserver=nameserver) set_external_routing(q, get_ci_subnet(q, networks['ci']), public_network) ci_subnet_index = ci_subnet_index + 1 with open(data_path + '/nodes/' + scenario + '.yaml') as scenario_yaml_file: scenario_yaml = yaml.load(scenario_yaml_file.read()) # Find needed internal networks for node, props in scenario_yaml['nodes'].items(): for network in props['networks']: if network != 'ci': # build network with NAT services networks[network] = False # Create internal networks for network, gate in networks.items(): if network != 'ci': networks[network] = make_network(q, 'ci-' + network + '-' + test_id) subnets[network] = make_subnet(q, 'ci-' + network + '-' + test_id, networks[network], index=ci_subnet_index, gateway=gate) ci_subnet_index = ci_subnet_index + 1 # There seems to be a bug in neutron where networks are not scheduled a dhcp agent unless a VM # boots on that network without a pre-made port. So we boot an instance that will do this # on all our networks dummynets = [network for network in networks.values()] dummy = boot_puppetised_instance(n, 'dummy', image, build_nic_net_list(dummynets), deploy=cloud_init, meta={'ci_test_id' : test_id}, os_flavor=u'm1.small' ) while dummy.status != u'ACTIVE': dummy = n.servers.get(dummy) dprint('dummy status: ' + str(dummy.status)) dummy.delete() # Allocate ports for node, props in scenario_yaml['nodes'].items(): for network in props['networks']: if node not in ports: ports[node] = {} ports[node][network] = allocate_ports(q, networks[network]['id'], test_id) else: ports[node][network] = allocate_ports(q, networks[network]['id'], test_id) dprint("networks") for net, value in networks.items(): dprint (net + str(value)) dprint ("subnets") for snet, value in subnets.items(): dprint (snet + str(value)) dprint ("ports") dprint (json.dumps(ports,sort_keys=True, indent=4)) # config is a dictionary updated from env vars and user supplied # yaml files to serve as input to hiera and build scripts initial_config_meta = build_metadata(data_path, scenario, 'config') hiera_config_meta = build_metadata(data_path, scenario, 'user') global_config_meta = build_metadata(data_path, scenario, 'global') meta_update = metadata_update(scenario_yaml, ports) hiera_config_meta.update(meta_update) initial_config_meta.update(meta_update) # Bootstrap script composition deploy_files = {} for node, props in scenario_yaml['nodes'].items(): deploy_files[node] = bootstrap.compose(node, data_path, scenario, initial_config_meta) dprint(node + 'deploy:\n' + deploy_files[node]) dprint("Deploy Files: " + str(deploy_files)) user_config_yaml = yaml.dump(hiera_config_meta, default_flow_style=False) initial_config_yaml = yaml.dump(initial_config_meta, default_flow_style=False) global_config_yaml = yaml.dump(global_config_meta, default_flow_style=False) dprint('Config Yaml: \n' + str(initial_config_yaml)) dprint('User Yaml: \n' + str(user_config_yaml)) dprint('Global Yaml: \n' + str(global_config_yaml)) port_list = {} for node, props in scenario_yaml['nodes'].items(): nics = [] for network in props['networks']: nics.append(ports[node][network][0]['id']) port_list[node] = build_nic_port_list(nics) for node, props in scenario_yaml['nodes'].items(): boot_puppetised_instance(n, node, image, port_list[node], deploy=cloud_init, files={ u'/root/deploy' : deploy_files[node], u'/root/user.yaml' : user_config_yaml, u'/root/config.yaml' : initial_config_yaml, u'/root/global.yaml' : global_config_yaml}, meta={'ci_test_id' : test_id} )
def make(n, q, k, args): image = args.image ci_subnet_index = 123 # TODO fix inital setup stuff scenario = args.scenario data_path = args.data_path fragment_path = args.fragment_path public_network = args.public_network nameserver = args.nameserver if args.debug: debug.debug = True test_id = uuid.uuid4().hex print test_id networks = {} subnets = {} ports = {} # Ci network with external route # There can be only one of these per tenant # because overlapping subnets + router doesn't work networks["ci"] = make_network(q, "ci") subnets["ci"] = make_subnet(q, "ci", networks["ci"], ci_subnet_index, gateway=True, dns_nameserver=nameserver) set_external_routing(q, get_ci_subnet(q, networks["ci"]), public_network) ci_subnet_index = ci_subnet_index + 1 with open(data_path + "/nodes/" + scenario + ".yaml") as scenario_yaml_file: scenario_yaml = yaml.load(scenario_yaml_file.read()) # Find needed internal networks for node, props in scenario_yaml["nodes"].items(): for network in props["networks"]: if network != "ci": # build network with NAT services networks[network] = False # Create internal networks for network, gate in networks.items(): if network != "ci": networks[network] = make_network(q, "ci-" + network + "-" + test_id) subnets[network] = make_subnet( q, "ci-" + network + "-" + test_id, networks[network], index=ci_subnet_index, gateway=gate ) ci_subnet_index = ci_subnet_index + 1 # There seems to be a bug in quantum where networks are not scheduled a dhcp agent unless a VM # boots on that network without a pre-made port. So we boot an instance that will do this # on all our networks dummynets = [network for network in networks.values()] dummy = boot_puppetised_instance( n, "dummy", image, build_nic_net_list(dummynets), deploy=cloud_init, meta={"ci_test_id": test_id}, os_flavor=u"m1.small", ) while dummy.status != u"ACTIVE": dummy = n.servers.get(dummy) dprint("dummy status: " + str(dummy.status)) dummy.delete() # Allocate ports for node, props in scenario_yaml["nodes"].items(): for network in props["networks"]: if node not in ports: ports[node] = {} ports[node][network] = allocate_ports(q, networks[network]["id"], test_id) else: ports[node][network] = allocate_ports(q, networks[network]["id"], test_id) dprint("networks") for net, value in networks.items(): dprint(net + str(value)) dprint("subnets") for snet, value in subnets.items(): dprint(snet + str(value)) dprint("ports") dprint(json.dumps(ports, sort_keys=True, indent=4)) # config is a dictionary updated from env vars and user supplied # yaml files to serve as input to hiera and build scripts initial_config_meta = build_metadata(data_path, scenario, "config") hiera_config_meta = build_metadata(data_path, scenario, "user") global_config_meta = build_metadata(data_path, scenario, "global") meta_update = metadata_update(scenario_yaml, ports) hiera_config_meta.update(meta_update) initial_config_meta.update(meta_update) # fragment composition deploy_files = {} for node, props in scenario_yaml["nodes"].items(): deploy_files[node] = fragment.compose(node, data_path, fragment_path, scenario, initial_config_meta) dprint(node + "deploy:\n" + deploy_files[node]) user_config_yaml = yaml.dump(hiera_config_meta, default_flow_style=False) initial_config_yaml = yaml.dump(initial_config_meta, default_flow_style=False) global_config_yaml = yaml.dump(global_config_meta, default_flow_style=False) dprint("Config Yaml: \n" + str(initial_config_yaml)) dprint("User Yaml: \n" + str(user_config_yaml)) dprint("Global Yaml: \n" + str(global_config_yaml)) port_list = {} for node, props in scenario_yaml["nodes"].items(): nics = [] for network in props["networks"]: nics.append(ports[node][network][0]["id"]) port_list[node] = build_nic_port_list(nics) for node, props in scenario_yaml["nodes"].items(): boot_puppetised_instance( n, node, image, port_list[node], deploy=cloud_init, files={ u"/root/deploy": deploy_files[node], u"/root/user.yaml": user_config_yaml, u"/root/config.yaml": initial_config_yaml, u"/root/global.yaml": global_config_yaml, }, meta={"ci_test_id": test_id}, )
def make(n, q, k, args): image = args.image ci_subnet_index = 123 # TODO fix inital setup stuff scenario = args.scenario data_path = args.data_path fragment_path = args.fragment_path public_network = args.public_network nameserver = args.nameserver if args.debug: debug.debug = True test_id = uuid.uuid4().hex print test_id networks = {} subnets = {} ports = {} # Ci network with external route # There can be only one of these per tenant # because overlapping subnets + router doesn't work networks['ci'] = make_network(q, 'ci') subnets['ci'] = make_subnet(q, 'ci', networks['ci'], ci_subnet_index, gateway=True, dns_nameserver=nameserver) set_external_routing(q, get_ci_subnet(q, networks['ci']), public_network) ci_subnet_index = ci_subnet_index + 1 with open(data_path + '/nodes/' + scenario + '.yaml') as scenario_yaml_file: scenario_yaml = yaml.load(scenario_yaml_file.read()) # Find needed internal networks for node, props in scenario_yaml['nodes'].items(): for network in props['networks']: if network != 'ci': # build network with NAT services networks[network] = False # Create internal networks for network, gate in networks.items(): if network != 'ci': networks[network] = make_network(q, 'ci-' + network + '-' + test_id) subnets[network] = make_subnet(q, 'ci-' + network + '-' + test_id, networks[network], index=ci_subnet_index, gateway=gate) ci_subnet_index = ci_subnet_index + 1 # There seems to be a bug in quantum where networks are not scheduled a dhcp agent unless a VM # boots on that network without a pre-made port. So we boot an instance that will do this # on all our networks dummynets = [network for network in networks.values()] dummy = boot_puppetised_instance(n, 'dummy', image, build_nic_net_list(dummynets), deploy=cloud_init, meta={'ci_test_id': test_id}, os_flavor=u'm1.small') while dummy.status != u'ACTIVE': dummy = n.servers.get(dummy) dprint('dummy status: ' + str(dummy.status)) dummy.delete() # Allocate ports for node, props in scenario_yaml['nodes'].items(): for network in props['networks']: if node not in ports: ports[node] = {} ports[node][network] = allocate_ports(q, networks[network]['id'], test_id) else: ports[node][network] = allocate_ports(q, networks[network]['id'], test_id) dprint("networks") for net, value in networks.items(): dprint(net + str(value)) dprint("subnets") for snet, value in subnets.items(): dprint(snet + str(value)) dprint("ports") dprint(json.dumps(ports, sort_keys=True, indent=4)) # config is a dictionary updated from env vars and user supplied # yaml files to serve as input to hiera and build scripts initial_config_meta = build_metadata(data_path, scenario, 'config') hiera_config_meta = build_metadata(data_path, scenario, 'user') global_config_meta = build_metadata(data_path, scenario, 'global') meta_update = metadata_update(scenario_yaml, ports) hiera_config_meta.update(meta_update) initial_config_meta.update(meta_update) # fragment composition deploy_files = {} for node, props in scenario_yaml['nodes'].items(): deploy_files[node] = fragment.compose(node, data_path, fragment_path, scenario, initial_config_meta) dprint(node + 'deploy:\n' + deploy_files[node]) user_config_yaml = yaml.dump(hiera_config_meta, default_flow_style=False) initial_config_yaml = yaml.dump(initial_config_meta, default_flow_style=False) global_config_yaml = yaml.dump(global_config_meta, default_flow_style=False) dprint('Config Yaml: \n' + str(initial_config_yaml)) dprint('User Yaml: \n' + str(user_config_yaml)) dprint('Global Yaml: \n' + str(global_config_yaml)) port_list = {} for node, props in scenario_yaml['nodes'].items(): nics = [] for network in props['networks']: nics.append(ports[node][network][0]['id']) port_list[node] = build_nic_port_list(nics) for node, props in scenario_yaml['nodes'].items(): boot_puppetised_instance(n, node, image, port_list[node], deploy=cloud_init, files={ u'/root/deploy': deploy_files[node], u'/root/user.yaml': user_config_yaml, u'/root/config.yaml': initial_config_yaml, u'/root/global.yaml': global_config_yaml }, meta={'ci_test_id': test_id})