def make(n, q, args):
image = args.image
ci_subnet_index = 123 # TODO fix inital setup stuff
scenario = args.scenario
data_path = args.data_path
fragment_path = args.fragment_path
if args.debug:
debug.debug = True
test_id = uuid.uuid4().hex
print test_id
networks = {}
subnets = {}
ports = {}
# Ci network with external route
# There can be only one of these per tenant
# because overlapping subnets + router doesn't work
networks['ci'] = make_network(q, 'ci')
subnets['ci'] = make_subnet(q, 'ci', networks['ci'], ci_subnet_index, gateway=True)
set_external_routing(q, get_ci_subnet(q))
ci_subnet_index = ci_subnet_index + 1
with open(data_path + '/nodes/' + scenario + '.yaml') as scenario_yaml_file:
scenario_yaml = yaml.load(scenario_yaml_file.read())
# Find needed internal networks
for node, props in scenario_yaml['nodes'].items():
for network in props['networks']:
if network != 'ci': # build network with NAT services
networks[network] = False
# Create internal networks
for network, gate in networks.items():
if network != 'ci':
networks[network] = make_network(q, 'ci-' + network + '-' + test_id)
subnets[network] = make_subnet(q, 'ci-' + network + '-' + test_id,
networks[network], index=ci_subnet_index, gateway=gate)
ci_subnet_index = ci_subnet_index + 1
# Allocate ports
for node, props in scenario_yaml['nodes'].items():
for network in props['networks']:
if node not in ports:
ports[node] = {}
ports[node][network] = allocate_ports(q, networks[network]['id'], test_id)
else:
ports[node][network] = allocate_ports(q, networks[network]['id'], test_id)
dprint("networks")
for net, value in networks.items():
dprint (net + str(value))
dprint ("subnets")
for snet, value in subnets.items():
dprint (snet + str(value))
dprint ("ports")
dprint (json.dumps(ports,sort_keys=True, indent=4))
# Quantum won't schedule dhcp agents to networks when using
# port preallocation. This requires the admin role
agent = q.list_dhcp_agent_hosting_networks(networks['ci']['id'])
for name, network in networks.items():
if name != 'ci':
q.add_network_to_dhcp_agent(agent['agents'][0]['id'], {"network_id" : network['id']})
# config is a dictionary updated from env vars and user supplied
# yaml files to serve as input to hiera and build scripts
initial_config_meta = build_metadata(data_path, scenario, 'config')
hiera_config_meta = build_metadata(data_path, scenario, 'user')
# IP addresses of particular nodes mapped to specified config
# values to go into hiera + build scripts. See data/nodes/2_role.yaml
meta_update = {}
for node, props in scenario_yaml['nodes'].items():
for network, mappings in props['networks'].items():
if mappings != None:
for mapping in mappings:
meta_update[mapping] = str(ports[node][network][0]['fixed_ips'][0]['ip_address'])
hiera_config_meta.update(meta_update)
initial_config_meta.update(meta_update)
# fragment composition
deploy_files = {}
for node, props in scenario_yaml['nodes'].items():
deploy_files[node] = fragment.compose(node, data_path, fragment_path, scenario, initial_config_meta)
dprint(node + 'deploy:\n' + deploy_files[node])
user_config_yaml = yaml.dump(hiera_config_meta, default_flow_style=False)
initial_config_yaml = yaml.dump(initial_config_meta, default_flow_style=False)
dprint('Config Yaml: \n' + str(initial_config_yaml))
dprint('User Yaml: \n' + str(user_config_yaml))
port_list = {}
for node, props in scenario_yaml['nodes'].items():
nics = []
for network in props['networks']:
nics.append(ports[node][network][0]['id'])
port_list[node] = build_nic_port_list(nics)
for node, props in scenario_yaml['nodes'].items():
boot_puppetised_instance(n,
node,
image,
port_list[node],
deploy=cloud_init,
files={
u'/root/deploy' : deploy_files[node],
u'/root/user.yaml' : user_config_yaml,
u'/root/config.yaml' : initial_config_yaml},
meta={'ci_test_id' : test_id}
)
def make(n, q, k, args):
image = args.image
ci_subnet_index = 123 # TODO fix inital setup stuff
scenario = args.scenario
data_path = args.data_path
fragment_path = args.fragment_path
public_network = args.public_network
nameserver = args.nameserver
if args.debug:
debug.debug = True
test_id = uuid.uuid4().hex
print test_id
networks = {}
subnets = {}
ports = {}
# Ci network with external route
# There can be only one of these per tenant
# because overlapping subnets + router doesn't work
networks['ci'] = make_network(q, 'ci')
subnets['ci'] = make_subnet(q, 'ci', networks['ci'], ci_subnet_index, gateway=True, dns_nameserver=nameserver)
set_external_routing(q, get_ci_subnet(q, networks['ci']), public_network)
ci_subnet_index = ci_subnet_index + 1
with open(data_path + '/nodes/' + scenario + '.yaml') as scenario_yaml_file:
scenario_yaml = yaml.load(scenario_yaml_file.read())
# Find needed internal networks
for node, props in scenario_yaml['nodes'].items():
for network in props['networks']:
if network != 'ci': # build network with NAT services
networks[network] = False
# Create internal networks
for network, gate in networks.items():
if network != 'ci':
networks[network] = make_network(q, 'ci-' + network + '-' + test_id)
subnets[network] = make_subnet(q, 'ci-' + network + '-' + test_id,
networks[network], index=ci_subnet_index, gateway=gate)
ci_subnet_index = ci_subnet_index + 1
# There seems to be a bug in quantum where networks are not scheduled a dhcp agent unless a VM
# boots on that network without a pre-made port. So we boot an instance that will do this
# on all our networks
dummynets = [network for network in networks.values()]
dummy = boot_puppetised_instance(n,
'dummy',
image,
build_nic_net_list(dummynets),
deploy=cloud_init,
meta={'ci_test_id' : test_id},
os_flavor=u'm1.small'
)
while dummy.status != u'ACTIVE':
dummy = n.servers.get(dummy)
dprint('dummy status: ' + str(dummy.status))
dummy.delete()
# Allocate ports
for node, props in scenario_yaml['nodes'].items():
for network in props['networks']:
if node not in ports:
ports[node] = {}
ports[node][network] = allocate_ports(q, networks[network]['id'], test_id)
else:
ports[node][network] = allocate_ports(q, networks[network]['id'], test_id)
dprint("networks")
for net, value in networks.items():
dprint (net + str(value))
dprint ("subnets")
for snet, value in subnets.items():
dprint (snet + str(value))
dprint ("ports")
dprint (json.dumps(ports,sort_keys=True, indent=4))
# config is a dictionary updated from env vars and user supplied
# yaml files to serve as input to hiera and build scripts
initial_config_meta = build_metadata(data_path, scenario, 'config')
hiera_config_meta = build_metadata(data_path, scenario, 'user')
global_config_meta = build_metadata(data_path, scenario, 'global')
meta_update = metadata_update(scenario_yaml, ports)
hiera_config_meta.update(meta_update)
initial_config_meta.update(meta_update)
# fragment composition
deploy_files = {}
for node, props in scenario_yaml['nodes'].items():
deploy_files[node] = fragment.compose(node, data_path, fragment_path, scenario, initial_config_meta)
dprint(node + 'deploy:\n' + deploy_files[node])
user_config_yaml = yaml.dump(hiera_config_meta, default_flow_style=False)
initial_config_yaml = yaml.dump(initial_config_meta, default_flow_style=False)
global_config_yaml = yaml.dump(global_config_meta, default_flow_style=False)
dprint('Config Yaml: \n' + str(initial_config_yaml))
dprint('User Yaml: \n' + str(user_config_yaml))
dprint('Global Yaml: \n' + str(global_config_yaml))
port_list = {}
for node, props in scenario_yaml['nodes'].items():
nics = []
for network in props['networks']:
nics.append(ports[node][network][0]['id'])
port_list[node] = build_nic_port_list(nics)
for node, props in scenario_yaml['nodes'].items():
boot_puppetised_instance(n,
node,
image,
port_list[node],
deploy=cloud_init,
files={
u'/root/deploy' : deploy_files[node],
u'/root/user.yaml' : user_config_yaml,
u'/root/config.yaml' : initial_config_yaml,
u'/root/global.yaml' : global_config_yaml},
meta={'ci_test_id' : test_id}
)
def rebuild(n,q,k,args):
image = args.image
ci_subnet_index = 123 # TODO fix inital setup stuff
scenario = args.scenario
data_path = args.data_path
fragment_path = args.fragment_path
test_id = args.test_id
if args.debug:
debug.debug = True
with open(data_path + '/nodes/' + scenario + '.yaml') as scenario_yaml_file:
scenario_yaml = yaml.load(scenario_yaml_file.read())
current_instances = build.get(n,q,k,args)[unicode(test_id)]
print current_instances
# determine which instances are missing
current_instance_names = [instance.name for instance in current_instances]
missing_instance_names = []
for instance in scenario_yaml['nodes'].keys():
if instance not in current_instance_names:
missing_instance_names.append(instance)
dprint("present instances: " + str(current_instance_names))
dprint("missing instances: " + str(missing_instance_names))
networks = {}
subnets = {}
ports = {}
networks['ci'] = build.get_ci_network(q,k)
all_nets = q.list_networks()
# build networks list, assume they're all there
for node, props in scenario_yaml['nodes'].items():
for network in props['networks']:
if network != 'ci':
networks[network] = get_network(all_nets, network, test_id)
dprint('networks ' + str(networks))
# New ports for the missing instances
for node, props in scenario_yaml['nodes'].items():
if node in missing_instance_names:
for network in props['networks']:
if node not in ports:
ports[node] = {}
dprint('creating port for node' + str(node) + ' on network ' + network)
ports[node][network] = build.allocate_ports(q, networks[network]['id'], test_id)
else:
dprint('creating port for node' + str(node) + ' on network ' + network)
ports[node][network] = build.allocate_ports(q, networks[network]['id'], test_id)
# Recreate port dictionary for existing instances
for node in current_instances:
nodename = str(node.name)
for network, ips in node.networks.items():
if nodename not in ports:
ports[nodename] = {}
ports[nodename][network] = [{'fixed_ips' : [{'ip_address' : ips[0]}]}]
else:
ports[nodename][network] = [{'fixed_ips' : [{'ip_address' : ips[0]}]}]
dprint("Ports" + str(ports))
# Re-create metadata and deploy
initial_config_meta = build_metadata(data_path, scenario, 'config')
hiera_config_meta = build_metadata(data_path, scenario, 'user')
meta_update = build.metadata_update(scenario_yaml, ports)
dprint('runtime metadata: ' + str(meta_update))
hiera_config_meta.update(meta_update)
initial_config_meta.update(meta_update)
# fragment composition
deploy_files = {}
for node, props in scenario_yaml['nodes'].items():
deploy_files[node] = fragment.compose(node, data_path, fragment_path, scenario, initial_config_meta)
dprint(node + 'deploy:\n' + deploy_files[node])
user_config_yaml = yaml.dump(hiera_config_meta, default_flow_style=False)
initial_config_yaml = yaml.dump(initial_config_meta, default_flow_style=False)
# Create missing instances
port_list = {}
for node, props in scenario_yaml['nodes'].items():
if node in missing_instance_names:
nics = []
for network in props['networks']:
nics.append(ports[node][network][0]['id'])
port_list[node] = build.build_nic_port_list(nics)
for node, props in scenario_yaml['nodes'].items():
if node in missing_instance_names:
dprint("booting " + str(node))
build.boot_puppetised_instance(n,
node,
image,
port_list[node],
deploy=build.cloud_init,
files={
u'/root/deploy' : deploy_files[node],
u'/root/user.yaml' : user_config_yaml,
u'/root/config.yaml' : initial_config_yaml},
meta={'ci_test_id' : test_id}
)
def make(n, q, k, args):
image = args.image
ci_subnet_index = 123 # TODO fix inital setup stuff
scenario = args.scenario
data_path = args.data_path
fragment_path = args.fragment_path
public_network = args.public_network
nameserver = args.nameserver
if args.debug:
debug.debug = True
test_id = uuid.uuid4().hex
print test_id
networks = {}
subnets = {}
ports = {}
# Ci network with external route
# There can be only one of these per tenant
# because overlapping subnets + router doesn't work
networks['ci'] = make_network(q, 'ci')
subnets['ci'] = make_subnet(q,
'ci',
networks['ci'],
ci_subnet_index,
gateway=True,
dns_nameserver=nameserver)
set_external_routing(q, get_ci_subnet(q, networks['ci']), public_network)
ci_subnet_index = ci_subnet_index + 1
with open(data_path + '/nodes/' + scenario +
'.yaml') as scenario_yaml_file:
scenario_yaml = yaml.load(scenario_yaml_file.read())
# Find needed internal networks
for node, props in scenario_yaml['nodes'].items():
for network in props['networks']:
if network != 'ci': # build network with NAT services
networks[network] = False
# Create internal networks
for network, gate in networks.items():
if network != 'ci':
networks[network] = make_network(q,
'ci-' + network + '-' + test_id)
subnets[network] = make_subnet(q,
'ci-' + network + '-' + test_id,
networks[network],
index=ci_subnet_index,
gateway=gate)
ci_subnet_index = ci_subnet_index + 1
# There seems to be a bug in quantum where networks are not scheduled a dhcp agent unless a VM
# boots on that network without a pre-made port. So we boot an instance that will do this
# on all our networks
dummynets = [network for network in networks.values()]
dummy = boot_puppetised_instance(n,
'dummy',
image,
build_nic_net_list(dummynets),
deploy=cloud_init,
meta={'ci_test_id': test_id},
os_flavor=u'm1.small')
while dummy.status != u'ACTIVE':
dummy = n.servers.get(dummy)
dprint('dummy status: ' + str(dummy.status))
dummy.delete()
# Allocate ports
for node, props in scenario_yaml['nodes'].items():
for network in props['networks']:
if node not in ports:
ports[node] = {}
ports[node][network] = allocate_ports(q,
networks[network]['id'],
test_id)
else:
ports[node][network] = allocate_ports(q,
networks[network]['id'],
test_id)
dprint("networks")
for net, value in networks.items():
dprint(net + str(value))
dprint("subnets")
for snet, value in subnets.items():
dprint(snet + str(value))
dprint("ports")
dprint(json.dumps(ports, sort_keys=True, indent=4))
# config is a dictionary updated from env vars and user supplied
# yaml files to serve as input to hiera and build scripts
initial_config_meta = build_metadata(data_path, scenario, 'config')
hiera_config_meta = build_metadata(data_path, scenario, 'user')
global_config_meta = build_metadata(data_path, scenario, 'global')
meta_update = metadata_update(scenario_yaml, ports)
hiera_config_meta.update(meta_update)
initial_config_meta.update(meta_update)
# fragment composition
deploy_files = {}
for node, props in scenario_yaml['nodes'].items():
deploy_files[node] = fragment.compose(node, data_path, fragment_path,
scenario, initial_config_meta)
dprint(node + 'deploy:\n' + deploy_files[node])
user_config_yaml = yaml.dump(hiera_config_meta, default_flow_style=False)
initial_config_yaml = yaml.dump(initial_config_meta,
default_flow_style=False)
global_config_yaml = yaml.dump(global_config_meta,
default_flow_style=False)
dprint('Config Yaml: \n' + str(initial_config_yaml))
dprint('User Yaml: \n' + str(user_config_yaml))
dprint('Global Yaml: \n' + str(global_config_yaml))
port_list = {}
for node, props in scenario_yaml['nodes'].items():
nics = []
for network in props['networks']:
nics.append(ports[node][network][0]['id'])
port_list[node] = build_nic_port_list(nics)
for node, props in scenario_yaml['nodes'].items():
boot_puppetised_instance(n,
node,
image,
port_list[node],
deploy=cloud_init,
files={
u'/root/deploy': deploy_files[node],
u'/root/user.yaml': user_config_yaml,
u'/root/config.yaml': initial_config_yaml,
u'/root/global.yaml': global_config_yaml
},
meta={'ci_test_id': test_id})
