def ensure_mesh(self): with util.RecordedOperation('ensure mesh', self) as _: instances = [] for iface in db.get_network_interfaces(self.uuid): if not iface['instance_uuid'] in instances: instances.append(iface['instance_uuid']) node_fqdns = [] for inst in instances: i = db.get_instance(inst) if not i['node'] in node_fqdns: node_fqdns.append(i['node']) # NOTE(mikal): why not use DNS here? Well, DNS might be outside # the control of the deployer if we're running in a public cloud # as an overlay cloud... node_ips = [config.parsed.get('NETWORK_NODE_IP')] for fqdn in node_fqdns: ip = db.get_node(fqdn)['ip'] if ip not in node_ips: node_ips.append(ip) discovered = list(self.discover_mesh()) LOG.debug('%s: Discovered mesh elements %s' % (self, discovered)) for node in discovered: if node in node_ips: node_ips.remove(node) else: self._remove_mesh_element(node) for node in node_ips: self._add_mesh_element(node)
def _make_hosts(self): if not os.path.exists(self.subst['config_dir']): os.makedirs(self.subst['config_dir']) t = self._read_template('dhcphosts.tmpl') instances = [] for ni in list(db.get_network_interfaces(self.network_uuid)): instance = db.get_instance(ni['instance_uuid']) if not instance: continue instances.append({ 'uuid': ni['instance_uuid'], 'macaddr': ni['macaddr'], 'ipv4': ni['ipv4'], 'name': instance.get('name', 'instance').replace(',', '') }) self.subst['instances'] = instances c = t.render(self.subst) with open(os.path.join(self.subst['config_dir'], 'hosts'), 'w') as f: f.write(c)
def wrapper(*args, **kwargs): if 'instance_uuid' in kwargs: kwargs['instance_from_db'] = db.get_instance( kwargs['instance_uuid']) if not kwargs.get('instance_from_db'): return error(404, 'instance not found') return func(*args, **kwargs)
def ensure_mesh(self): with db.get_object_lock(self, ttl=120, op='Network ensure mesh'): # Ensure network was not deleted whilst waiting for the lock. if self.is_dead(): raise DeadNetwork('network=%s' % self) removed = [] added = [] instances = [] for iface in db.get_network_interfaces(self.db_entry['uuid']): if not iface['instance_uuid'] in instances: instances.append(iface['instance_uuid']) node_fqdns = [] for inst in instances: i = db.get_instance(inst) if not i: continue if not i['node']: continue if not i['node'] in node_fqdns: node_fqdns.append(i['node']) # NOTE(mikal): why not use DNS here? Well, DNS might be outside # the control of the deployer if we're running in a public cloud # as an overlay cloud... node_ips = [config.NETWORK_NODE_IP] for fqdn in node_fqdns: ip = db.get_node(fqdn)['ip'] if ip not in node_ips: node_ips.append(ip) discovered = list(self.discover_mesh()) LOG.withObj(self).withField( 'discovered', discovered).debug('Discovered mesh elements') for node in discovered: if node in node_ips: node_ips.remove(node) else: self._remove_mesh_element(node) removed.append(node) for node in node_ips: self._add_mesh_element(node) added.append(node) if removed: db.add_event('network', self.db_entry['uuid'], 'remove mesh elements', None, None, ' '.join(removed)) if added: db.add_event('network', self.db_entry['uuid'], 'add mesh elements', None, None, ' '.join(added))
def wrapper(*args, **kwargs): if 'instance_uuid' in kwargs: kwargs['instance_from_db'] = db.get_instance( kwargs['instance_uuid']) if not kwargs.get('instance_from_db'): logutil.info([virt.ThinInstance(kwargs['instance_uuid'])], 'Instance not found, genuinely missing') return error(404, 'instance not found') return func(*args, **kwargs)
def wrapper(*args, **kwargs): if 'instance_uuid' in kwargs: kwargs['instance_from_db'] = db.get_instance( kwargs['instance_uuid']) if not kwargs.get('instance_from_db'): LOG.info('instance(%s): instance not found, genuinely missing' % kwargs.get('instance_uuid')) return error(404, 'instance not found') return func(*args, **kwargs)
def delete(self, confirm=False, namespace=None): """Delete all instances in the namespace.""" if confirm is not True: return error(400, 'parameter confirm is not set true') if get_jwt_identity() == 'system': if not isinstance(namespace, str): # A client using a system key must specify the namespace. This # ensures that deleting all instances in the cluster (by # specifying namespace='system') is a deliberate act. return error(400, 'system user must specify parameter namespace') else: if namespace and namespace != get_jwt_identity(): return error(401, 'you cannot delete other namespaces') namespace = get_jwt_identity() instances_del = [] tasks_by_node = {} for instance in list(db.get_instances(all=all, namespace=namespace)): if instance['state'] in ['deleted', 'error']: continue # If this instance is not on a node, just do the DB cleanup locally if not instance['node']: node = config.parsed.get('NODE_NAME') else: node = instance['node'] tasks_by_node.setdefault(node, []) tasks_by_node[node].append({ 'type': 'instance_delete', 'instance_uuid': instance['uuid'], 'next_state': 'deleted', 'next_state_message': None }) instances_del.append(instance['uuid']) for node in tasks_by_node: db.enqueue(node, {'tasks': tasks_by_node[node]}) waiting_for = copy.copy(instances_del) start_time = time.time() while ( waiting_for and (time.time() - start_time < config.parsed.get('API_ASYNC_WAIT'))): for instance_uuid in copy.copy(waiting_for): i = db.get_instance(instance_uuid) if i['state'] in ['deleted', 'error']: waiting_for.remove(instance_uuid) return instances_del
def ensure_mesh(self): with db.get_lock('network', None, self.uuid, ttl=120): removed = [] added = [] instances = [] for iface in db.get_network_interfaces(self.uuid): if not iface['instance_uuid'] in instances: instances.append(iface['instance_uuid']) node_fqdns = [] for inst in instances: i = db.get_instance(inst) if not i: continue if not i['node']: continue if not i['node'] in node_fqdns: node_fqdns.append(i['node']) # NOTE(mikal): why not use DNS here? Well, DNS might be outside # the control of the deployer if we're running in a public cloud # as an overlay cloud... node_ips = [config.parsed.get('NETWORK_NODE_IP')] for fqdn in node_fqdns: ip = db.get_node(fqdn)['ip'] if ip not in node_ips: node_ips.append(ip) discovered = list(self.discover_mesh()) logutil.debug([self], 'Discovered mesh elements %s' % discovered) for node in discovered: if node in node_ips: node_ips.remove(node) else: self._remove_mesh_element(node) removed.append(node) for node in node_ips: self._add_mesh_element(node) added.append(node) if removed: db.add_event('network', self.uuid, 'remove mesh elements', None, None, ' '.join(removed)) if added: db.add_event('network', self.uuid, 'add mesh elements', None, None, ' '.join(added))
def from_db(uuid): if not uuid: return None db_data = db.get_instance(uuid) if not db_data: return None # Handle pre-versioning DB entries if 'version' not in db_data: # Handle old version that also does not have video information if 'video' not in db_data: db_data['video'] = {'model': 'cirrus', 'memory': 16384} version = 1 else: version = db_data['version'] del db_data['version'] if version == 1: return Instance(**db_data) # Version number is unknown raise exceptions.BadMetadataPacket( 'Unknown version - Instance: %s', db_data)
def delete(self, instance_uuid=None, instance_from_db=None): # Check if instance has already been deleted if instance_from_db['state'] == 'deleted': return error(404, 'instance not found') # If this instance is not on a node, just do the DB cleanup locally if not instance_from_db['node']: node = config.parsed.get('NODE_NAME') else: node = instance_from_db['node'] db.enqueue_instance_delete(node, instance_from_db['uuid'], 'deleted', None) start_time = time.time() while time.time() - start_time < config.parsed.get('API_ASYNC_WAIT'): i = db.get_instance(instance_uuid) if i['state'] in ['deleted', 'error']: return time.sleep(0.5) return
def from_db(uuid): db_entry = db.get_instance(uuid) if not db_entry: return None return Instance(db_entry)
def _update_power_states(self): libvirt = util.get_libvirt() conn = libvirt.open(None) try: seen = [] # Active VMs have an ID. Active means running in libvirt # land. for domain_id in conn.listDomainsID(): domain = conn.lookupByID(domain_id) if not domain.name().startswith('sf:'): continue instance_uuid = domain.name().split(':')[1] log_ctx = LOG.withInstance(instance_uuid) instance = db.get_instance(instance_uuid) if not instance: # Instance is SF but not in database. Kill to reduce load. log_ctx.warning('Destroying unknown instance') util.execute(None, 'virsh destroy "sf:%s"' % instance_uuid) continue db.place_instance(instance_uuid, config.NODE_NAME) seen.append(domain.name()) if instance.get('state') == 'deleted': # NOTE(mikal): a delete might be in-flight in the queue. # We only worry about instances which should have gone # away five minutes ago. if time.time() - instance['state_updated'] < 300: continue db.instance_enforced_deletes_increment(instance_uuid) attempts = instance.get('enforced_deletes', 0) if attempts > 5: # Sometimes we just can't delete the VM. Try the big hammer instead. log_ctx.warning( 'Attempting alternate delete method for instance') util.execute(None, 'virsh destroy "sf:%s"' % instance_uuid) db.add_event('instance', instance_uuid, 'enforced delete', 'complete', None, None) else: i = virt.from_db(instance_uuid) i.delete() i.update_instance_state('deleted') log_ctx.withField( 'attempt', attempts).warning('Deleting stray instance') continue state = util.extract_power_state(libvirt, domain) db.update_instance_power_state(instance_uuid, state) if state == 'crashed': db.update_instance_state(instance_uuid, 'error') # Inactive VMs just have a name, and are powered off # in our state system. for domain_name in conn.listDefinedDomains(): if not domain_name.startswith('sf:'): continue if domain_name not in seen: instance_uuid = domain_name.split(':')[1] log_ctx = LOG.withInstance(instance_uuid) instance = db.get_instance(instance_uuid) if not instance: # Instance is SF but not in database. Kill because unknown. log_ctx.warning('Removing unknown inactive instance') domain = conn.lookupByName(domain_name) domain.undefine() continue if instance.get('state') == 'deleted': # NOTE(mikal): a delete might be in-flight in the queue. # We only worry about instances which should have gone # away five minutes ago. if time.time() - instance['state_updated'] < 300: continue domain = conn.lookupByName(domain_name) domain.undefine() log_ctx.info('Detected stray instance') db.add_event('instance', instance_uuid, 'deleted stray', 'complete', None, None) continue db.place_instance(instance_uuid, config.NODE_NAME) instance_path = os.path.join(config.get('STORAGE_PATH'), 'instances', instance_uuid) if not os.path.exists(instance_path): # If we're inactive and our files aren't on disk, # we have a problem. log_ctx.info('Detected error state for instance') db.update_instance_state(instance_uuid, 'error') elif instance.get('power_state') != 'off': log_ctx.info('Detected power off for instance') db.update_instance_power_state(instance_uuid, 'off') db.add_event('instance', instance_uuid, 'detected poweroff', 'complete', None, None) except libvirt.libvirtError as e: LOG.error('Failed to lookup all domains: %s' % e)
def post(self, name=None, cpus=None, memory=None, network=None, disk=None, ssh_key=None, user_data=None, placed_on=None, namespace=None, instance_uuid=None, video=None): global SCHEDULER # Check that the instance name is safe for use as a DNS host name if name != re.sub(r'([^a-zA-Z0-9_\-])', '', name) or len(name) > 63: return error(400, 'instance name must be useable as a DNS host name') # Sanity check if not disk: return error(400, 'instance must specify at least one disk') for d in disk: if not isinstance(d, dict): return error(400, 'disk specification should contain JSON objects') if network: for n in network: if not isinstance(n, dict): return error( 400, 'network specification should contain JSON objects') if 'network_uuid' not in n: return error( 400, 'network specification is missing network_uuid') if not video: video = {'model': 'cirrus', 'memory': 16384} if not namespace: namespace = get_jwt_identity() # Only system can specify a uuid if instance_uuid and get_jwt_identity() != 'system': return error(401, 'only system can specify an instance uuid') # If accessing a foreign namespace, we need to be an admin if get_jwt_identity() not in [namespace, 'system']: return error( 401, 'only admins can create resources in a different namespace') # The instance needs to exist in the DB before network interfaces are created if not instance_uuid: instance_uuid = str(uuid.uuid4()) db.add_event('instance', instance_uuid, 'uuid allocated', None, None, None) # Create instance object instance = virt.from_db(instance_uuid) if instance: if get_jwt_identity() not in [ instance.db_entry['namespace'], 'system' ]: logutil.info([virt.ThinInstance(instance_uuid)], 'Instance not found, ownership test') return error(404, 'instance not found') if not instance: instance = virt.from_definition(uuid=instance_uuid, name=name, disks=disk, memory_mb=memory, vcpus=cpus, ssh_key=ssh_key, user_data=user_data, owner=namespace, video=video, requested_placement=placed_on) # Initialise metadata db.persist_metadata('instance', instance_uuid, {}) # Allocate IP addresses order = 0 if network: for netdesc in network: n = net.from_db(netdesc['network_uuid']) if not n: db.enqueue_instance_delete( config.parsed.get('NODE_NAME'), instance_uuid, 'error', 'missing network %s during IP allocation phase' % netdesc['network_uuid']) return error( 404, 'network %s not found' % netdesc['network_uuid']) with db.get_lock('ipmanager', None, netdesc['network_uuid'], ttl=120): db.add_event('network', netdesc['network_uuid'], 'allocate address', None, None, instance_uuid) ipm = db.get_ipmanager(netdesc['network_uuid']) if 'address' not in netdesc or not netdesc['address']: netdesc['address'] = ipm.get_random_free_address() else: if not ipm.reserve(netdesc['address']): db.enqueue_instance_delete( config.parsed.get('NODE_NAME'), instance_uuid, 'error', 'failed to reserve an IP on network %s' % netdesc['network_uuid']) return error( 409, 'address %s in use' % netdesc['address']) db.persist_ipmanager(netdesc['network_uuid'], ipm.save()) if 'model' not in netdesc or not netdesc['model']: netdesc['model'] = 'virtio' db.create_network_interface(str(uuid.uuid4()), netdesc, instance_uuid, order) if not SCHEDULER: SCHEDULER = scheduler.Scheduler() try: # Have we been placed? if not placed_on: candidates = SCHEDULER.place_instance(instance, network) placement = candidates[0] else: SCHEDULER.place_instance(instance, network, candidates=[placed_on]) placement = placed_on except exceptions.LowResourceException as e: db.add_event('instance', instance_uuid, 'schedule', 'failed', None, 'insufficient resources: ' + str(e)) db.enqueue_instance_delete(config.parsed.get('NODE_NAME'), instance_uuid, 'error', 'scheduling failed') return error(507, str(e)) except exceptions.CandidateNodeNotFoundException as e: db.add_event('instance', instance_uuid, 'schedule', 'failed', None, 'candidate node not found: ' + str(e)) db.enqueue_instance_delete(config.get.parsed('NODE_NAME'), instance_uuid, 'error', 'scheduling failed') return error(404, 'node not found: %s' % e) # Record placement db.place_instance(instance_uuid, placement) db.add_event('instance', instance_uuid, 'placement', None, None, placement) # Create a queue entry for the instance start tasks = [{ 'type': 'instance_preflight', 'instance_uuid': instance_uuid, 'network': network }] for disk in instance.db_entry['block_devices']['devices']: if 'base' in disk and disk['base']: tasks.append({ 'type': 'image_fetch', 'instance_uuid': instance_uuid, 'url': disk['base'] }) tasks.append({ 'type': 'instance_start', 'instance_uuid': instance_uuid, 'network': network }) # Enqueue creation tasks on desired node task queue db.enqueue(placement, {'tasks': tasks}) db.add_event('instance', instance_uuid, 'create', 'enqueued', None, None) # Watch for a while and return results if things are fast, give up # after a while and just return the current state start_time = time.time() while time.time() - start_time < config.parsed.get('API_ASYNC_WAIT'): i = db.get_instance(instance_uuid) if i['state'] in ['created', 'deleted', 'error']: return i time.sleep(0.5) return i
def _maintain_networks(self): LOG.info('Maintaining networks') # Discover what networks are present _, _, vxid_to_mac = util.discover_interfaces() # Determine what networks we should be on host_networks = [] seen_vxids = [] if not util.is_network_node(): # For normal nodes, just the ones we have instances for for inst in list(db.get_instances(only_node=config.parsed.get('NODE_NAME'))): for iface in db.get_instance_interfaces(inst['uuid']): if not iface['network_uuid'] in host_networks: host_networks.append(iface['network_uuid']) else: # For network nodes, its all networks for n in db.get_networks(): host_networks.append(n['uuid']) # Network nodes also look for interfaces for absent instances # and delete them for ni in db.get_network_interfaces(n['uuid']): inst = db.get_instance(ni['instance_uuid']) if (not inst or inst.get('state', 'unknown') in ['deleted', 'error', 'unknown']): db.hard_delete_network_interface(ni['uuid']) LOG.withInstance( ni['instance_uuid']).withNetworkInterface( ni['uuid']).info('Hard deleted stray network interface') # Ensure we are on every network we have a host for for network in host_networks: try: n = net.from_db(network) if not n: continue if n.db_entry['state_updated'] - time.time() < 60: # Network state changed in the last minute, punt for now continue if not n.is_okay(): LOG.withObj(n).info('Recreating not okay network') n.create() n.ensure_mesh() seen_vxids.append(n.vxlan_id) except exceptions.LockException as e: LOG.warning( 'Failed to acquire lock while maintaining networks: %s' % e) # Determine if there are any extra vxids extra_vxids = set(vxid_to_mac.keys()) - set(seen_vxids) # Delete "deleted" SF networks and log unknown vxlans if extra_vxids: LOG.withField('vxids', extra_vxids).warning( 'Extra vxlans present!') # Determine the network uuids for those vxids # vxid_to_uuid = {} # for n in db.get_networks(): # vxid_to_uuid[n['vxid']] = n['uuid'] # for extra in extra_vxids: # if extra in vxid_to_uuid: # with db.get_lock('network', None, vxid_to_uuid[extra], # ttl=120, op='Network reap VXLAN'): # n = net.from_db(vxid_to_uuid[extra]) # n.delete() # LOG.info('Extra vxlan %s (network %s) removed.' # % (extra, vxid_to_uuid[extra])) # else: # LOG.error('Extra vxlan %s does not map to any network.' # % extra) # And record vxids in the database db.persist_node_vxid_mapping( config.parsed.get('NODE_NAME'), vxid_to_mac)
def post(self, name=None, cpus=None, memory=None, network=None, disk=None, ssh_key=None, user_data=None, placed_on=None, namespace=None, instance_uuid=None): global SCHEDULER # We need to sanitise the name so its safe for DNS name = re.sub(r'([^a-zA-Z0-9_\-])', '', name) if not namespace: namespace = get_jwt_identity() # If accessing a foreign namespace, we need to be an admin if get_jwt_identity() not in [namespace, 'system']: return error( 401, 'only admins can create resources in a different namespace') # The instance needs to exist in the DB before network interfaces are created if not instance_uuid: instance_uuid = str(uuid.uuid4()) db.add_event('instance', instance_uuid, 'uuid allocated', None, None, None) # Create instance object instance = virt.from_db(instance_uuid) if instance: if get_jwt_identity() not in [ instance.db_entry['namespace'], 'system' ]: LOG.info('instance(%s): instance not found, ownership test' % instance_uuid) return error(404, 'instance not found') if not instance: instance = virt.from_definition(uuid=instance_uuid, name=name, disks=disk, memory_mb=memory, vcpus=cpus, ssh_key=ssh_key, user_data=user_data, owner=namespace) if not SCHEDULER: SCHEDULER = scheduler.Scheduler() # Have we been placed? if not placed_on: candidates = SCHEDULER.place_instance(instance, network) if len(candidates) == 0: db.add_event('instance', instance_uuid, 'schedule', 'failed', None, 'insufficient resources') db.update_instance_state(instance_uuid, 'error') return error(507, 'insufficient capacity') placed_on = candidates[0] db.place_instance(instance_uuid, placed_on) db.add_event('instance', instance_uuid, 'placement', None, None, placed_on) else: try: candidates = SCHEDULER.place_instance(instance, network, candidates=[placed_on]) if len(candidates) == 0: db.add_event('instance', instance_uuid, 'schedule', 'failed', None, 'insufficient resources') db.update_instance_state(instance_uuid, 'error') return error(507, 'insufficient capacity') except scheduler.CandidateNodeNotFoundException as e: return error(404, 'node not found: %s' % e) # Have we been placed on a different node? if not placed_on == config.parsed.get('NODE_NAME'): body = flask_get_post_body() body['placed_on'] = placed_on body['instance_uuid'] = instance_uuid body['namespace'] = namespace token = util.get_api_token( 'http://%s:%d' % (placed_on, config.parsed.get('API_PORT')), namespace=namespace) r = requests.request('POST', 'http://%s:%d/instances' % (placed_on, config.parsed.get('API_PORT')), data=json.dumps(body), headers={ 'Authorization': token, 'User-Agent': util.get_user_agent() }) LOG.info('Returning proxied request: %d, %s' % (r.status_code, r.text)) resp = flask.Response(r.text, mimetype='application/json') resp.status_code = r.status_code return resp # Check we can get the required IPs nets = {} allocations = {} def error_with_cleanup(status_code, message): for network_uuid in allocations: n = net.from_db(network_uuid) for addr, _ in allocations[network_uuid]: with db.get_lock('sf/ipmanager/%s' % n.uuid, ttl=120) as _: ipm = db.get_ipmanager(n.uuid) ipm.release(addr) db.persist_ipmanager(n.uuid, ipm.save()) return error(status_code, message) order = 0 if network: for netdesc in network: if 'network_uuid' not in netdesc or not netdesc['network_uuid']: return error_with_cleanup(404, 'network not specified') if netdesc['network_uuid'] not in nets: n = net.from_db(netdesc['network_uuid']) if not n: return error_with_cleanup( 404, 'network %s not found' % netdesc['network_uuid']) nets[netdesc['network_uuid']] = n n.create() with db.get_lock('sf/ipmanager/%s' % netdesc['network_uuid'], ttl=120) as _: db.add_event('network', netdesc['network_uuid'], 'allocate address', None, None, instance_uuid) allocations.setdefault(netdesc['network_uuid'], []) ipm = db.get_ipmanager(netdesc['network_uuid']) if 'address' not in netdesc or not netdesc['address']: netdesc['address'] = ipm.get_random_free_address() else: if not ipm.reserve(netdesc['address']): return error_with_cleanup( 409, 'address %s in use' % netdesc['address']) db.persist_ipmanager(netdesc['network_uuid'], ipm.save()) allocations[netdesc['network_uuid']].append( (netdesc['address'], order)) if 'model' not in netdesc or not netdesc['model']: netdesc['model'] = 'virtio' db.create_network_interface(str(uuid.uuid4()), netdesc, instance_uuid, order) order += 1 # Initialise metadata db.persist_metadata('instance', instance_uuid, {}) # Now we can start the instance with db.get_lock('sf/instance/%s' % instance.db_entry['uuid'], ttl=900) as lock: with util.RecordedOperation('ensure networks exist', instance) as _: for network_uuid in nets: n = nets[network_uuid] n.ensure_mesh() n.update_dhcp() with util.RecordedOperation('instance creation', instance) as _: instance.create(lock=lock) for iface in db.get_instance_interfaces(instance.db_entry['uuid']): db.update_network_interface_state(iface['uuid'], 'created') return db.get_instance(instance_uuid)
def post(self, name=None, cpus=None, memory=None, network=None, disk=None, ssh_key=None, user_data=None): # The instance needs to exist in the DB before network interfaces are created new_instance_uuid = str(uuid.uuid4()) db.add_event('instance', new_instance_uuid, 'API CREATE', None, None, None) instance = virt.from_definition( uuid=new_instance_uuid, name=name, disks=disk, memory_mb=memory * 1024, vcpus=cpus, ssh_key=ssh_key, user_data=user_data ) # Check we can get the required IPs nets = {} allocations = {} def error_with_cleanup(status_code, message): for network_uuid in allocations: n = net.from_db(network_uuid) for addr, _ in allocations[network_uuid]: n.ipmanager.release(addr) n.persist_ipmanager() return error(status_code, message) order = 0 for netdesc in network: if not 'network_uuid' in netdesc or not netdesc['network_uuid']: error_with_cleanup(404, 'network not specified') if not netdesc['network_uuid'] in nets: n = net.from_db(netdesc['network_uuid']) if not n: error_with_cleanup( 404, 'network %s not found' % netdesc['network_uuid']) nets[netdesc['network_uuid']] = n n.create() allocations.setdefault(netdesc['network_uuid'], []) if not 'address' in netdesc or not netdesc['address']: netdesc['address'] = nets[netdesc['network_uuid'] ].ipmanager.get_random_free_address() else: if not nets[netdesc['network_uuid']].ipmanager.reserve(netdesc['address']): error_with_cleanup(409, 'address %s in use' % netdesc['address']) nets[netdesc['network_uuid']].persist_ipmanager() allocations[netdesc['network_uuid']].append( (netdesc['address'], order)) if not 'macaddress' in netdesc or not netdesc['macaddress']: netdesc['macaddress'] = str(randmac.RandMac( '00:00:00:00:00:00', True)).lstrip('\'').rstrip('\'') db.create_network_interface( str(uuid.uuid4()), netdesc['network_uuid'], new_instance_uuid, netdesc['macaddress'], netdesc['address'], order) order += 1 # Now we can start the instance with util.RecordedOperation('ensure networks exist', instance) as _: for network_uuid in nets: n = nets[network_uuid] n.ensure_mesh() n.update_dhcp() with util.RecordedOperation('instance creation', instance) as _: instance.create() return db.get_instance(new_instance_uuid)