def delete(self): with util.RecordedOperation('delete domain', self) as _: try: self.power_off() instance = self._get_domain() instance.undefine() except Exception: pass with util.RecordedOperation('delete disks', self) as _: try: shutil.rmtree(self.instance_path) except Exception: pass with util.RecordedOperation('release network addreses', self) as _: for ni in db.get_instance_interfaces(self.db_entry['uuid']): with db.get_lock('sf/ipmanager/%s' % ni['network_uuid'], ttl=120) as _: ipm = db.get_ipmanager(ni['network_uuid']) ipm.release(ni['ipv4']) db.persist_ipmanager(ni['network_uuid'], ipm.save()) db.update_instance_state(self.db_entry['uuid'], 'deleted') db.free_console_port(self.db_entry['console_port']) db.free_console_port(self.db_entry['vdi_port'])
def post(self, interface_uuid=None): ni = db.get_interface(interface_uuid) if not ni: return error(404, 'network interface not found') if not ni['floating']: return error(409, 'this interface does not have a floating ip') n = net.from_db(ni['network_uuid']) if not n: LOG.info('network(%s): network not found, genuinely missing' % ni['network_uuid']) return error(404, 'network not found') if get_jwt_identity() not in [n.namespace, 'system']: LOG.info('%s: network not found, ownership test' % n) return error(404, 'network not found') i = virt.from_db(ni['instance_uuid']) if get_jwt_identity() not in [i.db_entry['namespace'], 'system']: LOG.info('%s: instance not found, ownership test' % i) return error(404, 'instance not found') float_net = net.from_db('floating') if not float_net: return error(404, 'floating network not found') db.add_event('interface', interface_uuid, 'api', 'defloat', None, None) with db.get_lock('sf/ipmanager/floating', ttl=120) as _: ipm = db.get_ipmanager('floating') ipm.release(ni['floating']) db.persist_ipmanager('floating', ipm.save()) db.remove_floating_from_interface(ni['uuid']) n.remove_floating_ip(ni['floating'], ni['ipv4'])
def delete(self): with util.RecordedOperation('delete domain', self): try: self.power_off() instance = self._get_domain() if instance: instance.undefine() except Exception as e: util.ignore_exception('instance delete', e) with util.RecordedOperation('delete disks', self): try: if os.path.exists(self.instance_path): shutil.rmtree(self.instance_path) except Exception as e: util.ignore_exception('instance delete', e) with util.RecordedOperation('release network addresses', self): for ni in db.get_instance_interfaces(self.db_entry['uuid']): db.update_network_interface_state(ni['uuid'], 'deleted') with db.get_lock('ipmanager', None, ni['network_uuid'], ttl=120, op='Instance delete'): ipm = db.get_ipmanager(ni['network_uuid']) ipm.release(ni['ipv4']) db.persist_ipmanager(ni['network_uuid'], ipm.save()) db.free_console_port(self.db_entry['console_port']) db.free_console_port(self.db_entry['vdi_port'])
def delete(self, network_uuid=None, network_from_db=None): db.add_event('network', network_uuid, 'api', 'delete', None, None) if network_uuid == 'floating': return error(403, 'you cannot delete the floating network') # We only delete unused networks if len(list(db.get_network_interfaces(network_uuid))) > 0: return error(403, 'you cannot delete an in use network') # Check if network has already been deleted if network_from_db['state'] == 'deleted': return error(404, 'network not found') with db.get_lock('sf/network/%s' % network_uuid, ttl=900) as _: n = net.from_db(network_uuid) n.remove_dhcp() n.delete() if n.floating_gateway: with db.get_lock('sf/ipmanager/floating', ttl=120) as _: ipm = db.get_ipmanager('floating') ipm.release(n.floating_gateway) db.persist_ipmanager('floating', ipm.save()) db.update_network_state(network_uuid, 'deleted')
def __init__(self, uuid=None, vxlan_id=1, provide_dhcp=False, provide_nat=False, physical_nic='eth0', ipblock=None, floating_gateway=None, namespace=None): self.uuid = uuid self.vxlan_id = vxlan_id self.provide_dhcp = provide_dhcp self.provide_nat = provide_nat self.physical_nic = physical_nic self.floating_gateway = floating_gateway self.namespace = namespace with db.get_lock('ipmanager', None, self.uuid, ttl=120): ipm = db.get_ipmanager(self.uuid) self.ipblock = ipm.network_address self.router = ipm.get_address_at_index(1) self.dhcp_start = ipm.get_address_at_index(2) self.netmask = ipm.netmask self.broadcast = ipm.broadcast_address self.network_address = ipm.network_address ipm.reserve(self.router) db.persist_ipmanager(self.uuid, ipm.save())
def error_with_cleanup(status_code, message): for network_uuid in allocations: n = net.from_db(network_uuid) for addr, _ in allocations[network_uuid]: with db.get_lock('sf/ipmanager/%s' % n.uuid, ttl=120) as _: ipm = db.get_ipmanager(n.uuid) ipm.release(addr) db.persist_ipmanager(n.uuid, ipm.save()) return error(status_code, message)
def persist(self): d = { 'ipmanager.v3': { 'ipblock': self.ipblock, 'in_use': self.in_use, 'uuid': self.uuid } } db.persist_ipmanager(self.uuid, d)
def deploy_nat(self): if not self.provide_nat: return subst = self.subst_dict() if not self.floating_gateway: with db.get_lock('ipmanager', None, 'floating', ttl=120): ipm = db.get_ipmanager('floating') self.floating_gateway = ipm.get_random_free_address() db.persist_ipmanager('floating', ipm.save()) self.persist_floating_gateway() # No lock because no data changing ipm = db.get_ipmanager('floating') subst['floating_router'] = ipm.get_address_at_index(1) subst['floating_gateway'] = self.floating_gateway subst['floating_netmask'] = ipm.netmask with db.get_lock('network', None, self.uuid, ttl=120): if not subst['floating_gateway'] in list( util.get_interface_addresses( subst['netns'], subst['physical_veth_inner'])): with util.RecordedOperation('enable virtual routing', self): util.execute( None, '%(in_netns)s ip addr add %(floating_gateway)s/%(floating_netmask)s ' 'dev %(physical_veth_inner)s' % subst) util.execute( None, '%(in_netns)s ip link set %(physical_veth_inner)s up' % subst) util.execute( None, '%(in_netns)s route add default gw %(floating_router)s' % subst) if not util.nat_rules_for_ipblock(self.network_address): with util.RecordedOperation('enable nat', self): util.execute(None, 'echo 1 > /proc/sys/net/ipv4/ip_forward') util.execute( None, '%(in_netns)s iptables -A FORWARD -o %(physical_veth_inner)s ' '-i %(vx_veth_inner)s -j ACCEPT' % subst) util.execute( None, '%(in_netns)s iptables -A FORWARD -i %(physical_veth_inner)s ' '-o %(vx_veth_inner)s -j ACCEPT' % subst) util.execute( None, '%(in_netns)s iptables -t nat -A POSTROUTING -s %(ipblock)s/%(netmask)s ' '-o %(physical_veth_inner)s -j MASQUERADE' % subst)
def _delete_network(network_from_db): network_uuid = network_from_db['uuid'] db.add_event('network', network_uuid, 'api', 'delete', None, None) n = net.from_db(network_uuid) n.remove_dhcp() n.delete() if n.floating_gateway: with db.get_lock('ipmanager', None, 'floating', ttl=120): ipm = db.get_ipmanager('floating') ipm.release(n.floating_gateway) db.persist_ipmanager('floating', ipm.save()) db.update_network_state(network_uuid, 'deleted')
def post(self, interface_uuid=None): ni, n, err = _safe_get_network_interface(interface_uuid) if err: return err float_net = net.from_db('floating') if not float_net: return error(404, 'floating network not found') db.add_event('interface', interface_uuid, 'api', 'float', None, None) with db.get_lock('ipmanager', None, 'floating', ttl=120): ipm = db.get_ipmanager('floating') addr = ipm.get_random_free_address() db.persist_ipmanager('floating', ipm.save()) db.add_floating_to_interface(ni['uuid'], addr) n.add_floating_ip(addr, ni['ipv4'])
def post(self, interface_uuid=None): ni, n, err = _safe_get_network_interface(interface_uuid) if err: return err float_net = net.from_db('floating') if not float_net: return error(404, 'floating network not found') db.add_event('interface', interface_uuid, 'api', 'defloat', None, None) with db.get_lock('ipmanager', None, 'floating', ttl=120): ipm = db.get_ipmanager('floating') ipm.release(ni['floating']) db.persist_ipmanager('floating', ipm.save()) db.remove_floating_from_interface(ni['uuid']) n.remove_floating_ip(ni['floating'], ni['ipv4'])
def __init__(self, db_entry): self.db_entry = db_entry self.physical_nic = config.get('NODE_EGRESS_NIC') with db.get_lock('ipmanager', None, self.db_entry['uuid'], ttl=120, op='Network object initialization'): ipm = db.get_ipmanager(self.db_entry['uuid']) self.ipblock = ipm.network_address self.router = ipm.get_address_at_index(1) self.dhcp_start = ipm.get_address_at_index(2) self.netmask = ipm.netmask self.broadcast = ipm.broadcast_address self.network_address = ipm.network_address ipm.reserve(self.router) db.persist_ipmanager(self.db_entry['uuid'], ipm.save())
def delete(self): subst = self.subst_dict() LOG.withFields(subst).debug('net.delete()') # Cleanup local node with db.get_object_lock(self, ttl=120, op='Network delete'): if util.check_for_interface(subst['vx_bridge']): with util.RecordedOperation('delete vxlan bridge', self): util.execute(None, 'ip link delete %(vx_bridge)s' % subst) if util.check_for_interface(subst['vx_interface']): with util.RecordedOperation('delete vxlan interface', self): util.execute(None, 'ip link delete %(vx_interface)s' % subst) # If this is the network node do additional cleanup if util.is_network_node(): if util.check_for_interface(subst['vx_veth_outer']): with util.RecordedOperation('delete router veth', self): util.execute( None, 'ip link delete %(vx_veth_outer)s' % subst) if util.check_for_interface(subst['physical_veth_outer']): with util.RecordedOperation('delete physical veth', self): util.execute( None, 'ip link delete %(physical_veth_outer)s' % subst) if os.path.exists('/var/run/netns/%(netns)s' % subst): with util.RecordedOperation('delete netns', self): util.execute(None, 'ip netns del %(netns)s' % subst) if self.db_entry['floating_gateway']: with db.get_lock('ipmanager', None, 'floating', ttl=120, op='Network delete'): ipm = db.get_ipmanager('floating') ipm.release(self.db_entry['floating_gateway']) db.persist_ipmanager('floating', ipm.save())
def delete(self): subst = self.subst_dict() # Cleanup local node with db.get_lock('sf/net/%s' % self.uuid, ttl=120) as _: if util.check_for_interface(subst['vx_bridge']): with util.RecordedOperation('delete vxlan bridge', self) as _: processutils.execute('ip link delete %(vx_bridge)s' % subst, shell=True) if util.check_for_interface(subst['vx_interface']): with util.RecordedOperation('delete vxlan interface', self) as _: processutils.execute('ip link delete %(vx_interface)s' % subst, shell=True) # If this is the network node do additional cleanup if config.parsed.get('NODE_IP') == config.parsed.get('NETWORK_NODE_IP'): if util.check_for_interface(subst['vx_veth_outer']): with util.RecordedOperation('delete router veth', self) as _: processutils.execute('ip link delete %(vx_veth_outer)s' % subst, shell=True) if util.check_for_interface(subst['physical_veth_outer']): with util.RecordedOperation('delete physical veth', self) as _: processutils.execute('ip link delete %(physical_veth_outer)s' % subst, shell=True) if os.path.exists('/var/run/netns/%(netns)s' % subst): with util.RecordedOperation('delete netns', self) as _: processutils.execute('ip netns del %(netns)s' % subst, shell=True) if self.floating_gateway: with db.get_lock('sf/ipmanager/floating', ttl=120) as _: ipm = db.get_ipmanager('floating') ipm.release(self.floating_gateway) db.persist_ipmanager('floating', ipm.save())
def post(self, name=None, cpus=None, memory=None, network=None, disk=None, ssh_key=None, user_data=None, placed_on=None, namespace=None, instance_uuid=None): global SCHEDULER # We need to sanitise the name so its safe for DNS name = re.sub(r'([^a-zA-Z0-9_\-])', '', name) if not namespace: namespace = get_jwt_identity() # If accessing a foreign namespace, we need to be an admin if get_jwt_identity() not in [namespace, 'system']: return error( 401, 'only admins can create resources in a different namespace') # The instance needs to exist in the DB before network interfaces are created if not instance_uuid: instance_uuid = str(uuid.uuid4()) db.add_event('instance', instance_uuid, 'uuid allocated', None, None, None) # Create instance object instance = virt.from_db(instance_uuid) if instance: if get_jwt_identity() not in [ instance.db_entry['namespace'], 'system' ]: LOG.info('instance(%s): instance not found, ownership test' % instance_uuid) return error(404, 'instance not found') if not instance: instance = virt.from_definition(uuid=instance_uuid, name=name, disks=disk, memory_mb=memory, vcpus=cpus, ssh_key=ssh_key, user_data=user_data, owner=namespace) if not SCHEDULER: SCHEDULER = scheduler.Scheduler() # Have we been placed? if not placed_on: candidates = SCHEDULER.place_instance(instance, network) if len(candidates) == 0: db.add_event('instance', instance_uuid, 'schedule', 'failed', None, 'insufficient resources') db.update_instance_state(instance_uuid, 'error') return error(507, 'insufficient capacity') placed_on = candidates[0] db.place_instance(instance_uuid, placed_on) db.add_event('instance', instance_uuid, 'placement', None, None, placed_on) else: try: candidates = SCHEDULER.place_instance(instance, network, candidates=[placed_on]) if len(candidates) == 0: db.add_event('instance', instance_uuid, 'schedule', 'failed', None, 'insufficient resources') db.update_instance_state(instance_uuid, 'error') return error(507, 'insufficient capacity') except scheduler.CandidateNodeNotFoundException as e: return error(404, 'node not found: %s' % e) # Have we been placed on a different node? if not placed_on == config.parsed.get('NODE_NAME'): body = flask_get_post_body() body['placed_on'] = placed_on body['instance_uuid'] = instance_uuid body['namespace'] = namespace token = util.get_api_token( 'http://%s:%d' % (placed_on, config.parsed.get('API_PORT')), namespace=namespace) r = requests.request('POST', 'http://%s:%d/instances' % (placed_on, config.parsed.get('API_PORT')), data=json.dumps(body), headers={ 'Authorization': token, 'User-Agent': util.get_user_agent() }) LOG.info('Returning proxied request: %d, %s' % (r.status_code, r.text)) resp = flask.Response(r.text, mimetype='application/json') resp.status_code = r.status_code return resp # Check we can get the required IPs nets = {} allocations = {} def error_with_cleanup(status_code, message): for network_uuid in allocations: n = net.from_db(network_uuid) for addr, _ in allocations[network_uuid]: with db.get_lock('sf/ipmanager/%s' % n.uuid, ttl=120) as _: ipm = db.get_ipmanager(n.uuid) ipm.release(addr) db.persist_ipmanager(n.uuid, ipm.save()) return error(status_code, message) order = 0 if network: for netdesc in network: if 'network_uuid' not in netdesc or not netdesc['network_uuid']: return error_with_cleanup(404, 'network not specified') if netdesc['network_uuid'] not in nets: n = net.from_db(netdesc['network_uuid']) if not n: return error_with_cleanup( 404, 'network %s not found' % netdesc['network_uuid']) nets[netdesc['network_uuid']] = n n.create() with db.get_lock('sf/ipmanager/%s' % netdesc['network_uuid'], ttl=120) as _: db.add_event('network', netdesc['network_uuid'], 'allocate address', None, None, instance_uuid) allocations.setdefault(netdesc['network_uuid'], []) ipm = db.get_ipmanager(netdesc['network_uuid']) if 'address' not in netdesc or not netdesc['address']: netdesc['address'] = ipm.get_random_free_address() else: if not ipm.reserve(netdesc['address']): return error_with_cleanup( 409, 'address %s in use' % netdesc['address']) db.persist_ipmanager(netdesc['network_uuid'], ipm.save()) allocations[netdesc['network_uuid']].append( (netdesc['address'], order)) if 'model' not in netdesc or not netdesc['model']: netdesc['model'] = 'virtio' db.create_network_interface(str(uuid.uuid4()), netdesc, instance_uuid, order) order += 1 # Initialise metadata db.persist_metadata('instance', instance_uuid, {}) # Now we can start the instance with db.get_lock('sf/instance/%s' % instance.db_entry['uuid'], ttl=900) as lock: with util.RecordedOperation('ensure networks exist', instance) as _: for network_uuid in nets: n = nets[network_uuid] n.ensure_mesh() n.update_dhcp() with util.RecordedOperation('instance creation', instance) as _: instance.create(lock=lock) for iface in db.get_instance_interfaces(instance.db_entry['uuid']): db.update_network_interface_state(iface['uuid'], 'created') return db.get_instance(instance_uuid)
def deploy_nat(self): if not self.db_entry['provide_nat']: return subst = self.subst_dict() if not self.db_entry['floating_gateway']: with db.get_lock('ipmanager', None, 'floating', ttl=120, op='Network deploy NAT'): ipm = db.get_ipmanager('floating') self.db_entry[ 'floating_gateway'] = ipm.get_random_free_address() db.persist_ipmanager('floating', ipm.save()) self.persist_floating_gateway() # No lock because no data changing ipm = db.get_ipmanager('floating') subst['floating_router'] = ipm.get_address_at_index(1) subst['floating_gateway'] = self.db_entry['floating_gateway'] subst['floating_netmask'] = ipm.netmask with db.get_object_lock(self, ttl=120, op='Network deploy NAT'): # Ensure network was not deleted whilst waiting for the lock. if self.is_dead(): raise DeadNetwork('network=%s' % self) with util.RecordedOperation('enable virtual routing', self): addresses = util.get_interface_addresses( subst['netns'], subst['physical_veth_inner']) if not subst['floating_gateway'] in list(addresses): util.execute( None, '%(in_netns)s ip addr add ' '%(floating_gateway)s/%(floating_netmask)s ' 'dev %(physical_veth_inner)s' % subst) util.execute( None, '%(in_netns)s ip link set ' '%(physical_veth_inner)s up' % subst) default_routes = util.get_default_routes(subst['netns']) if default_routes != [subst['floating_router']]: if default_routes: for default_route in default_routes: util.execute( None, '%s route del default gw %s' % (subst['in_netns'], default_route)) util.execute( None, '%(in_netns)s route add default ' 'gw %(floating_router)s' % subst) if not util.nat_rules_for_ipblock(self.network_address): with util.RecordedOperation('enable nat', self): util.execute(None, 'echo 1 > /proc/sys/net/ipv4/ip_forward') util.execute( None, '%(in_netns)s iptables -A FORWARD ' '-o %(physical_veth_inner)s ' '-i %(vx_veth_inner)s -j ACCEPT' % subst) util.execute( None, '%(in_netns)s iptables -A FORWARD ' '-i %(physical_veth_inner)s ' '-o %(vx_veth_inner)s -j ACCEPT' % subst) util.execute( None, '%(in_netns)s iptables -t nat -A POSTROUTING ' '-s %(ipblock)s/%(netmask)s ' '-o %(physical_veth_inner)s ' '-j MASQUERADE' % subst)
def persist_ipmanager(self): db.persist_ipmanager(self.uuid, self.ipmanager.save())
def post(self, name=None, cpus=None, memory=None, network=None, disk=None, ssh_key=None, user_data=None, placed_on=None, namespace=None, instance_uuid=None, video=None): global SCHEDULER # Check that the instance name is safe for use as a DNS host name if name != re.sub(r'([^a-zA-Z0-9_\-])', '', name) or len(name) > 63: return error(400, 'instance name must be useable as a DNS host name') # Sanity check if not disk: return error(400, 'instance must specify at least one disk') for d in disk: if not isinstance(d, dict): return error(400, 'disk specification should contain JSON objects') if network: for n in network: if not isinstance(n, dict): return error( 400, 'network specification should contain JSON objects') if 'network_uuid' not in n: return error( 400, 'network specification is missing network_uuid') if not video: video = {'model': 'cirrus', 'memory': 16384} if not namespace: namespace = get_jwt_identity() # Only system can specify a uuid if instance_uuid and get_jwt_identity() != 'system': return error(401, 'only system can specify an instance uuid') # If accessing a foreign namespace, we need to be an admin if get_jwt_identity() not in [namespace, 'system']: return error( 401, 'only admins can create resources in a different namespace') # The instance needs to exist in the DB before network interfaces are created if not instance_uuid: instance_uuid = str(uuid.uuid4()) db.add_event('instance', instance_uuid, 'uuid allocated', None, None, None) # Create instance object instance = virt.from_db(instance_uuid) if instance: if get_jwt_identity() not in [ instance.db_entry['namespace'], 'system' ]: logutil.info([virt.ThinInstance(instance_uuid)], 'Instance not found, ownership test') return error(404, 'instance not found') if not instance: instance = virt.from_definition(uuid=instance_uuid, name=name, disks=disk, memory_mb=memory, vcpus=cpus, ssh_key=ssh_key, user_data=user_data, owner=namespace, video=video, requested_placement=placed_on) # Initialise metadata db.persist_metadata('instance', instance_uuid, {}) # Allocate IP addresses order = 0 if network: for netdesc in network: n = net.from_db(netdesc['network_uuid']) if not n: db.enqueue_instance_delete( config.parsed.get('NODE_NAME'), instance_uuid, 'error', 'missing network %s during IP allocation phase' % netdesc['network_uuid']) return error( 404, 'network %s not found' % netdesc['network_uuid']) with db.get_lock('ipmanager', None, netdesc['network_uuid'], ttl=120): db.add_event('network', netdesc['network_uuid'], 'allocate address', None, None, instance_uuid) ipm = db.get_ipmanager(netdesc['network_uuid']) if 'address' not in netdesc or not netdesc['address']: netdesc['address'] = ipm.get_random_free_address() else: if not ipm.reserve(netdesc['address']): db.enqueue_instance_delete( config.parsed.get('NODE_NAME'), instance_uuid, 'error', 'failed to reserve an IP on network %s' % netdesc['network_uuid']) return error( 409, 'address %s in use' % netdesc['address']) db.persist_ipmanager(netdesc['network_uuid'], ipm.save()) if 'model' not in netdesc or not netdesc['model']: netdesc['model'] = 'virtio' db.create_network_interface(str(uuid.uuid4()), netdesc, instance_uuid, order) if not SCHEDULER: SCHEDULER = scheduler.Scheduler() try: # Have we been placed? if not placed_on: candidates = SCHEDULER.place_instance(instance, network) placement = candidates[0] else: SCHEDULER.place_instance(instance, network, candidates=[placed_on]) placement = placed_on except exceptions.LowResourceException as e: db.add_event('instance', instance_uuid, 'schedule', 'failed', None, 'insufficient resources: ' + str(e)) db.enqueue_instance_delete(config.parsed.get('NODE_NAME'), instance_uuid, 'error', 'scheduling failed') return error(507, str(e)) except exceptions.CandidateNodeNotFoundException as e: db.add_event('instance', instance_uuid, 'schedule', 'failed', None, 'candidate node not found: ' + str(e)) db.enqueue_instance_delete(config.get.parsed('NODE_NAME'), instance_uuid, 'error', 'scheduling failed') return error(404, 'node not found: %s' % e) # Record placement db.place_instance(instance_uuid, placement) db.add_event('instance', instance_uuid, 'placement', None, None, placement) # Create a queue entry for the instance start tasks = [{ 'type': 'instance_preflight', 'instance_uuid': instance_uuid, 'network': network }] for disk in instance.db_entry['block_devices']['devices']: if 'base' in disk and disk['base']: tasks.append({ 'type': 'image_fetch', 'instance_uuid': instance_uuid, 'url': disk['base'] }) tasks.append({ 'type': 'instance_start', 'instance_uuid': instance_uuid, 'network': network }) # Enqueue creation tasks on desired node task queue db.enqueue(placement, {'tasks': tasks}) db.add_event('instance', instance_uuid, 'create', 'enqueued', None, None) # Watch for a while and return results if things are fast, give up # after a while and just return the current state start_time = time.time() while time.time() - start_time < config.parsed.get('API_ASYNC_WAIT'): i = db.get_instance(instance_uuid) if i['state'] in ['created', 'deleted', 'error']: return i time.sleep(0.5) return i