def delete(self, network_uuid=None, network_from_db=None):
db.add_event('network', network_uuid, 'api', 'delete', None, None)
if network_uuid == 'floating':
return error(403, 'you cannot delete the floating network')
# We only delete unused networks
if len(list(db.get_network_interfaces(network_uuid))) > 0:
return error(403, 'you cannot delete an in use network')
# Check if network has already been deleted
if network_from_db['state'] == 'deleted':
return error(404, 'network not found')
with db.get_lock('sf/network/%s' % network_uuid, ttl=900) as _:
n = net.from_db(network_uuid)
n.remove_dhcp()
n.delete()
if n.floating_gateway:
with db.get_lock('sf/ipmanager/floating', ttl=120) as _:
ipm = db.get_ipmanager('floating')
ipm.release(n.floating_gateway)
db.persist_ipmanager('floating', ipm.save())
db.update_network_state(network_uuid, 'deleted')
def deploy_nat(self):
if not self.provide_nat:
return
subst = self.subst_dict()
if not self.floating_gateway:
with db.get_lock('ipmanager', None, 'floating', ttl=120):
ipm = db.get_ipmanager('floating')
self.floating_gateway = ipm.get_random_free_address()
db.persist_ipmanager('floating', ipm.save())
self.persist_floating_gateway()
# No lock because no data changing
ipm = db.get_ipmanager('floating')
subst['floating_router'] = ipm.get_address_at_index(1)
subst['floating_gateway'] = self.floating_gateway
subst['floating_netmask'] = ipm.netmask
with db.get_lock('network', None, self.uuid, ttl=120):
if not subst['floating_gateway'] in list(
util.get_interface_addresses(
subst['netns'], subst['physical_veth_inner'])):
with util.RecordedOperation('enable virtual routing', self):
util.execute(
None,
'%(in_netns)s ip addr add %(floating_gateway)s/%(floating_netmask)s '
'dev %(physical_veth_inner)s' % subst)
util.execute(
None,
'%(in_netns)s ip link set %(physical_veth_inner)s up' %
subst)
util.execute(
None,
'%(in_netns)s route add default gw %(floating_router)s'
% subst)
if not util.nat_rules_for_ipblock(self.network_address):
with util.RecordedOperation('enable nat', self):
util.execute(None,
'echo 1 > /proc/sys/net/ipv4/ip_forward')
util.execute(
None,
'%(in_netns)s iptables -A FORWARD -o %(physical_veth_inner)s '
'-i %(vx_veth_inner)s -j ACCEPT' % subst)
util.execute(
None,
'%(in_netns)s iptables -A FORWARD -i %(physical_veth_inner)s '
'-o %(vx_veth_inner)s -j ACCEPT' % subst)
util.execute(
None,
'%(in_netns)s iptables -t nat -A POSTROUTING -s %(ipblock)s/%(netmask)s '
'-o %(physical_veth_inner)s -j MASQUERADE' % subst)
def post(self, interface_uuid=None):
ni = db.get_interface(interface_uuid)
if not ni:
return error(404, 'network interface not found')
if not ni['floating']:
return error(409, 'this interface does not have a floating ip')
n = net.from_db(ni['network_uuid'])
if not n:
LOG.info('network(%s): network not found, genuinely missing' %
ni['network_uuid'])
return error(404, 'network not found')
if get_jwt_identity() not in [n.namespace, 'system']:
LOG.info('%s: network not found, ownership test' % n)
return error(404, 'network not found')
i = virt.from_db(ni['instance_uuid'])
if get_jwt_identity() not in [i.db_entry['namespace'], 'system']:
LOG.info('%s: instance not found, ownership test' % i)
return error(404, 'instance not found')
float_net = net.from_db('floating')
if not float_net:
return error(404, 'floating network not found')
db.add_event('interface', interface_uuid, 'api', 'defloat', None, None)
with db.get_lock('sf/ipmanager/floating', ttl=120) as _:
ipm = db.get_ipmanager('floating')
ipm.release(ni['floating'])
db.persist_ipmanager('floating', ipm.save())
db.remove_floating_from_interface(ni['uuid'])
n.remove_floating_ip(ni['floating'], ni['ipv4'])
def __init__(self,
uuid=None,
vxlan_id=1,
provide_dhcp=False,
provide_nat=False,
physical_nic='eth0',
ipblock=None,
floating_gateway=None,
namespace=None):
self.uuid = uuid
self.vxlan_id = vxlan_id
self.provide_dhcp = provide_dhcp
self.provide_nat = provide_nat
self.physical_nic = physical_nic
self.floating_gateway = floating_gateway
self.namespace = namespace
with db.get_lock('ipmanager', None, self.uuid, ttl=120):
ipm = db.get_ipmanager(self.uuid)
self.ipblock = ipm.network_address
self.router = ipm.get_address_at_index(1)
self.dhcp_start = ipm.get_address_at_index(2)
self.netmask = ipm.netmask
self.broadcast = ipm.broadcast_address
self.network_address = ipm.network_address
ipm.reserve(self.router)
db.persist_ipmanager(self.uuid, ipm.save())
def delete(self):
with util.RecordedOperation('delete domain', self):
try:
self.power_off()
instance = self._get_domain()
if instance:
instance.undefine()
except Exception as e:
util.ignore_exception('instance delete', e)
with util.RecordedOperation('delete disks', self):
try:
if os.path.exists(self.instance_path):
shutil.rmtree(self.instance_path)
except Exception as e:
util.ignore_exception('instance delete', e)
with util.RecordedOperation('release network addresses', self):
for ni in db.get_instance_interfaces(self.db_entry['uuid']):
db.update_network_interface_state(ni['uuid'], 'deleted')
with db.get_lock('ipmanager',
None,
ni['network_uuid'],
ttl=120,
op='Instance delete'):
ipm = db.get_ipmanager(ni['network_uuid'])
ipm.release(ni['ipv4'])
db.persist_ipmanager(ni['network_uuid'], ipm.save())
db.free_console_port(self.db_entry['console_port'])
db.free_console_port(self.db_entry['vdi_port'])
def delete(self):
with util.RecordedOperation('delete domain', self) as _:
try:
self.power_off()
instance = self._get_domain()
instance.undefine()
except Exception:
pass
with util.RecordedOperation('delete disks', self) as _:
try:
shutil.rmtree(self.instance_path)
except Exception:
pass
with util.RecordedOperation('release network addreses', self) as _:
for ni in db.get_instance_interfaces(self.db_entry['uuid']):
with db.get_lock('sf/ipmanager/%s' % ni['network_uuid'],
ttl=120) as _:
ipm = db.get_ipmanager(ni['network_uuid'])
ipm.release(ni['ipv4'])
db.persist_ipmanager(ni['network_uuid'], ipm.save())
db.update_instance_state(self.db_entry['uuid'], 'deleted')
db.free_console_port(self.db_entry['console_port'])
db.free_console_port(self.db_entry['vdi_port'])
def post(self, namespace=None, key_name=None, key=None):
if not namespace:
return api_base.error(400, 'no namespace specified')
with db.get_lock('namespace', None, 'all', op='Namespace update'):
rec = db.get_namespace(namespace)
if not rec:
rec = {'name': namespace, 'keys': {}}
# Allow shortcut of creating key at same time as the namespace
if key_name:
if not key:
return api_base.error(400, 'no key specified')
if not isinstance(key, str):
# Must be a string to encode()
return api_base.error(400, 'key is not a string')
if key_name == 'service_key':
return api_base.error(403, 'illegal key name')
encoded = str(
base64.b64encode(
bcrypt.hashpw(key.encode('utf-8'), bcrypt.gensalt())),
'utf-8')
rec['keys'][key_name] = encoded
# Initialise metadata
db.persist_metadata('namespace', namespace, {})
db.persist_namespace(namespace, rec)
return namespace
def _get(self, locks, related_object):
"""Fetch image if not downloaded and return image path."""
with db.get_lock('image', config.parsed.get('NODE_NAME'),
self.hashed_image_url) as image_lock:
with util.RecordedOperation('fetch image', related_object):
dirty_fields, resp = self._requires_fetch()
if dirty_fields:
logutil.info([self],
'Starting fetch due to dirty fields %s' %
dirty_fields)
if related_object:
t, u = related_object.get_describing_tuple()
dirty_fields_pretty = []
for field in dirty_fields:
dirty_fields_pretty.append(
'%s: %s -> %s' %
(field, dirty_fields[field]['before'],
dirty_fields[field]['after']))
db.add_event(t, u, 'image requires fetch', None, None,
'\n'.join(dirty_fields_pretty))
actual_image = self._fetch(resp,
locks=locks.append(image_lock))
else:
actual_image = '%s.v%03d' % (self.hashed_image_path,
self.info['version'])
_transcode(locks, actual_image, related_object)
return actual_image
def get_api_token(base_url, namespace='system'):
with db.get_lock('namespace', None, namespace):
auth_url = base_url + '/auth'
logutil.info(None,
'Fetching %s auth token from %s' % (namespace, auth_url))
ns = db.get_namespace(namespace)
if 'service_key' in ns:
key = ns['service_key']
else:
key = ''.join(
random.choice(string.ascii_lowercase) for i in range(50))
ns['service_key'] = key
db.persist_namespace(namespace, ns)
r = requests.request('POST',
auth_url,
data=json.dumps({
'namespace': namespace,
'key': key
}),
headers={
'Content-Type': 'application/json',
'User-Agent': get_user_agent()
})
if r.status_code != 200:
raise Exception('Unauthorized')
return 'Bearer %s' % r.json()['access_token']
def image_fetch(url, instance_uuid):
instance = None
if instance_uuid:
instance = virt.from_db(instance_uuid)
try:
# TODO(andy): Wait up to 15 mins for another queue process to download
# the required image. This will be changed to queue on a
# "waiting_image_fetch" queue but this works now.
with db.get_lock('image',
config.NODE_NAME,
Image.calc_unique_ref(url),
timeout=15 * 60,
op='Image fetch') as lock:
img = Image.from_url(url)
img.get([lock], instance)
db.add_event('image', url, 'fetch', None, None, 'success')
except (exceptions.HTTPError, requests.exceptions.RequestException) as e:
LOG.withField('image', url).info('Failed to fetch image')
if instance_uuid:
db.enqueue_instance_error(instance_uuid,
'Image fetch failed: %s' % e)
# Clean common problems to store in events
msg = str(e)
re_conn_err = re.compile(r'.*NewConnectionError\(\'\<.*\>: (.*)\'')
m = re_conn_err.match(msg)
if m:
msg = m.group(1)
db.add_event('image', url, 'fetch', None, None, 'Error: ' + msg)
raise exceptions.ImageFetchTaskFailedException(
'Failed to fetch image %s' % url)
def post(self,
netblock=None,
provide_dhcp=None,
provide_nat=None,
name=None,
namespace=None):
try:
ipaddress.ip_network(netblock)
except ValueError as e:
return error(400, 'cannot parse netblock: %s' % e)
if not namespace:
namespace = get_jwt_identity()
# If accessing a foreign name namespace, we need to be an admin
if get_jwt_identity() not in [namespace, 'system']:
return error(
401,
'only admins can create resources in a different namespace')
network = db.allocate_network(netblock, provide_dhcp, provide_nat,
name, namespace)
db.add_event('network', network['uuid'], 'api', 'create', None, None)
# Networks should immediately appear on the network node
with db.get_lock('sf/network/%s' % network['uuid'], ttl=900) as _:
if config.parsed.get('NODE_IP') == config.parsed.get(
'NETWORK_NODE_IP'):
n = net.from_db(network['uuid'])
if not n:
LOG.info(
'network(%s): network not found, genuinely missing' %
network['uuid'])
return error(404, 'network not found')
n.create()
n.ensure_mesh()
else:
admin_token = util.get_api_token(
'http://%s:%d' % (config.parsed.get('NETWORK_NODE_IP'),
config.parsed.get('API_PORT')),
namespace=namespace)
requests.request('put', ('http://%s:%d/deploy_network_node' %
(config.parsed.get('NETWORK_NODE_IP'),
config.parsed.get('API_PORT'))),
data=json.dumps({'uuid': network['uuid']}),
headers={
'Authorization': admin_token,
'User-Agent': util.get_user_agent()
})
db.add_event('network', network['uuid'], 'api', 'created', None,
None)
db.update_network_state(network['uuid'], 'created')
# Initialise metadata
db.persist_metadata('network', network['uuid'], {})
return network
def error_with_cleanup(status_code, message):
for network_uuid in allocations:
n = net.from_db(network_uuid)
for addr, _ in allocations[network_uuid]:
with db.get_lock('sf/ipmanager/%s' % n.uuid, ttl=120) as _:
ipm = db.get_ipmanager(n.uuid)
ipm.release(addr)
db.persist_ipmanager(n.uuid, ipm.save())
return error(status_code, message)
def instance_start(instance_uuid, network):
log = LOG.withField('instance', instance_uuid)
with db.get_lock('instance',
None,
instance_uuid,
ttl=900,
timeout=120,
op='Instance start') as lock:
instance = virt.from_db(instance_uuid)
# Collect the networks
nets = {}
for netdesc in network:
if netdesc['network_uuid'] not in nets:
n = net.from_db(netdesc['network_uuid'])
if not n:
db.enqueue_instance_error(instance_uuid, 'missing network')
return
nets[netdesc['network_uuid']] = n
# Create the networks
with util.RecordedOperation('ensure networks exist', instance):
for network_uuid in nets:
n = nets[network_uuid]
try:
n.create()
n.ensure_mesh()
n.update_dhcp()
except exceptions.DeadNetwork as e:
log.withField(
'network',
n).warning('Instance tried to use dead network')
db.enqueue_instance_error(
instance_uuid, 'tried to use dead network: %s' % e)
return
# Allocate console and VDI ports
instance.allocate_instance_ports()
# Now we can start the instance
libvirt = util.get_libvirt()
try:
with util.RecordedOperation('instance creation', instance):
instance.create(lock=lock)
except libvirt.libvirtError as e:
code = e.get_error_code()
if code in (libvirt.VIR_ERR_CONFIG_UNSUPPORTED,
libvirt.VIR_ERR_XML_ERROR):
db.enqueue_instance_error(instance_uuid,
'instance failed to start: %s' % e)
return
for iface in db.get_instance_interfaces(instance_uuid):
db.update_network_interface_state(iface['uuid'], 'created')
def delete(self, namespace, key=None, value=None):
if not key:
return error(400, 'no key specified')
with db.get_lock('metadata', 'namespace', namespace):
md = db.get_metadata('namespace', namespace)
if md is None or key not in md:
return error(404, 'key not found')
del md[key]
db.persist_metadata('namespace', namespace, md)
def delete(self, network_uuid=None, key=None, network_from_db=None):
if not key:
return error(400, 'no key specified')
with db.get_lock('metadata', 'network', network_uuid):
md = db.get_metadata('network', network_uuid)
if md is None or key not in md:
return error(404, 'key not found')
del md[key]
db.persist_metadata('network', network_uuid, md)
def delete(self, instance_uuid=None, key=None, instance_from_db=None):
if not key:
return error(400, 'no key specified')
with db.get_lock('sf/metadata/instance/%s' % instance_uuid) as _:
md = db.get_metadata('instance', instance_uuid)
if md is None or key not in md:
return error(404, 'key not found')
del md[key]
db.persist_metadata('instance', instance_uuid, md)
def new(uuid, ipblock):
with db.get_lock('ipmanager',
None,
uuid,
ttl=120,
op='Network object initialization'):
ipm = IPManager(uuid, ipblock)
# Reserve first IP address
ipm.reserve(ipm.get_address_at_index(1), ipm.unique_label())
ipm.persist()
return ipm
def remove_nat(self):
if config.NODE_IS_NETWORK_NODE:
if self.floating_gateway:
with db.get_lock('ipmanager', None, 'floating', ttl=120,
op='Remove NAT'):
ipm = IPManager.from_db('floating')
ipm.release(self.floating_gateway)
ipm.persist()
self.update_floating_gateway(None)
else:
etcd.enqueue('networknode', RemoveNATNetworkTask(self.uuid))
def delete(self, namespace, key_name):
if not namespace:
return error(400, 'no namespace specified')
if not key_name:
return error(400, 'no key name specified')
with db.get_lock('namespace', None, namespace):
ns = db.get_namespace(namespace)
if ns.get('keys') and key_name in ns['keys']:
del ns['keys'][key_name]
else:
return error(404, 'key name not found in namespace')
db.persist_namespace(namespace, ns)
def assign_floating_ip(ni):
float_net = net.Network.from_db('floating')
if not float_net:
return api_base.error(404, 'floating network not found')
# Address is allocated and added to the record here, so the job has it later.
db.add_event('interface', ni.uuid, 'api', 'float', None, None)
with db.get_lock('ipmanager', None, 'floating', ttl=120, op='Interface float'):
ipm = IPManager.from_db('floating')
addr = ipm.get_random_free_address(ni.unique_label())
ipm.persist()
ni.floating = addr
def delete(self):
if self.floating['floating_address']:
etcd.enqueue(
'networknode',
DefloatNetworkInterfaceTask(self.network_uuid, self.uuid))
with db.get_lock('ipmanager', None, self.network_uuid,
ttl=120, op='Release fixed IP'):
ipm = IPManager.from_db(self.network_uuid)
ipm.release(self.ipv4)
ipm.persist()
self.state = dbo.STATE_DELETED
def _metadata_putpost(meta_type, owner, key, value):
if meta_type not in ['namespace', 'instance', 'network']:
return error(500, 'invalid meta_type %s' % meta_type)
if not key:
return error(400, 'no key specified')
if not value:
return error(400, 'no value specified')
with db.get_lock('metadata', meta_type, owner):
md = db.get_metadata(meta_type, owner)
if md is None:
md = {}
md[key] = value
db.persist_metadata(meta_type, owner, md)
def remove_dhcp(self):
if util.is_network_node():
subst = self.subst_dict()
with util.RecordedOperation('remove dhcp', self):
with db.get_lock('network', None, self.uuid, ttl=120):
d = dhcp.DHCP(self.uuid, subst['vx_veth_inner'])
d.remove_dhcpd()
else:
db.enqueue('networknode', {
'type': 'remove_dhcp',
'network_uuid': self.uuid
})
db.add_event('network', self.uuid, 'remove dhcp', 'enqueued', None,
None)
def delete(self):
subst = self.subst_dict()
# Cleanup local node
with db.get_lock('sf/net/%s' % self.uuid, ttl=120) as _:
if util.check_for_interface(subst['vx_bridge']):
with util.RecordedOperation('delete vxlan bridge', self) as _:
processutils.execute('ip link delete %(vx_bridge)s' % subst,
shell=True)
if util.check_for_interface(subst['vx_interface']):
with util.RecordedOperation('delete vxlan interface', self) as _:
processutils.execute('ip link delete %(vx_interface)s' % subst,
shell=True)
# If this is the network node do additional cleanup
if config.parsed.get('NODE_IP') == config.parsed.get('NETWORK_NODE_IP'):
if util.check_for_interface(subst['vx_veth_outer']):
with util.RecordedOperation('delete router veth', self) as _:
processutils.execute('ip link delete %(vx_veth_outer)s' % subst,
shell=True)
if util.check_for_interface(subst['physical_veth_outer']):
with util.RecordedOperation('delete physical veth', self) as _:
processutils.execute('ip link delete %(physical_veth_outer)s' % subst,
shell=True)
if os.path.exists('/var/run/netns/%(netns)s' % subst):
with util.RecordedOperation('delete netns', self) as _:
processutils.execute('ip netns del %(netns)s' % subst,
shell=True)
if self.floating_gateway:
with db.get_lock('sf/ipmanager/floating', ttl=120) as _:
ipm = db.get_ipmanager('floating')
ipm.release(self.floating_gateway)
db.persist_ipmanager('floating', ipm.save())
def ensure_mesh(self):
with db.get_lock('network', None, self.uuid, ttl=120):
removed = []
added = []
instances = []
for iface in db.get_network_interfaces(self.uuid):
if not iface['instance_uuid'] in instances:
instances.append(iface['instance_uuid'])
node_fqdns = []
for inst in instances:
i = db.get_instance(inst)
if not i:
continue
if not i['node']:
continue
if not i['node'] in node_fqdns:
node_fqdns.append(i['node'])
# NOTE(mikal): why not use DNS here? Well, DNS might be outside
# the control of the deployer if we're running in a public cloud
# as an overlay cloud...
node_ips = [config.parsed.get('NETWORK_NODE_IP')]
for fqdn in node_fqdns:
ip = db.get_node(fqdn)['ip']
if ip not in node_ips:
node_ips.append(ip)
discovered = list(self.discover_mesh())
logutil.debug([self], 'Discovered mesh elements %s' % discovered)
for node in discovered:
if node in node_ips:
node_ips.remove(node)
else:
self._remove_mesh_element(node)
removed.append(node)
for node in node_ips:
self._add_mesh_element(node)
added.append(node)
if removed:
db.add_event('network', self.uuid, 'remove mesh elements',
None, None, ' '.join(removed))
if added:
db.add_event('network', self.uuid, 'add mesh elements', None,
None, ' '.join(added))
def delete(self):
subst = self.subst_dict()
# Cleanup local node
with db.get_lock('network', None, self.uuid, ttl=120):
if util.check_for_interface(subst['vx_bridge']):
with util.RecordedOperation('delete vxlan bridge', self):
util.execute(None, 'ip link delete %(vx_bridge)s' % subst)
if util.check_for_interface(subst['vx_interface']):
with util.RecordedOperation('delete vxlan interface', self):
util.execute(None,
'ip link delete %(vx_interface)s' % subst)
# If this is the network node do additional cleanup
if util.is_network_node():
if util.check_for_interface(subst['vx_veth_outer']):
with util.RecordedOperation('delete router veth', self):
util.execute(
None, 'ip link delete %(vx_veth_outer)s' % subst)
if util.check_for_interface(subst['physical_veth_outer']):
with util.RecordedOperation('delete physical veth', self):
util.execute(
None,
'ip link delete %(physical_veth_outer)s' % subst)
if os.path.exists('/var/run/netns/%(netns)s' % subst):
with util.RecordedOperation('delete netns', self):
util.execute(None, 'ip netns del %(netns)s' % subst)
if self.floating_gateway:
with db.get_lock('ipmanager', None, 'floating', ttl=120):
ipm = db.get_ipmanager('floating')
ipm.release(self.floating_gateway)
db.persist_ipmanager('floating', ipm.save())
def _delete_network(network_from_db):
network_uuid = network_from_db['uuid']
db.add_event('network', network_uuid, 'api', 'delete', None, None)
n = net.from_db(network_uuid)
n.remove_dhcp()
n.delete()
if n.floating_gateway:
with db.get_lock('ipmanager', None, 'floating', ttl=120):
ipm = db.get_ipmanager('floating')
ipm.release(n.floating_gateway)
db.persist_ipmanager('floating', ipm.save())
db.update_network_state(network_uuid, 'deleted')
def post(self, interface_uuid=None):
ni, n, err = _safe_get_network_interface(interface_uuid)
if err:
return err
float_net = net.from_db('floating')
if not float_net:
return error(404, 'floating network not found')
db.add_event('interface', interface_uuid, 'api', 'defloat', None, None)
with db.get_lock('ipmanager', None, 'floating', ttl=120):
ipm = db.get_ipmanager('floating')
ipm.release(ni['floating'])
db.persist_ipmanager('floating', ipm.save())
db.remove_floating_from_interface(ni['uuid'])
n.remove_floating_ip(ni['floating'], ni['ipv4'])
def post(self, interface_uuid=None):
ni, n, err = _safe_get_network_interface(interface_uuid)
if err:
return err
float_net = net.from_db('floating')
if not float_net:
return error(404, 'floating network not found')
db.add_event('interface', interface_uuid, 'api', 'float', None, None)
with db.get_lock('ipmanager', None, 'floating', ttl=120):
ipm = db.get_ipmanager('floating')
addr = ipm.get_random_free_address()
db.persist_ipmanager('floating', ipm.save())
db.add_floating_to_interface(ni['uuid'], addr)
n.add_floating_ip(addr, ni['ipv4'])
def get_lock(self,
subtype=None,
ttl=60,
relatedobjects=None,
log_ctx=None,
op=None,
timeout=constants.ETCD_ATTEMPT_TIMEOUT):
if not log_ctx:
log_ctx = self.log
return db.get_lock(self.object_type,
subtype,
self.uuid,
ttl=ttl,
relatedobjects=relatedobjects,
log_ctx=log_ctx,
op=op,
timeout=timeout)