def _remove_mesh_element(self, node):
LOG.withObj(self).info('Removing excess mesh element %s' % node)
subst = self.subst_dict()
subst['node'] = node
util.execute(
None, 'bridge fdb del to 00:00:00:00:00:00 dst %(node)s '
'dev %(vx_interface)s' % subst)
def create_flat(locks, cache_file, disk_file):
"""Make a flat copy of the disk from the image cache."""
if os.path.exists(disk_file):
return
util.execute(locks, 'cp %s %s' % (cache_file, disk_file))
def _fetch(self, resp, locks=None):
"""Download the image if the latest version is not in the cache."""
fetched = 0
self.file_version += 1
last_refresh = 0
with open(self.version_image_path(), 'wb') as f:
for chunk in resp.iter_content(chunk_size=8192):
fetched += len(chunk)
f.write(chunk)
if time.time() - last_refresh > 5:
db.refresh_locks(locks)
last_refresh = time.time()
LOG.withImage(self).withField('bytes_fetched',
fetched).info('Fetch complete')
# Check if decompression not required
fn = self.version_image_path()
if not self.url.endswith('.gz'):
return fn
# Check if already decompressed
if not os.path.exists(f + '.orig'):
util.execute(locks, 'gunzip -k -q -c %s > %s.orig' % (fn, fn))
return fn + '.orig'
def _add_mesh_element(self, node):
LOG.withObj(self).info('Adding new mesh element %s' % node)
subst = self.subst_dict()
subst['node'] = node
util.execute(
None, 'bridge fdb append to 00:00:00:00:00:00 '
'dst %(node)s dev %(vx_interface)s' % subst)
def create_cow(locks, cache_file, disk_file):
"""Create a COW layer on top of the image cache."""
if os.path.exists(disk_file):
return
util.execute(locks,
'qemu-img create -b %s -f qcow2 %s' % (cache_file, disk_file))
def restart_dhcpd(self):
self._make_config()
self._make_hosts()
if not self._send_signal(signal.SIGHUP):
util.execute(
None,
'%(in_netns)s dnsmasq --conf-file=%(config_dir)s/config' %
self.subst)
def run(self):
LOG.info('Starting')
util.execute(None, (config.get('API_COMMAND_LINE') % {
'port': config.get('API_PORT'),
'timeout': config.get('API_TIMEOUT'),
'name': daemon.process_name('api')
}),
env_variables=os.environ)
def create_raw(locks, cache_file, disk_file):
"""Make a raw copy of the disk from the image cache."""
if os.path.exists(disk_file):
return
util.execute(
locks,
'qemu-img convert -t none -O raw %s %s' % (cache_file, disk_file))
def create_cow(locks, cache_file, disk_file, disk_size):
"""Create a COW layer on top of the image cache.
disk_size is specified in Gigabytes.
"""
if os.path.exists(disk_file):
return
util.execute(
locks, 'qemu-img create -b %s -f qcow2 %s %dG' %
(cache_file, disk_file, int(disk_size)))
def _transcode(locks, actual_image, related_object):
with util.RecordedOperation('transcode image', related_object):
if os.path.exists(actual_image + '.qcow2'):
return
current_format = identify(actual_image).get('file format')
if current_format == 'qcow2':
os.link(actual_image, actual_image + '.qcow2')
return
util.execute(
locks, 'qemu-img convert -t none -O qcow2 %s %s.qcow2' %
(actual_image, actual_image))
def remove_floating_ip(self, floating_address, inner_address):
LOG.withObj(self).info('Removing floating ip %s -> %s' %
(floating_address, inner_address))
subst = self.subst_dict()
subst['floating_address'] = floating_address
subst['inner_address'] = inner_address
util.execute(
None, 'ip addr del %(floating_address)s/%(netmask)s '
'dev %(physical_veth_outer)s' % subst)
util.execute(
None, '%(in_netns)s iptables -t nat -D PREROUTING '
'-d %(floating_address)s '
'-j DNAT --to-destination %(inner_address)s' % subst)
def add_floating_ip(self, floating_address, inner_address):
logutil.info([self], 'Adding floating ip %s -> %s' %
(floating_address, inner_address))
subst = self.subst_dict()
subst['floating_address'] = floating_address
subst['inner_address'] = inner_address
util.execute(
None, 'ip addr add %(floating_address)s/%(netmask)s '
'dev %(physical_veth_outer)s' % subst)
util.execute(
None, '%(in_netns)s iptables -t nat -A PREROUTING '
'-d %(floating_address)s -j DNAT --to-destination %(inner_address)s'
% subst)
def identify(path):
"""Work out what an image is."""
if not os.path.exists(path):
return {}
out, _ = util.execute(None, 'qemu-img info %s' % path)
data = {}
for line in out.split('\n'):
line = line.lstrip().rstrip()
elems = line.split(': ')
if len(elems) > 1:
key = elems[0]
value = ': '.join(elems[1:])
m = VALUE_WITH_BRACKETS_RE.match(value)
if m:
value = float(m.group(1))
elif value.endswith('K'):
value = float(value[:-1]) * 1024
elif value.endswith('M'):
value = float(value[:-1]) * 1024 * 1024
elif value.endswith('G'):
value = float(value[:-1]) * 1024 * 1024 * 1024
elif value.endswith('T'):
value = float(value[:-1]) * 1024 * 1024 * 1024 * 1024
try:
data[key] = float(value)
except Exception:
data[key] = value
return data
def clean_events_mesh_operations(etcd_client):
# TODO(andy): This can be removed when older versions do not exist
# We probably need to cleanup excess network mesh events. We also need to
# try and fetch small batches because of limits in the amount of data etcd3
# can return at one time.
# Save time and use the already available etcdctl client.
net_keys, stderr = util.execute(
None,
'etcdctl get --prefix /sf/event/network/ | grep sf/event',
check_exit_code=[0, 1])
if stderr:
print('ERROR: Unable to retrieve network keys:%s' % stderr)
return
# Split network events into networks
network_events = {}
for key in net_keys.split('\n'):
if not key:
continue
_blank, _sf, _event, _network, uuid, _time = key.split('/')
network_events.setdefault(uuid, []).append(key)
# Delete all but last 50 events
count = 0
for keys in network_events.values():
for k in keys[:-50]:
print('--> Removing verbose network event %s' % k)
etcd_client.delete(k)
count += 1
print(' - Cleaned up %d old network mesh events' % count)
def resize(self, locks, size):
"""Resize the image to the specified size."""
image_path = self.version_image_path()
backing_file = image_path + '.qcow2' + '.' + str(size) + 'G'
if os.path.exists(backing_file):
return backing_file
current_size = identify(image_path).get('virtual size')
if current_size == size * 1024 * 1024 * 1024:
os.link(image_path, backing_file)
return backing_file
util.execute(
locks, 'qemu-img create -b %s.qcow2 -f qcow2 %s %dG' %
(image_path, backing_file, size))
return backing_file
def resize(locks, hashed_image_path, size):
"""Resize the image to the specified size."""
backing_file = hashed_image_path + '.qcow2' + '.' + str(size) + 'G'
if os.path.exists(backing_file):
return backing_file
current_size = identify(hashed_image_path).get('virtual size')
if current_size == size * 1024 * 1024 * 1024:
os.link(hashed_image_path, backing_file)
return backing_file
util.execute(locks,
'cp %s %s' % (hashed_image_path + '.qcow2', backing_file))
util.execute(locks, 'qemu-img resize %s %sG' % (backing_file, size))
return backing_file
def discover_mesh(self):
mesh_re = re.compile(r'00:00:00:00:00:00 dst (.*) self permanent')
stdout, _ = util.execute(
None,
'bridge fdb show brport %(vx_interface)s' % self.subst_dict())
for line in stdout.split('\n'):
m = mesh_re.match(line)
if m:
yield m.group(1)
def delete(self):
subst = self.subst_dict()
LOG.withFields(subst).debug('net.delete()')
# Cleanup local node
with db.get_object_lock(self, ttl=120, op='Network delete'):
if util.check_for_interface(subst['vx_bridge']):
with util.RecordedOperation('delete vxlan bridge', self):
util.execute(None, 'ip link delete %(vx_bridge)s' % subst)
if util.check_for_interface(subst['vx_interface']):
with util.RecordedOperation('delete vxlan interface', self):
util.execute(None,
'ip link delete %(vx_interface)s' % subst)
# If this is the network node do additional cleanup
if util.is_network_node():
if util.check_for_interface(subst['vx_veth_outer']):
with util.RecordedOperation('delete router veth', self):
util.execute(
None, 'ip link delete %(vx_veth_outer)s' % subst)
if util.check_for_interface(subst['physical_veth_outer']):
with util.RecordedOperation('delete physical veth', self):
util.execute(
None,
'ip link delete %(physical_veth_outer)s' % subst)
if os.path.exists('/var/run/netns/%(netns)s' % subst):
with util.RecordedOperation('delete netns', self):
util.execute(None, 'ip netns del %(netns)s' % subst)
if self.db_entry['floating_gateway']:
with db.get_lock('ipmanager',
None,
'floating',
ttl=120,
op='Network delete'):
ipm = db.get_ipmanager('floating')
ipm.release(self.db_entry['floating_gateway'])
db.persist_ipmanager('floating', ipm.save())
def _fetch(self, resp, locks=None):
"""Download the image if we don't already have the latest version in cache."""
# Do the fetch
fetched = 0
self.info['version'] += 1
self.info['fetched_at'] = email.utils.formatdate()
for field in VALIDATED_IMAGE_FIELDS:
self.info[field] = resp.headers.get(field)
last_refresh = 0
with open(self.hashed_image_path + '.v%03d' % self.info['version'],
'wb') as f:
for chunk in resp.iter_content(chunk_size=8192):
fetched += len(chunk)
f.write(chunk)
if time.time() - last_refresh > 10:
db.refresh_locks(locks)
if fetched > 0:
self._persist_info()
logutil.info([self], 'Fetch complete (%d bytes)' % fetched)
# Decompress if required
if self.info['url'].endswith('.gz'):
if not os.path.exists(self.hashed_image_path +
'.v%03d.orig' % self.info['version']):
util.execute(
locks, 'gunzip -k -q -c %(img)s > %(img)s.orig' % {
'img':
self.hashed_image_path +
'.v%03d' % self.info['version']
})
return '%s.v%03d.orig' % (self.hashed_image_path,
self.info['version'])
return '%s.v%03d' % (self.hashed_image_path, self.info['version'])
def _fetch(self, resp, locks=None):
"""Download the image if we don't already have the latest version in
cache.
"""
# Do the fetch
fetched = 0
self.file_version += 1
self.fetched = email.utils.formatdate()
self.modified = resp.headers.get('Last-Modified')
self.size = resp.headers.get('Content-Length')
last_refresh = 0
with open(self.version_image_path(), 'wb') as f:
for chunk in resp.iter_content(chunk_size=8192):
fetched += len(chunk)
f.write(chunk)
if time.time() - last_refresh > 5:
db.refresh_locks(locks)
last_refresh = time.time()
if fetched > 0:
self.persist()
LOG.withImage(self).withField('bytes_fetched',
fetched).info('Fetch complete')
# Check if decompression not required
fn = self.version_image_path()
if not self.url.endswith('.gz'):
return fn
# Check if already decompressed
if not os.path.exists(f + '.orig'):
util.execute(locks, 'gunzip -k -q -c %s > %s.orig' % (fn, fn))
return fn + '.orig'
def _update_power_states(self):
libvirt = util.get_libvirt()
conn = libvirt.open(None)
try:
seen = []
# Active VMs have an ID. Active means running in libvirt
# land.
for domain_id in conn.listDomainsID():
domain = conn.lookupByID(domain_id)
if not domain.name().startswith('sf:'):
continue
instance_uuid = domain.name().split(':')[1]
instance = db.get_instance(instance_uuid)
if not instance:
# Instance is SF but not in database. Kill to reduce load.
logutil.warning([virt.ThinInstance(instance_uuid)],
'Destroying unknown instance')
util.execute(None, 'virsh destroy "sf:%s"' % instance_uuid)
continue
db.place_instance(instance_uuid,
config.parsed.get('NODE_NAME'))
seen.append(domain.name())
if instance.get('state') == 'deleted':
# NOTE(mikal): a delete might be in-flight in the queue.
# We only worry about instances which should have gone
# away five minutes ago.
if time.time() - instance['state_updated'] < 300:
continue
db.instance_enforced_deletes_increment(instance_uuid)
attempts = instance.get('enforced_deletes', 0)
if attempts > 5:
# Sometimes we just can't delete the VM. Try the big hammer instead.
logutil.warning(
[virt.ThinInstance(instance_uuid)],
'Attempting alternate delete method for instance')
util.execute(None,
'virsh destroy "sf:%s"' % instance_uuid)
db.add_event('instance', instance_uuid,
'enforced delete', 'complete', None, None)
else:
i = virt.from_db(instance_uuid)
i.delete()
i.update_instance_state('deleted')
logutil.warning([virt.ThinInstance(instance_uuid)],
'Deleting stray instance (attempt %d)' %
attempts)
continue
state = util.extract_power_state(libvirt, domain)
db.update_instance_power_state(instance_uuid, state)
if state == 'crashed':
db.update_instance_state(instance_uuid, 'error')
# Inactive VMs just have a name, and are powered off
# in our state system.
for domain_name in conn.listDefinedDomains():
if not domain_name.startswith('sf:'):
continue
if domain_name not in seen:
instance_uuid = domain_name.split(':')[1]
instance = db.get_instance(instance_uuid)
if instance.get('state') == 'deleted':
# NOTE(mikal): a delete might be in-flight in the queue.
# We only worry about instances which should have gone
# away five minutes ago.
if time.time() - instance['state_updated'] < 300:
continue
domain = conn.lookupByName(domain_name)
domain.undefine()
logutil.info([virt.ThinInstance(instance_uuid)],
'Detected stray instance')
db.add_event('instance', instance_uuid,
'deleted stray', 'complete', None, None)
continue
db.place_instance(instance_uuid,
config.parsed.get('NODE_NAME'))
instance_path = os.path.join(
config.parsed.get('STORAGE_PATH'), 'instances',
instance_uuid)
if not os.path.exists(instance_path):
# If we're inactive and our files aren't on disk,
# we have a problem.
logutil.info([virt.ThinInstance(instance_uuid)],
'Detected error state for instance')
db.update_instance_state(instance_uuid, 'error')
elif instance.get('power_state') != 'off':
logutil.info([virt.ThinInstance(instance_uuid)],
'Detected power off for instance')
db.update_instance_power_state(instance_uuid, 'off')
db.add_event('instance', instance_uuid,
'detected poweroff', 'complete', None,
None)
except libvirt.libvirtError as e:
logutil.error(None, 'Failed to lookup all domains: %s' % e)
def snapshot(locks, source, destination):
"""Convert a possibly COW layered disk file into a snapshot."""
util.execute(
locks, 'qemu-img convert --force-share -O qcow2 %s %s' %
(source, destination))
def create(self):
subst = self.subst_dict()
with db.get_lock('network', None, self.uuid, ttl=120):
if not util.check_for_interface(subst['vx_interface']):
with util.RecordedOperation('create vxlan interface', self):
util.execute(
None,
'ip link add %(vx_interface)s type vxlan id %(vx_id)s '
'dev %(physical_interface)s dstport 0' % subst)
util.execute(
None,
'sysctl -w net.ipv4.conf.%(vx_interface)s.arp_notify=1'
% subst)
if not util.check_for_interface(subst['vx_bridge']):
with util.RecordedOperation('create vxlan bridge', self):
util.execute(
None, 'ip link add %(vx_bridge)s type bridge' % subst)
util.execute(
None,
'ip link set %(vx_interface)s master %(vx_bridge)s' %
subst)
util.execute(None,
'ip link set %(vx_interface)s up' % subst)
util.execute(None, 'ip link set %(vx_bridge)s up' % subst)
util.execute(
None,
'sysctl -w net.ipv4.conf.%(vx_bridge)s.arp_notify=1' %
subst)
util.execute(None, 'brctl setfd %(vx_bridge)s 0' % subst)
util.execute(None, 'brctl stp %(vx_bridge)s off' % subst)
util.execute(None,
'brctl setageing %(vx_bridge)s 0' % subst)
if util.is_network_node():
if not os.path.exists('/var/run/netns/%(netns)s' % subst):
with util.RecordedOperation('create netns', self):
util.execute(None, 'ip netns add %(netns)s' % subst)
if not util.check_for_interface(subst['vx_veth_outer']):
with util.RecordedOperation('create router veth', self):
util.execute(
None,
'ip link add %(vx_veth_outer)s type veth peer name %(vx_veth_inner)s'
% subst)
util.execute(
None, 'ip link set %(vx_veth_inner)s netns %(netns)s' %
subst)
util.execute(
None,
'brctl addif %(vx_bridge)s %(vx_veth_outer)s' % subst)
util.execute(None,
'ip link set %(vx_veth_outer)s up' % subst)
util.execute(
None, '%(in_netns)s ip link set %(vx_veth_inner)s up' %
subst)
util.execute(
None,
'%(in_netns)s ip addr add %(router)s/%(netmask)s dev %(vx_veth_inner)s'
% subst)
if not util.check_for_interface(subst['physical_veth_outer']):
with util.RecordedOperation('create physical veth', self):
util.execute(
None,
'ip link add %(physical_veth_outer)s type veth peer name '
'%(physical_veth_inner)s' % subst)
util.execute(
None,
'brctl addif %(physical_bridge)s %(physical_veth_outer)s'
% subst)
util.execute(
None, 'ip link set %(physical_veth_outer)s up' % subst)
util.execute(
None,
'ip link set %(physical_veth_inner)s netns %(netns)s' %
subst)
self.deploy_nat()
self.update_dhcp()
else:
db.enqueue('networknode', {
'type': 'deploy',
'network_uuid': self.uuid
})
db.add_event('network', self.uuid, 'deploy', 'enqueued', None,
None)
def deploy_nat(self):
if not self.provide_nat:
return
subst = self.subst_dict()
if not self.floating_gateway:
with db.get_lock('ipmanager', None, 'floating', ttl=120):
ipm = db.get_ipmanager('floating')
self.floating_gateway = ipm.get_random_free_address()
db.persist_ipmanager('floating', ipm.save())
self.persist_floating_gateway()
# No lock because no data changing
ipm = db.get_ipmanager('floating')
subst['floating_router'] = ipm.get_address_at_index(1)
subst['floating_gateway'] = self.floating_gateway
subst['floating_netmask'] = ipm.netmask
with db.get_lock('network', None, self.uuid, ttl=120):
if not subst['floating_gateway'] in list(
util.get_interface_addresses(
subst['netns'], subst['physical_veth_inner'])):
with util.RecordedOperation('enable virtual routing', self):
util.execute(
None,
'%(in_netns)s ip addr add %(floating_gateway)s/%(floating_netmask)s '
'dev %(physical_veth_inner)s' % subst)
util.execute(
None,
'%(in_netns)s ip link set %(physical_veth_inner)s up' %
subst)
util.execute(
None,
'%(in_netns)s route add default gw %(floating_router)s'
% subst)
if not util.nat_rules_for_ipblock(self.network_address):
with util.RecordedOperation('enable nat', self):
util.execute(None,
'echo 1 > /proc/sys/net/ipv4/ip_forward')
util.execute(
None,
'%(in_netns)s iptables -A FORWARD -o %(physical_veth_inner)s '
'-i %(vx_veth_inner)s -j ACCEPT' % subst)
util.execute(
None,
'%(in_netns)s iptables -A FORWARD -i %(physical_veth_inner)s '
'-o %(vx_veth_inner)s -j ACCEPT' % subst)
util.execute(
None,
'%(in_netns)s iptables -t nat -A POSTROUTING -s %(ipblock)s/%(netmask)s '
'-o %(physical_veth_inner)s -j MASQUERADE' % subst)
def _update_power_states(self):
libvirt = util.get_libvirt()
conn = libvirt.open(None)
try:
seen = []
# Active VMs have an ID. Active means running in libvirt
# land.
for domain_id in conn.listDomainsID():
domain = conn.lookupByID(domain_id)
if not domain.name().startswith('sf:'):
continue
instance_uuid = domain.name().split(':')[1]
log_ctx = LOG.withInstance(instance_uuid)
instance = virt.Instance.from_db(instance_uuid)
if not instance:
# Instance is SF but not in database. Kill to reduce load.
log_ctx.warning('Destroying unknown instance')
util.execute(None,
'virsh destroy "sf:%s"' % instance_uuid)
continue
instance.place_instance(config.NODE_NAME)
seen.append(domain.name())
if instance.state == 'deleted':
# NOTE(mikal): a delete might be in-flight in the queue.
# We only worry about instances which should have gone
# away five minutes ago.
if time.time() - instance.state_updated < 300:
continue
instance.enforced_deletes_increment()
attempts = instance.enforced_deletes
if attempts > 5:
# Sometimes we just can't delete the VM. Try the big
# hammer instead.
log_ctx.warning(
'Attempting alternate delete method for instance')
util.execute(None,
'virsh destroy "sf:%s"' % instance_uuid)
instance.add_event('enforced delete', 'complete')
else:
instance.delete()
log_ctx.withField('attempt', attempts).warning(
'Deleting stray instance')
continue
state = util.extract_power_state(libvirt, domain)
instance.update_power_state(state)
if state == 'crashed':
instance.update_instance_state('error')
# Inactive VMs just have a name, and are powered off
# in our state system.
for domain_name in conn.listDefinedDomains():
if not domain_name.startswith('sf:'):
continue
if domain_name not in seen:
instance_uuid = domain_name.split(':')[1]
log_ctx = LOG.withInstance(instance_uuid)
instance = virt.Instance.from_db(instance_uuid)
if not instance:
# Instance is SF but not in database. Kill because
# unknown.
log_ctx.warning('Removing unknown inactive instance')
domain = conn.lookupByName(domain_name)
domain.undefine()
continue
if instance.state == 'deleted':
# NOTE(mikal): a delete might be in-flight in the queue.
# We only worry about instances which should have gone
# away five minutes ago.
if time.time() - instance['state_updated'] < 300:
continue
domain = conn.lookupByName(domain_name)
domain.undefine()
log_ctx.info('Detected stray instance')
instance.add_event('deleted stray', 'complete')
continue
instance.place_instance(config.NODE_NAME)
if not os.path.exists(instance.instance_path()):
# If we're inactive and our files aren't on disk,
# we have a problem.
log_ctx.info('Detected error state for instance')
instance.update_instance_state('error')
elif instance.power_state != 'off':
log_ctx.info('Detected power off for instance')
instance.update_power_state('off')
instance.add_event('detected poweroff', 'complete')
except libvirt.libvirtError as e:
LOG.error('Failed to lookup all domains: %s' % e)
def main():
global DAEMON_IMPLEMENTATIONS
global DAEMON_PIDS
setproctitle.setproctitle(daemon.process_name('main'))
# Log configuration on startup
for key, value in config.dict().items():
LOG.info('Configuration item %s = %s' % (key, value))
daemon.set_log_level(LOG, 'main')
# Check in early and often, also reset processing queue items
db.clear_stale_locks()
db.see_this_node()
db.restart_queues()
def _start_daemon(d):
pid = os.fork()
if pid == 0:
DAEMON_IMPLEMENTATIONS[d].Monitor(d).run()
DAEMON_PIDS[pid] = d
LOG.withField('pid', pid).info('Started %s' % d)
# Resource usage publisher, we need this early because scheduling decisions
# might happen quite early on.
_start_daemon('resources')
# If I am the network node, I need some setup
if util.is_network_node():
# Bootstrap the floating network in the Networks table
floating_network = db.get_network('floating')
if not floating_network:
db.create_floating_network(config.get('FLOATING_NETWORK'))
floating_network = net.from_db('floating')
subst = {
'physical_bridge':
util.get_safe_interface_name('phy-br-%s' %
config.get('NODE_EGRESS_NIC')),
'physical_nic':
config.get('NODE_EGRESS_NIC')
}
if not util.check_for_interface(subst['physical_bridge']):
# NOTE(mikal): Adding the physical interface to the physical bridge
# is considered outside the scope of the orchestration software as
# it will cause the node to lose network connectivity. So instead
# all we do is create a bridge if it doesn't exist and the wire
# everything up to it. We can do egress NAT in that state, even if
# floating IPs don't work.
with util.RecordedOperation('create physical bridge', None):
# No locking as read only
ipm = db.get_ipmanager('floating')
subst['master_float'] = ipm.get_address_at_index(1)
subst['netmask'] = ipm.netmask
util.create_interface(subst['physical_bridge'], 'bridge', '')
util.execute(None,
'ip link set %(physical_bridge)s up' % subst)
util.execute(
None, 'ip addr add %(master_float)s/%(netmask)s '
'dev %(physical_bridge)s' % subst)
util.execute(
None, 'iptables -A FORWARD -o %(physical_nic)s '
'-i %(physical_bridge)s -j ACCEPT' % subst)
util.execute(
None, 'iptables -A FORWARD -i %(physical_nic)s '
'-o %(physical_bridge)s -j ACCEPT' % subst)
util.execute(
None, 'iptables -t nat -A POSTROUTING '
'-o %(physical_nic)s -j MASQUERADE' % subst)
def _audit_daemons():
running_daemons = []
for pid in DAEMON_PIDS:
running_daemons.append(DAEMON_PIDS[pid])
for d in DAEMON_IMPLEMENTATIONS:
if d not in running_daemons:
_start_daemon(d)
for d in DAEMON_PIDS:
if not psutil.pid_exists(d):
LOG.warning('%s pid is missing, restarting' % DAEMON_PIDS[d])
_start_daemon(DAEMON_PIDS[d])
_audit_daemons()
restore_instances()
while True:
time.sleep(10)
wpid, _ = os.waitpid(-1, os.WNOHANG)
while wpid != 0:
LOG.warning('%s died (pid %d)' %
(DAEMON_PIDS.get(wpid, 'unknown'), wpid))
del DAEMON_PIDS[wpid]
wpid, _ = os.waitpid(-1, os.WNOHANG)
_audit_daemons()
db.see_this_node()
def create(self):
subst = self.subst_dict()
with db.get_object_lock(self, ttl=120, op='Network create'):
# Ensure network was not deleted whilst waiting for the lock.
if self.is_dead():
raise DeadNetwork('network=%s' % self)
if not util.check_for_interface(subst['vx_interface']):
with util.RecordedOperation('create vxlan interface', self):
util.create_interface(
subst['vx_interface'], 'vxlan',
'id %(vx_id)s dev %(physical_interface)s dstport 0' %
subst)
util.execute(
None, 'sysctl -w net.ipv4.conf.'
'%(vx_interface)s.arp_notify=1' % subst)
if not util.check_for_interface(subst['vx_bridge']):
with util.RecordedOperation('create vxlan bridge', self):
util.create_interface(subst['vx_bridge'], 'bridge', '')
util.execute(
None, 'ip link set %(vx_interface)s '
'master %(vx_bridge)s' % subst)
util.execute(None,
'ip link set %(vx_interface)s up' % subst)
util.execute(None, 'ip link set %(vx_bridge)s up' % subst)
util.execute(
None, 'sysctl -w net.ipv4.conf.'
'%(vx_bridge)s.arp_notify=1' % subst)
util.execute(None, 'brctl setfd %(vx_bridge)s 0' % subst)
util.execute(None, 'brctl stp %(vx_bridge)s off' % subst)
util.execute(None,
'brctl setageing %(vx_bridge)s 0' % subst)
if util.is_network_node():
if not os.path.exists('/var/run/netns/%(netns)s' % subst):
with util.RecordedOperation('create netns', self):
util.execute(None, 'ip netns add %(netns)s' % subst)
if not util.check_for_interface(subst['vx_veth_outer']):
with util.RecordedOperation('create router veth', self):
util.create_interface(
subst['vx_veth_outer'], 'veth',
'peer name %(vx_veth_inner)s' % subst)
util.execute(
None, 'ip link set %(vx_veth_inner)s netns %(netns)s' %
subst)
util.execute(
None,
'brctl addif %(vx_bridge)s %(vx_veth_outer)s' % subst)
util.execute(None,
'ip link set %(vx_veth_outer)s up' % subst)
util.execute(
None, '%(in_netns)s ip link set %(vx_veth_inner)s up' %
subst)
util.execute(
None,
'%(in_netns)s ip addr add %(router)s/%(netmask)s '
'dev %(vx_veth_inner)s' % subst)
if not util.check_for_interface(subst['physical_veth_outer']):
with util.RecordedOperation('create physical veth', self):
util.create_interface(
subst['physical_veth_outer'], 'veth',
'peer name %(physical_veth_inner)s' % subst)
util.execute(
None, 'brctl addif %(physical_bridge)s '
'%(physical_veth_outer)s' % subst)
util.execute(
None, 'ip link set %(physical_veth_outer)s up' % subst)
util.execute(
None, 'ip link set %(physical_veth_inner)s '
'netns %(netns)s' % subst)
self.deploy_nat()
self.update_dhcp()
else:
db.enqueue('networknode', DeployNetworkTask(self.db_entry['uuid']))
db.add_event('network', self.db_entry['uuid'], 'deploy',
'enqueued', None, None)
def create(self, lock=None):
db.update_instance_state(self.db_entry['uuid'], 'creating')
# Ensure we have state on disk
if not os.path.exists(self.instance_path):
LOG.withObj(self).debug('Creating instance storage at %s' %
self.instance_path)
os.makedirs(self.instance_path)
# Generate a config drive
with util.RecordedOperation('make config drive', self):
self._make_config_drive(
os.path.join(
self.instance_path,
self.db_entry['block_devices']['devices'][1]['path']))
# Prepare disks
if not self.db_entry['block_devices']['finalized']:
modified_disks = []
for disk in self.db_entry['block_devices']['devices']:
if disk.get('base'):
img = images.Image.from_url(disk['base'])
hashed_image_path = img.version_image_path()
with util.RecordedOperation('detect cdrom images', self):
try:
cd = pycdlib.PyCdlib()
cd.open(hashed_image_path)
disk['present_as'] = 'cdrom'
except Exception:
pass
if disk.get('present_as', 'cdrom') == 'cdrom':
# There is no point in resizing or COW'ing a cdrom
disk['path'] = disk['path'].replace('.qcow2', '.raw')
disk['type'] = 'raw'
disk['snapshot_ignores'] = True
try:
os.link(hashed_image_path, disk['path'])
except OSError:
# Different filesystems
util.execute([lock], 'cp %s %s' %
(hashed_image_path, disk['path']))
# Due to limitations in some installers, cdroms are always on IDE
disk['device'] = 'hd%s' % disk['device'][-1]
disk['bus'] = 'ide'
else:
if config.parsed.get('DISK_FORMAT') == 'qcow':
with util.RecordedOperation(
'create copy on write layer', self):
images.create_cow([lock], hashed_image_path,
disk['path'], disk['size'])
# Record the backing store for modern libvirts
disk['backing'] = (
'<backingStore type=\'file\'>\n'
' <format type=\'qcow2\'/>\n'
' <source file=\'%s\'/>\n'
' </backingStore>\n' %
(hashed_image_path))
elif config.parsed.get('DISK_FORMAT') == 'qcow_flat':
with util.RecordedOperation('resize image', self):
resized_image_path = img.resize([lock],
disk['size'])
with util.RecordedOperation(
'create flat layer', self):
images.create_flat([lock], resized_image_path,
disk['path'])
elif config.parsed.get('DISK_FORMAT') == 'flat':
with util.RecordedOperation('resize image', self):
resized_image_path = img.resize([lock],
disk['size'])
with util.RecordedOperation(
'create raw disk', self):
images.create_raw([lock], resized_image_path,
disk['path'])
else:
raise Exception('Unknown disk format')
elif not os.path.exists(disk['path']):
util.execute(
None, 'qemu-img create -f qcow2 %s %sG' %
(disk['path'], disk['size']))
modified_disks.append(disk)
self.db_entry['block_devices']['devices'] = modified_disks
self.db_entry['block_devices']['finalized'] = True
db.persist_block_devices(self.db_entry['uuid'],
self.db_entry['block_devices'])
# Create the actual instance
with util.RecordedOperation('create domain XML', self):
self._create_domain_xml()
# Sometimes on Ubuntu 20.04 we need to wait for port binding to work.
# Revisiting this is tracked by issue 320 on github.
with util.RecordedOperation('create domain', self):
if not self.power_on():
attempts = 0
while not self.power_on() and attempts < 100:
LOG.withObj(self).warning(
'Instance required an additional attempt to power on')
time.sleep(5)
attempts += 1
if self.is_powered_on():
LOG.withObj(self).info('Instance now powered on')
else:
LOG.withObj(self).info('Instance failed to power on')
db.update_instance_state(self.db_entry['uuid'], 'created')
def deploy_nat(self):
if not self.db_entry['provide_nat']:
return
subst = self.subst_dict()
if not self.db_entry['floating_gateway']:
with db.get_lock('ipmanager',
None,
'floating',
ttl=120,
op='Network deploy NAT'):
ipm = db.get_ipmanager('floating')
self.db_entry[
'floating_gateway'] = ipm.get_random_free_address()
db.persist_ipmanager('floating', ipm.save())
self.persist_floating_gateway()
# No lock because no data changing
ipm = db.get_ipmanager('floating')
subst['floating_router'] = ipm.get_address_at_index(1)
subst['floating_gateway'] = self.db_entry['floating_gateway']
subst['floating_netmask'] = ipm.netmask
with db.get_object_lock(self, ttl=120, op='Network deploy NAT'):
# Ensure network was not deleted whilst waiting for the lock.
if self.is_dead():
raise DeadNetwork('network=%s' % self)
with util.RecordedOperation('enable virtual routing', self):
addresses = util.get_interface_addresses(
subst['netns'], subst['physical_veth_inner'])
if not subst['floating_gateway'] in list(addresses):
util.execute(
None, '%(in_netns)s ip addr add '
'%(floating_gateway)s/%(floating_netmask)s '
'dev %(physical_veth_inner)s' % subst)
util.execute(
None, '%(in_netns)s ip link set '
'%(physical_veth_inner)s up' % subst)
default_routes = util.get_default_routes(subst['netns'])
if default_routes != [subst['floating_router']]:
if default_routes:
for default_route in default_routes:
util.execute(
None, '%s route del default gw %s' %
(subst['in_netns'], default_route))
util.execute(
None, '%(in_netns)s route add default '
'gw %(floating_router)s' % subst)
if not util.nat_rules_for_ipblock(self.network_address):
with util.RecordedOperation('enable nat', self):
util.execute(None,
'echo 1 > /proc/sys/net/ipv4/ip_forward')
util.execute(
None, '%(in_netns)s iptables -A FORWARD '
'-o %(physical_veth_inner)s '
'-i %(vx_veth_inner)s -j ACCEPT' % subst)
util.execute(
None, '%(in_netns)s iptables -A FORWARD '
'-i %(physical_veth_inner)s '
'-o %(vx_veth_inner)s -j ACCEPT' % subst)
util.execute(
None, '%(in_netns)s iptables -t nat -A POSTROUTING '
'-s %(ipblock)s/%(netmask)s '
'-o %(physical_veth_inner)s '
'-j MASQUERADE' % subst)
