def run(self):
LOG.info('Starting')
util.execute(None, (config.get('API_COMMAND_LINE') % {
'port': config.get('API_PORT'),
'timeout': config.get('API_TIMEOUT'),
'name': daemon.process_name('api')
}),
env_variables=os.environ)
def __init__(self, db_entry):
self.db_entry = db_entry
self.instance_path = os.path.join(config.get('STORAGE_PATH'),
'instances', self.db_entry['uuid'])
self.snapshot_path = os.path.join(config.get('STORAGE_PATH'),
'snapshots')
self.xml_file = os.path.join(self.instance_path, 'libvirt.xml')
if not self.db_entry['block_devices']:
self._populate_block_devices()
def run(self):
LOG.info('Starting Monitor Daemon')
observers = {}
while True:
# Cleanup terminated observers
all_observers = list(observers.keys())
for instance_uuid in all_observers:
if not observers[instance_uuid].is_alive():
# Reap process
observers[instance_uuid].join(1)
LOG.withInstance(instance_uuid).info(
'Trigger observer has terminated')
db.add_event('instance', instance_uuid, 'trigger monitor',
'crashed', None, None)
del observers[instance_uuid]
# Start missing observers
extra_instances = list(observers.keys())
for inst in db.get_instances(only_node=config.NODE_NAME):
if inst['uuid'] in extra_instances:
extra_instances.remove(inst['uuid'])
if inst['state'] != 'created':
continue
if inst['uuid'] not in observers:
console_path = os.path.join(config.get('STORAGE_PATH'),
'instances', inst['uuid'],
'console.log')
p = multiprocessing.Process(
target=observe,
args=(console_path, inst['uuid']),
name='%s-%s' %
(daemon.process_name('triggers'), inst['uuid']))
p.start()
observers[inst['uuid']] = p
LOG.withInstance(
inst['uuid']).info('Started trigger observer')
db.add_event('instance', inst['uuid'], 'trigger monitor',
'started', None, None)
# Cleanup extra observers
for instance_uuid in extra_instances:
p = observers[instance_uuid]
try:
os.kill(p.pid, signal.SIGKILL)
observers[instance_uuid].join(1)
except Exception:
pass
del observers[instance_uuid]
LOG.withInstance(instance_uuid).info(
'Finished trigger observer')
db.add_event('instance', instance_uuid, 'trigger monitor',
'finished', None, None)
time.sleep(1)
def _has_sufficient_cpu(self, cpus, node):
max_cpu = (self.metrics[node].get('cpu_max', 0) *
config.get('CPU_OVERCOMMIT_RATIO'))
current_cpu = self.metrics[node].get('cpu_total_instance_vcpus', 0)
if current_cpu + cpus > max_cpu:
return False
return True
def run(self):
LOG.info('Starting')
# Delay first compaction until system startup load has reduced
last_compaction = time.time() - random.randint(1, 20*60)
while True:
# Update power state of all instances on this hypervisor
LOG.info('Updating power states')
self._update_power_states()
# Cleanup soft deleted instances and networks
delay = config.get('CLEANER_DELAY')
for i in db.get_stale_instances(delay):
LOG.withInstance(i['uuid']).info('Hard deleting instance')
db.hard_delete_instance(i['uuid'])
for n in db.get_stale_networks(delay):
LOG.withNetwork(n['uuid']).info('Hard deleting network')
db.hard_delete_network(n['uuid'])
for ni in db.get_stale_network_interfaces(delay):
LOG.withNetworkInterface(
ni['uuid']).info('Hard deleting network interface')
db.hard_delete_network_interface(ni['uuid'])
# Perform etcd maintenance
if time.time() - last_compaction > 1800:
LOG.info('Compacting etcd')
self._compact_etcd()
last_compaction = time.time()
time.sleep(60)
def _create_domain_xml(self):
"""Create the domain XML for the instance."""
if os.path.exists(self.xml_file):
return
with open(os.path.join(config.get('STORAGE_PATH'),
'libvirt.tmpl')) as f:
t = jinja2.Template(f.read())
networks = []
for iface in list(db.get_instance_interfaces(self.db_entry['uuid'])):
n = net.from_db(iface['network_uuid'])
networks.append({
'macaddr': iface['macaddr'],
'bridge': n.subst_dict()['vx_bridge'],
'model': iface['model']
})
# NOTE(mikal): the database stores memory allocations in MB, but the
# domain XML takes them in KB. That wouldn't be worth a comment here if
# I hadn't spent _ages_ finding a bug related to it.
xml = t.render(uuid=self.db_entry['uuid'],
memory=self.db_entry['memory'] * 1024,
vcpus=self.db_entry['cpus'],
disks=self.db_entry['block_devices']['devices'],
networks=networks,
instance_path=self.instance_path,
console_port=self.db_entry['console_port'],
vdi_port=self.db_entry['vdi_port'],
video_model=self.db_entry['video']['model'],
video_memory=self.db_entry['video']['memory'])
with open(self.xml_file, 'w') as f:
f.write(xml)
def _get_cache_path():
image_cache_path = os.path.join(config.get('STORAGE_PATH'), 'image_cache')
if not os.path.exists(image_cache_path):
LOG.withField('image_cache_path',
image_cache_path).debug('Creating image cache')
os.makedirs(image_cache_path, exist_ok=True)
return image_cache_path
def _populate_block_devices(self):
if not self.disk_spec:
# This should not occur since the API will filter for zero disks.
LOG.withObj(self).error('Found disk spec empty')
# Stop continuous crashing by falsely claiming disks are configured.
self.block_devices = {'finalized': True}
return
bus = _get_defaulted_disk_bus(self.disk_spec[0])
root_device = _get_disk_device_base(bus) + 'a'
config_device = _get_disk_device_base(bus) + 'b'
disk_type = 'qcow2'
if config.get('DISK_FORMAT') == 'flat':
disk_type = 'raw'
self.block_devices = {
'devices': [
{
'type': disk_type,
'size': _safe_int_cast(self.disk_spec[0].get('size')),
'device': root_device,
'bus': bus,
'path': os.path.join(self.instance_path(), root_device),
'base': self.disk_spec[0].get('base'),
'present_as': _get_defaulted_disk_type(self.disk_spec[0]),
'snapshot_ignores': False
},
{
'type': 'raw',
'device': config_device,
'bus': bus,
'path': os.path.join(self.instance_path(), config_device),
'present_as': 'disk',
'snapshot_ignores': True
}
]
}
i = 0
for d in self.disk_spec[1:]:
bus = _get_defaulted_disk_bus(d)
device = _get_disk_device_base(bus) + chr(ord('c') + i)
self.block_devices['devices'].append({
'type': disk_type,
'size': _safe_int_cast(d.get('size')),
'device': device,
'bus': bus,
'path': os.path.join(self.instance_path(), device),
'base': d.get('base'),
'present_as': _get_defaulted_disk_type(d),
'snapshot_ignores': False
})
i += 1
self.block_devices['finalized'] = False
def _has_sufficient_ram(self, memory, node):
# There are two things to track here... We must always have
# RAM_SYSTEM_RESERVATION gb of RAM for operating system tasks -- assume
# there is no overlap with existing VMs when checking this. Note as
# well that metrics are in MB...
available = (self.metrics[node].get('memory_available', 0) -
(config.get('RAM_SYSTEM_RESERVATION') * 1024))
if available - memory < 0.0:
return False
# ...Secondly, if we're using KSM and over committing memory, we
# shouldn't overcommit more than by RAM_OVERCOMMIT_RATIO
instance_memory = (
self.metrics[node].get('memory_total_instance_actual', 0) + memory)
if (instance_memory / self.metrics[node].get('memory_max', 0) >
config.get('RAM_OVERCOMMIT_RATIO')):
return False
return True
def resolve(name):
resp = requests.get(CIRROS_URL,
headers={'User-Agent': util.get_user_agent()})
if resp.status_code != 200:
raise exceptions.HTTPError(
'Failed to fetch http://download.cirros-cloud.net/, '
'status code %d' % resp.status_code)
if name == 'cirros':
versions = []
dir_re = re.compile(r'.*<a href="([0-9]+\.[0-9]+\.[0-9]+)/">.*/</a>.*')
for line in resp.text.split('\n'):
m = dir_re.match(line)
if m:
versions.append(m.group(1))
LOG.withField('versions', versions).info('Found cirros versions')
vernum = versions[-1]
else:
try:
# Name is assumed to be in the form cirros:0.4.0
_, vernum = name.split(':')
except Exception:
raise exceptions.VersionSpecificationError(
'Cannot parse version: %s' % name)
url = config.get('DOWNLOAD_URL_CIRROS') % {'vernum': vernum}
log = LOG.withField('url', url)
# Retrieve check sum file
checksum_url = CIRROS_URL + '/' + vernum + '/MD5SUMS'
resp = requests.get(checksum_url,
headers={'User-Agent': util.get_user_agent()})
log.withField('checksum_url', checksum_url).withField(
'resp', resp).debug("Checksum request response")
if resp.status_code != 200:
# Cirros does not always have a checksum file available
log.info('Unable to retrieve MD5SUMS for cirros image')
return url, None
sum_re = re.compile(r'^([0-9a-f]+) .*' + 'cirros-' + vernum +
'-x86_64-disk.img')
checksum = None
for line in resp.text.split('\n'):
m = sum_re.match(line)
if m:
checksum = m.group(1)
break
if not checksum_url:
log.warning('Did not find checksum')
log.withField('checksum', checksum).info('Checksum retrieval')
return (url, checksum)
def __init__(self, network, interface):
self.network_uuid = network.db_entry['uuid']
self.subst = {
'config_dir':
os.path.join(config.get('STORAGE_PATH'), 'dhcp',
self.network_uuid),
'zone':
config.get('ZONE'),
'router':
network.router,
'dhcp_start':
network.dhcp_start,
'netmask':
network.netmask,
'broadcast':
network.broadcast,
'in_netns':
'ip netns exec %s' % self.network_uuid,
'interface':
interface
}
def set_log_level(log, name):
# Check that id is a valid name
process_name(name)
# Check for configuration override
level = config.get('LOGLEVEL_' + name.upper())
if level:
numeric_level = getattr(logging, level.upper(), None)
if not isinstance(numeric_level, int):
raise ValueError('Invalid log level: %s' % level)
else:
numeric_level = logging.INFO
log.setLevel(numeric_level)
def process(self, msg, kwargs):
msg = '%s[%s] %s' % (setproctitle.getproctitle(), os.getpid(), msg)
kwargs["extra"] = self.extra
if config.get('LOG_METHOD_TRACE'):
# Determine the name of the calling method
filename = traceback.extract_stack()[-4].filename
f_match = self.FILENAME_RE.match(filename)
if f_match:
filename = f_match.group(1)
caller = '%s:%s:%s()' % (filename,
traceback.extract_stack()[-4].lineno,
traceback.extract_stack()[-4].name)
self._extra['method'] = caller
return msg, kwargs
def __enter__(self):
start_time = time.time()
slow_warned = False
threshold = int(config.get('SLOW_LOCK_THRESHOLD'))
while time.time() - start_time < self.timeout:
res = self.acquire()
if res:
duration = time.time() - start_time
if duration > threshold:
db.add_event(self.objecttype, self.objectname,
'lock', 'acquired', None,
'Waited %d seconds for lock' % duration)
self.log_ctx.withField('duration', duration
).info('Acquiring a lock was slow')
return self
duration = time.time() - start_time
if (duration > threshold and not slow_warned):
db.add_event(self.objecttype, self.objectname,
'lock', 'acquire', None,
'Waiting for lock more than threshold')
node, pid = self.get_holder()
self.log_ctx.withFields({'duration': duration,
'threshold': threshold,
'holder-pid': pid,
'holder-node': node,
}).info('Waiting for lock')
slow_warned = True
time.sleep(1)
duration = time.time() - start_time
db.add_event(self.objecttype, self.objectname,
'lock', 'failed', None,
'Failed to acquire lock after %.02f seconds' % duration)
node, pid = self.get_holder()
self.log_ctx.withFields({'duration': duration,
'holder-pid': pid,
'holder-node': node,
}).info('Failed to acquire lock')
raise exceptions.LockException(
'Cannot acquire lock %s, timed out after %.02f seconds'
% (self.name, duration))
def subst_dict(self):
retval = {
'vx_id': self.db_entry['vxid'],
'vx_interface': 'vxlan-%s' % self.db_entry['vxid'],
'vx_bridge': 'br-vxlan-%s' % self.db_entry['vxid'],
'vx_veth_outer': 'veth-%s-o' % self.db_entry['vxid'],
'vx_veth_inner': 'veth-%s-i' % self.db_entry['vxid'],
'physical_interface': self.physical_nic,
'physical_bridge': 'phy-br-%s' % config.get('NODE_EGRESS_NIC'),
'physical_veth_outer': 'phy-%s-o' % self.db_entry['vxid'],
'physical_veth_inner': 'phy-%s-i' % self.db_entry['vxid'],
'netns': self.db_entry['uuid'],
'in_netns': 'ip netns exec %s' % self.db_entry['uuid'],
'ipblock': self.ipblock,
'netmask': self.netmask,
'router': self.router,
'broadcast': self.broadcast,
}
return retval
def __init__(self, db_entry):
self.db_entry = db_entry
self.physical_nic = config.get('NODE_EGRESS_NIC')
with db.get_lock('ipmanager',
None,
self.db_entry['uuid'],
ttl=120,
op='Network object initialization'):
ipm = db.get_ipmanager(self.db_entry['uuid'])
self.ipblock = ipm.network_address
self.router = ipm.get_address_at_index(1)
self.dhcp_start = ipm.get_address_at_index(2)
self.netmask = ipm.netmask
self.broadcast = ipm.broadcast_address
self.network_address = ipm.network_address
ipm.reserve(self.router)
db.persist_ipmanager(self.db_entry['uuid'], ipm.save())
def run(self):
LOG.info('Starting')
gauges = {
'updated_at': Gauge('updated_at',
'The last time metrics were updated')
}
last_metrics = 0
def update_metrics():
global last_metrics
stats = _get_stats()
for metric in stats:
if metric not in gauges:
gauges[metric] = Gauge(metric, '')
gauges[metric].set(stats[metric])
db.update_metrics_bulk(stats)
gauges['updated_at'].set_to_current_time()
while True:
try:
jobname, _ = db.dequeue('%s-metrics' % config.NODE_NAME)
if jobname:
if time.time() - last_metrics > 2:
update_metrics()
last_metrics = time.time()
db.resolve('%s-metrics' % config.NODE_NAME, jobname)
else:
time.sleep(0.2)
timer = time.time() - last_metrics
if timer > config.get('SCHEDULER_CACHE_TIMEOUT'):
update_metrics()
last_metrics = time.time()
except Exception as e:
util.ignore_exception('resource statistics', e)
def main():
global DAEMON_IMPLEMENTATIONS
global DAEMON_PIDS
setproctitle.setproctitle(daemon.process_name('main'))
# Log configuration on startup
for key, value in config.dict().items():
LOG.info('Configuration item %s = %s' % (key, value))
daemon.set_log_level(LOG, 'main')
# Check in early and often, also reset processing queue items
db.clear_stale_locks()
db.see_this_node()
db.restart_queues()
def _start_daemon(d):
pid = os.fork()
if pid == 0:
DAEMON_IMPLEMENTATIONS[d].Monitor(d).run()
DAEMON_PIDS[pid] = d
LOG.withField('pid', pid).info('Started %s' % d)
# Resource usage publisher, we need this early because scheduling decisions
# might happen quite early on.
_start_daemon('resources')
# If I am the network node, I need some setup
if util.is_network_node():
# Bootstrap the floating network in the Networks table
floating_network = db.get_network('floating')
if not floating_network:
db.create_floating_network(config.get('FLOATING_NETWORK'))
floating_network = net.from_db('floating')
subst = {
'physical_bridge':
util.get_safe_interface_name('phy-br-%s' %
config.get('NODE_EGRESS_NIC')),
'physical_nic':
config.get('NODE_EGRESS_NIC')
}
if not util.check_for_interface(subst['physical_bridge']):
# NOTE(mikal): Adding the physical interface to the physical bridge
# is considered outside the scope of the orchestration software as
# it will cause the node to lose network connectivity. So instead
# all we do is create a bridge if it doesn't exist and the wire
# everything up to it. We can do egress NAT in that state, even if
# floating IPs don't work.
with util.RecordedOperation('create physical bridge', None):
# No locking as read only
ipm = db.get_ipmanager('floating')
subst['master_float'] = ipm.get_address_at_index(1)
subst['netmask'] = ipm.netmask
util.create_interface(subst['physical_bridge'], 'bridge', '')
util.execute(None,
'ip link set %(physical_bridge)s up' % subst)
util.execute(
None, 'ip addr add %(master_float)s/%(netmask)s '
'dev %(physical_bridge)s' % subst)
util.execute(
None, 'iptables -A FORWARD -o %(physical_nic)s '
'-i %(physical_bridge)s -j ACCEPT' % subst)
util.execute(
None, 'iptables -A FORWARD -i %(physical_nic)s '
'-o %(physical_bridge)s -j ACCEPT' % subst)
util.execute(
None, 'iptables -t nat -A POSTROUTING '
'-o %(physical_nic)s -j MASQUERADE' % subst)
def _audit_daemons():
running_daemons = []
for pid in DAEMON_PIDS:
running_daemons.append(DAEMON_PIDS[pid])
for d in DAEMON_IMPLEMENTATIONS:
if d not in running_daemons:
_start_daemon(d)
for d in DAEMON_PIDS:
if not psutil.pid_exists(d):
LOG.warning('%s pid is missing, restarting' % DAEMON_PIDS[d])
_start_daemon(DAEMON_PIDS[d])
_audit_daemons()
restore_instances()
while True:
time.sleep(10)
wpid, _ = os.waitpid(-1, os.WNOHANG)
while wpid != 0:
LOG.warning('%s died (pid %d)' %
(DAEMON_PIDS.get(wpid, 'unknown'), wpid))
del DAEMON_PIDS[wpid]
wpid, _ = os.waitpid(-1, os.WNOHANG)
_audit_daemons()
db.see_this_node()
def _get_stats():
libvirt = util.get_libvirt()
retval = {}
conn = libvirt.open(None)
# CPU info
present_cpus, _, available_cpus = conn.getCPUMap()
retval.update({
'cpu_max': present_cpus,
'cpu_available': available_cpus,
})
retval['cpu_max_per_instance'] = conn.getMaxVcpus(None)
# This is disabled as data we don't currently use
# for i in range(present_cpus):
# per_cpu_stats = conn.getCPUStats(i)
# for key in per_cpu_stats:
# retval['cpu_core%d_%s' % (i, key)] = per_cpu_stats[key]
try:
load_1, load_5, load_15 = psutil.getloadavg()
retval.update({
'cpu_load_1': load_1,
'cpu_load_5': load_5,
'cpu_load_15': load_15,
})
except Exception as e:
util.ignore_exception('load average', e)
# System memory info, converting bytes to mb
stats = psutil.virtual_memory()
retval.update({
'memory_max': stats.total // 1024 // 1024,
'memory_available': stats.available // 1024 // 1024
})
# libvirt memory info, converting kb to mb
memory_status = conn.getMemoryStats(
libvirt.VIR_NODE_MEMORY_STATS_ALL_CELLS)
retval.update({
'memory_max_libvirt': memory_status['total'] // 1024,
'memory_available_libvirt': memory_status['free'] // 1024,
})
# Kernel Shared Memory (KSM) information
ksm_details = {}
for ent in os.listdir('/sys/kernel/mm/ksm'):
with open('/sys/kernel/mm/ksm/%s' % ent) as f:
ksm_details['memory_ksm_%s' % ent] = int(f.read().rstrip())
retval.update(ksm_details)
# Disk info
s = os.statvfs(config.get('STORAGE_PATH'))
disk_counters = psutil.disk_io_counters()
retval.update({
'disk_total': s.f_frsize * s.f_blocks,
'disk_free': s.f_frsize * s.f_bavail,
'disk_used': s.f_frsize * (s.f_blocks - s.f_bfree),
'disk_read_bytes': disk_counters.read_bytes,
'disk_write_bytes': disk_counters.write_bytes,
})
# Network info
net_counters = psutil.net_io_counters()
retval.update({
'network_read_bytes': net_counters.bytes_recv,
'network_write_bytes': net_counters.bytes_sent,
})
# Virtual machine consumption info
total_instances = 0
total_active_instances = 0
total_instance_max_memory = 0
total_instance_actual_memory = 0
total_instance_vcpus = 0
total_instance_cpu_time = 0
for guest in conn.listAllDomains():
try:
active = guest.isActive() == 1
if active:
_, maxmem, mem, cpus, cpu_time = guest.info()
except libvirt.libvirtError as e:
LOG.debug('During resource calc ignored libvirt error: %s' % e)
active = False
if active:
total_instances += 1
total_active_instances += 1
total_instance_max_memory += maxmem
total_instance_actual_memory += mem
total_instance_vcpus += cpus
total_instance_cpu_time += cpu_time
# Queue health statistics
node_queue_processing, node_queue_waiting = db.get_queue_length(
config.NODE_NAME)
retval.update({
'cpu_total_instance_vcpus':
total_instance_vcpus,
'cpu_total_instance_cpu_time':
total_instance_cpu_time,
'memory_total_instance_max':
total_instance_max_memory // 1024,
'memory_total_instance_actual':
total_instance_actual_memory // 1024,
'instances_total':
total_instances,
'instances_active':
total_active_instances,
'node_queue_processing':
node_queue_processing,
'node_queue_waiting':
node_queue_waiting,
})
if util.is_network_node():
network_queue_processing, network_queue_waiting = db.get_queue_length(
'networknode')
retval.update({
'network_queue_processing': network_queue_processing,
'network_queue_waiting': network_queue_waiting,
})
return retval
def create(self, lock=None):
self.update_instance_state('creating')
# Ensure we have state on disk
if not os.path.exists(self.instance_path()):
LOG.withObj(self).debug(
'Creating instance storage at %s' % self.instance_path())
os.makedirs(self.instance_path(), exist_ok=True)
# Generate a config drive
with util.RecordedOperation('make config drive', self):
self._make_config_drive(os.path.join(
self.instance_path(), self.block_devices['devices'][1]['path']))
# Prepare disks
if not self.block_devices['finalized']:
modified_disks = []
for disk in self.block_devices['devices']:
if disk.get('base'):
img = images.Image.from_url(disk['base'])
hashed_image_path = img.version_image_path()
with util.RecordedOperation('detect cdrom images', self):
try:
cd = pycdlib.PyCdlib()
cd.open(hashed_image_path)
disk['present_as'] = 'cdrom'
except Exception:
pass
if disk.get('present_as', 'cdrom') == 'cdrom':
# There is no point in resizing or COW'ing a cdrom
disk['path'] = disk['path'].replace('.qcow2', '.raw')
disk['type'] = 'raw'
disk['snapshot_ignores'] = True
try:
os.link(hashed_image_path, disk['path'])
except OSError:
# Different filesystems
util.execute(
[lock], 'cp %s %s' % (hashed_image_path, disk['path']))
# Due to limitations in some installers, cdroms are always on IDE
disk['device'] = 'hd%s' % disk['device'][-1]
disk['bus'] = 'ide'
else:
if config.get('DISK_FORMAT') == 'qcow':
with util.RecordedOperation('create copy on write layer', self):
images.create_cow([lock], hashed_image_path,
disk['path'], disk['size'])
# Record the backing store for modern libvirts
disk['backing'] = (
'<backingStore type=\'file\'>\n'
' <format type=\'qcow2\'/>\n'
' <source file=\'%s\'/>\n'
' </backingStore>\n'
% (hashed_image_path))
elif config.get('DISK_FORMAT') == 'qcow_flat':
with util.RecordedOperation('resize image', self):
resized_image_path = img.resize(
[lock], disk['size'])
with util.RecordedOperation('create flat layer', self):
images.create_flat(
[lock], resized_image_path, disk['path'])
elif config.get('DISK_FORMAT') == 'flat':
with util.RecordedOperation('resize image', self):
resized_image_path = img.resize(
[lock], disk['size'])
with util.RecordedOperation('create raw disk', self):
images.create_raw(
[lock], resized_image_path, disk['path'])
else:
raise Exception('Unknown disk format')
elif not os.path.exists(disk['path']):
util.execute(None, 'qemu-img create -f qcow2 %s %sG'
% (disk['path'], disk['size']))
modified_disks.append(disk)
self.block_devices['devices'] = modified_disks
self.block_devices['finalized'] = True
self.persist()
# Create the actual instance
with util.RecordedOperation('create domain XML', self):
self._create_domain_xml()
# Sometimes on Ubuntu 20.04 we need to wait for port binding to work.
# Revisiting this is tracked by issue 320 on github.
with util.RecordedOperation('create domain', self):
if not self.power_on():
attempts = 0
while not self.power_on() and attempts < 100:
LOG.withObj(self).warning(
'Instance required an additional attempt to power on')
time.sleep(5)
attempts += 1
if self.is_powered_on():
LOG.withObj(self).info('Instance now powered on')
else:
LOG.withObj(self).info('Instance failed to power on')
self.update_instance_state('created')
def _update_power_states(self):
libvirt = util.get_libvirt()
conn = libvirt.open(None)
try:
seen = []
# Active VMs have an ID. Active means running in libvirt
# land.
for domain_id in conn.listDomainsID():
domain = conn.lookupByID(domain_id)
if not domain.name().startswith('sf:'):
continue
instance_uuid = domain.name().split(':')[1]
log_ctx = LOG.withInstance(instance_uuid)
instance = db.get_instance(instance_uuid)
if not instance:
# Instance is SF but not in database. Kill to reduce load.
log_ctx.warning('Destroying unknown instance')
util.execute(None, 'virsh destroy "sf:%s"' % instance_uuid)
continue
db.place_instance(instance_uuid, config.NODE_NAME)
seen.append(domain.name())
if instance.get('state') == 'deleted':
# NOTE(mikal): a delete might be in-flight in the queue.
# We only worry about instances which should have gone
# away five minutes ago.
if time.time() - instance['state_updated'] < 300:
continue
db.instance_enforced_deletes_increment(instance_uuid)
attempts = instance.get('enforced_deletes', 0)
if attempts > 5:
# Sometimes we just can't delete the VM. Try the big hammer instead.
log_ctx.warning(
'Attempting alternate delete method for instance')
util.execute(None,
'virsh destroy "sf:%s"' % instance_uuid)
db.add_event('instance', instance_uuid,
'enforced delete', 'complete', None, None)
else:
i = virt.from_db(instance_uuid)
i.delete()
i.update_instance_state('deleted')
log_ctx.withField(
'attempt', attempts).warning('Deleting stray instance')
continue
state = util.extract_power_state(libvirt, domain)
db.update_instance_power_state(instance_uuid, state)
if state == 'crashed':
db.update_instance_state(instance_uuid, 'error')
# Inactive VMs just have a name, and are powered off
# in our state system.
for domain_name in conn.listDefinedDomains():
if not domain_name.startswith('sf:'):
continue
if domain_name not in seen:
instance_uuid = domain_name.split(':')[1]
log_ctx = LOG.withInstance(instance_uuid)
instance = db.get_instance(instance_uuid)
if not instance:
# Instance is SF but not in database. Kill because unknown.
log_ctx.warning('Removing unknown inactive instance')
domain = conn.lookupByName(domain_name)
domain.undefine()
continue
if instance.get('state') == 'deleted':
# NOTE(mikal): a delete might be in-flight in the queue.
# We only worry about instances which should have gone
# away five minutes ago.
if time.time() - instance['state_updated'] < 300:
continue
domain = conn.lookupByName(domain_name)
domain.undefine()
log_ctx.info('Detected stray instance')
db.add_event('instance', instance_uuid,
'deleted stray', 'complete', None, None)
continue
db.place_instance(instance_uuid, config.NODE_NAME)
instance_path = os.path.join(config.get('STORAGE_PATH'),
'instances', instance_uuid)
if not os.path.exists(instance_path):
# If we're inactive and our files aren't on disk,
# we have a problem.
log_ctx.info('Detected error state for instance')
db.update_instance_state(instance_uuid, 'error')
elif instance.get('power_state') != 'off':
log_ctx.info('Detected power off for instance')
db.update_instance_power_state(instance_uuid, 'off')
db.add_event('instance', instance_uuid,
'detected poweroff', 'complete', None,
None)
except libvirt.libvirtError as e:
LOG.error('Failed to lookup all domains: %s' % e)
def _read_template(self, template):
with open(os.path.join(config.get('STORAGE_PATH'), template)) as f:
return jinja2.Template(f.read())
def place_instance(self, instance, network, candidates=None):
with util.RecordedOperation('schedule', instance):
log_ctx = LOG.withObj(instance)
diff = time.time() - self.metrics_updated
if diff > config.get('SCHEDULER_CACHE_TIMEOUT'):
self.refresh_metrics()
if candidates:
log_ctx.info('Scheduling %s forced as candidates' % candidates)
instance.add_event('schedule', 'Forced candidates', None,
str(candidates))
for node in candidates:
if node not in self.metrics:
raise exceptions.CandidateNodeNotFoundException(node)
else:
candidates = []
for node in self.metrics.keys():
candidates.append(node)
log_ctx.info('Scheduling %s start as candidates' % candidates)
instance.add_event('schedule', 'Initial candidates', None,
str(candidates))
if not candidates:
raise exceptions.LowResourceException('No nodes with metrics')
# Can we host that many vCPUs?
for node in copy.copy(candidates):
max_cpu = self.metrics[node].get('cpu_max_per_instance', 0)
if instance.cpus > max_cpu:
candidates.remove(node)
log_ctx.info('Scheduling %s have enough actual CPU' % candidates)
instance.add_event('schedule', 'Have enough actual CPU', None,
str(candidates))
if not candidates:
raise exceptions.LowResourceException(
'Requested vCPUs exceeds vCPU limit')
# Do we have enough idle CPU?
for node in copy.copy(candidates):
if not self._has_sufficient_cpu(instance.cpus, node):
candidates.remove(node)
log_ctx.info('Scheduling %s have enough idle CPU' % candidates)
instance.add_event('schedule', 'Have enough idle CPU', None,
str(candidates))
if not candidates:
raise exceptions.LowResourceException(
'No nodes with enough idle CPU')
# Do we have enough idle RAM?
for node in copy.copy(candidates):
if not self._has_sufficient_ram(instance.memory, node):
candidates.remove(node)
log_ctx.info('Scheduling %s have enough idle RAM' % candidates)
instance.add_event('schedule', 'Have enough idle RAM', None,
str(candidates))
if not candidates:
raise exceptions.LowResourceException(
'No nodes with enough idle RAM')
# Do we have enough idle disk?
for node in copy.copy(candidates):
if not self._has_sufficient_disk(instance, node):
candidates.remove(node)
log_ctx.info('Scheduling %s have enough idle disk' % candidates)
instance.add_event('schedule', 'Have enough idle disk', None,
str(candidates))
if not candidates:
raise exceptions.LowResourceException(
'No nodes with enough disk space')
# What nodes have the highest number of networks already present?
if network:
requested_networks = []
for net in network:
network_uuid = net['network_uuid']
if network_uuid not in requested_networks:
requested_networks.append(network_uuid)
candidates = self._find_most_matching_networks(
requested_networks, candidates)
log_ctx.info('Scheduling %s have most matching networks' %
candidates)
instance.add_event('schedule', 'Have most matching networks',
None, str(candidates))
# What nodes have the base image already?
requested_images = []
for disk in instance.block_devices['devices']:
if disk.get('base'):
requested_images = disk.get('base')
candidates = self._find_most_matching_images(
requested_images, candidates)
log_ctx.info('Scheduling %s have most matching images' %
candidates)
instance.add_event('schedule', 'Have most matching images', None,
str(candidates))
# Avoid allocating to network node if possible
net_node = db.get_network_node()
if len(candidates) > 1 and net_node['fqdn'] in candidates:
candidates.remove(net_node['fqdn'])
log_ctx.info('Scheduling %s are non-network nodes' %
candidates)
instance.add_event('schedule', 'Are non-network nodes', None,
str(candidates))
# Return a shuffled list of options
random.shuffle(candidates)
return candidates
def resolve(name):
resp = requests.get(UBUNTU_URL,
headers={'User-Agent': util.get_user_agent()})
if resp.status_code != 200:
raise exceptions.HTTPError('Failed to fetch %s, status code %d' %
(UBUNTU_URL, resp.status_code))
num_to_name = {}
name_to_num = {}
dir_re = re.compile(
r'.*<a href="(.*)/">.*Ubuntu Server ([0-9]+\.[0-9]+).*')
for line in resp.text.split('\n'):
m = dir_re.match(line)
if m:
num_to_name[m.group(2)] = m.group(1)
name_to_num[m.group(1)] = m.group(2)
LOG.withField('versions', num_to_name).info('Found ubuntu versions')
vernum = None
vername = None
if name == 'ubuntu':
vernum = sorted(num_to_name.keys())[-1]
vername = num_to_name[vernum]
else:
try:
# Name is assumed to be in the form ubuntu:18.04 or ubuntu:bionic
_, version = name.split(':')
if version in num_to_name:
vernum = version
vername = num_to_name[version]
else:
vername = version
vernum = name_to_num[version]
except Exception:
raise exceptions.VersionSpecificationError(
'Cannot parse version: %s' % name)
url = (config.get('DOWNLOAD_URL_UBUNTU') % {
'vernum': vernum,
'vername': vername
})
log = LOG.withField('url', url)
# Retrieve check sum file
checksum_url = UBUNTU_URL + '/' + vername + '/current/MD5SUMS'
resp = requests.get(checksum_url,
headers={'User-Agent': util.get_user_agent()})
if resp.status_code != 200:
raise exceptions.HTTPError('Failed to fetch %s, status code %d' %
(checksum_url, resp.status_code))
sum_re = re.compile(r'^([0-9a-f]+) .*' + vername +
'-server-cloudimg-amd64.img')
checksum = None
for line in resp.text.split('\n'):
m = sum_re.match(line)
if m:
checksum = m.group(1)
break
if not checksum_url:
log.warning('Did not find checksum')
checksum = checksum.strip()
log.withField('checksum', checksum).info('Checksum check')
return (url, checksum)
def _get_defaulted_disk_bus(disk):
bus = disk.get('bus')
if bus:
return bus
return config.get('DISK_BUS')
def instance_path(self):
return os.path.join(config.get('STORAGE_PATH'), 'instances', self.uuid)
def __init__(self, id):
super(Monitor, self).__init__(id)
start_http_server(config.get('PROMETHEUS_METRICS_PORT'))
def _make_config_drive(self, disk_path):
"""Create a config drive"""
# NOTE(mikal): with a big nod at https://gist.github.com/pshchelo/378f3c4e7d18441878b9652e9478233f
iso = pycdlib.PyCdlib()
iso.new(interchange_level=4,
joliet=True,
rock_ridge='1.09',
vol_ident='config-2')
# We're only going to pretend to do the most recent OpenStack version
iso.add_directory('/openstack',
rr_name='openstack',
joliet_path='/openstack')
iso.add_directory('/openstack/2017-02-22',
rr_name='2017-02-22',
joliet_path='/openstack/2017-02-22')
iso.add_directory('/openstack/latest',
rr_name='latest',
joliet_path='/openstack/latest')
# meta_data.json
md = json.dumps({
'random_seed': base64.b64encode(os.urandom(512)).decode('ascii'),
'uuid': self.uuid,
'availability_zone': config.get('ZONE'),
'hostname': '%s.local' % self.name,
'launch_index': 0,
'devices': [],
'project_id': None,
'name': self.name,
'public_keys': {
'mykey': self.ssh_key
}
}).encode('ascii')
iso.add_fp(io.BytesIO(md), len(md), '/openstack/latest/meta_data.json;1',
rr_name='meta_data.json',
joliet_path='/openstack/latest/meta_data.json')
iso.add_fp(io.BytesIO(md), len(md), '/openstack/2017-02-22/meta_data.json;2',
rr_name='meta_data.json',
joliet_path='/openstack/2017-02-22/meta_data.json')
# user_data
if self.user_data:
user_data = base64.b64decode(self.user_data)
iso.add_fp(io.BytesIO(user_data), len(user_data), '/openstack/latest/user_data',
rr_name='user_data',
joliet_path='/openstack/latest/user_data.json')
iso.add_fp(io.BytesIO(user_data), len(user_data), '/openstack/2017-02-22/user_data',
rr_name='user_data',
joliet_path='/openstack/2017-02-22/user_data.json')
# network_data.json
nd = {
'links': [],
'networks': [],
'services': [
{
'address': '8.8.8.8',
'type': 'dns'
}
]
}
seen_networks = []
for iface in db.get_instance_interfaces(self.uuid):
devname = 'eth%d' % iface['order']
nd['links'].append(
{
'ethernet_mac_address': iface['macaddr'],
'id': devname,
'name': devname,
'mtu': 1450,
'type': 'vif',
'vif_id': iface['uuid']
}
)
if not iface['network_uuid'] in seen_networks:
n = net.from_db(iface['network_uuid'])
nd['networks'].append(
{
'id': iface['network_uuid'],
'link': devname,
'type': 'ipv4',
'ip_address': iface['ipv4'],
'netmask': str(n.netmask),
'routes': [
{
'network': '0.0.0.0',
'netmask': '0.0.0.0',
'gateway': str(n.router)
}
],
'network_id': iface['network_uuid']
}
)
seen_networks.append(iface['network_uuid'])
nd_encoded = json.dumps(nd).encode('ascii')
iso.add_fp(io.BytesIO(nd_encoded), len(nd_encoded),
'/openstack/latest/network_data.json;3',
rr_name='network_data.json',
joliet_path='/openstack/latest/vendor_data.json')
iso.add_fp(io.BytesIO(nd_encoded), len(nd_encoded),
'/openstack/2017-02-22/network_data.json;4',
rr_name='network_data.json',
joliet_path='/openstack/2017-02-22/vendor_data.json')
# empty vendor_data.json and vendor_data2.json
vd = '{}'.encode('ascii')
iso.add_fp(io.BytesIO(vd), len(vd),
'/openstack/latest/vendor_data.json;5',
rr_name='vendor_data.json',
joliet_path='/openstack/latest/vendor_data.json')
iso.add_fp(io.BytesIO(vd), len(vd),
'/openstack/2017-02-22/vendor_data.json;6',
rr_name='vendor_data.json',
joliet_path='/openstack/2017-02-22/vendor_data.json')
iso.add_fp(io.BytesIO(vd), len(vd),
'/openstack/latest/vendor_data2.json;7',
rr_name='vendor_data2.json',
joliet_path='/openstack/latest/vendor_data2.json')
iso.add_fp(io.BytesIO(vd), len(vd),
'/openstack/2017-02-22/vendor_data2.json;8',
rr_name='vendor_data2.json',
joliet_path='/openstack/2017-02-22/vendor_data2.json')
# Dump to disk
iso.write(disk_path)
iso.close()
def snapshot_path(self):
return os.path.join(config.get('STORAGE_PATH'), 'snapshots')
