def power_on(self):
if not os.path.exists(self.xml_file):
db.enqueue_instance_error(self.db_entry['uuid'],
'missing domain file in power on')
libvirt = util.get_libvirt()
with open(self.xml_file) as f:
xml = f.read()
instance = self._get_domain()
if not instance:
conn = libvirt.open(None)
instance = conn.defineXML(xml)
if not instance:
db.enqueue_instance_error(self.db_entry['uuid'],
'power on failed to create domain')
raise exceptions.NoDomainException()
try:
instance.create()
except libvirt.libvirtError as e:
if not str(e).startswith(
'Requested operation is not valid: domain is already running'
):
LOG.withObj(self).warning('Instance start error: %s' % e)
return False
instance.setAutostart(1)
db.update_instance_power_state(
self.db_entry['uuid'], util.extract_power_state(libvirt, instance))
db.add_event('instance', self.db_entry['uuid'], 'poweron', 'complete',
None, None)
return True
def delete(self):
with util.RecordedOperation('delete domain', self):
try:
self.power_off()
instance = self._get_domain()
if instance:
instance.undefine()
except Exception as e:
util.ignore_exception('instance delete', e)
with util.RecordedOperation('delete disks', self):
try:
if os.path.exists(self.instance_path):
shutil.rmtree(self.instance_path)
except Exception as e:
util.ignore_exception('instance delete', e)
with util.RecordedOperation('release network addresses', self):
for ni in db.get_instance_interfaces(self.db_entry['uuid']):
db.update_network_interface_state(ni['uuid'], 'deleted')
with db.get_lock('ipmanager', None, ni['network_uuid'],
ttl=120):
ipm = db.get_ipmanager(ni['network_uuid'])
ipm.release(ni['ipv4'])
db.persist_ipmanager(ni['network_uuid'], ipm.save())
db.update_instance_power_state(self.db_entry['uuid'], 'off')
db.free_console_port(self.db_entry['console_port'])
db.free_console_port(self.db_entry['vdi_port'])
def unpause(self):
libvirt = util.get_libvirt()
instance = self._get_domain()
instance.resume()
db.update_instance_power_state(
self.db_entry['uuid'], util.extract_power_state(libvirt, instance))
db.add_event('instance', self.db_entry['uuid'], 'unpause', 'complete',
None, None)
def power_off(self):
libvirt = util.get_libvirt()
instance = self._get_domain()
if not instance:
return
try:
instance.destroy()
except libvirt.libvirtError as e:
LOG.withObj(self).error('Failed to delete domain: %s' % e)
db.update_instance_power_state(self.db_entry['uuid'], 'off')
db.add_event('instance', self.db_entry['uuid'], 'poweroff', 'complete',
None, None)
def test_update_instance_power_state(self, mock_put, mock_get):
db.update_instance_power_state('uuid42', 'off')
mock_get.assert_called()
etcd_write = mock_put.mock_calls[0][1]
self.assertEqual(('instance', None, 'uuid42'), etcd_write[0:3])
self.assertTrue(time.time() - etcd_write[3]['power_state_updated'] < 3)
del etcd_write[3]['power_state_updated']
self.assertEqual(
{
'power_state': 'off',
'power_state_previous': 'on',
'uuid': 'uuid42',
'video': {
'memory': 16384,
'model': 'cirrus'
},
'error_message': None,
}, etcd_write[3])
def test_update_instance_power_state_transition_new(
self, mock_put, mock_get):
db.update_instance_power_state('uuid42', 'on')
mock_get.assert_called()
mock_put.assert_not_called()
def test_update_instance_power_state_duplicate(self, mock_put, mock_get):
db.update_instance_power_state('uuid42', 'on')
mock_get.assert_called()
mock_put.assert_not_called()
def _update_power_states(self):
libvirt = util.get_libvirt()
conn = libvirt.open(None)
try:
seen = []
# Active VMs have an ID. Active means running in libvirt
# land.
for domain_id in conn.listDomainsID():
domain = conn.lookupByID(domain_id)
if not domain.name().startswith('sf:'):
continue
instance_uuid = domain.name().split(':')[1]
log_ctx = LOG.withInstance(instance_uuid)
instance = db.get_instance(instance_uuid)
if not instance:
# Instance is SF but not in database. Kill to reduce load.
log_ctx.warning('Destroying unknown instance')
util.execute(None, 'virsh destroy "sf:%s"' % instance_uuid)
continue
db.place_instance(instance_uuid, config.NODE_NAME)
seen.append(domain.name())
if instance.get('state') == 'deleted':
# NOTE(mikal): a delete might be in-flight in the queue.
# We only worry about instances which should have gone
# away five minutes ago.
if time.time() - instance['state_updated'] < 300:
continue
db.instance_enforced_deletes_increment(instance_uuid)
attempts = instance.get('enforced_deletes', 0)
if attempts > 5:
# Sometimes we just can't delete the VM. Try the big hammer instead.
log_ctx.warning(
'Attempting alternate delete method for instance')
util.execute(None,
'virsh destroy "sf:%s"' % instance_uuid)
db.add_event('instance', instance_uuid,
'enforced delete', 'complete', None, None)
else:
i = virt.from_db(instance_uuid)
i.delete()
i.update_instance_state('deleted')
log_ctx.withField(
'attempt', attempts).warning('Deleting stray instance')
continue
state = util.extract_power_state(libvirt, domain)
db.update_instance_power_state(instance_uuid, state)
if state == 'crashed':
db.update_instance_state(instance_uuid, 'error')
# Inactive VMs just have a name, and are powered off
# in our state system.
for domain_name in conn.listDefinedDomains():
if not domain_name.startswith('sf:'):
continue
if domain_name not in seen:
instance_uuid = domain_name.split(':')[1]
log_ctx = LOG.withInstance(instance_uuid)
instance = db.get_instance(instance_uuid)
if not instance:
# Instance is SF but not in database. Kill because unknown.
log_ctx.warning('Removing unknown inactive instance')
domain = conn.lookupByName(domain_name)
domain.undefine()
continue
if instance.get('state') == 'deleted':
# NOTE(mikal): a delete might be in-flight in the queue.
# We only worry about instances which should have gone
# away five minutes ago.
if time.time() - instance['state_updated'] < 300:
continue
domain = conn.lookupByName(domain_name)
domain.undefine()
log_ctx.info('Detected stray instance')
db.add_event('instance', instance_uuid,
'deleted stray', 'complete', None, None)
continue
db.place_instance(instance_uuid, config.NODE_NAME)
instance_path = os.path.join(config.get('STORAGE_PATH'),
'instances', instance_uuid)
if not os.path.exists(instance_path):
# If we're inactive and our files aren't on disk,
# we have a problem.
log_ctx.info('Detected error state for instance')
db.update_instance_state(instance_uuid, 'error')
elif instance.get('power_state') != 'off':
log_ctx.info('Detected power off for instance')
db.update_instance_power_state(instance_uuid, 'off')
db.add_event('instance', instance_uuid,
'detected poweroff', 'complete', None,
None)
except libvirt.libvirtError as e:
LOG.error('Failed to lookup all domains: %s' % e)
