def _emit_event(self, instance_name, instance_uuid, instance_state):
virt_event = self._get_virt_event(instance_uuid,
instance_state)
utils.spawn_n(self._state_change_callback, virt_event)
utils.spawn_n(self._handle_serial_console_workers,
instance_name, instance_state)
def rebuild_instance(
self,
context,
instance,
orig_image_ref,
image_ref,
injected_files,
new_pass,
orig_sys_metadata,
bdms,
recreate=False,
on_shared_storage=False,
preserve_ephemeral=False,
host=None,
request_spec=None,
kwargs=None,
):
# kwargs unused but required for cell compatibility.
utils.spawn_n(
self._manager.rebuild_instance,
context,
instance=instance,
new_pass=new_pass,
injected_files=injected_files,
image_ref=image_ref,
orig_image_ref=orig_image_ref,
orig_sys_metadata=orig_sys_metadata,
bdms=bdms,
recreate=recreate,
on_shared_storage=on_shared_storage,
host=host,
preserve_ephemeral=preserve_ephemeral,
request_spec=request_spec,
)
def _emit_event(self, instance_name, instance_uuid, instance_state):
virt_event = self._get_virt_event(instance_uuid, instance_state)
utils.spawn_n(self._state_change_callback, virt_event)
if instance_state == constants.HYPERV_VM_STATE_ENABLED:
utils.spawn_n(self._running_state_callback, instance_name,
instance_uuid)
def start_failover_listener_daemon(self):
"""Start the daemon failover listener."""
listener = self._clustutils.get_vm_owner_change_listener()
cbk = functools.partial(utils.spawn_n, self._failover_migrate)
utils.spawn_n(listener, cbk)
def _emit_event(self, instance_name, instance_uuid, instance_state):
virt_event = self._get_virt_event(instance_uuid,
instance_state)
utils.spawn_n(self._state_change_callback, virt_event)
utils.spawn_n(self._handle_serial_console_workers,
instance_name, instance_state)
def build_instances(
self,
context,
instances,
image,
filter_properties,
admin_password,
injected_files,
requested_networks,
security_groups,
block_device_mapping,
legacy_bdm=True,
):
utils.spawn_n(
self._manager.build_instances,
context,
instances=instances,
image=image,
filter_properties=filter_properties,
admin_password=admin_password,
injected_files=injected_files,
requested_networks=requested_networks,
security_groups=security_groups,
block_device_mapping=block_device_mapping,
legacy_bdm=legacy_bdm,
)
def rebuild_instance(self,
context,
instance,
orig_image_ref,
image_ref,
injected_files,
new_pass,
orig_sys_metadata,
bdms,
recreate=False,
on_shared_storage=False,
preserve_ephemeral=False,
host=None,
kwargs=None):
# kwargs unused but required for cell compatibility.
utils.spawn_n(self._manager.rebuild_instance,
context,
instance=instance,
new_pass=new_pass,
injected_files=injected_files,
image_ref=image_ref,
orig_image_ref=orig_image_ref,
orig_sys_metadata=orig_sys_metadata,
bdms=bdms,
recreate=recreate,
on_shared_storage=on_shared_storage,
host=host,
preserve_ephemeral=preserve_ephemeral)
def _emit_event(self, instance_name, instance_uuid, instance_state):
virt_event = self._get_virt_event(instance_uuid,
instance_state)
utils.spawn_n(self._state_change_callback, virt_event)
if instance_state == constants.HYPERV_VM_STATE_ENABLED:
utils.spawn_n(self._running_state_callback,
instance_name, instance_uuid)
def _init_instance_info(self, compute_nodes=None):
"""Creates the initial view of instances for all hosts.
As this initial population of instance information may take some time,
we don't wish to block the scheduler's startup while this completes.
The async method allows us to simply mock out the _init_instance_info()
method in tests.
:param compute_nodes: a list of nodes to populate instances info for
if is None, compute_nodes will be looked up in database
"""
def _async_init_instance_info(compute_nodes):
context = context_module.RequestContext()
LOG.debug("START:_async_init_instance_info")
self._instance_info = {}
if not compute_nodes:
compute_nodes = objects.ComputeNodeList.get_all(
context).objects
LOG.debug("Total number of compute nodes: %s", len(compute_nodes))
# Break the queries into batches of 10 to reduce the total number
# of calls to the DB.
batch_size = 10
start_node = 0
end_node = batch_size
while start_node <= len(compute_nodes):
curr_nodes = compute_nodes[start_node:end_node]
start_node += batch_size
end_node += batch_size
filters = {
"host": [curr_node.host for curr_node in curr_nodes],
"deleted": False
}
result = objects.InstanceList.get_by_filters(
context.elevated(), filters)
instances = result.objects
LOG.debug("Adding %s instances for hosts %s-%s",
len(instances), start_node, end_node)
for instance in instances:
host = instance.host
if host not in self._instance_info:
self._instance_info[host] = {
"instances": {},
"updated": False
}
inst_dict = self._instance_info[host]
inst_dict["instances"][instance.uuid] = instance
# Call sleep() to cooperatively yield
time.sleep(0)
LOG.debug("END:_async_init_instance_info")
# Run this async so that we don't block the scheduler start-up
utils.spawn_n(_async_init_instance_info, compute_nodes)
def build_instances(self, context, instances, image,
filter_properties, admin_password, injected_files,
requested_networks, security_groups, block_device_mapping,
legacy_bdm=True):
utils.spawn_n(self._manager.build_instances, context,
instances=instances, image=image,
filter_properties=filter_properties,
admin_password=admin_password, injected_files=injected_files,
requested_networks=requested_networks,
security_groups=security_groups,
block_device_mapping=block_device_mapping,
legacy_bdm=legacy_bdm)
def _init_instance_info(self, compute_nodes=None):
"""Creates the initial view of instances for all hosts.
As this initial population of instance information may take some time,
we don't wish to block the scheduler's startup while this completes.
The async method allows us to simply mock out the _init_instance_info()
method in tests.
:param compute_nodes: a list of nodes to populate instances info for
if is None, compute_nodes will be looked up in database
"""
def _async_init_instance_info(compute_nodes):
context = context_module.RequestContext()
LOG.debug("START:_async_init_instance_info")
self._instance_info = {}
if not compute_nodes:
compute_nodes = objects.ComputeNodeList.get_all(
context).objects
LOG.debug("Total number of compute nodes: %s", len(compute_nodes))
# Break the queries into batches of 10 to reduce the total number
# of calls to the DB.
batch_size = 10
start_node = 0
end_node = batch_size
while start_node <= len(compute_nodes):
curr_nodes = compute_nodes[start_node:end_node]
start_node += batch_size
end_node += batch_size
filters = {"host": [curr_node.host
for curr_node in curr_nodes],
"deleted": False}
result = objects.InstanceList.get_by_filters(
context.elevated(), filters)
instances = result.objects
LOG.debug("Adding %s instances for hosts %s-%s",
len(instances), start_node, end_node)
for instance in instances:
host = instance.host
if host not in self._instance_info:
self._instance_info[host] = {"instances": {},
"updated": False}
inst_dict = self._instance_info[host]
inst_dict["instances"][instance.uuid] = instance
# Call sleep() to cooperatively yield
time.sleep(0)
LOG.debug("END:_async_init_instance_info")
# Run this async so that we don't block the scheduler start-up
utils.spawn_n(_async_init_instance_info, compute_nodes)
def test_spawn_n_no_context(self):
self.assertIsNone(common_context.get_current())
def _fake_spawn(func, *args, **kwargs):
# call the method to ensure no error is raised
func(*args, **kwargs)
self.assertEqual('test', args[0])
def fake(arg):
pass
with mock.patch.object(eventlet, 'spawn_n', _fake_spawn):
utils.spawn_n(fake, 'test')
self.assertIsNone(common_context.get_current())
def test_spawn_n_no_context(self):
self.assertIsNone(common_context.get_current())
def _fake_spawn(func, *args, **kwargs):
# call the method to ensure no error is raised
func(*args, **kwargs)
self.assertEqual('test', args[0])
def fake(arg):
pass
with mock.patch.object(eventlet, 'spawn_n', _fake_spawn):
utils.spawn_n(fake, 'test')
self.assertIsNone(common_context.get_current())
def test_spawn_n_context(self):
self.assertIsNone(common_context.get_current())
ctxt = context.RequestContext('user', 'project')
def _fake_spawn(func, *args, **kwargs):
# call the method to ensure no error is raised
func(*args, **kwargs)
self.assertEqual(ctxt, args[0])
self.assertEqual('test', kwargs['kwarg1'])
def fake(context, kwarg1=None):
pass
with mock.patch.object(eventlet, 'spawn_n', _fake_spawn):
utils.spawn_n(fake, ctxt, kwarg1='test')
self.assertEqual(ctxt, common_context.get_current())
def test_spawn_n_context(self):
self.assertIsNone(common_context.get_current())
ctxt = context.RequestContext('user', 'project')
def _fake_spawn(func, *args, **kwargs):
# call the method to ensure no error is raised
func(*args, **kwargs)
self.assertEqual(ctxt, args[0])
self.assertEqual('test', kwargs['kwarg1'])
def fake(context, kwarg1=None):
pass
with mock.patch.object(eventlet, 'spawn_n', _fake_spawn):
utils.spawn_n(fake, ctxt, kwarg1='test')
self.assertEqual(ctxt, common_context.get_current())
def test_spawn_n_context(self):
self.assertIsNone(common_context.get_current())
ctxt = context.RequestContext("user", "project")
def _fake_spawn(func, *args, **kwargs):
# call the method to ensure no error is raised
func(*args, **kwargs)
self.assertEqual(ctxt, args[0])
self.assertEqual("test", kwargs["kwarg1"])
def fake(context, kwarg1=None):
pass
with mock.patch.object(eventlet, "spawn_n", _fake_spawn):
utils.spawn_n(fake, ctxt, kwarg1="test")
self.assertEqual(ctxt, common_context.get_current())
def reclaim_failovered_instances(self):
# NOTE(claudiub): some instances might have failovered while the
# nova-compute service was down. Those instances will have to be
# reclaimed by this node.
expected_attrs = ['id', 'uuid', 'name', 'host']
host_instance_uuids = self._vmops.list_instance_uuids()
nova_instances = self._get_nova_instances(expected_attrs,
host_instance_uuids)
# filter out instances that are known to be on this host.
nova_instances = [instance for instance in nova_instances if
self._this_node.upper() != instance.host.upper()]
for instance in nova_instances:
utils.spawn_n(self._failover_migrate,
instance.name, instance.host,
self._this_node)
def _check_inactive_enabled_compute_hosts(self, ctxt):
"""Check inactive enabled compute hosts periodically"""
ctxt = context.get_admin_context()
LOG.info("Checking inactive enabled compute hosts started")
# Get enabled compute services from DB
compute_services = db.service_get_all_by_topic(ctxt,
CONF.compute_topic)
# Get inactive enabled compute services
inactive_services = [
service for service in compute_services
if not self.servicegroup_api.service_is_up(service)]
LOG.info("%s inactive enabled compute host(s) found",
len(inactive_services))
# Prepare a new retry counter
new_retry_counter = {}
# Heal each inactive compute service
for service in inactive_services:
# Get the host name
host = service['host']
# Create a new retry_counter
if host in self.retry_counter:
new_retry_counter[host] = self.retry_counter[host] + 1
else:
new_retry_counter[host] = 1
# Do host evacuation if retry count of the host is exceeded
if new_retry_counter[host] > CONF.saver.retry_count:
utils.spawn_n(self._host_evacuate, ctxt, service)
# Remove service from new retry counter
del(new_retry_counter[host])
# Replace an old retry counter with new one
self.retry_counter = new_retry_counter
LOG.info("Checking inactive enabled compute hosts finished")
def test_spawn_n_has_link(self):
self.useFixture(fixtures.SpawnIsSynchronousFixture())
gt = utils.spawn_n(mock.MagicMock)
passed_arg = 'test'
call_count = []
def fake(thread, param):
self.assertEqual(gt, thread)
self.assertEqual(passed_arg, param)
call_count.append(1)
gt.link(fake, passed_arg)
self.assertEqual(1, len(call_count))
def test_spawn_n_has_link(self):
self.useFixture(fixtures.SpawnIsSynchronousFixture())
gt = utils.spawn_n(mock.MagicMock)
passed_arg = 'test'
call_count = []
def fake(thread, param):
self.assertEqual(gt, thread)
self.assertEqual(passed_arg, param)
call_count.append(1)
gt.link(fake, passed_arg)
self.assertEqual(1, len(call_count))
class LocalComputeTaskAPI(object):
def __init__(self):
# TODO(danms): This needs to be something more generic for
# other/future users of this sort of functionality.
self._manager = utils.ExceptionHelper(manager.ComputeTaskManager())
def resize_instance(self,
context,
instance,
extra_instance_updates,
scheduler_hint,
flavor,
reservations,
clean_shutdown=True,
request_spec=None):
# NOTE(comstud): 'extra_instance_updates' is not used here but is
# needed for compatibility with the cells_rpcapi version of this
# method.
self._manager.migrate_server(context,
instance,
scheduler_hint,
live=False,
rebuild=False,
flavor=flavor,
block_migration=None,
disk_over_commit=None,
reservations=reservations,
clean_shutdown=clean_shutdown,
request_spec=request_spec)
def live_migrate_instance(self,
context,
instance,
host_name,
block_migration,
disk_over_commit,
request_spec=None,
async=False):
scheduler_hint = {'host': host_name}
if async:
wrap = lambda *args: utils.spawn_n(*args)
else:
wrap = lambda *args: args[0](*args[1:])
wrap(self._manager.live_migrate_instance, context, instance,
scheduler_hint, block_migration, disk_over_commit, request_spec)
def test_spawn_n_return_has_wait(self):
self.useFixture(fixtures.SpawnIsSynchronousFixture())
gt = utils.spawn_n(lambda x: '%s' % x, 'foo')
foo = gt.wait()
self.assertEqual('foo', foo)
def test_spawn_n_return_has_wait(self):
self.useFixture(fixtures.SpawnIsSynchronousFixture())
gt = utils.spawn_n(lambda x: '%s' % x, 'foo')
foo = gt.wait()
self.assertEqual('foo', foo)
def test_spawn_passes_through(self):
self.useFixture(fixtures.SpawnIsSynchronousFixture())
tester = mock.MagicMock()
utils.spawn_n(tester.function, 'foo', bar='bar')
tester.function.assert_called_once_with('foo', bar='bar')
def unshelve_instance(self, context, instance):
utils.spawn_n(self._manager.unshelve_instance, context, instance=instance)
def unshelve_instance(self, context, instance):
utils.spawn_n(self._manager.unshelve_instance,
context,
instance=instance)
def start_listener(self):
utils.spawn_n(self._listener, self._event_callback)
def _s3_create(self, context, metadata):
"""Gets a manifest from s3 and makes an image."""
image_path = tempfile.mkdtemp(dir=CONF.image_decryption_dir)
image_location = metadata['properties']['image_location'].lstrip('/')
bucket_name = image_location.split('/')[0]
manifest_path = image_location[len(bucket_name) + 1:]
bucket = self._conn(context).get_bucket(bucket_name)
key = bucket.get_key(manifest_path)
manifest = key.get_contents_as_string()
manifest, image, image_uuid = self._s3_parse_manifest(context,
metadata,
manifest)
def delayed_create():
"""This handles the fetching and decrypting of the part files."""
log_vars = {'image_location': image_location,
'image_path': image_path}
def _update_image_state(context, image_uuid, image_state):
metadata = {'properties': {'image_state': image_state}}
self.service.update(context, image_uuid, metadata,
purge_props=False)
def _update_image_data(context, image_uuid, image_data):
metadata = {}
self.service.update(context, image_uuid, metadata, image_data,
purge_props=False)
try:
_update_image_state(context, image_uuid, 'downloading')
try:
parts = []
elements = manifest.find('image').getiterator('filename')
for fn_element in elements:
part = self._download_file(bucket,
fn_element.text,
image_path)
parts.append(part)
# NOTE(vish): this may be suboptimal, should we use cat?
enc_filename = os.path.join(image_path, 'image.encrypted')
with open(enc_filename, 'w') as combined:
for filename in parts:
with open(filename) as part:
shutil.copyfileobj(part, combined)
except Exception:
LOG.exception(_LE("Failed to download %(image_location)s "
"to %(image_path)s"), log_vars)
_update_image_state(context, image_uuid, 'failed_download')
return
_update_image_state(context, image_uuid, 'decrypting')
try:
hex_key = manifest.find('image/ec2_encrypted_key').text
encrypted_key = binascii.a2b_hex(hex_key)
hex_iv = manifest.find('image/ec2_encrypted_iv').text
encrypted_iv = binascii.a2b_hex(hex_iv)
dec_filename = os.path.join(image_path, 'image.tar.gz')
self._decrypt_image(context, enc_filename, encrypted_key,
encrypted_iv, dec_filename)
except Exception:
LOG.exception(_LE("Failed to decrypt %(image_location)s "
"to %(image_path)s"), log_vars)
_update_image_state(context, image_uuid, 'failed_decrypt')
return
_update_image_state(context, image_uuid, 'untarring')
try:
unz_filename = self._untarzip_image(image_path,
dec_filename)
except Exception:
LOG.exception(_LE("Failed to untar %(image_location)s "
"to %(image_path)s"), log_vars)
_update_image_state(context, image_uuid, 'failed_untar')
return
_update_image_state(context, image_uuid, 'uploading')
try:
with open(unz_filename) as image_file:
_update_image_data(context, image_uuid, image_file)
except Exception:
LOG.exception(_LE("Failed to upload %(image_location)s "
"to %(image_path)s"), log_vars)
_update_image_state(context, image_uuid, 'failed_upload')
return
metadata = {'status': 'active',
'properties': {'image_state': 'available'}}
self.service.update(context, image_uuid, metadata,
purge_props=False)
shutil.rmtree(image_path)
except exception.ImageNotFound:
LOG.info(_LI("Image %s was deleted underneath us"), image_uuid)
return
utils.spawn_n(delayed_create)
return image
def _s3_create(self, context, metadata):
"""Gets a manifest from s3 and makes an image."""
image_path = tempfile.mkdtemp(dir=CONF.image_decryption_dir)
image_location = metadata['properties']['image_location'].lstrip('/')
bucket_name = image_location.split('/')[0]
manifest_path = image_location[len(bucket_name) + 1:]
bucket = self._conn(context).get_bucket(bucket_name)
key = bucket.get_key(manifest_path)
manifest = key.get_contents_as_string()
manifest, image, image_uuid = self._s3_parse_manifest(context,
metadata,
manifest)
def delayed_create():
"""This handles the fetching and decrypting of the part files."""
log_vars = {'image_location': image_location,
'image_path': image_path}
def _update_image_state(context, image_uuid, image_state):
metadata = {'properties': {'image_state': image_state}}
self.service.update(context, image_uuid, metadata,
purge_props=False)
def _update_image_data(context, image_uuid, image_data):
metadata = {}
self.service.update(context, image_uuid, metadata, image_data,
purge_props=False)
try:
_update_image_state(context, image_uuid, 'downloading')
try:
parts = []
elements = manifest.find('image').getiterator('filename')
for fn_element in elements:
part = self._download_file(bucket,
fn_element.text,
image_path)
parts.append(part)
# NOTE(vish): this may be suboptimal, should we use cat?
enc_filename = os.path.join(image_path, 'image.encrypted')
with open(enc_filename, 'w') as combined:
for filename in parts:
with open(filename) as part:
shutil.copyfileobj(part, combined)
except Exception:
LOG.exception(_LE("Failed to download %(image_location)s "
"to %(image_path)s"), log_vars)
_update_image_state(context, image_uuid, 'failed_download')
return
_update_image_state(context, image_uuid, 'decrypting')
try:
hex_key = manifest.find('image/ec2_encrypted_key').text
encrypted_key = binascii.a2b_hex(hex_key)
hex_iv = manifest.find('image/ec2_encrypted_iv').text
encrypted_iv = binascii.a2b_hex(hex_iv)
dec_filename = os.path.join(image_path, 'image.tar.gz')
self._decrypt_image(context, enc_filename, encrypted_key,
encrypted_iv, dec_filename)
except Exception:
LOG.exception(_LE("Failed to decrypt %(image_location)s "
"to %(image_path)s"), log_vars)
_update_image_state(context, image_uuid, 'failed_decrypt')
return
_update_image_state(context, image_uuid, 'untarring')
try:
unz_filename = self._untarzip_image(image_path,
dec_filename)
except Exception:
LOG.exception(_LE("Failed to untar %(image_location)s "
"to %(image_path)s"), log_vars)
_update_image_state(context, image_uuid, 'failed_untar')
return
_update_image_state(context, image_uuid, 'uploading')
try:
with open(unz_filename) as image_file:
_update_image_data(context, image_uuid, image_file)
except Exception:
LOG.exception(_LE("Failed to upload %(image_location)s "
"to %(image_path)s"), log_vars)
_update_image_state(context, image_uuid, 'failed_upload')
return
metadata = {'status': 'active',
'properties': {'image_state': 'available'}}
self.service.update(context, image_uuid, metadata,
purge_props=False)
shutil.rmtree(image_path)
except exception.ImageNotFound:
LOG.info(_LI("Image %s was deleted underneath us"), image_uuid)
return
utils.spawn_n(delayed_create)
return image
def unshelve_instance(self, context, instance, request_spec=None):
utils.spawn_n(self._manager.unshelve_instance, context,
instance=instance, request_spec=request_spec)
def hot_extend_vdisk(self, context, instance, disk, size, granularity):
utils.spawn_n(self._manager.build_instances, context, instance, disk,
size, granularity)
def unshelve_instance(self, context, instance, request_spec=None):
utils.spawn_n(self._manager.unshelve_instance, context, instance=instance, request_spec=request_spec)
def start_listener(self):
utils.spawn_n(self._poll_events)
def test_spawn_passes_through(self):
self.useFixture(fixtures.SpawnIsSynchronousFixture())
tester = mock.MagicMock()
utils.spawn_n(tester.function, 'foo', bar='bar')
tester.function.assert_called_once_with('foo', bar='bar')
def start_listener(self):
utils.spawn_n(self._listener, self._event_callback)
