def find_orphaned_instances(session, verbose=False):
"""Find and return a list of orphaned instances."""
ctxt = context.get_admin_context(read_deleted="only")
orphaned_instances = []
for vm_ref, vm_rec in _get_applicable_vm_recs(session):
try:
uuid = vm_rec['other_config']['nova_uuid']
instance = db.api.instance_get_by_uuid(ctxt, uuid)
except (KeyError, exception.InstanceNotFound):
# NOTE(jk0): Err on the side of caution here. If we don't know
# anything about the particular instance, ignore it.
print_xen_object("INFO: Ignoring VM",
vm_rec,
indent_level=0,
verbose=verbose)
continue
# NOTE(jk0): This would be triggered if a VM was deleted but the
# actual deletion process failed somewhere along the line.
is_active_and_deleting = (instance.vm_state == "active"
and instance.task_state == "deleting")
# NOTE(jk0): A zombie VM is an instance that is not active and hasn't
# been updated in over the specified period.
is_zombie_vm = (instance.vm_state != "active" and utils.is_older_than(
instance.updated_at, FLAGS.zombie_instance_updated_at_window))
if is_active_and_deleting or is_zombie_vm:
orphaned_instances.append((vm_ref, vm_rec, instance))
return orphaned_instances
def find_orphaned_instances(session, verbose=False):
"""Find and return a list of orphaned instances."""
ctxt = context.get_admin_context(read_deleted="only")
orphaned_instances = []
for vm_rec in _get_applicable_vm_recs(session):
try:
uuid = vm_rec['other_config']['nova_uuid']
instance = db.api.instance_get_by_uuid(ctxt, uuid)
except (KeyError, exception.InstanceNotFound):
# NOTE(jk0): Err on the side of caution here. If we don't know
# anything about the particular instance, ignore it.
print_xen_object("INFO: Ignoring VM", vm_rec, indent_level=0,
verbose=verbose)
continue
# NOTE(jk0): This would be triggered if a VM was deleted but the
# actual deletion process failed somewhere along the line.
is_active_and_deleting = (instance.vm_state == "active" and
instance.task_state == "deleting")
# NOTE(jk0): A zombie VM is an instance that is not active and hasn't
# been updated in over the specified period.
is_zombie_vm = (instance.vm_state != "active"
and utils.is_older_than(instance.updated_at,
FLAGS.zombie_instance_updated_at_window))
if is_active_and_deleting or is_zombie_vm:
orphaned_instances.append(instance)
return orphaned_instances
def find_orphaned_instances(session, verbose=False):
"""Find and return a list of orphaned instances."""
ctxt = context.get_admin_context()
orphaned_instances = []
for vm_rec in _get_applicable_vm_recs(session):
try:
instance_id = get_instance_id_from_name_label(
vm_rec["name_label"], FLAGS.instance_name_template)
except UnrecognizedNameLabel, exc:
print_xen_object("WARNING: Unrecognized VM", vm_rec,
indent_level=0, verbose=verbose)
continue
try:
instance = db.api.instance_get(ctxt, instance_id)
except exception.InstanceNotFound:
# NOTE(jk0): Err on the side of caution here. If we don't know
# anything about the particular instance, print a warning and let
# the operator handle it manually.
print >> sys.stderr, "Instance %s not found" % instance_id
# NOTE(jk0): A zombie VM is an instance that is not active and hasn't
# been updated in over the specified period.
is_zombie_vm = (instance.vm_state != "active"
and utils.is_older_than(instance.updated_at,
FLAGS.zombie_instance_updated_at_window))
if is_zombie_vm:
orphaned_instances.append(instance)
def poll_rescued_instances(self, timeout):
"""Look for expirable rescued instances.
- forcibly exit rescue mode for any instances that have been
in rescue mode for >= the provided timeout
"""
last_ran = self.poll_rescue_last_ran
if not last_ran:
# We need a base time to start tracking.
self.poll_rescue_last_ran = utils.utcnow()
return
if not utils.is_older_than(last_ran, timeout):
# Do not run. Let's bail.
return
# Update the time tracker and proceed.
self.poll_rescue_last_ran = utils.utcnow()
rescue_vms = []
for instance in self.list_instances():
if instance.endswith("-rescue"):
rescue_vms.append(dict(name=instance,
vm_ref=VMHelper.lookup(self._session,
instance)))
for vm in rescue_vms:
rescue_vm_ref = vm["vm_ref"]
self._destroy_rescue_instance(rescue_vm_ref)
original_name = vm["name"].split("-rescue", 1)[0]
original_vm_ref = VMHelper.lookup(self._session, original_name)
self._release_bootlock(original_vm_ref)
self._session.call_xenapi("VM.start", original_vm_ref, False,
False)
def find_orphaned_instances(session, verbose=False):
"""Find and return a list of orphaned instances."""
ctxt = context.get_admin_context()
ctxt.read_deleted = True
orphaned_instances = []
for vm_rec in _get_applicable_vm_recs(session):
try:
instance_id = get_instance_id_from_name_label(
vm_rec["name_label"], FLAGS.instance_name_template)
except UnrecognizedNameLabel, exc:
print_xen_object("WARNING: Unrecognized VM", vm_rec,
indent_level=0, verbose=verbose)
continue
try:
instance = db.api.instance_get(ctxt, instance_id)
except exception.InstanceNotFound:
# NOTE(jk0): Err on the side of caution here. If we don't know
# anything about the particular instance, ignore it.
print_xen_object("INFO: Ignoring VM", vm_rec, indent_level=0,
verbose=verbose)
continue
# NOTE(jk0): This would be triggered if a VM was deleted but the
# actual deletion process failed somewhere along the line.
is_active_and_deleting = (instance.vm_state == "active" and
instance.task_state == "deleting")
# NOTE(jk0): A zombie VM is an instance that is not active and hasn't
# been updated in over the specified period.
is_zombie_vm = (instance.vm_state != "active"
and utils.is_older_than(instance.updated_at,
FLAGS.zombie_instance_updated_at_window))
if is_active_and_deleting or is_zombie_vm:
orphaned_instances.append(instance)
def poll_rescued_instances(self, timeout):
"""Look for expirable rescued instances.
- forcibly exit rescue mode for any instances that have been
in rescue mode for >= the provided timeout
"""
last_ran = self.poll_rescue_last_ran
if not last_ran:
# We need a base time to start tracking.
self.poll_rescue_last_ran = utils.utcnow()
return
if not utils.is_older_than(last_ran, timeout):
# Do not run. Let's bail.
return
# Update the time tracker and proceed.
self.poll_rescue_last_ran = utils.utcnow()
rescue_vms = []
for instance in self.list_instances():
if instance.endswith("-rescue"):
rescue_vms.append(dict(name=instance,
vm_ref=VMHelper.lookup(self._session,
instance)))
for vm in rescue_vms:
rescue_vm_ref = vm["vm_ref"]
self._destroy_rescue_instance(rescue_vm_ref)
original_name = vm["name"].split("-rescue", 1)[0]
original_vm_ref = VMHelper.lookup(self._session, original_name)
self._release_bootlock(original_vm_ref)
self._session.call_xenapi("VM.start", original_vm_ref, False,
False)