def vm_migrate_cb(result, task_id, vm_uuid=None, slave_vm_uuid=None):
"""
A callback function for api.vm.migrate.views.vm_migrate.
"""
ghost_vm = SlaveVm.get_by_uuid(slave_vm_uuid)
msg = result.get('message', '')
if result['returncode'] == 0 and msg and 'Successfully migrated' in msg:
# Save node and delete placeholder VM first
node = ghost_vm.vm.node
nss = set(ghost_vm.vm.get_node_storages())
ghost_vm.delete(
) # post_delete signal will update node and storage resources
# Fetch VM after ghost_vm is deleted, because it updates vm.slave_vms array
vm = Vm.objects.select_related('node', 'dc').get(uuid=vm_uuid)
changing_node = vm.node != ghost_vm.vm.node
json = result.pop('json', None)
try: # save json from smartos
json_active = vm.json.load(json)
vm.json_active = json_active
vm.json = json_active
except Exception as e:
logger.exception(e)
logger.error(
'Could not parse json output from vm_migrate(%s). Error: %s',
vm_uuid, e)
raise TaskException(result, 'Could not parse json output')
nss.update(list(vm.get_node_storages()))
# Revert status and set new node (should trigger node resource update)
vm.revert_notready(save=False)
if changing_node:
vm.set_node(node)
vm.save(update_node_resources=True, update_storage_resources=nss)
SlaveVm.switch_vm_snapshots_node_storages(vm, nss=nss)
vm_node_changed.send(task_id, vm=vm, force_update=True) # Signal!
else:
vm = Vm.objects.get(uuid=vm_uuid)
_vm_migrate_cb_failed(result, task_id, vm, ghost_vm)
logger.error(
'Found nonzero returncode in result from vm_migrate(%s). Error: %s',
vm_uuid, msg)
raise TaskException(
result, 'Got bad return code (%s). Error: %s' %
(result['returncode'], msg))
task_log_cb_success(result, task_id, vm=vm, **result['meta'])
if vm.json_changed():
vm_update(vm)
return result
def vm_replica_cb_failed_cleanup(sender, apiview, result, task_id, status, obj,
**kwargs):
"""Signal receiver emitted after task is revoked."""
if sender == 'vm_replica':
slave_vm = SlaveVm.get_by_uuid(apiview['slave_vm_uuid'])
_vm_replica_cb_failed(result, task_id, obj, slave_vm,
apiview['method'])
def vm_replica_reinit_cb(result, task_id, vm_uuid=None, slave_vm_uuid=None):
"""
A callback function for api.vm.replica.views.vm_replica_reinit.
"""
slave_vm = SlaveVm.get_by_uuid(slave_vm_uuid)
vm = slave_vm.master_vm
assert vm.uuid == vm_uuid
action = result['meta']['apiview']['method']
result, jsons = _parse_vm_replica_result(result,
vm,
slave_vm,
action,
key_json_idx=0,
cb_name='vm_replica_reinit')
if result['returncode'] != 0:
if jsons and jsons[0].get('success',
False): # Successfully reversed replication
slave_vm.last_sync = _parse_last_sync(jsons[0])
slave_vm.rep_reinit_required = False
slave_vm.save()
msg = result['detail']
logger.error(
'Found nonzero returncode in result from %s vm_replica_reinit(%s, %s). Error: %s',
action, vm_uuid, slave_vm_uuid, msg)
errmsg = _update_task_result_failure(result, msg)
raise TaskException(result, errmsg)
slave_vm.rep_reinit_required = False
slave_vm.last_sync = _parse_last_sync(jsons[0])
_save_svc_state(slave_vm, jsons)
_save_svc_params(slave_vm, jsons)
slave_vm.save()
msg = 'Server replica was successfully reinitialized'
_update_task_result_success(result, slave_vm, action, msg)
task_log_cb_success(result, task_id, vm=vm, **result['meta'])
return result
def vm_replica_failover_cb(result, task_id, vm_uuid=None, slave_vm_uuid=None):
"""
A callback function for api.vm.replica.views.vm_replica_failover.
"""
slave_vm = SlaveVm.get_by_uuid(slave_vm_uuid,
sr=(
'vm',
'master_vm',
'vm__node',
'vm__dc',
))
vm = slave_vm.master_vm
assert vm.uuid == vm_uuid
action = result['meta']['apiview']['method']
force = result['meta']['apiview']['force']
result, jsons = _parse_vm_replica_result(result,
vm,
slave_vm,
action,
key_json_idx=-1,
cb_name='vm_replica_failover')
sync_status = _save_svc_state(slave_vm, jsons)
if result['returncode'] != 0:
if sync_status is not None:
slave_vm.save(update_fields=('sync_status', ))
vm.revert_notready()
msg = result['detail']
logger.error(
'Found nonzero returncode in result from %s vm_replica_failover(%s, %s). Error: %s',
action, vm_uuid, slave_vm_uuid, msg)
errmsg = _update_task_result_failure(result, msg)
raise TaskException(result, errmsg)
# New master VM was born
# Delete tasks for old master
if force:
tasks = list(vm.tasks.keys())
try:
tasks.remove(task_id)
except ValueError:
pass
_delete_tasks(vm, tasks)
# Create internal shutdown task of old master VM
old_vm_status = result['meta']['apiview']['orig_status']
_vm_shutdown(vm)
# Save new master, degrade old master
slave_vm.master_vm.revert_notready(save=False)
new_vm = slave_vm.fail_over()
# Re-check status of old master (current degraded slave) because it was shut down,
# but the state wasn't save (it was notready back then)
vm_status_one(task_id, vm)
# Continue with prompting of new master and degradation of old
SlaveVm.switch_vm_snapshots_node_storages(new_vm,
nss=vm.get_node_storages())
# Force update of zabbix
vm_json_active_changed.send(task_id,
vm=new_vm,
old_json_active={},
force_update=True) # Signal!
if new_vm.node != vm.node:
vm_node_changed.send(task_id, vm=new_vm, force_update=True) # Signal!
msg = 'Server replica was successfully promoted to master'
_update_task_result_success(result, slave_vm, action, msg)
task_log_cb_success(result, task_id, vm=new_vm, **result['meta'])
request = get_dummy_request(vm.dc, method='PUT', system_user=True)
# Mark pending backups as "lost" :( TODO: implement vm_backup_sync
new_vm.backup_set.filter(status=Backup.PENDING).update(status=Backup.LOST)
# Sync snapshots on new master VM (mark missing snapshots as "lost")
for disk_id, _ in enumerate(new_vm.json_active_get_disks(), start=1):
call_api_view(request,
'PUT',
vm_snapshot_list,
new_vm.hostname,
data={'disk_id': disk_id},
log_response=True)
if old_vm_status == Vm.RUNNING:
# Start new master VM
call_api_view(request,
'PUT',
vm_status,
new_vm.hostname,
action='start',
log_response=True)
return result
def vm_replica_cb(result, task_id, vm_uuid=None, slave_vm_uuid=None):
"""
A callback function for api.vm.replica.views.vm_replica.
"""
slave_vm = SlaveVm.get_by_uuid(slave_vm_uuid)
vm = slave_vm.master_vm
assert vm.uuid == vm_uuid
action = result['meta']['apiview']['method']
result, jsons = _parse_vm_replica_result(result, vm, slave_vm, action)
if action == 'POST':
vm.revert_notready()
if jsons and jsons[0].get('success', False):
esrep_init = jsons[0]
# New slave VM was successfully created on target node
# noinspection PyTypeChecker
json_active = pickle.loads(
base64.decodestring(esrep_init.pop('slave_json')))
slave_vm.vm.json = slave_vm.vm.json_active = json_active
slave_vm.vm.status = Vm.STOPPED
slave_vm.vm.save(update_fields=('status', 'status_change',
'enc_json', 'enc_json_active',
'changed'))
slave_vm.last_sync = _parse_last_sync(esrep_init)
else:
slave_vm.delete()
sync_status = _save_svc_state(slave_vm, jsons)
msg = result['detail']
if result['returncode'] == 0 and jsons:
if action == 'POST':
_save_svc_params(slave_vm, jsons)
slave_vm.save()
msg = 'Server replica was successfully initialized'
elif action == 'PUT':
_save_svc_params(slave_vm, jsons)
slave_vm.save()
msg = 'Server replication service was successfully updated'
elif action == 'DELETE':
slave_vm.delete()
msg = 'Server replica was successfully destroyed'
# noinspection PyTypeChecker
if len(jsons[-1]['master_cleaned_disks']) != len(
vm.json_active_get_disks()):
warning = "WARNING: Master server's disks were not cleaned properly"
result['detail'] += ' msg=' + warning
msg += '; ' + warning
else:
if sync_status is not None:
slave_vm.save(update_fields=('sync_status', ))
logger.error(
'Found nonzero returncode in result from %s vm_replica(%s, %s). Error: %s',
action, vm_uuid, slave_vm_uuid, msg)
errmsg = _update_task_result_failure(result, msg)
raise TaskException(result, errmsg)
_update_task_result_success(result, slave_vm, action, msg)
task_log_cb_success(result, task_id, vm=vm, **result['meta'])
return result
def _task_cleanup(result, task_id, task_status, obj, **kwargs):
"""
Cleanup after task is revoked.
"""
apiview = result['meta']['apiview']
view = apiview['view']
if view == 'vm_snapshot':
from vms.models import Snapshot
from api.vm.snapshot.tasks import _vm_snapshot_cb_failed
snap = Snapshot.objects.get(vm=obj,
disk_id=Snapshot.get_disk_id(
obj, apiview['disk_id']),
name=apiview['snapname'])
_vm_snapshot_cb_failed(result, task_id, snap, apiview['method'])
elif view == 'vm_snapshot_list':
from vms.models import Snapshot
from api.vm.snapshot.tasks import _vm_snapshot_list_cb_failed
snaps = Snapshot.objects.filter(vm=obj,
disk_id=Snapshot.get_disk_id(
obj, apiview['disk_id']),
name__in=apiview['snapnames'])
_vm_snapshot_list_cb_failed(result, task_id, snaps, apiview['method'])
elif view == 'vm_backup':
from vms.models import Backup
from api.vm.backup.tasks import _vm_backup_cb_failed
bkp = Backup.objects.get(vm_hostname=apiview['hostname'],
vm_disk_id=apiview['disk_id'] - 1,
name=apiview['bkpname'])
_vm_backup_cb_failed(result, task_id, bkp, apiview['method'], vm=obj)
elif view == 'vm_backup_list':
from vms.models import Backup
from api.vm.backup.tasks import _vm_backup_list_cb_failed
bkps = Backup.objects.filter(vm_hostname=apiview['hostname'],
vm_disk_id=apiview['disk_id'] - 1,
name__in=apiview['bkpnames'])
_vm_backup_list_cb_failed(result, task_id, bkps, apiview['method'])
elif view == 'vm_manage':
if apiview['method'] == 'POST':
from api.vm.base.tasks import _vm_create_cb_failed
result['message'] = ''
_vm_create_cb_failed(result, task_id, obj)
elif apiview['method'] == 'DELETE':
from api.vm.base.tasks import _vm_delete_cb_failed
_vm_delete_cb_failed(result, task_id, obj)
elif apiview['method'] == 'PUT':
from api.vm.base.tasks import _vm_update_cb_done
_vm_update_cb_done(result, task_id, obj)
elif view == 'vm_status':
from api.vm.status.tasks import _vm_status_cb_failed
if apiview['method'] == 'PUT':
_vm_status_cb_failed(result, task_id, obj)
elif view == 'vm_migrate':
from vms.models import SlaveVm
from api.vm.migrate.tasks import _vm_migrate_cb_failed
ghost_vm = SlaveVm.get_by_uuid(apiview['slave_vm_uuid'])
_vm_migrate_cb_failed(result, task_id, obj, ghost_vm)
elif view == 'image_manage' or view == 'image_snapshot':
# obj = Image
from vms.models import Snapshot
from api.image.base.tasks import _image_manage_cb_failed
method = apiview['method']
snap_id = obj.src_snap_id
if method == 'POST' and snap_id:
snap = Snapshot.objects.get(id=snap_id)
else:
snap = None
_image_manage_cb_failed(result, task_id, obj, method, snap=snap)
elif view == 'node_image':
# obj = NodeStorage
from vms.models import Image
from api.node.image.tasks import _node_image_cb_failed
img = Image.objects.get(name=apiview['name'])
_node_image_cb_failed(result, task_id, obj, img)
else:
task_cleanup_signal.send(sender=view,
apiview=apiview,
result=result,
task_id=task_id,
status=task_status,
obj=obj)
def validate(self, attrs):
vm = self.vm
node = attrs.get('node', vm.node)
changing_node = attrs.get('node', vm.node) != vm.node
# Ghost VM is a copy of a VM used to take up place in DB.
# When node is changing we have to have all disks in a ghost VM.
# When changing only disk pools, only the changed disks have to be in a ghost VM.
ghost_vm = SlaveVm(_master_vm=vm)
ghost_vm.reserve_resources = changing_node
ghost_vm.set_migration_hostname()
ghost_vm.node = node
ghost_vm_define = SlaveVmDefine(ghost_vm)
# Validate root_zpool (we can do this after we know the new node)
root_zpool = attrs.get('root_zpool', None)
# Every pool must be validated when changing node
try:
root_zpool = ghost_vm_define.save_root_zpool(
root_zpool, save_same_zpool=changing_node)
except APIValidationError as exc:
self._errors['node'] = exc.api_errors
return attrs
# Validate disk_zpools (we can do this after we know the new node)
if ghost_vm.vm.is_kvm():
disk_zpools = attrs.get('disk_zpools', {})
try:
disk_zpools = ghost_vm_define.save_disk_zpools(
disk_zpools, save_same_zpool=changing_node)
except APIValidationError as exc:
self._errors['node'] = exc.api_errors
return attrs
else:
disk_zpools = {}
# Nothing changed, he?
if not changing_node and not (root_zpool or disk_zpools):
raise s.ValidationError(_('Nothing to do.'))
# Validate dc_node resources
try:
ghost_vm_define.validate_node_resources(
ignore_cpu_ram=not changing_node)
except APIValidationError as exc:
self._errors['node'] = exc.api_errors
return attrs
# Validate storage resources
try:
ghost_vm_define.validate_storage_resources()
except APIValidationError as exc:
self._errors['node'] = exc.api_errors
return attrs
# Validate images
self.img_required = ghost_vm_define.check_required_images()
# Save params
# noinspection PyAttributeOutsideInit
self._root_zpool = root_zpool
# noinspection PyAttributeOutsideInit
self._disk_zpools = disk_zpools
# noinspection PyAttributeOutsideInit
self.ghost_vm_define = ghost_vm_define
# noinspection PyAttributeOutsideInit
self.changing_node = changing_node
return attrs
def vm_migrate_cb(result, task_id, vm_uuid=None, slave_vm_uuid=None):
"""
A callback function for api.vm.migrate.views.vm_migrate.
"""
ghost_vm = SlaveVm.get_by_uuid(slave_vm_uuid)
msg = result.get('message', '')
if result['returncode'] == 0 and msg and 'Successfully migrated' in msg:
# Save node and delete placeholder VM first
node = ghost_vm.vm.node
nss = set(ghost_vm.vm.get_node_storages())
ghost_vm.delete(
) # post_delete signal will update node and storage resources
# Fetch VM after ghost_vm is deleted, because it updates vm.slave_vms array
vm = Vm.objects.select_related('node', 'dc').get(uuid=vm_uuid)
changing_node = vm.node != ghost_vm.vm.node
json = result.pop('json', None)
try: # save json from smartos
json_active = vm.json.load(json)
vm.json_active = json_active
vm.json = json_active
except Exception as e:
logger.exception(e)
logger.error(
'Could not parse json output from vm_migrate(%s). Error: %s',
vm_uuid, e)
raise TaskException(result, 'Could not parse json output')
nss.update(list(vm.get_node_storages()))
# Revert status and set new node (should trigger node resource update)
vm.revert_notready(save=False)
if changing_node:
vm.set_node(node)
vm.save(update_node_resources=True, update_storage_resources=nss)
SlaveVm.switch_vm_snapshots_node_storages(vm, nss=nss)
vm_node_changed.send(task_id, vm=vm, force_update=True) # Signal!
else:
vm = Vm.objects.get(uuid=vm_uuid)
_vm_migrate_cb_failed(result, task_id, vm, ghost_vm)
logger.error(
'Found nonzero returncode in result from vm_migrate(%s). Error: %s',
vm_uuid, msg)
raise TaskException(
result, 'Got bad return code (%s). Error: %s' %
(result['returncode'], msg))
task_log_cb_success(result, task_id, vm=vm, **result['meta'])
if vm.json_changed():
logger.info(
'Running PUT vm_manage(%s), because something (vnc port?) has changed changed',
vm)
from api.vm.base.views import vm_manage
from api.utils.request import get_dummy_request
from api.utils.views import call_api_view
request = get_dummy_request(vm.dc, method='PUT', system_user=True)
res = call_api_view(request, 'PUT', vm_manage, vm.hostname)
if res.status_code == 201:
logger.warn('PUT vm_manage(%s) was successful: %s', vm, res.data)
else:
logger.error('PUT vm_manage(%s) failed: %s (%s): %s', vm,
res.status_code, res.status_text, res.data)
return result