def node_vm_snapshot_sync_cb(result, task_id, nodestorage_id=None):
"""
A callback function for PUT api.node.snapshot.views.node_vm_snapshot_list a.k.a. node_vm_snapshot_sync.
"""
ns = NodeStorage.objects.select_related('node', 'storage').get(id=nodestorage_id)
node = ns.node
data = result.pop('data', '')
if result['returncode'] != 0:
msg = result.get('message', '') or data
logger.error('Found nonzero returncode in result from PUT node_vm_snapshot_list(%s@%s). Error: %s',
ns.zpool, node.hostname, msg)
raise TaskException(result, 'Got bad return code (%s). Error: %s' % (result['returncode'], msg))
node_snaps = parse_node_snaps(data)
logger.info('Found %d snapshots on node storage %s@%s', len(node_snaps), ns.zpool, node.hostname)
ns_snaps = ns.snapshot_set.select_related('vm').all()
lost = sync_snapshots(ns_snaps, node_snaps)
# Remaining snapshots on compute node are internal or old lost snapshots which do not exist in DB
# or replicated snapshots. Let's count all the remaining es- and as- snapshots sizes as replicated snapshots
snap_prefix = Snapshot.USER_PREFIX
rep_snaps_size = sum(t_long(node_snaps.pop(snap)[1]) for snap in tuple(node_snaps.keys())
if snap.startswith(snap_prefix))
ns.set_rep_snapshots_size(rep_snaps_size)
# The internal snapshots also include dataset backups on a backup node
if node.is_backup:
node_bkps = ns.backup_set.select_related('node', 'vm').filter(type=Backup.DATASET)
lost_bkps = sync_backups(node_bkps, node_snaps)
else:
lost_bkps = 0
logger.info('Node storage %s@%s has %s bytes of replicated snapshots', ns.zpool, node.hostname, rep_snaps_size)
logger.info('Node storage %s@%s has following internal/service snapshots: %s',
ns.zpool, node.hostname, node_snaps.keys())
# Recalculate snapshot counters for all DCs
ns.save(update_resources=True, update_dcnode_resources=True, recalculate_vms_size=False,
recalculate_snapshots_size=True, recalculate_images_size=False, recalculate_backups_size=False,
recalculate_dc_snapshots_size=ns.dc.all())
# Remove cached snapshot sum for each VM:
for vm_uuid in ns_snaps.values_list('vm__uuid', flat=True).distinct():
Snapshot.clear_total_vm_size(vm_uuid)
if not result['meta'].get('internal'):
msg = 'Snapshots successfully synced'
if lost:
msg += '; WARNING: %d snapshot(s) lost' % lost
if lost_bkps:
msg += '; WARNING: %d backup(s) lost' % lost_bkps
result['message'] = msg
task_log_cb_success(result, task_id, obj=ns, **result['meta'])
return result
def put(self):
request, vm = self.request, self.vm
if vm.locked:
raise VmIsLocked
if vm.status not in (vm.STOPPED, vm.RUNNING, vm.NOTCREATED):
raise VmIsNotOperational(
'VM is not stopped, running or notcreated')
if vm.json_changed():
raise PreconditionRequired(
'VM definition has changed; Update first')
ser = VmDcSerializer(request, vm, data=self.data)
if not ser.is_valid():
return FailureTaskResponse(request, ser.errors, vm=vm)
if vm.tasks:
raise VmHasPendingTasks
old_dc = vm.dc
dc = ser.dc
# Change DC for one VM, repeat this for other VM + Recalculate node & storage resources in target and source
vm.dc = dc
vm.save(update_node_resources=True, update_storage_resources=True)
# Change task log entries DC for target VM
TaskLogEntry.objects.filter(object_pk=vm.uuid).update(dc=dc)
# Change related VM backup's DC
Backup.objects.filter(vm=vm).update(dc=dc)
for ns in ser.nss: # Issue #chili-885
for i in (dc, old_dc):
Backup.update_resources(ns, vm, i)
Snapshot.update_resources(ns, vm, i)
detail = 'Successfully migrated VM %s from datacenter %s to datacenter %s' % (
vm.hostname, old_dc.name, dc.name)
# Will create task log entry in old DC
res = SuccessTaskResponse(request,
detail,
vm=vm,
msg=LOG_MIGRATE_DC,
detail=detail)
# Create task log entry in new DC too
task_log_success(task_id_from_task_id(res.data.get('task_id'),
dc_id=dc.id),
LOG_MIGRATE_DC,
obj=vm,
detail=detail,
update_user_tasks=False)
return res
def _check_snap_size_limit(self):
"""Issue #chili-848"""
limit = self.vm.snapshot_size_quota_value
if limit is not None:
total = Snapshot.get_total_vm_size(self.vm)
if total >= limit:
raise ExpectationFailed('VM snapshot size limit reached')
def _check_snap_dc_size_limit(self):
"""Issue #chili-848"""
limit = self.vm.dc.settings.VMS_VM_SNAPSHOT_DC_SIZE_LIMIT
if limit is not None:
limit = int(limit)
total = Snapshot.get_total_dc_size(self.vm.dc)
if total >= limit:
raise ExpectationFailed('DC snapshot size limit reached')
def _check_snap_size_limit(self):
"""Issue #chili-848"""
try:
limit = int(self.vm.json_active['internal_metadata']
['snapshot_size_limit'])
except (TypeError, KeyError, IndexError):
pass
else:
total = Snapshot.get_total_vm_size(self.vm)
if total >= limit:
raise ExpectationFailed('VM snapshot size limit reached')
def get_disk_id(request, vm, data, key='disk_id', default=1):
"""Get disk_id from data and return additional disk information"""
disk_id = data.get(key, default)
# noinspection PyBroadException
try:
disk_id = int(disk_id)
if not disk_id > 0:
raise ValueError
disk = vm.json_active_get_disks()[disk_id - 1]
zfs_filesystem = disk['zfs_filesystem']
real_disk_id = Snapshot.get_real_disk_id(disk)
except:
raise InvalidInput('Invalid %s' % key)
return disk_id, real_disk_id, zfs_filesystem
def filter_disk_id(vm, query_filter, data, default=None):
"""Validate disk_id and update dictionary used for queryset filtering"""
disk_id = data.get('disk_id', default)
if disk_id is not None:
# noinspection PyBroadException
try:
disk_id = int(disk_id)
if not disk_id > 0:
raise ValueError
if vm:
query_filter['disk_id'] = Snapshot.get_disk_id(vm, disk_id)
else:
query_filter['vm_disk_id'] = disk_id - 1
except:
raise InvalidInput('Invalid disk_id')
return query_filter
def vm_delete_snapshots_of_removed_disks(vm):
"""
This helper function deletes snapshots for VM with changing disk IDs. Bug #chili-363
++ Bug #chili-220 - removing snapshot and backup definitions for removed disks.
"""
removed_disk_ids = [
Snapshot.get_real_disk_id(i)
for i in vm.create_json_update_disks().get('remove_disks', [])
]
if removed_disk_ids:
Snapshot.objects.filter(vm=vm, disk_id__in=removed_disk_ids).delete()
SnapshotDefine.objects.filter(vm=vm,
disk_id__in=removed_disk_ids).delete()
Backup.objects.filter(vm=vm, disk_id__in=removed_disk_ids,
last=True).update(last=False)
BackupDefine.objects.filter(vm=vm,
disk_id__in=removed_disk_ids).delete()
return removed_disk_ids
def _get_real_disk_id(self):
return Snapshot.get_disk_id(self._vm, self.cleaned_data['disk_id'])
def vm_backup_cb(result, task_id, vm_uuid=None, node_uuid=None, bkp_id=None):
"""
A callback function for api.vm.backup.views.vm_backup.
"""
bkp = Backup.objects.select_related('vm', 'dc').get(id=bkp_id)
action = result['meta']['apiview']['method']
json = result.pop('json', '')
message = result.get('message', json)
data = {}
obj_id = vm_uuid or node_uuid
success = False
try: # save json from esbackup
data = bkp.json.load(json)
except Exception as e:
logger.error(
'Could not parse json output from %s vm_backup(%s, %s). Error: %s',
action, obj_id, bkp, e)
result['detail'] = message or json
else:
success = data.get('success', False)
try:
result['detail'] = _vm_backup_cb_detail(data)
except Exception as ex:
logger.exception(ex)
result['detail'] = json.replace('\n', '')
msg = data.get('msg', message)
if action == 'PUT':
vm = Vm.objects.get(uuid=vm_uuid)
obj = vm
else:
vm = None
obj = bkp.vm or bkp.node
if bkp.type == Backup.DATASET:
if action == 'POST':
_vm_backup_update_snapshots(
data, 'new_name',
'file_path') # Update file_path of archived backups
_vm_backup_deleted_last_snapshot_names(
data) # Remove last flag from deleted snapshots
elif action == 'DELETE':
_vm_backup_update_snapshots(
data, 'written',
'size') # Update size of remaining backups
_vm_backup_deleted_last_snapshot_names(
data) # Remove last flag from deleted snapshots
if result['returncode'] == 0 and success:
if action == 'POST':
if bkp.type == Backup.DATASET:
bkp.file_path = data.get('backup_snapshot', '')
bkp.size = data.get('backup_snapshot_size', None)
if data.get('last_snapshot_name', None):
bkp.last = True
else:
bkp.file_path = data.get('file', '')
bkp.size = data.get('size', None)
bkp.checksum = data.get('checksum', '')
result['message'] = 'Backup successfully created'
if bkp.fsfreeze:
if 'freeze failed' in msg:
bkp.fsfreeze = False
result['message'] += ' (filesystem freeze failed)'
MonitoringBackend.vm_send_alert(
bkp.vm,
'Backup %s of server %s@disk-%s was created, but filesystem freeze '
'failed.' %
(bkp.name, bkp.vm.hostname, bkp.array_disk_id),
priority=MonitoringBackend.WARNING)
bkp.manifest_path = data.get('metadata_file', '')
bkp.time = data.get('time_elapsed', None)
bkp.status = bkp.OK
bkp.save()
if bkp.define and bkp.define.retention: # Retention - delete oldest snapshot
assert bkp.vm == bkp.define.vm
assert bkp.disk_id == bkp.define.disk_id
from api.vm.backup.views import vm_backup_list
_delete_oldest(Backup, bkp.define, vm_backup_list, 'bkpnames',
task_id, LOG_BKPS_DELETE)
bkp.update_zpool_resources()
elif action == 'PUT':
bkp.status = bkp.OK
bkp.save_status()
if result['meta']['apiview']['force']: # Remove all snapshots
disk = vm.json_active_get_disks()[
result['meta']['apiview']['target_disk_id'] - 1]
real_disk_id = Snapshot.get_real_disk_id(disk)
# TODO: check indexes
Snapshot.objects.filter(vm=vm, disk_id=real_disk_id).delete()
vm.revert_notready()
result['message'] = 'Backup successfully restored'
elif action == 'DELETE':
bkp.delete()
bkp.update_zpool_resources()
result['message'] = 'Backup successfully deleted'
else:
_vm_backup_cb_failed(result, task_id, bkp, action,
vm=vm) # Delete backup or update backup status
logger.error(
'Found nonzero returncode in result from %s vm_backup(%s, %s). Error: %s',
action, obj_id, bkp, msg)
raise TaskException(result,
'Got bad return code (%s). Error: %s' %
(result['returncode'], msg),
bkp=bkp)
task_log_cb_success(result, task_id, obj=obj, **result['meta'])
return result
def _task_cleanup(result, task_id, task_status, obj, **kwargs):
"""
Cleanup after task is revoked.
"""
apiview = result['meta']['apiview']
view = apiview['view']
if view == 'vm_snapshot':
from vms.models import Snapshot
from api.vm.snapshot.tasks import _vm_snapshot_cb_failed
snap = Snapshot.objects.get(vm=obj,
disk_id=Snapshot.get_disk_id(
obj, apiview['disk_id']),
name=apiview['snapname'])
_vm_snapshot_cb_failed(result, task_id, snap, apiview['method'])
elif view == 'vm_snapshot_list':
from vms.models import Snapshot
from api.vm.snapshot.tasks import _vm_snapshot_list_cb_failed
snaps = Snapshot.objects.filter(vm=obj,
disk_id=Snapshot.get_disk_id(
obj, apiview['disk_id']),
name__in=apiview['snapnames'])
_vm_snapshot_list_cb_failed(result, task_id, snaps, apiview['method'])
elif view == 'vm_backup':
from vms.models import Backup
from api.vm.backup.tasks import _vm_backup_cb_failed
bkp = Backup.objects.get(vm_hostname=apiview['hostname'],
vm_disk_id=apiview['disk_id'] - 1,
name=apiview['bkpname'])
_vm_backup_cb_failed(result, task_id, bkp, apiview['method'], vm=obj)
elif view == 'vm_backup_list':
from vms.models import Backup
from api.vm.backup.tasks import _vm_backup_list_cb_failed
bkps = Backup.objects.filter(vm_hostname=apiview['hostname'],
vm_disk_id=apiview['disk_id'] - 1,
name__in=apiview['bkpnames'])
_vm_backup_list_cb_failed(result, task_id, bkps, apiview['method'])
elif view == 'vm_manage':
if apiview['method'] == 'POST':
from api.vm.base.tasks import _vm_create_cb_failed
result['message'] = ''
_vm_create_cb_failed(result, task_id, obj)
elif apiview['method'] == 'DELETE':
from api.vm.base.tasks import _vm_delete_cb_failed
_vm_delete_cb_failed(result, task_id, obj)
elif apiview['method'] == 'PUT':
from api.vm.base.tasks import _vm_update_cb_done
_vm_update_cb_done(result, task_id, obj)
elif view == 'vm_status':
from api.vm.status.tasks import _vm_status_cb_failed
if apiview['method'] == 'PUT':
_vm_status_cb_failed(result, task_id, obj)
elif view == 'vm_migrate':
from vms.models import SlaveVm
from api.vm.migrate.tasks import _vm_migrate_cb_failed
ghost_vm = SlaveVm.get_by_uuid(apiview['slave_vm_uuid'])
_vm_migrate_cb_failed(result, task_id, obj, ghost_vm)
elif view == 'image_manage' or view == 'image_snapshot':
# obj = Image
from vms.models import Snapshot
from api.image.base.tasks import _image_manage_cb_failed
method = apiview['method']
snap_id = obj.src_snap_id
if method == 'POST' and snap_id:
snap = Snapshot.objects.get(id=snap_id)
else:
snap = None
_image_manage_cb_failed(result, task_id, obj, method, snap=snap)
elif view == 'node_image':
# obj = NodeStorage
from vms.models import Image
from api.node.image.tasks import _node_image_cb_failed
img = Image.objects.get(name=apiview['name'])
_node_image_cb_failed(result, task_id, obj, img)
else:
task_cleanup_signal.send(sender=view,
apiview=apiview,
result=result,
task_id=task_id,
status=task_status,
obj=obj)
def vm_snapshot_sync_cb(result, task_id, vm_uuid=None, disk_id=None):
"""
A callback function for PUT api.vm.snapshot.views.vm_snapshot_list a.k.a. vm_snapshot_sync.
"""
vm = Vm.objects.select_related('dc').get(uuid=vm_uuid)
data = result.pop('data', '')
if result['returncode'] != 0:
msg = result.get('message', '') or data
logger.error(
'Found nonzero returncode in result from PUT vm_snapshot_list(%s). Error: %s',
vm_uuid, msg)
raise TaskException(
result, 'Got bad return code (%s). Error: %s' %
(result['returncode'], msg))
node_snaps = parse_node_snaps(data)
logger.info('Found %d snapshots for VM %s on disk ID %s', len(node_snaps),
vm, disk_id)
lost = sync_snapshots(
vm.snapshot_set.select_related('vm').filter(disk_id=disk_id).all(),
node_snaps)
# Remaining snapshots on compute node are internal or old lost snapshots which do not exist in DB
# remaining es- and as- snapshots must be created in DB; some is- and rs- could be probably removed, but
# these are hard to determine, so we are ignoring them
snap_prefix = Snapshot.USER_PREFIX
new_snaps = {
snap: node_snaps.pop(snap)
for snap in tuple(node_snaps.keys()) if snap.startswith(snap_prefix)
}
ns = vm.get_node_storage(disk_id)
if new_snaps:
logger.warn(
'VM %s has following snapshots on disk ID %s, which are not defined in DB: %s',
vm, disk_id, new_snaps.keys())
for zfs_name, info in new_snaps.items():
try:
name = info[2]
if not name:
raise IndexError
except IndexError:
name = info[0]
try:
Snapshot.create_from_zfs_name(zfs_name,
name=name,
timestamp=int(info[0]),
vm=vm,
disk_id=disk_id,
zpool=ns,
size=t_long(info[1]),
note='Found by snapshot sync')
except Exception as exc:
logger.error(
'Could not recreate snapshot %s (vm=%s, disk_id=%s). Error: %s',
zfs_name, vm, disk_id, exc)
else:
logger.warn('Recreated snapshot %s (vm=%s, disk_id=%s)',
zfs_name, vm, disk_id)
logger.info(
'VM %s has following internal/service snapshots on disk ID %s: %s', vm,
disk_id, node_snaps.keys())
# Update node storage snapshot size counters
Snapshot.update_resources(ns, vm)
try:
# Update last flag on dataset backups
bkp_ids = [snap[3:] for snap in node_snaps if snap.startswith('is-')]
if bkp_ids:
vm.backup_set.filter(disk_id=disk_id,
id__in=bkp_ids).update(last=True)
vm.backup_set.filter(
disk_id=disk_id,
last=True).exclude(id__in=bkp_ids).update(last=False)
else:
vm.backup_set.filter(disk_id=disk_id, last=True).update(last=False)
except Exception as exc:
logger.exception(exc)
msg = 'Snapshots successfully synced'
if lost:
msg += '; WARNING: %d snapshot(s) lost' % lost
if new_snaps:
msg += '; WARNING: %d snapshot(s) found' % len(new_snaps)
result['message'] = msg
task_log_cb_success(result, task_id, vm=vm, **result['meta'])
return result
def vm_snapshot_cb(result, task_id, vm_uuid=None, snap_id=None):
"""
A callback function for api.vm.snapshot.views.vm_snapshot.
"""
snap = Snapshot.objects.select_related('vm').get(id=snap_id)
vm = Vm.objects.get(uuid=vm_uuid)
action = result['meta']['apiview']['method']
msg = result.get('message', '')
if result['returncode'] == 0:
if msg:
# noinspection PyBroadException
try:
result['detail'] = _vm_snapshot_cb_detail(json.loads(msg))
except Exception:
result['detail'] = 'msg=' + to_string(msg)
else:
result['detail'] = ''
if action == 'POST':
assert vm == snap.vm
snap.status = snap.OK
result['message'] = 'Snapshot successfully created'
if snap.fsfreeze:
if 'freeze failed' in msg:
snap.fsfreeze = False
result['message'] += ' (filesystem freeze failed)'
MonitoringBackend.vm_send_alert(
vm,
'Snapshot %s of server %s@disk-%s was created, but filesystem '
'freeze failed.' %
(snap.name, vm.hostname, snap.array_disk_id),
priority=MonitoringBackend.WARNING)
snap.save(update_fields=('status', 'fsfreeze'))
if snap.define and snap.define.retention: # Retention - delete oldest snapshot
assert vm == snap.define.vm
assert snap.disk_id == snap.define.disk_id
from api.vm.snapshot.views import vm_snapshot_list
_delete_oldest(Snapshot, snap.define, vm_snapshot_list,
'snapnames', task_id, LOG_SNAPS_DELETE)
elif action == 'PUT':
if vm != snap.vm:
snap.vm.revert_notready()
vm.revert_notready()
snap.status = snap.OK
snap.save_status()
if result['meta']['apiview']['force']:
if snap.vm == vm:
# TODO: check indexes
Snapshot.objects.filter(vm=vm,
disk_id=snap.disk_id,
id__gt=snap.id).delete()
else:
disk = vm.json_active_get_disks()[
result['meta']['apiview']['target_disk_id'] - 1]
real_disk_id = Snapshot.get_real_disk_id(disk)
# TODO: check indexes
Snapshot.objects.filter(vm=vm,
disk_id=real_disk_id).delete()
result['message'] = 'Snapshot successfully restored'
elif action == 'DELETE':
assert vm == snap.vm
snap.delete()
result['message'] = 'Snapshot successfully deleted'
else:
_vm_snapshot_cb_failed(
result, task_id, snap, action,
vm=vm) # Delete snapshot or update snapshot status
logger.error(
'Found nonzero returncode in result from %s vm_snapshot(%s, %s). Error: %s',
action, vm_uuid, snap, msg)
raise TaskException(result,
'Got bad return code (%s). Error: %s' %
(result['returncode'], msg),
snap=snap)
task_log_cb_success(result, task_id, vm=vm, **result['meta'])
return result
