def node_vm_snapshot_sync_cb(result, task_id, nodestorage_id=None): """ A callback function for PUT api.node.snapshot.views.node_vm_snapshot_list a.k.a. node_vm_snapshot_sync. """ ns = NodeStorage.objects.select_related('node', 'storage').get(id=nodestorage_id) node = ns.node data = result.pop('data', '') if result['returncode'] != 0: msg = result.get('message', '') or data logger.error('Found nonzero returncode in result from PUT node_vm_snapshot_list(%s@%s). Error: %s', ns.zpool, node.hostname, msg) raise TaskException(result, 'Got bad return code (%s). Error: %s' % (result['returncode'], msg)) node_snaps = parse_node_snaps(data) logger.info('Found %d snapshots on node storage %s@%s', len(node_snaps), ns.zpool, node.hostname) ns_snaps = ns.snapshot_set.select_related('vm').all() lost = sync_snapshots(ns_snaps, node_snaps) # Remaining snapshots on compute node are internal or old lost snapshots which do not exist in DB # or replicated snapshots. Let's count all the remaining es- and as- snapshots sizes as replicated snapshots snap_prefix = Snapshot.USER_PREFIX rep_snaps_size = sum(t_long(node_snaps.pop(snap)[1]) for snap in tuple(node_snaps.keys()) if snap.startswith(snap_prefix)) ns.set_rep_snapshots_size(rep_snaps_size) # The internal snapshots also include dataset backups on a backup node if node.is_backup: node_bkps = ns.backup_set.select_related('node', 'vm').filter(type=Backup.DATASET) lost_bkps = sync_backups(node_bkps, node_snaps) else: lost_bkps = 0 logger.info('Node storage %s@%s has %s bytes of replicated snapshots', ns.zpool, node.hostname, rep_snaps_size) logger.info('Node storage %s@%s has following internal/service snapshots: %s', ns.zpool, node.hostname, node_snaps.keys()) # Recalculate snapshot counters for all DCs ns.save(update_resources=True, update_dcnode_resources=True, recalculate_vms_size=False, recalculate_snapshots_size=True, recalculate_images_size=False, recalculate_backups_size=False, recalculate_dc_snapshots_size=ns.dc.all()) # Remove cached snapshot sum for each VM: for vm_uuid in ns_snaps.values_list('vm__uuid', flat=True).distinct(): Snapshot.clear_total_vm_size(vm_uuid) if not result['meta'].get('internal'): msg = 'Snapshots successfully synced' if lost: msg += '; WARNING: %d snapshot(s) lost' % lost if lost_bkps: msg += '; WARNING: %d backup(s) lost' % lost_bkps result['message'] = msg task_log_cb_success(result, task_id, obj=ns, **result['meta']) return result
def put(self): request, vm = self.request, self.vm if vm.locked: raise VmIsLocked if vm.status not in (vm.STOPPED, vm.RUNNING, vm.NOTCREATED): raise VmIsNotOperational( 'VM is not stopped, running or notcreated') if vm.json_changed(): raise PreconditionRequired( 'VM definition has changed; Update first') ser = VmDcSerializer(request, vm, data=self.data) if not ser.is_valid(): return FailureTaskResponse(request, ser.errors, vm=vm) if vm.tasks: raise VmHasPendingTasks old_dc = vm.dc dc = ser.dc # Change DC for one VM, repeat this for other VM + Recalculate node & storage resources in target and source vm.dc = dc vm.save(update_node_resources=True, update_storage_resources=True) # Change task log entries DC for target VM TaskLogEntry.objects.filter(object_pk=vm.uuid).update(dc=dc) # Change related VM backup's DC Backup.objects.filter(vm=vm).update(dc=dc) for ns in ser.nss: # Issue #chili-885 for i in (dc, old_dc): Backup.update_resources(ns, vm, i) Snapshot.update_resources(ns, vm, i) detail = 'Successfully migrated VM %s from datacenter %s to datacenter %s' % ( vm.hostname, old_dc.name, dc.name) # Will create task log entry in old DC res = SuccessTaskResponse(request, detail, vm=vm, msg=LOG_MIGRATE_DC, detail=detail) # Create task log entry in new DC too task_log_success(task_id_from_task_id(res.data.get('task_id'), dc_id=dc.id), LOG_MIGRATE_DC, obj=vm, detail=detail, update_user_tasks=False) return res
def _check_snap_size_limit(self): """Issue #chili-848""" limit = self.vm.snapshot_size_quota_value if limit is not None: total = Snapshot.get_total_vm_size(self.vm) if total >= limit: raise ExpectationFailed('VM snapshot size limit reached')
def _check_snap_dc_size_limit(self): """Issue #chili-848""" limit = self.vm.dc.settings.VMS_VM_SNAPSHOT_DC_SIZE_LIMIT if limit is not None: limit = int(limit) total = Snapshot.get_total_dc_size(self.vm.dc) if total >= limit: raise ExpectationFailed('DC snapshot size limit reached')
def _check_snap_size_limit(self): """Issue #chili-848""" try: limit = int(self.vm.json_active['internal_metadata'] ['snapshot_size_limit']) except (TypeError, KeyError, IndexError): pass else: total = Snapshot.get_total_vm_size(self.vm) if total >= limit: raise ExpectationFailed('VM snapshot size limit reached')
def get_disk_id(request, vm, data, key='disk_id', default=1): """Get disk_id from data and return additional disk information""" disk_id = data.get(key, default) # noinspection PyBroadException try: disk_id = int(disk_id) if not disk_id > 0: raise ValueError disk = vm.json_active_get_disks()[disk_id - 1] zfs_filesystem = disk['zfs_filesystem'] real_disk_id = Snapshot.get_real_disk_id(disk) except: raise InvalidInput('Invalid %s' % key) return disk_id, real_disk_id, zfs_filesystem
def filter_disk_id(vm, query_filter, data, default=None): """Validate disk_id and update dictionary used for queryset filtering""" disk_id = data.get('disk_id', default) if disk_id is not None: # noinspection PyBroadException try: disk_id = int(disk_id) if not disk_id > 0: raise ValueError if vm: query_filter['disk_id'] = Snapshot.get_disk_id(vm, disk_id) else: query_filter['vm_disk_id'] = disk_id - 1 except: raise InvalidInput('Invalid disk_id') return query_filter
def vm_delete_snapshots_of_removed_disks(vm): """ This helper function deletes snapshots for VM with changing disk IDs. Bug #chili-363 ++ Bug #chili-220 - removing snapshot and backup definitions for removed disks. """ removed_disk_ids = [ Snapshot.get_real_disk_id(i) for i in vm.create_json_update_disks().get('remove_disks', []) ] if removed_disk_ids: Snapshot.objects.filter(vm=vm, disk_id__in=removed_disk_ids).delete() SnapshotDefine.objects.filter(vm=vm, disk_id__in=removed_disk_ids).delete() Backup.objects.filter(vm=vm, disk_id__in=removed_disk_ids, last=True).update(last=False) BackupDefine.objects.filter(vm=vm, disk_id__in=removed_disk_ids).delete() return removed_disk_ids
def _get_real_disk_id(self): return Snapshot.get_disk_id(self._vm, self.cleaned_data['disk_id'])
def vm_backup_cb(result, task_id, vm_uuid=None, node_uuid=None, bkp_id=None): """ A callback function for api.vm.backup.views.vm_backup. """ bkp = Backup.objects.select_related('vm', 'dc').get(id=bkp_id) action = result['meta']['apiview']['method'] json = result.pop('json', '') message = result.get('message', json) data = {} obj_id = vm_uuid or node_uuid success = False try: # save json from esbackup data = bkp.json.load(json) except Exception as e: logger.error( 'Could not parse json output from %s vm_backup(%s, %s). Error: %s', action, obj_id, bkp, e) result['detail'] = message or json else: success = data.get('success', False) try: result['detail'] = _vm_backup_cb_detail(data) except Exception as ex: logger.exception(ex) result['detail'] = json.replace('\n', '') msg = data.get('msg', message) if action == 'PUT': vm = Vm.objects.get(uuid=vm_uuid) obj = vm else: vm = None obj = bkp.vm or bkp.node if bkp.type == Backup.DATASET: if action == 'POST': _vm_backup_update_snapshots( data, 'new_name', 'file_path') # Update file_path of archived backups _vm_backup_deleted_last_snapshot_names( data) # Remove last flag from deleted snapshots elif action == 'DELETE': _vm_backup_update_snapshots( data, 'written', 'size') # Update size of remaining backups _vm_backup_deleted_last_snapshot_names( data) # Remove last flag from deleted snapshots if result['returncode'] == 0 and success: if action == 'POST': if bkp.type == Backup.DATASET: bkp.file_path = data.get('backup_snapshot', '') bkp.size = data.get('backup_snapshot_size', None) if data.get('last_snapshot_name', None): bkp.last = True else: bkp.file_path = data.get('file', '') bkp.size = data.get('size', None) bkp.checksum = data.get('checksum', '') result['message'] = 'Backup successfully created' if bkp.fsfreeze: if 'freeze failed' in msg: bkp.fsfreeze = False result['message'] += ' (filesystem freeze failed)' MonitoringBackend.vm_send_alert( bkp.vm, 'Backup %s of server %s@disk-%s was created, but filesystem freeze ' 'failed.' % (bkp.name, bkp.vm.hostname, bkp.array_disk_id), priority=MonitoringBackend.WARNING) bkp.manifest_path = data.get('metadata_file', '') bkp.time = data.get('time_elapsed', None) bkp.status = bkp.OK bkp.save() if bkp.define and bkp.define.retention: # Retention - delete oldest snapshot assert bkp.vm == bkp.define.vm assert bkp.disk_id == bkp.define.disk_id from api.vm.backup.views import vm_backup_list _delete_oldest(Backup, bkp.define, vm_backup_list, 'bkpnames', task_id, LOG_BKPS_DELETE) bkp.update_zpool_resources() elif action == 'PUT': bkp.status = bkp.OK bkp.save_status() if result['meta']['apiview']['force']: # Remove all snapshots disk = vm.json_active_get_disks()[ result['meta']['apiview']['target_disk_id'] - 1] real_disk_id = Snapshot.get_real_disk_id(disk) # TODO: check indexes Snapshot.objects.filter(vm=vm, disk_id=real_disk_id).delete() vm.revert_notready() result['message'] = 'Backup successfully restored' elif action == 'DELETE': bkp.delete() bkp.update_zpool_resources() result['message'] = 'Backup successfully deleted' else: _vm_backup_cb_failed(result, task_id, bkp, action, vm=vm) # Delete backup or update backup status logger.error( 'Found nonzero returncode in result from %s vm_backup(%s, %s). Error: %s', action, obj_id, bkp, msg) raise TaskException(result, 'Got bad return code (%s). Error: %s' % (result['returncode'], msg), bkp=bkp) task_log_cb_success(result, task_id, obj=obj, **result['meta']) return result
def _task_cleanup(result, task_id, task_status, obj, **kwargs): """ Cleanup after task is revoked. """ apiview = result['meta']['apiview'] view = apiview['view'] if view == 'vm_snapshot': from vms.models import Snapshot from api.vm.snapshot.tasks import _vm_snapshot_cb_failed snap = Snapshot.objects.get(vm=obj, disk_id=Snapshot.get_disk_id( obj, apiview['disk_id']), name=apiview['snapname']) _vm_snapshot_cb_failed(result, task_id, snap, apiview['method']) elif view == 'vm_snapshot_list': from vms.models import Snapshot from api.vm.snapshot.tasks import _vm_snapshot_list_cb_failed snaps = Snapshot.objects.filter(vm=obj, disk_id=Snapshot.get_disk_id( obj, apiview['disk_id']), name__in=apiview['snapnames']) _vm_snapshot_list_cb_failed(result, task_id, snaps, apiview['method']) elif view == 'vm_backup': from vms.models import Backup from api.vm.backup.tasks import _vm_backup_cb_failed bkp = Backup.objects.get(vm_hostname=apiview['hostname'], vm_disk_id=apiview['disk_id'] - 1, name=apiview['bkpname']) _vm_backup_cb_failed(result, task_id, bkp, apiview['method'], vm=obj) elif view == 'vm_backup_list': from vms.models import Backup from api.vm.backup.tasks import _vm_backup_list_cb_failed bkps = Backup.objects.filter(vm_hostname=apiview['hostname'], vm_disk_id=apiview['disk_id'] - 1, name__in=apiview['bkpnames']) _vm_backup_list_cb_failed(result, task_id, bkps, apiview['method']) elif view == 'vm_manage': if apiview['method'] == 'POST': from api.vm.base.tasks import _vm_create_cb_failed result['message'] = '' _vm_create_cb_failed(result, task_id, obj) elif apiview['method'] == 'DELETE': from api.vm.base.tasks import _vm_delete_cb_failed _vm_delete_cb_failed(result, task_id, obj) elif apiview['method'] == 'PUT': from api.vm.base.tasks import _vm_update_cb_done _vm_update_cb_done(result, task_id, obj) elif view == 'vm_status': from api.vm.status.tasks import _vm_status_cb_failed if apiview['method'] == 'PUT': _vm_status_cb_failed(result, task_id, obj) elif view == 'vm_migrate': from vms.models import SlaveVm from api.vm.migrate.tasks import _vm_migrate_cb_failed ghost_vm = SlaveVm.get_by_uuid(apiview['slave_vm_uuid']) _vm_migrate_cb_failed(result, task_id, obj, ghost_vm) elif view == 'image_manage' or view == 'image_snapshot': # obj = Image from vms.models import Snapshot from api.image.base.tasks import _image_manage_cb_failed method = apiview['method'] snap_id = obj.src_snap_id if method == 'POST' and snap_id: snap = Snapshot.objects.get(id=snap_id) else: snap = None _image_manage_cb_failed(result, task_id, obj, method, snap=snap) elif view == 'node_image': # obj = NodeStorage from vms.models import Image from api.node.image.tasks import _node_image_cb_failed img = Image.objects.get(name=apiview['name']) _node_image_cb_failed(result, task_id, obj, img) else: task_cleanup_signal.send(sender=view, apiview=apiview, result=result, task_id=task_id, status=task_status, obj=obj)
def vm_snapshot_sync_cb(result, task_id, vm_uuid=None, disk_id=None): """ A callback function for PUT api.vm.snapshot.views.vm_snapshot_list a.k.a. vm_snapshot_sync. """ vm = Vm.objects.select_related('dc').get(uuid=vm_uuid) data = result.pop('data', '') if result['returncode'] != 0: msg = result.get('message', '') or data logger.error( 'Found nonzero returncode in result from PUT vm_snapshot_list(%s). Error: %s', vm_uuid, msg) raise TaskException( result, 'Got bad return code (%s). Error: %s' % (result['returncode'], msg)) node_snaps = parse_node_snaps(data) logger.info('Found %d snapshots for VM %s on disk ID %s', len(node_snaps), vm, disk_id) lost = sync_snapshots( vm.snapshot_set.select_related('vm').filter(disk_id=disk_id).all(), node_snaps) # Remaining snapshots on compute node are internal or old lost snapshots which do not exist in DB # remaining es- and as- snapshots must be created in DB; some is- and rs- could be probably removed, but # these are hard to determine, so we are ignoring them snap_prefix = Snapshot.USER_PREFIX new_snaps = { snap: node_snaps.pop(snap) for snap in tuple(node_snaps.keys()) if snap.startswith(snap_prefix) } ns = vm.get_node_storage(disk_id) if new_snaps: logger.warn( 'VM %s has following snapshots on disk ID %s, which are not defined in DB: %s', vm, disk_id, new_snaps.keys()) for zfs_name, info in new_snaps.items(): try: name = info[2] if not name: raise IndexError except IndexError: name = info[0] try: Snapshot.create_from_zfs_name(zfs_name, name=name, timestamp=int(info[0]), vm=vm, disk_id=disk_id, zpool=ns, size=t_long(info[1]), note='Found by snapshot sync') except Exception as exc: logger.error( 'Could not recreate snapshot %s (vm=%s, disk_id=%s). Error: %s', zfs_name, vm, disk_id, exc) else: logger.warn('Recreated snapshot %s (vm=%s, disk_id=%s)', zfs_name, vm, disk_id) logger.info( 'VM %s has following internal/service snapshots on disk ID %s: %s', vm, disk_id, node_snaps.keys()) # Update node storage snapshot size counters Snapshot.update_resources(ns, vm) try: # Update last flag on dataset backups bkp_ids = [snap[3:] for snap in node_snaps if snap.startswith('is-')] if bkp_ids: vm.backup_set.filter(disk_id=disk_id, id__in=bkp_ids).update(last=True) vm.backup_set.filter( disk_id=disk_id, last=True).exclude(id__in=bkp_ids).update(last=False) else: vm.backup_set.filter(disk_id=disk_id, last=True).update(last=False) except Exception as exc: logger.exception(exc) msg = 'Snapshots successfully synced' if lost: msg += '; WARNING: %d snapshot(s) lost' % lost if new_snaps: msg += '; WARNING: %d snapshot(s) found' % len(new_snaps) result['message'] = msg task_log_cb_success(result, task_id, vm=vm, **result['meta']) return result
def vm_snapshot_cb(result, task_id, vm_uuid=None, snap_id=None): """ A callback function for api.vm.snapshot.views.vm_snapshot. """ snap = Snapshot.objects.select_related('vm').get(id=snap_id) vm = Vm.objects.get(uuid=vm_uuid) action = result['meta']['apiview']['method'] msg = result.get('message', '') if result['returncode'] == 0: if msg: # noinspection PyBroadException try: result['detail'] = _vm_snapshot_cb_detail(json.loads(msg)) except Exception: result['detail'] = 'msg=' + to_string(msg) else: result['detail'] = '' if action == 'POST': assert vm == snap.vm snap.status = snap.OK result['message'] = 'Snapshot successfully created' if snap.fsfreeze: if 'freeze failed' in msg: snap.fsfreeze = False result['message'] += ' (filesystem freeze failed)' MonitoringBackend.vm_send_alert( vm, 'Snapshot %s of server %s@disk-%s was created, but filesystem ' 'freeze failed.' % (snap.name, vm.hostname, snap.array_disk_id), priority=MonitoringBackend.WARNING) snap.save(update_fields=('status', 'fsfreeze')) if snap.define and snap.define.retention: # Retention - delete oldest snapshot assert vm == snap.define.vm assert snap.disk_id == snap.define.disk_id from api.vm.snapshot.views import vm_snapshot_list _delete_oldest(Snapshot, snap.define, vm_snapshot_list, 'snapnames', task_id, LOG_SNAPS_DELETE) elif action == 'PUT': if vm != snap.vm: snap.vm.revert_notready() vm.revert_notready() snap.status = snap.OK snap.save_status() if result['meta']['apiview']['force']: if snap.vm == vm: # TODO: check indexes Snapshot.objects.filter(vm=vm, disk_id=snap.disk_id, id__gt=snap.id).delete() else: disk = vm.json_active_get_disks()[ result['meta']['apiview']['target_disk_id'] - 1] real_disk_id = Snapshot.get_real_disk_id(disk) # TODO: check indexes Snapshot.objects.filter(vm=vm, disk_id=real_disk_id).delete() result['message'] = 'Snapshot successfully restored' elif action == 'DELETE': assert vm == snap.vm snap.delete() result['message'] = 'Snapshot successfully deleted' else: _vm_snapshot_cb_failed( result, task_id, snap, action, vm=vm) # Delete snapshot or update snapshot status logger.error( 'Found nonzero returncode in result from %s vm_snapshot(%s, %s). Error: %s', action, vm_uuid, snap, msg) raise TaskException(result, 'Got bad return code (%s). Error: %s' % (result['returncode'], msg), snap=snap) task_log_cb_success(result, task_id, vm=vm, **result['meta']) return result