def _vm_snapshot_cb_alert(result,
task_id,
snap_id=None,
task_exception=None,
**kwargs):
"""Alert function for failed snapshot creation"""
action = result['meta']['apiview']['method']
if action == 'POST':
action_msg = 'created'
elif action == 'DELETE':
action_msg = 'deleted'
else:
return # Alert only failed creation and deletion
snap = getattr(task_exception, 'snap', None)
if not snap:
snap = Snapshot.objects.select_related('vm').get(id=snap_id)
if snap.type == Snapshot.AUTO:
vm = snap.vm
if vm:
MonitoringBackend.vm_send_alert(
vm,
'Automatic snapshot %s of server %s@disk-%s could not be %s.' %
(snap.name, vm.hostname, snap.array_disk_id, action_msg))
def _delete_oldest(model, define, view_function, view_item, task_id, msg):
"""
Helper for finding oldest snapshots/backups and running DELETE view_function().
@type model: django.db.models.Model
"""
vm = define.vm
# TODO: check indexes
# noinspection PyUnresolvedReferences
total = model.objects.filter(vm=vm,
disk_id=define.disk_id,
define=define,
status=model.OK).count()
to_delete = total - define.retention
if to_delete < 1:
return None
# List of snapshot or backup names to delete TODO: check indexes
# noinspection PyUnresolvedReferences
oldest = model.objects.filter(vm=vm, disk_id=define.disk_id, define=define, status=model.OK)\
.values_list('name', flat=True).order_by('id')[:to_delete]
view_name = view_function.__name__
view_data = {'disk_id': define.array_disk_id, view_item: tuple(oldest)}
request = get_dummy_request(vm.dc, method='DELETE', system_user=True)
request.define_id = define.id # Automatic task
# Go!
logger.info('Running DELETE %s(%s, %s), because %s>%s', view_name, vm,
view_data, total, define.retention)
res = call_api_view(request,
'DELETE',
view_function,
vm.hostname,
data=view_data)
if res.status_code in (200, 201):
logger.warn('DELETE %s(%s, %s) was successful: %s', view_name, vm,
view_data, res.data)
else:
logger.error('Running DELETE %s(%s, %s) failed: %s (%s): %s',
view_name, vm, view_data, res.status_code,
res.status_text, res.data)
MonitoringBackend.vm_send_alert(
vm, 'Automatic deletion of old %ss %s/disk-%s failed to start.' %
(model.__name__.lower(), vm.hostname, define.array_disk_id))
# Need to log this, because nobody else does (+ there is no PENDING task)
detail = 'hostname=%s, %s=%s, disk_id=%s, Error: %s' % (
vm.hostname, view_item, ','.join(oldest), define.array_disk_id,
get_task_error_message(res.data))
task_log_error(task_id,
msg,
vm=vm,
detail=detail,
update_user_tasks=False)
return res
def vm_snapshot_beat(snap_define_id):
"""
This is a periodic beat task. Run POST vm_snapshot according to snapshot definition.
"""
from api.vm.snapshot.views import vm_snapshot
snap_define = SnapshotDefine.objects.get(id=snap_define_id)
snap_name = snap_define.generate_snapshot_name()
vm = snap_define.vm
disk_id = snap_define.array_disk_id
request = get_dummy_request(vm.dc, method='POST', system_user=True)
request.define_id = snap_define.id # Automatic task
# Go!
res = call_api_view(request,
'POST',
vm_snapshot,
vm.hostname,
snap_name,
data={
'disk_id': disk_id,
'fsfreeze': snap_define.fsfreeze
})
if res.status_code == 201:
logger.info(
'POST vm_snapshot(%s, %s, {disk_id=%s}) was successful: %s', vm,
snap_name, disk_id, res.data)
else:
# Need to log this, because nobody else does (+ there is no PENDING task)
detail = 'snapname=%s, disk_id=%s, type=%s. Error: %s' % (
snap_name, disk_id, Snapshot.AUTO, get_task_error_message(
res.data))
task_log_error(task_id_from_task_id(vm_snapshot_beat.request.id,
dc_id=vm.dc.id),
LOG_SNAP_CREATE,
vm=vm,
detail=detail,
update_user_tasks=False)
if res.status_code == HTTP_423_LOCKED:
logger.warning(
'Running POST vm_snapshot(%s, %s, {disk_id=%s}) failed: %s (%s): %s',
vm, snap_name, disk_id, res.status_code, res.status_text,
res.data)
else:
logger.error(
'Running POST vm_snapshot(%s, %s, {disk_id=%s}) failed: %s (%s): %s',
vm, snap_name, disk_id, res.status_code, res.status_text,
res.data)
MonitoringBackend.vm_send_alert(
vm, 'Automatic snapshot %s/disk-%s@%s failed to start.' %
(vm.hostname, disk_id, snap_define.name))
def _vm_backup_cb_alert(result,
task_id,
bkp_id=None,
task_exception=None,
**kwargs):
"""Alert function for failed backup creation"""
action = result['meta']['apiview']['method']
if action == 'POST':
action_msg = 'created'
elif action == 'DELETE':
action_msg = 'deleted'
else:
return # Alert only failed creation and deletion
bkp = getattr(task_exception, 'bkp', None)
if not bkp:
bkp = Backup.objects.select_related('vm').get(id=bkp_id)
vm = bkp.vm
if vm:
MonitoringBackend.vm_send_alert(
vm, 'Backup %s of server %s@disk-%s could not be %s.' %
(bkp.name, vm.hostname, bkp.array_disk_id, action_msg))
def vm_backup_cb(result, task_id, vm_uuid=None, node_uuid=None, bkp_id=None):
"""
A callback function for api.vm.backup.views.vm_backup.
"""
bkp = Backup.objects.select_related('vm', 'dc').get(id=bkp_id)
action = result['meta']['apiview']['method']
json = result.pop('json', '')
message = result.get('message', json)
data = {}
obj_id = vm_uuid or node_uuid
success = False
try: # save json from esbackup
data = bkp.json.load(json)
except Exception as e:
logger.error(
'Could not parse json output from %s vm_backup(%s, %s). Error: %s',
action, obj_id, bkp, e)
result['detail'] = message or json
else:
success = data.get('success', False)
try:
result['detail'] = _vm_backup_cb_detail(data)
except Exception as ex:
logger.exception(ex)
result['detail'] = json.replace('\n', '')
msg = data.get('msg', message)
if action == 'PUT':
vm = Vm.objects.get(uuid=vm_uuid)
obj = vm
else:
vm = None
obj = bkp.vm or bkp.node
if bkp.type == Backup.DATASET:
if action == 'POST':
_vm_backup_update_snapshots(
data, 'new_name',
'file_path') # Update file_path of archived backups
_vm_backup_deleted_last_snapshot_names(
data) # Remove last flag from deleted snapshots
elif action == 'DELETE':
_vm_backup_update_snapshots(
data, 'written',
'size') # Update size of remaining backups
_vm_backup_deleted_last_snapshot_names(
data) # Remove last flag from deleted snapshots
if result['returncode'] == 0 and success:
if action == 'POST':
if bkp.type == Backup.DATASET:
bkp.file_path = data.get('backup_snapshot', '')
bkp.size = data.get('backup_snapshot_size', None)
if data.get('last_snapshot_name', None):
bkp.last = True
else:
bkp.file_path = data.get('file', '')
bkp.size = data.get('size', None)
bkp.checksum = data.get('checksum', '')
result['message'] = 'Backup successfully created'
if bkp.fsfreeze:
if 'freeze failed' in msg:
bkp.fsfreeze = False
result['message'] += ' (filesystem freeze failed)'
MonitoringBackend.vm_send_alert(
bkp.vm,
'Backup %s of server %s@disk-%s was created, but filesystem freeze '
'failed.' %
(bkp.name, bkp.vm.hostname, bkp.array_disk_id),
priority=MonitoringBackend.WARNING)
bkp.manifest_path = data.get('metadata_file', '')
bkp.time = data.get('time_elapsed', None)
bkp.status = bkp.OK
bkp.save()
if bkp.define and bkp.define.retention: # Retention - delete oldest snapshot
assert bkp.vm == bkp.define.vm
assert bkp.disk_id == bkp.define.disk_id
from api.vm.backup.views import vm_backup_list
_delete_oldest(Backup, bkp.define, vm_backup_list, 'bkpnames',
task_id, LOG_BKPS_DELETE)
bkp.update_zpool_resources()
elif action == 'PUT':
bkp.status = bkp.OK
bkp.save_status()
if result['meta']['apiview']['force']: # Remove all snapshots
disk = vm.json_active_get_disks()[
result['meta']['apiview']['target_disk_id'] - 1]
real_disk_id = Snapshot.get_real_disk_id(disk)
# TODO: check indexes
Snapshot.objects.filter(vm=vm, disk_id=real_disk_id).delete()
vm.revert_notready()
result['message'] = 'Backup successfully restored'
elif action == 'DELETE':
bkp.delete()
bkp.update_zpool_resources()
result['message'] = 'Backup successfully deleted'
else:
_vm_backup_cb_failed(result, task_id, bkp, action,
vm=vm) # Delete backup or update backup status
logger.error(
'Found nonzero returncode in result from %s vm_backup(%s, %s). Error: %s',
action, obj_id, bkp, msg)
raise TaskException(result,
'Got bad return code (%s). Error: %s' %
(result['returncode'], msg),
bkp=bkp)
task_log_cb_success(result, task_id, obj=obj, **result['meta'])
return result
def vm_snapshot_cb(result, task_id, vm_uuid=None, snap_id=None):
"""
A callback function for api.vm.snapshot.views.vm_snapshot.
"""
snap = Snapshot.objects.select_related('vm').get(id=snap_id)
vm = snap.vm
action = result['meta']['apiview']['method']
msg = result.get('message', '')
if result['returncode'] == 0:
if msg:
result['detail'] = 'msg=' + to_string(msg)
else:
result['detail'] = ''
if action == 'POST':
snap.status = snap.OK
result['message'] = 'Snapshot successfully created'
if snap.fsfreeze:
if 'freeze failed' in msg:
snap.fsfreeze = False
result['message'] += ' (filesystem freeze failed)'
MonitoringBackend.vm_send_alert(
vm,
'Snapshot %s of server %s@disk-%s was created, but filesystem '
'freeze failed.' %
(snap.name, vm.hostname, snap.array_disk_id),
priority=MonitoringBackend.WARNING)
snap.save(update_fields=('status', 'fsfreeze'))
if snap.define and snap.define.retention: # Retention - delete oldest snapshot
assert vm == snap.define.vm
assert snap.disk_id == snap.define.disk_id
from api.vm.snapshot.views import vm_snapshot_list
_delete_oldest(Snapshot, snap.define, vm_snapshot_list,
'snapnames', task_id, LOG_SNAPS_DELETE)
elif action == 'PUT':
snap.status = snap.OK
snap.save_status()
if result['meta']['apiview']['force']:
# TODO: check indexes
Snapshot.objects.filter(vm=vm,
disk_id=snap.disk_id,
id__gt=snap.id).delete()
vm.revert_notready()
result['message'] = 'Snapshot successfully restored'
elif action == 'DELETE':
snap.delete()
result['message'] = 'Snapshot successfully deleted'
else:
_vm_snapshot_cb_failed(
result, task_id, snap,
action) # Delete snapshot or update snapshot status
logger.error(
'Found nonzero returncode in result from %s vm_snapshot(%s, %s). Error: %s',
action, vm_uuid, snap, msg)
raise TaskException(result,
'Got bad return code (%s). Error: %s' %
(result['returncode'], msg),
snap=snap)
task_log_cb_success(result, task_id, vm=vm, **result['meta'])
return result