def get_current_status(self, force_change=False):
"""Get current VM status"""
request, vm = self.request, self.vm
if vm.node.status not in vm.node.STATUS_OPERATIONAL:
raise NodeIsNotOperational
apiview = self.apiview
msg = LOG_STATUS_GET
cmd = 'vmadm list -p -H -o state uuid=' + vm.uuid
lock = 'vm_status_current vm:%s' % vm.uuid
meta = {
'output': {
'returncode': 'returncode',
'stdout': 'stdout',
'stderr': 'stderr',
'hostname': vm.hostname
},
'msg': msg,
'vm_uuid': vm.uuid,
'apiview': apiview,
'last_status': vm.status,
}
callback = ('api.vm.status.tasks.vm_status_current_cb', {
'vm_uuid': vm.uuid,
'force_change': force_change
})
tid, err = execute(request,
vm.owner.id,
cmd,
meta=meta,
callback=callback,
queue=vm.node.fast_queue,
lock=lock,
lock_timeout=5,
nolog=True)
if err:
return FailureTaskResponse(request, err, vm=vm)
else:
return TaskResponse(request,
tid,
vm=vm,
api_view=apiview,
data=self.data) # No msg
def _run_execute(self, msg, cmd, stdin=None, detail_dict=None, block_key=None, **apiview_kwargs):
request, master_vm, slave_vm, repname = self.request, self.vm, self.slave_vm, self.slave_vm.name
if detail_dict is None:
detail_dict = {'repname': repname}
# Prepare task data
apiview = {
'view': self._api_view_,
'method': request.method,
'hostname': master_vm.hostname,
'repname': repname
}
apiview.update(apiview_kwargs)
meta = {
'output': {'returncode': 'returncode', 'stdout': 'jsons', 'stderr': 'message'},
'replace_stdout': ((master_vm.uuid, master_vm.hostname), (slave_vm.uuid, repname)),
'msg': msg,
'vm_uuid': master_vm.uuid,
'slave_vm_uuid': slave_vm.uuid,
'apiview': apiview,
}
lock = 'vm_replica vm:%s' % master_vm.uuid
callback = (
'api.vm.replica.tasks.%s_cb' % self._api_view_,
{'vm_uuid': master_vm.uuid, 'slave_vm_uuid': slave_vm.uuid}
)
cmd = cmd.format(
master_uuid=master_vm.uuid,
slave_uuid=slave_vm.uuid,
master_node=master_vm.node.address,
id=slave_vm.rep_id,
)
self._check_slave_vm_node()
# Execute task
tid, err = execute(request, master_vm.owner.id, cmd, meta=meta, lock=lock, callback=callback, stdin=stdin,
queue=slave_vm.node.fast_queue, block_key=block_key)
if err:
return FailureTaskResponse(request, err, vm=master_vm)
else:
self._success = True
return TaskResponse(request, tid, msg=msg, vm=master_vm, api_view=apiview,
detail_dict=detail_dict, data=self.data)
def get_current_status(self):
"""Get current VM status"""
request, vm = self.request, self.vm
if vm.status not in (Vm.RUNNING, Vm.STOPPED, Vm.STOPPING, Vm.ERROR):
raise VmIsNotOperational
if vm.node.status not in vm.node.STATUS_OPERATIONAL:
raise NodeIsNotOperational
apiview = self.apiview
msg = LOG_STATUS_GET
cmd = 'vmadm list -p -H -o state,zoneid uuid=' + vm.uuid
meta = {
'output': {
'returncode': 'returncode',
'stdout': 'stdout',
'stderr': 'stderr',
'hostname': vm.hostname
},
'msg': msg,
'vm_uuid': vm.uuid,
'apiview': apiview,
'last_status': vm.status,
}
callback = ('api.vm.status.tasks.vm_status_current_cb', {
'vm_uuid': vm.uuid
})
tid, err = execute(request,
vm.owner.id,
cmd,
meta=meta,
callback=callback,
queue=vm.node.fast_queue,
nolog=True)
if err:
return FailureTaskResponse(request, err, vm=vm)
else:
return TaskResponse(request,
tid,
vm=vm,
api_view=apiview,
data=self.data) # No msg
def delete(self):
request, vm = self.request, self.vm
# only admin
if not (request.user and request.user.is_admin(request)):
raise PermissionDenied
if vm.uuid == ImageVm.get_uuid():
raise VmIsLocked('VM is image server')
if vm.locked:
raise VmIsLocked
if vm.status not in (vm.STOPPED, vm.FROZEN):
raise VmIsNotOperational('VM is not stopped')
if vm.tasks:
raise VmHasPendingTasks
apiview = self.apiview
msg = LOG_VM_DELETE
cmd = 'vmadm delete ' + vm.uuid
meta = {
'output': {'returncode': 'returncode', 'stderr': 'message'},
'replace_text': ((vm.uuid, vm.hostname),),
'msg': msg, 'vm_uuid': vm.uuid, 'apiview': apiview
}
callback = ('api.vm.base.tasks.vm_delete_cb', {'vm_uuid': vm.uuid})
logger.debug('Deleting VM %s from compute node', vm)
err = True
vm.set_notready()
try:
tid, err = execute(request, vm.owner.id, cmd, meta=meta, lock=self.lock, expires=VM_VM_EXPIRES,
callback=callback, queue=vm.node.slow_queue)
if err:
return FailureTaskResponse(request, err, vm=vm)
else:
return TaskResponse(request, tid, msg=msg, vm=vm, api_view=apiview, data=self.data)
finally:
if err:
vm.revert_notready()
def run_node_img_sources_sync(node,
new_img_sources=None,
node_img_sources=None):
"""
Update imgadm sources on compute node.
Always called by node_sysinfo_cb after the sysinfo data is processed (even if no node data is changed).
"""
if new_img_sources is None:
image_vm = ImageVm()
new_img_sources = image_vm.sources
if node_img_sources is not None and new_img_sources == node_img_sources:
logger.info(
'Image sources already synced for node %s - skipping update', node)
return
logger.warn(
'Image sources are not synchronized on node %s - creating imgadm sources synchronization task',
node)
stdin = ImageVm.get_imgadm_conf(new_img_sources).dump()
cmd = 'cat /dev/stdin > /var/imgadm/imgadm.conf'
lock = 'node %s imgadm_sources' % node.uuid
tid, err = execute(ERIGONES_TASK_USER,
None,
cmd,
stdin=stdin,
callback=False,
lock=lock,
queue=node.fast_queue,
expires=180,
nolog=True,
tg=TG_DC_UNBOUND,
ping_worker=False,
check_user_tasks=False)
if err:
logger.error(
'Got error (%s) when running task %s for updating imgadm sources on node %s',
err, tid, node)
else:
logger.info('Created task %s for updating imgadm sources on node %s',
tid, node)
def run_node_authorized_keys_sync():
"""
Create and update authorized_keys on every compute node.
"""
dc1_settings = DefaultDc().settings # erigones.conf.settings
if not dc1_settings.VMS_NODE_SSH_KEYS_SYNC:
logger.warn('Node authorized_keys synchronization is disabled!')
return
nodes = Node.objects.all().order_by('hostname')
# Create one authorized_keys list
authorized_keys = [node.sshkey for node in nodes if node.sshkey]
# Add user specified compute node SSH keys
authorized_keys.extend(dc1_settings.VMS_NODE_SSH_KEYS_DEFAULT)
# Save the authorized_keys file on every compute node to persistent /usbkey/config.inc/ and /root/.ssh locations
files = '/usbkey/config.inc/authorized_keys /root/.ssh/authorized_keys'
cmd = 'tee %s; chmod 640 %s' % (files, files)
stdin = '\n'.join(authorized_keys)
for node in nodes:
if node.authorized_keys == stdin:
logger.info('authorized_keys already synced for node %s - skipping update', node)
continue
# We update authorized_keys only on online nodes
# But we will also run this whenever node status is changed to online
if not node.is_online():
logger.warn('Excluding node %s from updating authorized_keys because it is not in online state', node)
continue
lock = 'node %s authorized_keys' % node.uuid
cb = ('api.node.sshkey.tasks.node_authorized_keys_sync_cb', {'node_uuid': node.uuid})
tid, err = execute(ERIGONES_TASK_USER, None, cmd, stdin=stdin, callback=cb, lock=lock, queue=node.fast_queue,
expires=180, nolog=True, tg=TG_DC_UNBOUND, ping_worker=False, check_user_tasks=False)
if err:
logger.error('Got error (%s) when running task %s for updating authorized_keys on node %s', err, tid, node)
else:
logger.info('Created task %s for updating authorized_keys on node %s', tid, node)
def delete(self):
self._check_vm_status()
self._check_snap_status(lost_ok=True)
request, vm, snap = self.request, self.vm, self.snap
apiview, detail = self._get_apiview_detail()
msg = LOG_SNAP_DELETE
if snap.status == Snapshot.LOST:
snap.delete()
res = {'message': 'Snapshot successfully deleted from DB'}
return SuccessTaskResponse(request,
res,
msg=msg,
vm=vm,
detail=detail)
SnapshotSerializer(request, snap)
lock = self.LOCK % (vm.uuid, snap.disk_id)
cmd = 'esnapshot destroy "%s@%s" 2>&1' % (self.zfs_filesystem,
snap.zfs_name)
tid, err = execute(request,
vm.owner.id,
cmd,
meta=snap_meta(vm, msg, apiview, detail),
lock=lock,
callback=snap_callback(vm, snap),
queue=vm.node.fast_queue,
tt=self.tt)
if err:
return FailureTaskResponse(request, err, vm=vm)
else:
snap.save_status(snap.PENDING)
return TaskResponse(request,
tid,
msg=msg,
vm=vm,
api_view=apiview,
detail=detail,
data=self.data)
def post(self):
request, vm = self.request, self.vm
if not self.vm.is_kvm():
raise OperationNotSupported
if vm.status not in (vm.RUNNING, vm.STOPPING):
raise VmIsNotOperational
apiview = {'view': 'vm_screenshot', 'method': request.method, 'hostname': vm.hostname}
cmd = 'vmadm sysrq %s nmi >&2 && sleep 0.5 && vmadm sysrq %s screenshot >&2 && \
cat /%s/%s/root/tmp/vm.ppm' % (vm.uuid, vm.uuid, vm.zpool, vm.uuid)
lock = 'vm_screenshot vm:%s' % vm.uuid
meta = {'output': {'returncode': 'returncode', 'stderr': 'message', 'stdout': 'image'},
'replace_stderr': ((vm.uuid, vm.hostname),),
'encode_stdout': True, 'compress_stdout': True, 'apiview': apiview}
callback = ('api.vm.other.tasks.vm_screenshot_cb', {'vm_uuid': vm.uuid})
tid, err = execute(request, vm.owner.id, cmd, meta=meta, lock=lock, callback=callback, queue=vm.node.fast_queue)
if err:
return FailureTaskResponse(request, err, vm=vm)
else:
return TaskResponse(request, tid, vm=vm, api_view=apiview, data=self.data) # No msg
def vm_deploy(vm, force_stop=False):
"""
Internal API call used for finishing VM deploy;
Actually cleaning the json and starting the VM.
"""
if force_stop: # VM is running without OS -> stop
cmd = 'vmadm stop %s -F >/dev/null 2>/dev/null; vmadm get %s 2>/dev/null' % (
vm.uuid, vm.uuid)
else: # VM is stopped and deployed -> start
cmd = 'vmadm start %s >/dev/null 2>/dev/null; vmadm get %s 2>/dev/null' % (
vm.uuid, vm.uuid)
msg = 'Deploy server'
lock = 'vmadm deploy ' + vm.uuid
meta = {
'output': {
'returncode': 'returncode',
'stderr': 'message',
'stdout': 'json'
},
'replace_stderr': ((vm.uuid, vm.hostname), ),
'msg': msg,
'vm_uuid': vm.uuid
}
callback = ('api.vm.base.tasks.vm_deploy_cb', {'vm_uuid': vm.uuid})
return execute(ERIGONES_TASK_USER,
None,
cmd,
meta=meta,
lock=lock,
callback=callback,
queue=vm.node.fast_queue,
nolog=True,
ping_worker=False,
check_user_tasks=False)
def put(self):
request, vm = self.request, self.vm
# only admin
if not (request.user and request.user.is_admin(request)):
raise PermissionDenied
node = vm.node
apiview = self.apiview
apiview['force'] = bool(ForceSerializer(data=self.data, default=False))
queue = vm.node.fast_queue
new_node_uuid = None
detail_dict = {}
if vm.status not in (vm.RUNNING, vm.STOPPED):
raise VmIsNotOperational('VM is not stopped or running')
if apiview['force']:
detail_dict['force'] = True
# final cmd and empty stdin
cmd = 'vmadm get %s 2>/dev/null' % vm.uuid
stdin = None
block_key = None
node_param = self.data.get('node')
if node_param:
if not request.user.is_staff:
raise PermissionDenied
node = get_node(request, node_param, dc=request.dc, exists_ok=True, noexists_fail=True)
if node.hostname == vm.node.hostname:
raise InvalidInput('VM already has the requested node set in DB')
apiview['node'] = detail_dict['node'] = node.hostname
queue = node.fast_queue
new_node_uuid = node.uuid
elif vm.json_changed():
if vm.locked:
raise VmIsLocked
json_update = vm.json_update()
self.check_update(json_update)
if (vm.json_disks_changed() or vm.json_nics_changed()) and vm.tasks:
raise VmHasPendingTasks
# create json suitable for update
stdin, cmd1 = self.fix_update(json_update)
self.validate_update(vm, stdin, cmd1)
stdin = stdin.dump()
# final cmd
cmd = cmd1 + 'vmadm update %s >&2; e=$?; vmadm get %s 2>/dev/null; exit $e' % (vm.uuid, vm.uuid)
# Possible node_image import task which will block this task on node worker
block_key = self.node_image_import(vm.node, json_update.get('add_disks', []))
else: # JSON unchanged and not force
detail = 'Successfully updated VM %s (locally)' % vm.hostname
res = SuccessTaskResponse(request, detail, msg=LOG_VM_UPDATE, vm=vm, detail=detail)
vm_updated.send(TaskID(res.data.get('task_id'), request=request), vm=vm) # Signal!
return res
# Check compute node status after we know which compute node the task is going to be run on
# The internal vm.node.status checking is disabled in get_vm() in __init__
if node.status != node.ONLINE:
raise NodeIsNotOperational
msg = LOG_VM_UPDATE
meta = {
'output': {'returncode': 'returncode', 'stderr': 'message', 'stdout': 'json'},
'replace_stderr': ((vm.uuid, vm.hostname),), 'msg': msg, 'vm_uuid': vm.uuid, 'apiview': apiview
}
callback = ('api.vm.base.tasks.vm_update_cb', {'vm_uuid': vm.uuid, 'new_node_uuid': new_node_uuid})
logger.debug('Updating VM %s with json: """%s"""', vm, stdin)
err = True
vm.set_notready()
try:
tid, err = execute(request, vm.owner.id, cmd, stdin=stdin, meta=meta, lock=self.lock, callback=callback,
queue=queue, block_key=block_key)
if err:
return FailureTaskResponse(request, err, vm=vm)
else:
return TaskResponse(request, tid, msg=msg, vm=vm, api_view=apiview, data=self.data,
detail_dict=detail_dict)
finally:
if err:
vm.revert_notready()
def put(self):
request, vm = self.request, self.vm
# only admin
if not (request.user and request.user.is_admin(request)):
raise PermissionDenied
apiview = self.apiview
apiview['force'] = bool(ForceSerializer(data=self.data, default=False))
if vm.status not in (vm.RUNNING, vm.STOPPED):
raise VmIsNotOperational('VM is not stopped or running')
if apiview['force']:
# final cmd and empty stdin
cmd = 'vmadm get %s 2>/dev/null' % vm.uuid
stdin = None
block_key = None
elif vm.json_changed():
if vm.locked:
raise VmIsLocked
json_update = vm.json_update()
self.check_update(json_update)
if (vm.json_disks_changed() or vm.json_nics_changed()) and vm.tasks:
raise VmHasPendingTasks
# create json suitable for update
stdin, cmd1 = self.fix_update(json_update)
stdin = stdin.dump()
# cmd = zfs set... >&2;
if cmd1 and vm.snapshot_set.exists():
raise ExpectationFailed('VM has snapshots')
# final cmd
cmd = cmd1 + 'vmadm update %s >&2; e=$?; vmadm get %s 2>/dev/null; exit $e' % (vm.uuid, vm.uuid)
# Possible node_image import task which will block this task on node worker
block_key = self.node_image_import(vm.node, json_update.get('add_disks', []))
else: # JSON unchanged and not force
detail = 'Successfully updated VM %s (locally)' % vm.hostname
res = SuccessTaskResponse(request, detail, msg=LOG_VM_UPDATE, vm=vm, detail=detail)
vm_updated.send(TaskID(res.data.get('task_id'), request=request), vm=vm) # Signal!
return res
msg = LOG_VM_UPDATE
meta = {
'output': {'returncode': 'returncode', 'stderr': 'message', 'stdout': 'json'},
'replace_stderr': ((vm.uuid, vm.hostname),), 'msg': msg, 'vm_uuid': vm.uuid, 'apiview': apiview
}
callback = ('api.vm.base.tasks.vm_update_cb', {'vm_uuid': vm.uuid})
logger.debug('Updating VM %s with json: """%s"""', vm, stdin)
err = True
vm.set_notready()
try:
tid, err = execute(request, vm.owner.id, cmd, stdin=stdin, meta=meta, lock=self.lock, callback=callback,
queue=vm.node.fast_queue, block_key=block_key)
if err:
return FailureTaskResponse(request, err, vm=vm)
else:
return TaskResponse(request, tid, msg=msg, vm=vm, api_view=apiview, data=self.data)
finally:
if err:
vm.revert_notready()
def post(self):
request, vm = self.request, self.vm
ser = VmCreateSerializer(data=self.data)
if not ser.is_valid():
return FailureTaskResponse(request, ser.errors, vm=vm)
if not vm.is_kvm():
if not (vm.dc.settings.VMS_VM_SSH_KEYS_DEFAULT or vm.owner.usersshkey_set.exists()):
raise PreconditionRequired('VM owner has no SSH keys available')
apiview = self.apiview
# noinspection PyTypeChecker
cmd = 'vmadm create >&2; e=$? %s; vmadm get %s 2>/dev/null; vmadm start %s >&2; exit $e' % (
self.fix_create(vm), vm.uuid, vm.uuid)
recreate = apiview['recreate'] = ser.data['recreate']
# noinspection PyAugmentAssignment
if recreate:
# recreate should be available to every vm owner
if not (request.user and request.user.is_authenticated()):
raise PermissionDenied
if vm.locked:
raise VmIsLocked
if vm.status != vm.STOPPED:
raise VmIsNotOperational('VM is not stopped')
if not ser.data['force']:
raise ExpectationFailed('Are you sure?')
msg = LOG_VM_RECREATE
# noinspection PyAugmentAssignment
cmd = 'vmadm delete ' + vm.uuid + ' >&2 && sleep 1; ' + cmd
elif vm.status == vm.NOTCREATED:
# only admin
if not (request.user and request.user.is_admin(request)):
raise PermissionDenied
if not vm.node: # we need to find a node for this vm now
logger.debug('VM %s has no compute node defined. Choosing node automatically', vm)
VmDefineView(request).choose_node(vm)
logger.info('New compute node %s for VM %s was chosen automatically.', vm.node, vm)
msg = LOG_VM_CREATE
else:
raise VmIsNotOperational('VM is already created')
# Check boot flag (KVM) or disk image (OS) (bug #chili-418)
if not vm.is_bootable():
raise PreconditionRequired('VM has no bootable disk')
if vm.tasks:
raise VmHasPendingTasks
old_status = vm.status
deploy = apiview['deploy'] = vm.is_deploy_needed()
resize = apiview['resize'] = vm.is_resize_needed()
if not vm.is_blank():
vm.set_root_pw()
# Set new status also for blank VM (where deployment is not needed)
# This status will be changed in vm_status_event_cb (if everything goes well).
vm.status = vm.CREATING
vm.save() # save status / node / vnc_port / root_pw
stdin = vm.fix_json(deploy=deploy, resize=resize, recreate=recreate).dump()
meta = {
'output': {'returncode': 'returncode', 'stderr': 'message', 'stdout': 'json'},
'replace_stderr': ((vm.uuid, vm.hostname),),
'msg': msg,
'vm_uuid': vm.uuid,
'apiview': apiview
}
callback = ('api.vm.base.tasks.vm_create_cb', {'vm_uuid': vm.uuid})
err = True
try:
# Possible node_image import task which will block this task on node worker
block_key = self.node_image_import(vm.node, vm.json_get_disks())
logger.debug('Creating new VM %s on node %s with json: """%s"""', vm, vm.node, stdin)
tid, err = execute(request, vm.owner.id, cmd, stdin=stdin, meta=meta, expires=VM_VM_EXPIRES, lock=self.lock,
callback=callback, queue=vm.node.slow_queue, block_key=block_key)
if err:
return FailureTaskResponse(request, err, vm=vm)
else:
# Inform user about creating
vm_status_changed(tid, vm, vm.CREATING, save_state=False)
return TaskResponse(request, tid, msg=msg, vm=vm, api_view=apiview, data=self.data)
finally:
if err: # Revert old status
vm.status = old_status
vm.save_status()
def put(self):
request, vm, action = self.request, self.vm, self.action
# Cannot change status unless the VM is created on node
if vm.status not in self.statuses:
raise VmIsNotOperational
if action not in self.actions:
raise ExpectationFailed('Bad action')
apiview = self.apiview
f_ser = VmStatusFreezeSerializer(data=self.data)
if f_ser.is_valid():
freeze = apiview['freeze'] = f_ser.data['freeze']
unfreeze = apiview['unfreeze'] = f_ser.data['unfreeze']
else:
return FailureTaskResponse(request, f_ser.errors, vm=vm)
if ((action == 'start' and vm.status == Vm.STOPPED and not freeze) or
(action == 'reboot' and vm.status == Vm.RUNNING and not freeze) or
(action == 'stop' and vm.status in (Vm.STOPPING, Vm.RUNNING))):
pass
elif action == 'stop' and vm.status == Vm.STOPPED and freeze:
if not request.user.is_admin(request):
raise PermissionDenied
tid = task_id_from_request(request,
owner_id=vm.owner.id,
dummy=True)
vm_status_changed(tid, vm, vm.FROZEN, save_state=True)
res = {
'message':
'VM %s is already stopped. Changing status to frozen.' %
vm.hostname
}
return SuccessTaskResponse(request, res, task_id=tid, vm=vm)
elif action == 'stop' and vm.status == Vm.FROZEN and unfreeze:
if not request.user.is_admin(request):
raise PermissionDenied
tid = task_id_from_request(request,
owner_id=vm.owner.id,
dummy=True)
vm_status_changed(tid, vm, vm.STOPPED, save_state=True)
res = {
'message': 'Removing frozen status for VM %s.' % vm.hostname
}
return SuccessTaskResponse(request, res, task_id=tid, vm=vm)
else:
raise ExpectationFailed('Bad action')
dc_settings = request.dc.settings
if action in ('stop',
'reboot') and vm.uuid in dc_settings.VMS_NO_SHUTDOWN:
raise PreconditionRequired('Internal VM can\'t be stopped')
lock = 'vm_status vm:%s' % vm.uuid
stdin = None
apiview['update'] = False
transition_to_stopping = False
if action == 'start':
msg = LOG_START
ser = VmStatusActionIsoSerializer(request, vm, data=self.data)
if not ser.is_valid():
return FailureTaskResponse(request, ser.errors, vm=vm)
if ser.data and ser.iso:
if not request.user.is_admin(request) and vm.is_installed() and \
(ser.iso.name != dc_settings.VMS_ISO_RESCUECD):
raise PreconditionRequired('VM is not installed')
msg = LOG_START_ISO
iso = ser.iso
cmd = self._start_cmd(iso=iso,
iso2=ser.iso2,
once=ser.data['cdimage_once'])
else:
iso = None
cmd = self._start_cmd()
ser_update = VmStatusUpdateJSONSerializer(data=self.data)
if ser_update.is_valid():
if vm.json_changed():
apiview['update'] = ser_update.data['update']
logger.info('VM %s json != json_active', vm)
if apiview['update']:
from api.vm.base.vm_manage import VmManage
stdin, os_cmd = VmManage.fix_update(vm.json_update())
stdin = stdin.dump()
if os_cmd: # Dangerous, explicit update needed
# TODO: fix in gui
raise PreconditionRequired(
'VM must be updated first')
if iso:
msg = LOG_START_UPDATE_ISO
else:
msg = LOG_START_UPDATE
cmd_update = 'vmadm update %s >&2; e=$?; vmadm get %s 2>/dev/null; ' % (
vm.uuid, vm.uuid)
cmd = cmd_update + cmd + '; exit $e'
# logger.info('VM %s json_active is going to be updated with json """%s"""', vm, stdin)
else:
logger.warning('VM %s json_active update disabled', vm)
else:
return FailureTaskResponse(request, ser_update.errors, vm=vm)
else:
force = ForceSerializer(data=self.data, default=False).is_true()
cmd = self._action_cmd(action, force=force)
if action == 'reboot':
msg = LOG_REBOOT
else:
msg = LOG_STOP
if force:
apiview['force'] = True
if action == 'reboot':
msg = LOG_REBOOT_FORCE
else:
lock += ' force'
msg = LOG_STOP_FORCE
elif vm.status == Vm.STOPPING:
raise VmIsNotOperational(
'VM is already stopping; try to use force')
else:
transition_to_stopping = True
meta = {
'output': {
'returncode': 'returncode',
'stderr': 'message',
'stdout': 'json'
},
'replace_stderr': ((vm.uuid, vm.hostname), ),
'detail': self.detail,
'msg': msg,
'vm_uuid': vm.uuid,
'apiview': apiview,
'last_status': vm.status,
}
callback = ('api.vm.status.tasks.vm_status_cb', {'vm_uuid': vm.uuid})
tid, err = execute(request,
vm.owner.id,
cmd,
stdin=stdin,
meta=meta,
lock=lock,
callback=callback,
queue=vm.node.fast_queue)
if err:
return FailureTaskResponse(request, err, vm=vm)
else:
if transition_to_stopping:
vm.save_status(Vm.STOPPING)
return TaskResponse(request,
tid,
msg=msg,
vm=vm,
api_view=apiview,
detail=self.detail,
data=self.data,
api_data={
'status': vm.status,
'status_display': vm.status_display()
})
def put(self):
assert self.request.dc.is_default()
ser = UpdateSerializer(self.request, data=self.data)
if not ser.is_valid():
return FailureTaskResponse(self.request,
ser.errors,
dc_bound=False)
node = self.node
version = ser.data['version']
key = ser.data.get('key')
cert = ser.data.get('cert')
del node.system_version # Request latest version in next command
node_version = node.system_version
if not (isinstance(node_version, text_type) and node_version):
raise NodeIsNotOperational(
'Node version information could not be retrieved')
node_version = node_version.split(':')[-1] # remove edition prefix
if version == ('v' + node_version) and not ser.data.get('force'):
raise PreconditionRequired('Node is already up-to-date')
if node.status != node.OFFLINE:
raise NodeIsNotOperational(
'Unable to perform update on node that is not in maintenance state'
)
if node_version.startswith('2.'):
# Old-style (pre 3.0) update mechanism
return self._update_v2(version, key=key, cert=cert)
# Upload key and cert and get command array
worker = node.worker(Q_FAST)
update_cmd = worker_command('system_update_command',
worker,
version=version,
key=key,
cert=cert,
force=ser.data.get('force'),
timeout=10)
if update_cmd is None:
raise GatewayTimeout('Node worker is not responding')
if not isinstance(update_cmd, list):
raise PreconditionRequired(
'Node update command could be retrieved')
msg = LOG_SYSTEM_UPDATE
_apiview_ = {
'view': 'system_node_update',
'method': self.request.method,
'hostname': node.hostname,
'version': version,
}
meta = {
'apiview': _apiview_,
'msg': msg,
'node_uuid': node.uuid,
'output': {
'returncode': 'returncode',
'stdout': 'message'
},
'check_returncode': True,
}
lock = self.LOCK % node.hostname
cmd = '%s 2>&1' % ' '.join(update_cmd)
tid, err = execute(self.request,
node.owner.id,
cmd,
meta=meta,
lock=lock,
queue=node.fast_queue,
tg=TG_DC_UNBOUND)
if err:
return FailureTaskResponse(self.request, err, dc_bound=False)
else:
return TaskResponse(self.request,
tid,
msg=msg,
obj=node,
api_view=_apiview_,
data=self.data,
dc_bound=False,
detail_dict=ser.detail_dict(force_full=True))
def put(self):
request, vm = self.request, self.vm
if vm.locked:
raise VmIsLocked
if vm.status not in (vm.STOPPED, vm.RUNNING):
raise VmIsNotOperational('VM is not stopped or running')
if vm.json_changed():
raise PreconditionRequired(
'VM definition has changed; Update first')
ser = VmMigrateSerializer(request, vm, data=self.data)
if not ser.is_valid():
return FailureTaskResponse(request, ser.errors, vm=vm)
if vm.tasks:
raise VmHasPendingTasks
err = True
ghost_vm = None
# Set VM to nonready (+"api lock")
vm.set_notready()
try:
# Create a dummy/placeholder VM
ghost_vm = ser.save_ghost_vm()
# Possible node_image import task which will block this task on node worker
block_key = ser.node_image_import()
# We have a custom detail dict with all necessary api view parameters
detail_dict = ser.detail_dict()
# Prepare task data
apiview = {
'view': 'vm_migrate',
'method': request.method,
'hostname': vm.hostname
}
apiview.update(detail_dict)
lock = 'vm_migrate vm:%s' % vm.uuid
meta = {
'output': {
'returncode': 'returncode',
'stderr': 'message',
'stdout': 'json'
},
'replace_stderr': ((vm.uuid, vm.hostname), ),
'msg': LOG_MIGRATE,
'vm_uuid': vm.uuid,
'slave_vm_uuid': ghost_vm.uuid,
'apiview': apiview,
}
callback = ('api.vm.migrate.tasks.vm_migrate_cb', {
'vm_uuid': vm.uuid,
'slave_vm_uuid': ghost_vm.uuid
})
# Execute task
tid, err = execute(request,
vm.owner.id,
ser.esmigrate_cmd,
meta=meta,
lock=lock,
callback=callback,
queue=vm.node.fast_queue,
block_key=block_key)
if err: # Error, revert VM status, delete placeholder VM
return FailureTaskResponse(request, err, vm=vm)
else: # Success, task is running
return TaskResponse(request,
tid,
msg=LOG_MIGRATE,
vm=vm,
api_view=apiview,
detail_dict=detail_dict,
data=self.data)
finally:
if err:
vm.revert_notready()
if ghost_vm:
ghost_vm.delete()
def delete(self):
"""Delete multiple snapshots"""
# TODO: not documented
request, data, vm = self.request, self.data, self.vm
disk_id, real_disk_id, zfs_filesystem = get_disk_id(request, vm, data)
# Parse data['snapnames']
snaps, __ = get_snapshots(request, vm, real_disk_id, data)
self._check_vm_status()
snaps_lost = snaps.filter(status=Snapshot.LOST)
msg = LOG_SNAPS_DELETE
if snaps_lost:
_result = {'message': 'Snapshots successfully deleted from DB'}
_detail = "snapnames='%s', disk_id=%s" % (','.join(
i.name for i in snaps_lost), disk_id)
snaps_lost.delete()
res = SuccessTaskResponse(request,
_result,
msg=msg,
vm=vm,
detail=_detail)
snaps = snaps.filter(
status=Snapshot.OK) # Work with OK snapshots from now on
if not snaps:
return res
elif any(i.status != Snapshot.OK for i in snaps):
raise ExpectationFailed('VM snapshot status is not OK')
# Task type (a = automatic, e = manual)
if getattr(request, 'define_id', None):
tt = TT_AUTO
else:
tt = TT_EXEC
snapnames = [i.name for i in snaps]
_apiview_ = {
'view': 'vm_snapshot_list',
'method': request.method,
'hostname': vm.hostname,
'disk_id': disk_id,
'snapnames': snapnames
}
_detail_ = "snapnames='%s', disk_id=%s" % (','.join(snapnames),
disk_id)
snap_ids = [snap.id for snap in snaps]
zfs_names = ','.join([snap.zfs_name for snap in snaps])
lock = self.LOCK % (vm.uuid, real_disk_id)
cmd = 'esnapshot destroy "%s@%s" 2>&1' % (zfs_filesystem, zfs_names)
callback = ('api.vm.snapshot.tasks.vm_snapshot_list_cb', {
'vm_uuid': vm.uuid,
'snap_ids': snap_ids
})
tid, err = execute(request,
vm.owner.id,
cmd,
meta=snap_meta(vm, msg, _apiview_, _detail_),
lock=lock,
callback=callback,
queue=vm.node.fast_queue,
tt=tt)
if err:
return FailureTaskResponse(request, err, vm=vm)
else:
snaps.update(status=Snapshot.PENDING)
return TaskResponse(request,
tid,
msg=msg,
vm=vm,
api_view=_apiview_,
detail=_detail_,
data=self.data)
def node_overlay_arp_file(task_id,
overlay_rule_name,
node_exclusive=None,
**kwargs):
"""
Task for generating ARP files for VMs and nodes connected to specific overlay.
It is called by various signals (see below).
"""
if node_exclusive:
# update rules only on a specific compute node
if node_exclusive.overlay_rules.get(overlay_rule_name,
{}).get('arp_file'):
overlay_nodes = [node_exclusive]
else:
return
else:
# list of nodes where the overlay rule is defined and uses the files search plugin
overlay_nodes = [
node for node in Node.objects.all()
if node.overlay_rules.get(overlay_rule_name, {}).get('arp_file')
]
# list of VM NICs (see VNIC namedtuple above) defined over the overlay
overlay_vnics = list(
_get_overlay_vm_vnics(
overlay_rule_name,
Vm.objects.select_related('node').filter(
slavevm__isnull=True,
node__in=overlay_nodes).exclude(status=Vm.NOTCREATED)))
# Add list of Node VNICs which are defined over the overlay
overlay_vnics += list(
_get_overlay_node_vnics(overlay_rule_name, overlay_nodes))
for node in overlay_nodes:
# We update arp files only on online nodes
# But we will also run this whenever node status is changed to online
if not node.is_online():
logger.warn(
'Excluding node %s from updating arp file for overlay "%s" because it is not in online state',
node, overlay_rule_name)
continue
overlay_arp_file = node.overlay_rules[overlay_rule_name]['arp_file']
arp_table = json.dumps(_get_overlay_arp_table(node, overlay_rule_name,
overlay_vnics),
indent=2)
cmd = ('cat /dev/stdin > {arp_file} && '
'chmod 0400 {arp_file} && '
'chown netadm:netadm {arp_file} && '
'svcadm restart network/varpd').format(
arp_file=overlay_arp_file)
lock = 'node:{node_uuid} overlay:{overlay_rule_name}'.format(
node_uuid=node.uuid, overlay_rule_name=overlay_rule_name)
queue = node.fast_queue
tid, err = execute(ERIGONES_TASK_USER,
None,
cmd,
stdin=arp_table,
callback=False,
lock=lock,
queue=queue,
expires=300,
nolog=True,
tg=TG_DC_UNBOUND,
ping_worker=False,
check_user_tasks=False)
if err:
logger.error(
'Got error (%s) when running task %s for updating overlay ARP file %s on node %s',
err, tid, overlay_arp_file, node)
else:
logger.info(
'Created task %s for updating overlay ARP file %s on node %s',
tid, overlay_arp_file, node)
def put(self):
if 'note' in self.data:
# Changing snapshot note instead of rollback (not logging)
return self._update_note()
request, vm, snap = self.request, self.vm, self.snap
ser = SnapshotRestoreSerializer(request, vm, data=self.data)
if not ser.is_valid():
return FailureTaskResponse(self.request, ser.errors)
target_vm, target_vm_disk_id = ser.target_vm, ser.target_vm_disk_id
if vm.node.status not in vm.node.STATUS_OPERATIONAL:
raise NodeIsNotOperational
if target_vm.locked:
raise VmIsLocked
if target_vm != vm:
if target_vm.node.status not in target_vm.node.STATUS_OPERATIONAL:
raise NodeIsNotOperational
self._check_vm_status(vm=target_vm)
if not vm.has_compatible_brand(target_vm.brand):
raise PreconditionRequired('VM brand mismatch')
source_disk = vm.json_active_get_disks()[self.disk_id - 1]
target_disk = target_vm.json_active_get_disks()[target_vm_disk_id -
1]
if target_disk['size'] != source_disk['size']:
raise PreconditionRequired('Disk size mismatch')
self._check_vm_status()
self._check_snap_status()
apiview, detail = self._get_apiview_detail()
apiview['force'] = ser.data['force']
if target_vm != vm:
detail += ", source_hostname='%s', target_hostname='%s', target_disk_id=%s" % (
vm.hostname, target_vm.hostname, target_vm_disk_id)
apiview['source_hostname'] = vm.hostname
apiview['target_hostname'] = target_vm.hostname
apiview['target_disk_id'] = target_vm_disk_id
if not apiview['force']:
if Snapshot.objects.only('id').filter(
vm=target_vm,
disk_id=ser.target_vm_real_disk_id).exists():
raise ExpectationFailed('Target VM has snapshots')
elif not apiview['force']:
snaplast = Snapshot.objects.only('id').filter(
vm=vm, disk_id=snap.disk_id).order_by('-id')[0]
if snap.id != snaplast.id:
raise ExpectationFailed('VM has more recent snapshots')
if target_vm.status != vm.STOPPED:
raise VmIsNotOperational('VM is not stopped')
if target_vm.tasks:
raise VmHasPendingTasks
msg = LOG_SNAP_UPDATE
lock = self.LOCK % (vm.uuid, snap.disk_id)
if target_vm == vm:
cmd = 'esnapshot rollback "%s@%s" 2>&1' % (self.zfs_filesystem,
snap.zfs_name)
else:
cmd = 'esbackup snap-restore -s %s@%s -d %s' % (
self.zfs_filesystem, snap.zfs_name,
ser.target_vm_disk_zfs_filesystem)
if vm.node != target_vm.node:
cmd += ' -H %s' % target_vm.node.address
vm.set_notready()
target_vm.set_notready()
tid, err = execute(request,
target_vm.owner.id,
cmd,
meta=snap_meta(target_vm, msg, apiview, detail),
lock=lock,
callback=snap_callback(target_vm, snap),
queue=vm.node.fast_queue)
if err:
target_vm.revert_notready()
if vm != target_vm:
vm.revert_notready()
return FailureTaskResponse(request, err, vm=target_vm)
else:
snap.save_status(snap.ROLLBACK)
return TaskResponse(request,
tid,
msg=msg,
vm=target_vm,
api_view=apiview,
detail=detail,
data=self.data)
def put(self): # noqa: R701
request, vm, action = self.request, self.vm, self.action
# Cannot change status unless the VM is created on node
if vm.status not in self.statuses and action != 'current':
raise VmIsNotOperational
if action not in self.actions:
raise ExpectationFailed('Bad action')
apiview = self.apiview
f_ser = VmStatusFreezeSerializer(data=self.data)
if f_ser.is_valid():
freeze = apiview['freeze'] = f_ser.data['freeze']
unfreeze = apiview['unfreeze'] = f_ser.data['unfreeze']
else:
return FailureTaskResponse(request, f_ser.errors, vm=vm)
if ((action == 'start' and vm.status == Vm.STOPPED and not freeze) or
(action == 'reboot' and vm.status == Vm.RUNNING and not freeze) or
(action == 'stop' and vm.status in (Vm.STOPPING, Vm.RUNNING))):
pass
elif action == 'stop' and vm.status == Vm.STOPPED and freeze:
if not request.user.is_admin(request):
raise PermissionDenied
tid = task_id_from_request(request,
owner_id=vm.owner.id,
dummy=True)
vm_status_changed(tid, vm, vm.FROZEN, save_state=True)
res = {
'message':
'VM %s is already stopped. Changing status to frozen.' %
vm.hostname
}
return SuccessTaskResponse(request, res, task_id=tid, vm=vm)
elif action == 'stop' and vm.status == Vm.FROZEN and unfreeze:
if not request.user.is_admin(request):
raise PermissionDenied
tid = task_id_from_request(request,
owner_id=vm.owner.id,
dummy=True)
vm_status_changed(tid, vm, vm.STOPPED, save_state=True)
res = {
'message': 'Removing frozen status for VM %s.' % vm.hostname
}
return SuccessTaskResponse(request, res, task_id=tid, vm=vm)
elif action == 'current':
# Limit PUT /current/ action to be Admins and SuperAdmins
if not request.user.is_admin(request):
raise PermissionDenied
if vm.status in self.statuses_force_change_allowed:
return self.get_current_status(force_change=True)
elif vm.status in self.stuck_statuses_force_change_allowed:
if vm.tasks:
raise VmHasPendingTasks
else:
return self.get_current_status(force_change=True)
else:
raise VmIsNotOperational
else:
raise ExpectationFailed('Bad action')
dc_settings = request.dc.settings
if action in ('stop',
'reboot') and vm.uuid in dc_settings.VMS_NO_SHUTDOWN:
raise PreconditionRequired('Internal VM can\'t be stopped')
lock = 'vm_status vm:%s' % vm.uuid
stdin = None
apiview['update'] = False
transition_to_stopping = False
# The update parameter is used by all actions (start, stop, reboot)
ser_update = VmStatusUpdateJSONSerializer(data=self.data,
default=(action
in ('start',
'reboot')))
if not ser_update.is_valid():
return FailureTaskResponse(request, ser_update.errors, vm=vm)
if vm.json_changed():
apiview['update'] = ser_update.data['update']
logger.info('VM %s json != json_active', vm)
if not apiview['update']:
logger.info('VM %s json_active update disabled', vm)
if action == 'start':
ser = VmStatusActionIsoSerializer(request, vm, data=self.data)
if not ser.is_valid():
return FailureTaskResponse(request, ser.errors, vm=vm)
if ser.data and ser.iso:
if not request.user.is_admin(request) and vm.is_installed() and \
(ser.iso.name != dc_settings.VMS_ISO_RESCUECD):
raise PreconditionRequired('VM is not installed')
msg = LOG_START_ISO
iso = ser.iso
cmd = self._start_cmd(iso=iso,
iso2=ser.iso2,
once=ser.data['cdimage_once'])
else:
msg = LOG_START
iso = None
cmd = self._start_cmd()
if apiview['update']:
if vm.tasks:
raise VmHasPendingTasks
cmd, stdin = self._add_update_cmd(cmd, os_cmd_allowed=False)
if iso:
msg = LOG_START_UPDATE_ISO
else:
msg = LOG_START_UPDATE
else:
ser_stop_reboot = VmStatusStopSerializer(request,
vm,
data=self.data)
if not ser_stop_reboot.is_valid():
return FailureTaskResponse(request,
ser_stop_reboot.errors,
vm=vm)
update = apiview.get('update',
False) # VmStatusUpdateJSONSerializer
force = apiview['force'] = ser_stop_reboot.data.get('force', False)
timeout = ser_stop_reboot.data.get('timeout', None)
if not force and timeout:
apiview['timeout'] = timeout
if update:
if vm.tasks:
raise VmHasPendingTasks
# This will always perform a vmadm stop command, followed by a vmadm update command and optionally
# followed by a vmadm start command (reboot)
pre_cmd = self._action_cmd('stop',
force=force,
timeout=timeout)
if action == 'reboot':
if force:
msg = LOG_REBOOT_FORCE_UPDATE
else:
msg = LOG_REBOOT_UPDATE
post_cmd = self._action_cmd('start')
else:
if force:
msg = LOG_STOP_FORCE_UPDATE
else:
msg = LOG_STOP_UPDATE
post_cmd = ''
cmd, stdin = self._add_update_cmd(post_cmd,
os_cmd_allowed=True,
pre_cmd=pre_cmd)
else:
cmd = self._action_cmd(action, force=force, timeout=timeout)
if force:
if action == 'reboot':
msg = LOG_REBOOT_FORCE
else:
lock += ' force'
msg = LOG_STOP_FORCE
else:
if action == 'reboot':
msg = LOG_REBOOT
else:
msg = LOG_STOP
if vm.status == Vm.STOPPING:
if update:
raise PreconditionRequired(
'Cannot perform update while VM is stopping')
if not force:
raise VmIsNotOperational(
'VM is already stopping; try to use force')
else:
transition_to_stopping = True
meta = {
'output': {
'returncode': 'returncode',
'stderr': 'message',
'stdout': 'json'
},
'replace_stderr': ((vm.uuid, vm.hostname), ),
'detail': self.detail,
'msg': msg,
'vm_uuid': vm.uuid,
'apiview': apiview,
'last_status': vm.status,
}
callback = ('api.vm.status.tasks.vm_status_cb', {'vm_uuid': vm.uuid})
tid, err = execute(request,
vm.owner.id,
cmd,
stdin=stdin,
meta=meta,
lock=lock,
callback=callback,
queue=vm.node.fast_queue)
if err:
return FailureTaskResponse(request, err, vm=vm)
else:
if transition_to_stopping:
vm.save_status(Vm.STOPPING)
return TaskResponse(request,
tid,
msg=msg,
vm=vm,
api_view=apiview,
detail=self.detail,
data=self.data,
api_data={
'status': vm.status,
'status_display': vm.status_display()
})
def execute(self, *args, **kwargs):
self.task_id, self.error = execute(self.request, self.obj.owner.id,
*args, **kwargs)
if self.error:
return None
return self.task_id
