def process_op_status(vm, etime, jobid, opcode, status, logmsg, nics=None): """Process a job progress notification from the backend Process an incoming message from the backend (currently Ganeti). Job notifications with a terminating status (sucess, error, or canceled), also update the operating state of the VM. """ # See #1492, #1031, #1111 why this line has been removed #if (opcode not in [x[0] for x in VirtualMachine.BACKEND_OPCODES] or if status not in [x[0] for x in BACKEND_STATUSES]: raise VirtualMachine.InvalidBackendMsgError(opcode, status) vm.backendjobid = jobid vm.backendjobstatus = status vm.backendopcode = opcode vm.backendlogmsg = logmsg # Notifications of success change the operating state state_for_success = VirtualMachine.OPER_STATE_FROM_OPCODE.get(opcode, None) if status == 'success' and state_for_success is not None: vm.operstate = state_for_success # Update the NICs of the VM if status == "success" and nics is not None: _process_net_status(vm, etime, nics) # Special case: if OP_INSTANCE_CREATE fails --> ERROR if opcode == 'OP_INSTANCE_CREATE' and status in ('canceled', 'error'): vm.operstate = 'ERROR' vm.backendtime = etime elif opcode == 'OP_INSTANCE_REMOVE': # Set the deleted flag explicitly, cater for admin-initiated removals # Special case: OP_INSTANCE_REMOVE fails for machines in ERROR, # when no instance exists at the Ganeti backend. # See ticket #799 for all the details. # if (status == 'success' or (status == 'error' and (vm.operstate == 'ERROR' or vm.action == 'DESTROY'))): _process_net_status(vm, etime, nics=[]) vm.deleted = True vm.operstate = state_for_success vm.backendtime = etime # Issue and accept commission to Quotaholder quotas.issue_and_accept_commission(vm, delete=True) # Update backendtime only for jobs that have been successfully completed, # since only these jobs update the state of the VM. Else a "race condition" # may occur when a successful job (e.g. OP_INSTANCE_REMOVE) completes # before an error job and messages arrive in reversed order. if status == 'success': vm.backendtime = etime vm.save()
def process_op_status(vm, etime, jobid, opcode, status, logmsg, nics=None, job_fields=None): """Process a job progress notification from the backend Process an incoming message from the backend (currently Ganeti). Job notifications with a terminating status (sucess, error, or canceled), also update the operating state of the VM. """ # See #1492, #1031, #1111 why this line has been removed #if (opcode not in [x[0] for x in VirtualMachine.BACKEND_OPCODES] or if status not in [x[0] for x in BACKEND_STATUSES]: raise VirtualMachine.InvalidBackendMsgError(opcode, status) vm.backendjobid = jobid vm.backendjobstatus = status vm.backendopcode = opcode vm.backendlogmsg = logmsg if status not in rapi.JOB_STATUS_FINALIZED: vm.save() return if job_fields is None: job_fields = {} new_operstate = None new_flavor = None state_for_success = VirtualMachine.OPER_STATE_FROM_OPCODE.get(opcode) if status == rapi.JOB_STATUS_SUCCESS: # If job succeeds, change operating state if needed if state_for_success is not None: new_operstate = state_for_success beparams = job_fields.get("beparams", None) if beparams: # Change the flavor of the VM new_flavor = _process_resize(vm, beparams) # Update backendtime only for jobs that have been successfully # completed, since only these jobs update the state of the VM. Else a # "race condition" may occur when a successful job (e.g. # OP_INSTANCE_REMOVE) completes before an error job and messages arrive # in reversed order. vm.backendtime = etime if status in rapi.JOB_STATUS_FINALIZED and nics is not None: # Update the NICs of the VM _process_net_status(vm, etime, nics) # Special case: if OP_INSTANCE_CREATE fails --> ERROR if opcode == 'OP_INSTANCE_CREATE' and status in (rapi.JOB_STATUS_CANCELED, rapi.JOB_STATUS_ERROR): new_operstate = "ERROR" vm.backendtime = etime # Update state of associated NICs vm.nics.all().update(state="ERROR") elif opcode == 'OP_INSTANCE_REMOVE': # Special case: OP_INSTANCE_REMOVE fails for machines in ERROR, # when no instance exists at the Ganeti backend. # See ticket #799 for all the details. if (status == rapi.JOB_STATUS_SUCCESS or (status == rapi.JOB_STATUS_ERROR and not vm_exists_in_backend(vm))): # VM has been deleted for nic in vm.nics.all(): # Release the IP remove_nic_ips(nic) # And delete the NIC. nic.delete() vm.deleted = True new_operstate = state_for_success vm.backendtime = etime status = rapi.JOB_STATUS_SUCCESS if status in rapi.JOB_STATUS_FINALIZED: # Job is finalized: Handle quotas/commissioning vm = handle_vm_quotas(vm, job_id=jobid, job_opcode=opcode, job_status=status, job_fields=job_fields) # and clear task fields if vm.task_job_id == jobid: vm.task = None vm.task_job_id = None if new_operstate is not None: vm.operstate = new_operstate if new_flavor is not None: vm.flavor = new_flavor vm.save()