def config_qos(self, req): cmd = jsonobject.loads(req[http.REQUEST_BODY]) rsp = AgentRsp() shell.run("modprobe ifb; ip link set %s up" % QOS_IFB) if cmd.vCenterIps: interface_setup_rule = [] def set_up_qos_rules(target_interface): # a bare number in tc class use bytes as unit config_qos_cmd = "tc qdisc add dev {0} ingress;" \ "tc filter add dev {0} parent ffff: protocol ip u32 match " \ "u32 0 0 flowid 1:1 action mirred egress redirect dev {1};" \ "tc qdisc del dev {1} root >/dev/null 2>&1;" \ "tc qdisc add dev {1} root handle 1: htb;" \ "tc class add dev {1} parent 1: classid 1:1 htb rate {2} burst 100m" \ .format(target_interface, QOS_IFB, cmd.inboundBandwidth) return shell.run(config_qos_cmd) for vcenter_ip in cmd.vCenterIps: interface = linux.find_route_interface_by_destination_ip(linux.get_host_by_name(vcenter_ip)) if interface and interface not in interface_setup_rule: if set_up_qos_rules(interface) == 0: interface_setup_rule.append(interface) else: logger.debug("Failed to set up qos rules on interface %s" % interface) continue list_url_cmd = shell.ShellCmd("ps aux | grep '[v]irt-v2v' | grep -v convert.ret | awk '{print $13}'") list_url_cmd(False) limited_interface = [] if list_url_cmd.return_code == 0 and list_url_cmd.stdout: # will get a url format like # vpx://administrator%[email protected]/Datacenter-xxx/Cluster-xxx/127.0.0.1?no_verify=1 for url in list_url_cmd.stdout.split('\n'): vmware_host_ip = linux.get_host_by_name(url.split('/')[-1].split('?')[0]) interface = linux.find_route_interface_by_destination_ip(vmware_host_ip) if interface: cmdstr = "tc filter replace dev %s protocol ip parent 1: prio 1 u32 match ip src %s/32 flowid 1:1" \ % (QOS_IFB, vmware_host_ip) if shell.run(cmdstr) != 0: logger.debug("Failed to set up tc filter on interface %s for ip %s" % (interface, vmware_host_ip)) else: limited_interface.append(interface) return jsonobject.dumps(rsp)
def clean_dnat(dnatInfo): vmware_host_ip = linux.get_host_by_name(dnatInfo.hostName) dnat_interface = linux.find_route_interface_by_destination_ip(dnatInfo.convertIp) if vmware_host_ip != dnatInfo.convertIp: if shell.run("route -n |grep %s |grep %s" % (vmware_host_ip, dnat_interface)) == 0: shell.run("route del -host %s dev %s" % (vmware_host_ip, dnat_interface)) if shell.run("iptables -t nat -nL --line-number | grep DNAT |grep %s | grep %s" % (vmware_host_ip, dnatInfo.convertIp)) == 0: nat_number = shell.call("iptables -t nat -nL --line-number | grep DNAT |grep %s | grep %s" % (vmware_host_ip, dnatInfo.convertIp)).split(" ")[0] shell.run("iptables -D OUTPUT %s -t nat" % nat_number)
def delete_qos(self, req): cmd = jsonobject.loads(req[http.REQUEST_BODY]) rsp = AgentRsp() if cmd.vCenterIps: def delete_qos_rules(target_interface): if target_interface: # delete ifb interface tc rules cmdstr = "tc qdisc del dev %s root >/dev/null 2>&1" % QOS_IFB shell.run(cmdstr) # delete target interface tc rules cmdstr = "tc qdisc del dev %s ingress >/dev/null 2>&1" % target_interface shell.run(cmdstr) for vcenter_ip in cmd.vCenterIps: interface = linux.find_route_interface_by_destination_ip(linux.get_host_by_name(vcenter_ip)) if interface: delete_qos_rules(interface) return jsonobject.dumps(rsp)
def config_qos(self, req): cmd = jsonobject.loads(req[http.REQUEST_BODY]) rsp = AgentRsp() shell.run("modprobe ifb; ip link set %s up" % QOS_IFB) if not cmd.sourceHosts: return jsonobject.dumps(rsp) interfaces_to_setup_rule = {} def set_up_qos_rules(target_interface): # a bare number in tc class use bytes as unit config_qos_cmd = "tc qdisc add dev {0} ingress;" \ "tc filter add dev {0} parent ffff: protocol ip u32 match " \ "u32 0 0 flowid 1:1 action mirred egress redirect dev {1};" \ "tc qdisc del dev {1} root >/dev/null 2>&1;" \ "tc qdisc add dev {1} root handle 1: htb;" \ "tc class add dev {1} parent 1: classid 1:1 htb rate {2} burst 100m" \ .format(target_interface, QOS_IFB, cmd.inboundBandwidth) return shell.run(config_qos_cmd) for host in cmd.sourceHosts: host_ip = linux.get_host_by_name(host) interface = linux.find_route_interface_by_destination_ip(host_ip) if interface: interfaces_to_setup_rule[host_ip] = interface for interface in set(interfaces_to_setup_rule.values()): if set_up_qos_rules(interface) != 0: logger.debug("Failed to set up qos rules on interface %s" % interface) for host_ip, interface in interfaces_to_setup_rule.items(): cmdstr = "tc filter replace dev %s protocol ip parent 1: prio 1 u32 match ip src %s/32 flowid 1:1" \ % (QOS_IFB, host_ip) if shell.run(cmdstr) != 0: logger.debug("Failed to set up tc filter on interface %s for ip %s" % (interface, host_ip)) return jsonobject.dumps(rsp)
def convert(self, req): def get_mount_command(cmd): timeout_str = "timeout 30" username = getUsername(cmd.libvirtURI) if username == 'root': return "{0} mount".format(timeout_str) if cmd.sshPassword: return "echo {0} | {1} sudo -S mount".format(cmd.sshPassword, timeout_str) return "{0} sudo mount".format(timeout_str) def validate_and_make_dir(_dir): exists = os.path.exists(_dir) if not exists: linux.mkdir(_dir) return exists def do_ssh_mount(cmd, local_mount_point, vm_v2v_dir, real_storage_path): mount_cmd = get_mount_command(cmd) mount_paths = "{}:{} {}".format(cmd.managementIp, real_storage_path, local_mount_point) alternative_mount = mount_cmd + " -o vers=3" with lock.NamedLock(local_mount_point): cmdstr = "mkdir -p {0} && ls {1} 2>/dev/null || {2} {3} || {4} {3}".format( local_mount_point, vm_v2v_dir, mount_cmd, mount_paths, alternative_mount) try: runSshCmd(cmd.libvirtURI, cmd.sshPrivKey, cmdstr) except shell.ShellError as ex: if "Stale file handle" in str(ex): cmdstr = "umount {0} && {1} {2} || {3} {2}".format( local_mount_point, mount_cmd, mount_paths, alternative_mount) runSshCmd(cmd.libvirtURI, cmd.sshPrivKey, cmdstr) cmd = jsonobject.loads(req[http.REQUEST_BODY]) real_storage_path = getRealStoragePath(cmd.storagePath) storage_dir = os.path.join(real_storage_path, cmd.srcVmUuid) rsp = ConvertRsp() last_task = self.load_and_save_task(req, rsp, validate_and_make_dir, storage_dir) if last_task and last_task.agent_pid == os.getpid(): rsp = self.wait_task_complete(last_task) return jsonobject.dumps(rsp) local_mount_point = os.path.join("/tmp/zs-v2v/", cmd.managementIp) vm_v2v_dir = os.path.join(local_mount_point, cmd.srcVmUuid) libvirtHost = getHostname(cmd.libvirtURI) try: do_ssh_mount(cmd, local_mount_point, vm_v2v_dir, real_storage_path) except shell.ShellError as ex: logger.info(str(ex)) raise Exception('host {} cannot access NFS on {}'.format(libvirtHost, cmd.managementIp)) if linux.find_route_interface_by_destination_ip(linux.get_host_by_name(cmd.managementIp)): cmdstr = "tc filter replace dev %s protocol ip parent 1: prio 1 u32 match ip src %s/32 flowid 1:1" \ % (QOS_IFB, cmd.managementIp) shell.run(cmdstr) volumes = None filters = buildFilterDict(cmd.volumeFilters) startTime = time.time() with LibvirtConn(cmd.libvirtURI, cmd.saslUser, cmd.saslPass, cmd.sshPrivKey) as c: dom = c.lookupByUUIDString(cmd.srcVmUuid) if not dom: raise Exception('VM not found: {}'.format(cmd.srcVmUuid)) xmlDesc = dom.XMLDesc(0) dxml = xmlobject.loads(xmlDesc) if dxml.os.hasattr('firmware_') and dxml.os.firmware_ == 'efi' or dxml.os.hasattr('loader'): rsp.bootMode = 'UEFI' volumes = filter(lambda v: not skipVolume(filters, v.name), getVolumes(dom, dxml)) oldstat, _ = dom.state() needResume = True if cmd.pauseVm and oldstat != libvirt.VIR_DOMAIN_PAUSED: dom.suspend() needResume = False # libvirt >= 3.7.0 ? flags = 0 if c.getLibVersion() < getVerNumber(3, 7) else libvirt.VIR_DOMAIN_BLOCK_COPY_TRANSIENT_JOB needDefine = False if flags == 0 and dom.isPersistent(): dom.undefine() needDefine = True for v in volumes: localpath = os.path.join(storage_dir, v.name) info = dom.blockJobInfo(v.name, 0) if os.path.exists(localpath) and not info: os.remove(localpath) if not os.path.exists(localpath) and info: raise Exception("blockjob already exists on disk: "+v.name) if info: continue logger.info("start copying {}/{} ...".format(cmd.srcVmUuid, v.name)) # c.f. https://github.com/OpenNebula/one/issues/2646 linux.touch_file(localpath) dom.blockCopy(v.name, "<disk type='file'><source file='{}'/><driver type='{}'/></disk>".format(os.path.join(vm_v2v_dir, v.name), cmd.format), None, flags) end_progress = 60 total_volume_size = sum(volume.size for volume in volumes) if cmd.sendCommandUrl: Report.url = cmd.sendCommandUrl report = Report(cmd.threadContext, cmd.threadContextStack) report.processType = "KVM-V2V" while True: current_progress = 0.0 job_canceled = False for v in volumes: if v.endTime: current_progress += 1.0 * float(v.size) / float(total_volume_size) continue info = dom.blockJobInfo(v.name, 0) if not info: err_msg = 'blockjob not found on disk %s, maybe job has been canceled' % v.name logger.warn(err_msg) job_canceled = True continue end = info['end'] cur = info['cur'] if cur == end : v.endTime = time.time() logger.info("completed copying {}/{} ...".format(cmd.srcVmUuid, v.name)) progress = 1.0 else: progress = float(cur) / float(end) current_progress += progress * float(v.size) / float(total_volume_size) report.progress_report(str(int(current_progress * float(end_progress))), "start") if all(map(lambda v: v.endTime, volumes)) or job_canceled: break time.sleep(5) if job_canceled: rsp.success = False rsp.error = "cannot find blockjob on vm %s, maybe it has been canceled" % cmd.srcVmUuid if not cmd.pauseVm and oldstat != libvirt.VIR_DOMAIN_PAUSED: dom.suspend() needResume = True try: for v in volumes: if dom.blockJobInfo(v.name, 0): dom.blockJobAbort(v.name) finally: if needResume: dom.resume() if needDefine: c.defineXML(xmlDesc) # TODO # - monitor progress def makeVolumeInfo(v, startTime, devId): return { "installPath": os.path.join(storage_dir, v.name), "actualSize": v.physicalSize, "virtualSize": v.size, "virtioScsi": v.bus == 'scsi', "deviceName": v.name, "downloadTime": int(v.endTime - startTime), "deviceId": devId } if not rsp.success: return jsonobject.dumps(rsp) idx = 1 rv, dvs = None, [] for v in volumes: if v.type == 'ROOT': rv = makeVolumeInfo(v, startTime, 0) else: dvs.append(makeVolumeInfo(v, startTime, idx)) idx += 1 rsp.rootVolumeInfo = rv rsp.dataVolumeInfos = dvs return jsonobject.dumps(rsp)
def convert(self, req): cmd = jsonobject.loads(req[http.REQUEST_BODY]) rsp = ConvertRsp() storage_dir = os.path.join(cmd.storagePath, cmd.srcVmUuid) def validate_and_make_dir(_dir): existing = os.path.exists(_dir) if not existing: shell.call("mkdir -p %s" % _dir) return existing last_task = self.load_and_save_task(req, rsp, validate_and_make_dir, storage_dir) if last_task and last_task.agent_pid == os.getpid(): rsp = self.wait_task_complete(last_task) return jsonobject.dumps(rsp) new_task = self.load_task(req) cmdstr = "echo '{1}' > {0}/passwd".format(storage_dir, cmd.vCenterPassword) if shell.run(cmdstr) != 0: rsp.success = False rsp.error = "failed to create passwd {} in v2v conversion host".format(storage_dir) return jsonobject.dumps(rsp) @thread.AsyncThread def save_pid(): linux.wait_callback_success(os.path.exists, v2v_pid_path) with open(v2v_pid_path, 'r') as fd: new_task.current_pid = fd.read().strip() new_task.current_process_cmd = echo_pid_cmd new_task.current_process_name = "virt_v2v_cmd" logger.debug("longjob[uuid:%s] saved process[pid:%s, name:%s]" % (cmd.longJobUuid, new_task.current_pid, new_task.current_process_name)) def get_v2v_cmd(cmd, rsp): extra_params = "" if cmd.extraParams: for k, v in cmd.extraParams.__dict__.items(): extra_params = ' '.join((extra_params, ("--%s" % k), v)) if cmd.vddkVersion == '6.5': return 'VIRTIO_WIN=/var/lib/zstack/v2v/zstack-windows-virtio-driver.iso \ virt-v2v -ic vpx://{0}?no_verify=1 {1} -it vddk \ --vddk-libdir=/var/lib/zstack/v2v/vmware-vix-disklib-distrib \ --vddk-thumbprint={3} -o local -os {2} --password-file {2}/passwd {5} \ -of {4} > {2}/virt_v2v_log 2>&1'.format(cmd.srcVmUri, shellquote(cmd.srcVmName), storage_dir, cmd.thumbprint, cmd.format, extra_params) if cmd.vddkVersion == '5.5': if not self._ndbkit_is_work(): rsp.success = False rsp.error = "nbdkit with vddk 5.5 is not work, try to reconnect conversion host" return jsonobject.dumps(rsp) return 'export PATH={5}:$PATH; \ VIRTIO_WIN=/var/lib/zstack/v2v/zstack-windows-virtio-driver.iso \ virt-v2v -ic vpx://{0}?no_verify=1 {1} -it vddk \ --vddk-libdir=/var/lib/zstack/v2v/nbdkit_build_lib/vmware-vix-disklib-distrib \ --vddk-thumbprint={3} -o local -os {2} --password-file {2}/passwd {6} \ -of {4} > {2}/virt_v2v_log 2>&1'.format(cmd.srcVmUri, shellquote(cmd.srcVmName), storage_dir, cmd.thumbprint, cmd.format, self._get_nbdkit_dir_path(), extra_params) def dnat_to_convert_interface(ip, dnat_ip, dnat_interface): if shell.run("route -n |grep %s |grep %s" % (ip, dnat_interface)) != 0: shell.run("route add -host %s dev %s" % (ip, dnat_interface)) if shell.run("iptables -t nat -nL --line-number | grep DNAT |grep %s | grep %s" % (ip, dnat_ip)) != 0: shell.run("iptables -t nat -A OUTPUT -d %s -j DNAT --to-destination %s" % (ip, dnat_ip)) virt_v2v_cmd = get_v2v_cmd(cmd, rsp) src_vm_uri = cmd.srcVmUri host_name = src_vm_uri.split('/')[-1] if not linux.is_valid_hostname(host_name): rsp.success = False rsp.error = "cannot resolve hostname %s, check the hostname configured on the DNS server" % host_name return jsonobject.dumps(rsp) v2v_pid_path = os.path.join(storage_dir, "convert.pid") v2v_cmd_ret_path = os.path.join(storage_dir, "convert.ret") echo_pid_cmd = "echo $$ > %s; %s; ret=$?; echo $ret > %s; exit $ret" % ( v2v_pid_path, virt_v2v_cmd, v2v_cmd_ret_path) vmware_host_ip = linux.get_host_by_name(host_name) interface = linux.find_route_interface_by_destination_ip(vmware_host_ip) convertInterface = linux.find_route_interface_by_destination_ip(cmd.srcHostIp) if vmware_host_ip != cmd.srcHostIp: dnat_to_convert_interface(vmware_host_ip, cmd.srcHostIp, convertInterface) if convertInterface: cmdstr = "tc filter replace dev %s protocol ip parent 1: prio 1 u32 match ip src %s/32 flowid 1:1" \ % (QOS_IFB, cmd.srcHostIp) shell.run(cmdstr) max_retry_times = 1 retry_counter = [0] @linux.retry(times=max_retry_times+1, sleep_time=1) def run_convert_if_need(): def do_run(): save_pid() ret = shell.run(echo_pid_cmd) new_task.current_process_return_code = ret retry_if_needed(ret) return ret def retry_if_needed(ret): if ret == 0: return if retry_counter[0] != max_retry_times and shell.run("grep -q 'guestfs_launch failed' %s" % log_path) == 0: retry_counter[0] += 1 raise RetryException( "launch guestfs failed, rerun v2v longjob %s" % cmd.longJobUuid) pid = linux.read_file(v2v_pid_path) log_path = "%s/virt_v2v_log" % storage_dir if not pid: return do_run() pid = int(pid.strip()) process_completed = os.path.exists(v2v_cmd_ret_path) process_has_been_killed = not os.path.exists(v2v_cmd_ret_path) and not os.path.exists('/proc/%d' % pid) process_still_running = not os.path.exists(v2v_cmd_ret_path) and os.path.exists('/proc/%d' % pid) if process_has_been_killed: return do_run() if process_still_running: linux.wait_callback_success(os.path.exists, v2v_cmd_ret_path, timeout=259200, interval=60) # delete password file passwd_file = os.path.join(storage_dir, "passwd") if os.path.exists(passwd_file): os.remove(passwd_file) ret = linux.read_file(v2v_cmd_ret_path) retry_if_needed(ret) return int(ret.strip() if ret else 126) if run_convert_if_need() != 0: v2v_log_file = "/tmp/v2v_log/%s-virt-v2v-log" % cmd.longJobUuid # create folder to save virt-v2v log tail_cmd = 'mkdir -p /tmp/v2v_log; tail -c 1M %s/virt_v2v_log > %s' % (storage_dir, v2v_log_file) shell.run(tail_cmd) with open(v2v_log_file, 'a') as fd: fd.write('\n>>> virt_v2v command: %s\n' % virt_v2v_cmd) rsp.success = False # Check if the v2v convert fails due to vCenter 5.5 bug: https://bugzilla.redhat.com/show_bug.cgi?id=1287681#c14 ret = shell.call("grep -io 'File name .* refers to non-existing datastore .*' %s | tail -n 1" % v2v_log_file) if ret: rsp.error = ret.strip("\n") + ". This may be a bug in vCenter 5.5, please detach ISO in vSphere client and try again" else: rsp.error = "failed to run virt-v2v command, log in conversion host: %s" % v2v_log_file return jsonobject.dumps(rsp) root_vol = (r"%s/%s-sda" % (storage_dir, cmd.srcVmName)).encode('utf-8') logger.debug(root_vol) if not os.path.exists(root_vol): rsp.success = False rsp.error = "failed to convert root volume of " + cmd.srcVmName return jsonobject.dumps(rsp) root_volume_actual_size, root_volume_virtual_size = self._get_qcow2_sizes(root_vol) rsp.rootVolumeInfo = {"installPath": root_vol, "actualSize": root_volume_actual_size, "virtualSize": root_volume_virtual_size, "deviceId": 0} rsp.dataVolumeInfos = [] for dev in 'bcdefghijklmnopqrstuvwxyz': data_vol = (r"%s/%s-sd%c" % (storage_dir, cmd.srcVmName, dev)).encode('utf-8') if os.path.exists(data_vol): aSize, vSize = self._get_qcow2_sizes(data_vol) rsp.dataVolumeInfos.append({"installPath": data_vol, "actualSize": aSize, "virtualSize": vSize, "deviceId": ord(dev) - ord('a')}) else: break xml = (r"%s/%s.xml" % (storage_dir, cmd.srcVmName)).encode('utf-8') if self._check_str_in_file(xml, "<nvram "): rsp.bootMode = 'UEFI' def collect_time_cost(): # [ 138.3] Copying disk 1/13 to # [ 408.1] Copying disk 2/13 to # ... # [1055.2] Copying disk 11/13 to # [1082.3] Copying disk 12/13 to # [1184.9] Copying disk 13/13 to # [1218.0] Finishing off # Copying disk is start time of copy, so also get finish off time for calculating last disk's time cost s = shell.ShellCmd("""awk -F"[][]" '/Copying disk|Finishing off/{print $2}' %s/virt_v2v_log""" % storage_dir) s(False) if s.return_code != 0: return times = s.stdout.split('\n') if len(times) < 2: return rsp.rootVolumeInfo['downloadTime'] = int(float(times[1]) - float(times[0])) times = times[1:] for i in xrange(0, len(rsp.dataVolumeInfos)): if i + 1 < len(times): rsp.dataVolumeInfos[i]["downloadTime"] = int(float(times[i + 1]) - float(times[i])) try: collect_time_cost() except Exception as e: logger.debug("Failed to collect time cost, because %s" % e.message) return jsonobject.dumps(rsp)