def config_qos(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
shell.run("modprobe ifb; ip link set %s up" % QOS_IFB)
if cmd.vCenterIps:
interface_setup_rule = []
def set_up_qos_rules(target_interface):
# a bare number in tc class use bytes as unit
config_qos_cmd = "tc qdisc add dev {0} ingress;" \
"tc filter add dev {0} parent ffff: protocol ip u32 match " \
"u32 0 0 flowid 1:1 action mirred egress redirect dev {1};" \
"tc qdisc del dev {1} root >/dev/null 2>&1;" \
"tc qdisc add dev {1} root handle 1: htb;" \
"tc class add dev {1} parent 1: classid 1:1 htb rate {2} burst 100m" \
.format(target_interface, QOS_IFB, cmd.inboundBandwidth)
return shell.run(config_qos_cmd)
for vcenter_ip in cmd.vCenterIps:
interface = linux.find_route_interface_by_destination_ip(linux.get_host_by_name(vcenter_ip))
if interface and interface not in interface_setup_rule:
if set_up_qos_rules(interface) == 0:
interface_setup_rule.append(interface)
else:
logger.debug("Failed to set up qos rules on interface %s" % interface)
continue
list_url_cmd = shell.ShellCmd("ps aux | grep '[v]irt-v2v' | grep -v convert.ret | awk '{print $13}'")
list_url_cmd(False)
limited_interface = []
if list_url_cmd.return_code == 0 and list_url_cmd.stdout:
# will get a url format like
# vpx://administrator%[email protected]/Datacenter-xxx/Cluster-xxx/127.0.0.1?no_verify=1
for url in list_url_cmd.stdout.split('\n'):
vmware_host_ip = linux.get_host_by_name(url.split('/')[-1].split('?')[0])
interface = linux.find_route_interface_by_destination_ip(vmware_host_ip)
if interface:
cmdstr = "tc filter replace dev %s protocol ip parent 1: prio 1 u32 match ip src %s/32 flowid 1:1" \
% (QOS_IFB, vmware_host_ip)
if shell.run(cmdstr) != 0:
logger.debug("Failed to set up tc filter on interface %s for ip %s"
% (interface, vmware_host_ip))
else:
limited_interface.append(interface)
return jsonobject.dumps(rsp)
def clean_dnat(dnatInfo):
vmware_host_ip = linux.get_host_by_name(dnatInfo.hostName)
dnat_interface = linux.find_route_interface_by_destination_ip(dnatInfo.convertIp)
if vmware_host_ip != dnatInfo.convertIp:
if shell.run("route -n |grep %s |grep %s" % (vmware_host_ip, dnat_interface)) == 0:
shell.run("route del -host %s dev %s" % (vmware_host_ip, dnat_interface))
if shell.run("iptables -t nat -nL --line-number | grep DNAT |grep %s | grep %s" % (vmware_host_ip, dnatInfo.convertIp)) == 0:
nat_number = shell.call("iptables -t nat -nL --line-number | grep DNAT |grep %s | grep %s" % (vmware_host_ip, dnatInfo.convertIp)).split(" ")[0]
shell.run("iptables -D OUTPUT %s -t nat" % nat_number)
def delete_qos(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
if cmd.vCenterIps:
def delete_qos_rules(target_interface):
if target_interface:
# delete ifb interface tc rules
cmdstr = "tc qdisc del dev %s root >/dev/null 2>&1" % QOS_IFB
shell.run(cmdstr)
# delete target interface tc rules
cmdstr = "tc qdisc del dev %s ingress >/dev/null 2>&1" % target_interface
shell.run(cmdstr)
for vcenter_ip in cmd.vCenterIps:
interface = linux.find_route_interface_by_destination_ip(linux.get_host_by_name(vcenter_ip))
if interface:
delete_qos_rules(interface)
return jsonobject.dumps(rsp)
def config_qos(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
shell.run("modprobe ifb; ip link set %s up" % QOS_IFB)
if not cmd.sourceHosts:
return jsonobject.dumps(rsp)
interfaces_to_setup_rule = {}
def set_up_qos_rules(target_interface):
# a bare number in tc class use bytes as unit
config_qos_cmd = "tc qdisc add dev {0} ingress;" \
"tc filter add dev {0} parent ffff: protocol ip u32 match " \
"u32 0 0 flowid 1:1 action mirred egress redirect dev {1};" \
"tc qdisc del dev {1} root >/dev/null 2>&1;" \
"tc qdisc add dev {1} root handle 1: htb;" \
"tc class add dev {1} parent 1: classid 1:1 htb rate {2} burst 100m" \
.format(target_interface, QOS_IFB, cmd.inboundBandwidth)
return shell.run(config_qos_cmd)
for host in cmd.sourceHosts:
host_ip = linux.get_host_by_name(host)
interface = linux.find_route_interface_by_destination_ip(host_ip)
if interface:
interfaces_to_setup_rule[host_ip] = interface
for interface in set(interfaces_to_setup_rule.values()):
if set_up_qos_rules(interface) != 0:
logger.debug("Failed to set up qos rules on interface %s" % interface)
for host_ip, interface in interfaces_to_setup_rule.items():
cmdstr = "tc filter replace dev %s protocol ip parent 1: prio 1 u32 match ip src %s/32 flowid 1:1" \
% (QOS_IFB, host_ip)
if shell.run(cmdstr) != 0:
logger.debug("Failed to set up tc filter on interface %s for ip %s"
% (interface, host_ip))
return jsonobject.dumps(rsp)
def convert(self, req):
def get_mount_command(cmd):
timeout_str = "timeout 30"
username = getUsername(cmd.libvirtURI)
if username == 'root':
return "{0} mount".format(timeout_str)
if cmd.sshPassword:
return "echo {0} | {1} sudo -S mount".format(cmd.sshPassword, timeout_str)
return "{0} sudo mount".format(timeout_str)
def validate_and_make_dir(_dir):
exists = os.path.exists(_dir)
if not exists:
linux.mkdir(_dir)
return exists
def do_ssh_mount(cmd, local_mount_point, vm_v2v_dir, real_storage_path):
mount_cmd = get_mount_command(cmd)
mount_paths = "{}:{} {}".format(cmd.managementIp, real_storage_path, local_mount_point)
alternative_mount = mount_cmd + " -o vers=3"
with lock.NamedLock(local_mount_point):
cmdstr = "mkdir -p {0} && ls {1} 2>/dev/null || {2} {3} || {4} {3}".format(
local_mount_point,
vm_v2v_dir,
mount_cmd,
mount_paths,
alternative_mount)
try:
runSshCmd(cmd.libvirtURI, cmd.sshPrivKey, cmdstr)
except shell.ShellError as ex:
if "Stale file handle" in str(ex):
cmdstr = "umount {0} && {1} {2} || {3} {2}".format(
local_mount_point,
mount_cmd,
mount_paths,
alternative_mount)
runSshCmd(cmd.libvirtURI, cmd.sshPrivKey, cmdstr)
cmd = jsonobject.loads(req[http.REQUEST_BODY])
real_storage_path = getRealStoragePath(cmd.storagePath)
storage_dir = os.path.join(real_storage_path, cmd.srcVmUuid)
rsp = ConvertRsp()
last_task = self.load_and_save_task(req, rsp, validate_and_make_dir, storage_dir)
if last_task and last_task.agent_pid == os.getpid():
rsp = self.wait_task_complete(last_task)
return jsonobject.dumps(rsp)
local_mount_point = os.path.join("/tmp/zs-v2v/", cmd.managementIp)
vm_v2v_dir = os.path.join(local_mount_point, cmd.srcVmUuid)
libvirtHost = getHostname(cmd.libvirtURI)
try:
do_ssh_mount(cmd, local_mount_point, vm_v2v_dir, real_storage_path)
except shell.ShellError as ex:
logger.info(str(ex))
raise Exception('host {} cannot access NFS on {}'.format(libvirtHost, cmd.managementIp))
if linux.find_route_interface_by_destination_ip(linux.get_host_by_name(cmd.managementIp)):
cmdstr = "tc filter replace dev %s protocol ip parent 1: prio 1 u32 match ip src %s/32 flowid 1:1" \
% (QOS_IFB, cmd.managementIp)
shell.run(cmdstr)
volumes = None
filters = buildFilterDict(cmd.volumeFilters)
startTime = time.time()
with LibvirtConn(cmd.libvirtURI, cmd.saslUser, cmd.saslPass, cmd.sshPrivKey) as c:
dom = c.lookupByUUIDString(cmd.srcVmUuid)
if not dom:
raise Exception('VM not found: {}'.format(cmd.srcVmUuid))
xmlDesc = dom.XMLDesc(0)
dxml = xmlobject.loads(xmlDesc)
if dxml.os.hasattr('firmware_') and dxml.os.firmware_ == 'efi' or dxml.os.hasattr('loader'):
rsp.bootMode = 'UEFI'
volumes = filter(lambda v: not skipVolume(filters, v.name), getVolumes(dom, dxml))
oldstat, _ = dom.state()
needResume = True
if cmd.pauseVm and oldstat != libvirt.VIR_DOMAIN_PAUSED:
dom.suspend()
needResume = False
# libvirt >= 3.7.0 ?
flags = 0 if c.getLibVersion() < getVerNumber(3, 7) else libvirt.VIR_DOMAIN_BLOCK_COPY_TRANSIENT_JOB
needDefine = False
if flags == 0 and dom.isPersistent():
dom.undefine()
needDefine = True
for v in volumes:
localpath = os.path.join(storage_dir, v.name)
info = dom.blockJobInfo(v.name, 0)
if os.path.exists(localpath) and not info:
os.remove(localpath)
if not os.path.exists(localpath) and info:
raise Exception("blockjob already exists on disk: "+v.name)
if info:
continue
logger.info("start copying {}/{} ...".format(cmd.srcVmUuid, v.name))
# c.f. https://github.com/OpenNebula/one/issues/2646
linux.touch_file(localpath)
dom.blockCopy(v.name,
"<disk type='file'><source file='{}'/><driver type='{}'/></disk>".format(os.path.join(vm_v2v_dir, v.name), cmd.format),
None,
flags)
end_progress = 60
total_volume_size = sum(volume.size for volume in volumes)
if cmd.sendCommandUrl:
Report.url = cmd.sendCommandUrl
report = Report(cmd.threadContext, cmd.threadContextStack)
report.processType = "KVM-V2V"
while True:
current_progress = 0.0
job_canceled = False
for v in volumes:
if v.endTime:
current_progress += 1.0 * float(v.size) / float(total_volume_size)
continue
info = dom.blockJobInfo(v.name, 0)
if not info:
err_msg = 'blockjob not found on disk %s, maybe job has been canceled' % v.name
logger.warn(err_msg)
job_canceled = True
continue
end = info['end']
cur = info['cur']
if cur == end :
v.endTime = time.time()
logger.info("completed copying {}/{} ...".format(cmd.srcVmUuid, v.name))
progress = 1.0
else:
progress = float(cur) / float(end)
current_progress += progress * float(v.size) / float(total_volume_size)
report.progress_report(str(int(current_progress * float(end_progress))), "start")
if all(map(lambda v: v.endTime, volumes)) or job_canceled:
break
time.sleep(5)
if job_canceled:
rsp.success = False
rsp.error = "cannot find blockjob on vm %s, maybe it has been canceled" % cmd.srcVmUuid
if not cmd.pauseVm and oldstat != libvirt.VIR_DOMAIN_PAUSED:
dom.suspend()
needResume = True
try:
for v in volumes:
if dom.blockJobInfo(v.name, 0):
dom.blockJobAbort(v.name)
finally:
if needResume:
dom.resume()
if needDefine:
c.defineXML(xmlDesc)
# TODO
# - monitor progress
def makeVolumeInfo(v, startTime, devId):
return { "installPath": os.path.join(storage_dir, v.name),
"actualSize": v.physicalSize,
"virtualSize": v.size,
"virtioScsi": v.bus == 'scsi',
"deviceName": v.name,
"downloadTime": int(v.endTime - startTime),
"deviceId": devId }
if not rsp.success:
return jsonobject.dumps(rsp)
idx = 1
rv, dvs = None, []
for v in volumes:
if v.type == 'ROOT':
rv = makeVolumeInfo(v, startTime, 0)
else:
dvs.append(makeVolumeInfo(v, startTime, idx))
idx += 1
rsp.rootVolumeInfo = rv
rsp.dataVolumeInfos = dvs
return jsonobject.dumps(rsp)
def convert(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = ConvertRsp()
storage_dir = os.path.join(cmd.storagePath, cmd.srcVmUuid)
def validate_and_make_dir(_dir):
existing = os.path.exists(_dir)
if not existing:
shell.call("mkdir -p %s" % _dir)
return existing
last_task = self.load_and_save_task(req, rsp, validate_and_make_dir, storage_dir)
if last_task and last_task.agent_pid == os.getpid():
rsp = self.wait_task_complete(last_task)
return jsonobject.dumps(rsp)
new_task = self.load_task(req)
cmdstr = "echo '{1}' > {0}/passwd".format(storage_dir, cmd.vCenterPassword)
if shell.run(cmdstr) != 0:
rsp.success = False
rsp.error = "failed to create passwd {} in v2v conversion host".format(storage_dir)
return jsonobject.dumps(rsp)
@thread.AsyncThread
def save_pid():
linux.wait_callback_success(os.path.exists, v2v_pid_path)
with open(v2v_pid_path, 'r') as fd:
new_task.current_pid = fd.read().strip()
new_task.current_process_cmd = echo_pid_cmd
new_task.current_process_name = "virt_v2v_cmd"
logger.debug("longjob[uuid:%s] saved process[pid:%s, name:%s]" %
(cmd.longJobUuid, new_task.current_pid, new_task.current_process_name))
def get_v2v_cmd(cmd, rsp):
extra_params = ""
if cmd.extraParams:
for k, v in cmd.extraParams.__dict__.items():
extra_params = ' '.join((extra_params, ("--%s" % k), v))
if cmd.vddkVersion == '6.5':
return 'VIRTIO_WIN=/var/lib/zstack/v2v/zstack-windows-virtio-driver.iso \
virt-v2v -ic vpx://{0}?no_verify=1 {1} -it vddk \
--vddk-libdir=/var/lib/zstack/v2v/vmware-vix-disklib-distrib \
--vddk-thumbprint={3} -o local -os {2} --password-file {2}/passwd {5} \
-of {4} > {2}/virt_v2v_log 2>&1'.format(cmd.srcVmUri, shellquote(cmd.srcVmName), storage_dir,
cmd.thumbprint, cmd.format, extra_params)
if cmd.vddkVersion == '5.5':
if not self._ndbkit_is_work():
rsp.success = False
rsp.error = "nbdkit with vddk 5.5 is not work, try to reconnect conversion host"
return jsonobject.dumps(rsp)
return 'export PATH={5}:$PATH; \
VIRTIO_WIN=/var/lib/zstack/v2v/zstack-windows-virtio-driver.iso \
virt-v2v -ic vpx://{0}?no_verify=1 {1} -it vddk \
--vddk-libdir=/var/lib/zstack/v2v/nbdkit_build_lib/vmware-vix-disklib-distrib \
--vddk-thumbprint={3} -o local -os {2} --password-file {2}/passwd {6} \
-of {4} > {2}/virt_v2v_log 2>&1'.format(cmd.srcVmUri, shellquote(cmd.srcVmName),
storage_dir,
cmd.thumbprint, cmd.format, self._get_nbdkit_dir_path(), extra_params)
def dnat_to_convert_interface(ip, dnat_ip, dnat_interface):
if shell.run("route -n |grep %s |grep %s" % (ip, dnat_interface)) != 0:
shell.run("route add -host %s dev %s" % (ip, dnat_interface))
if shell.run("iptables -t nat -nL --line-number | grep DNAT |grep %s | grep %s" % (ip, dnat_ip)) != 0:
shell.run("iptables -t nat -A OUTPUT -d %s -j DNAT --to-destination %s" % (ip, dnat_ip))
virt_v2v_cmd = get_v2v_cmd(cmd, rsp)
src_vm_uri = cmd.srcVmUri
host_name = src_vm_uri.split('/')[-1]
if not linux.is_valid_hostname(host_name):
rsp.success = False
rsp.error = "cannot resolve hostname %s, check the hostname configured on the DNS server" % host_name
return jsonobject.dumps(rsp)
v2v_pid_path = os.path.join(storage_dir, "convert.pid")
v2v_cmd_ret_path = os.path.join(storage_dir, "convert.ret")
echo_pid_cmd = "echo $$ > %s; %s; ret=$?; echo $ret > %s; exit $ret" % (
v2v_pid_path, virt_v2v_cmd, v2v_cmd_ret_path)
vmware_host_ip = linux.get_host_by_name(host_name)
interface = linux.find_route_interface_by_destination_ip(vmware_host_ip)
convertInterface = linux.find_route_interface_by_destination_ip(cmd.srcHostIp)
if vmware_host_ip != cmd.srcHostIp:
dnat_to_convert_interface(vmware_host_ip, cmd.srcHostIp, convertInterface)
if convertInterface:
cmdstr = "tc filter replace dev %s protocol ip parent 1: prio 1 u32 match ip src %s/32 flowid 1:1" \
% (QOS_IFB, cmd.srcHostIp)
shell.run(cmdstr)
max_retry_times = 1
retry_counter = [0]
@linux.retry(times=max_retry_times+1, sleep_time=1)
def run_convert_if_need():
def do_run():
save_pid()
ret = shell.run(echo_pid_cmd)
new_task.current_process_return_code = ret
retry_if_needed(ret)
return ret
def retry_if_needed(ret):
if ret == 0:
return
if retry_counter[0] != max_retry_times and shell.run("grep -q 'guestfs_launch failed' %s" % log_path) == 0:
retry_counter[0] += 1
raise RetryException(
"launch guestfs failed, rerun v2v longjob %s" % cmd.longJobUuid)
pid = linux.read_file(v2v_pid_path)
log_path = "%s/virt_v2v_log" % storage_dir
if not pid:
return do_run()
pid = int(pid.strip())
process_completed = os.path.exists(v2v_cmd_ret_path)
process_has_been_killed = not os.path.exists(v2v_cmd_ret_path) and not os.path.exists('/proc/%d' % pid)
process_still_running = not os.path.exists(v2v_cmd_ret_path) and os.path.exists('/proc/%d' % pid)
if process_has_been_killed:
return do_run()
if process_still_running:
linux.wait_callback_success(os.path.exists, v2v_cmd_ret_path, timeout=259200, interval=60)
# delete password file
passwd_file = os.path.join(storage_dir, "passwd")
if os.path.exists(passwd_file):
os.remove(passwd_file)
ret = linux.read_file(v2v_cmd_ret_path)
retry_if_needed(ret)
return int(ret.strip() if ret else 126)
if run_convert_if_need() != 0:
v2v_log_file = "/tmp/v2v_log/%s-virt-v2v-log" % cmd.longJobUuid
# create folder to save virt-v2v log
tail_cmd = 'mkdir -p /tmp/v2v_log; tail -c 1M %s/virt_v2v_log > %s' % (storage_dir, v2v_log_file)
shell.run(tail_cmd)
with open(v2v_log_file, 'a') as fd:
fd.write('\n>>> virt_v2v command: %s\n' % virt_v2v_cmd)
rsp.success = False
# Check if the v2v convert fails due to vCenter 5.5 bug: https://bugzilla.redhat.com/show_bug.cgi?id=1287681#c14
ret = shell.call("grep -io 'File name .* refers to non-existing datastore .*' %s | tail -n 1" % v2v_log_file)
if ret:
rsp.error = ret.strip("\n") + ". This may be a bug in vCenter 5.5, please detach ISO in vSphere client and try again"
else:
rsp.error = "failed to run virt-v2v command, log in conversion host: %s" % v2v_log_file
return jsonobject.dumps(rsp)
root_vol = (r"%s/%s-sda" % (storage_dir, cmd.srcVmName)).encode('utf-8')
logger.debug(root_vol)
if not os.path.exists(root_vol):
rsp.success = False
rsp.error = "failed to convert root volume of " + cmd.srcVmName
return jsonobject.dumps(rsp)
root_volume_actual_size, root_volume_virtual_size = self._get_qcow2_sizes(root_vol)
rsp.rootVolumeInfo = {"installPath": root_vol,
"actualSize": root_volume_actual_size,
"virtualSize": root_volume_virtual_size,
"deviceId": 0}
rsp.dataVolumeInfos = []
for dev in 'bcdefghijklmnopqrstuvwxyz':
data_vol = (r"%s/%s-sd%c" % (storage_dir, cmd.srcVmName, dev)).encode('utf-8')
if os.path.exists(data_vol):
aSize, vSize = self._get_qcow2_sizes(data_vol)
rsp.dataVolumeInfos.append({"installPath": data_vol,
"actualSize": aSize,
"virtualSize": vSize,
"deviceId": ord(dev) - ord('a')})
else:
break
xml = (r"%s/%s.xml" % (storage_dir, cmd.srcVmName)).encode('utf-8')
if self._check_str_in_file(xml, "<nvram "):
rsp.bootMode = 'UEFI'
def collect_time_cost():
# [ 138.3] Copying disk 1/13 to
# [ 408.1] Copying disk 2/13 to
# ...
# [1055.2] Copying disk 11/13 to
# [1082.3] Copying disk 12/13 to
# [1184.9] Copying disk 13/13 to
# [1218.0] Finishing off
# Copying disk is start time of copy, so also get finish off time for calculating last disk's time cost
s = shell.ShellCmd("""awk -F"[][]" '/Copying disk|Finishing off/{print $2}' %s/virt_v2v_log""" % storage_dir)
s(False)
if s.return_code != 0:
return
times = s.stdout.split('\n')
if len(times) < 2:
return
rsp.rootVolumeInfo['downloadTime'] = int(float(times[1]) - float(times[0]))
times = times[1:]
for i in xrange(0, len(rsp.dataVolumeInfos)):
if i + 1 < len(times):
rsp.dataVolumeInfos[i]["downloadTime"] = int(float(times[i + 1]) - float(times[i]))
try:
collect_time_cost()
except Exception as e:
logger.debug("Failed to collect time cost, because %s" % e.message)
return jsonobject.dumps(rsp)