def is_volume_on_pvs(volume_path, pvUuids, includingMissing=True):
files = linux.qcow2_get_file_chain(volume_path)
if len(files) == 0:
# could not read qcow2
logger.debug("can not read volume %s, return true" % volume_path)
return True
pv_names = []
for p in pvUuids:
name = get_pv_name_by_uuid(p)
if name != "":
pv_names.append(get_pv_name_by_uuid(p) + "(")
if includingMissing:
pv_names.append("unknown")
for f in files:
o = bash.bash_o(
"timeout -s SIGKILL 10 lvs --noheading --nolocking %s -odevices" % f).strip().lower() # type: str
logger.debug("volume %s is on pv %s" % (volume_path, o))
if len(filter(lambda n: o.find(n.lower()) > 0, pv_names)) > 0:
logger.debug("lv %s on pv %s(%s), return true" % (volume_path, pvUuids, pv_names))
return True
if o == "" and includingMissing:
logger.debug("pv of lv %s is missing, return true")
return True
return False
def commit_to_imagestore(self, cmd, req):
fpath = cmd.primaryStorageInstallPath
# Synchronize cached writes for 'fpath'
shell.call('/bin/sync ' + fpath)
# Add the image to registry
cmdstr = '%s -json -callbackurl %s -taskid %s -imageUuid %s add -desc \'%s\' -file %s' % (self.ZSTORE_CLI_PATH, req[http.REQUEST_HEADER].get(http.CALLBACK_URI),
req[http.REQUEST_HEADER].get(http.TASK_UUID), cmd.imageUuid, cmd.description, fpath)
logger.debug('adding %s to local image store' % fpath)
shell.call(cmdstr.encode(encoding="utf-8"))
logger.debug('%s added to local image store' % fpath)
name, imageid = self._get_image_reference(fpath)
rsp = kvmagent.AgentResponse()
rsp.backupStorageInstallPath = self._build_install_path(name, imageid)
rsp.size = linux.qcow2_size_and_actual_size(cmd.primaryStorageInstallPath)[0]
# we need to sum all the disk size within the chain ...
chain = linux.qcow2_get_file_chain(cmd.primaryStorageInstallPath)
rsp.actualSize = sum([ linux.qcow2_size_and_actual_size(f)[1] for f in chain ])
return jsonobject.dumps(rsp)
def is_volume_on_pvs(volume_path, pvUuids, includingMissing=True):
files = linux.qcow2_get_file_chain(volume_path)
if len(files) == 0:
# could not read qcow2
logger.debug("can not read volume %s, return true" % volume_path)
return True
pv_names = []
for p in pvUuids:
name = get_pv_name_by_uuid(p)
if name != "":
pv_names.append(get_pv_name_by_uuid(p) + "(")
if includingMissing:
pv_names.append("unknown")
for f in files:
o = bash.bash_o(
"timeout -s SIGKILL 10 lvs --noheading --nolocking %s -odevices" %
f).strip().lower() # type: str
logger.debug("volume %s is on pv %s" % (volume_path, o))
if len(filter(lambda n: o.find(n.lower()) > 0, pv_names)) > 0:
logger.debug("lv %s on pv %s(%s), return true" %
(volume_path, pvUuids, pv_names))
return True
if o == "" and includingMissing:
logger.debug("pv of lv %s is missing, return true")
return True
return False
def create_template_from_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
volume_abs_path = translate_absolute_path_from_install_path(cmd.volumePath)
install_abs_path = translate_absolute_path_from_install_path(cmd.installPath)
if cmd.sharedVolume:
lvm.do_active_lv(volume_abs_path, lvm.LvmlockdLockType.SHARE, True)
with lvm.RecursiveOperateLv(volume_abs_path, shared=cmd.sharedVolume, skip_deactivate_tags=[IMAGE_TAG]):
virtual_size = linux.qcow2_virtualsize(volume_abs_path)
total_size = 0
for qcow2 in linux.qcow2_get_file_chain(volume_abs_path):
total_size += int(lvm.get_lv_size(qcow2))
if total_size > virtual_size:
total_size = virtual_size
if not lvm.lv_exists(install_abs_path):
lvm.create_lv_from_absolute_path(install_abs_path, total_size,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.OperateLv(install_abs_path, shared=False, delete_when_exception=True):
linux.create_template(volume_abs_path, install_abs_path)
logger.debug('successfully created template[%s] from volume[%s]' % (cmd.installPath, cmd.volumePath))
if cmd.compareQcow2 is True:
logger.debug("comparing qcow2 between %s and %s")
bash.bash_errorout("time qemu-img compare %s %s" % (volume_abs_path, install_abs_path))
logger.debug("confirmed qcow2 %s and %s are identical" % (volume_abs_path, install_abs_path))
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
def commit_to_imagestore(self, cmd, req):
fpath = cmd.primaryStorageInstallPath
# Synchronize cached writes for 'fpath'
shell.call('/bin/sync ' + fpath)
# Add the image to registry
cmdstr = '%s -json -callbackurl %s -taskid %s -imageUuid %s add -file %s' % (self.ZSTORE_CLI_PATH, req[http.REQUEST_HEADER].get(http.CALLBACK_URI),
req[http.REQUEST_HEADER].get(http.TASK_UUID), cmd.imageUuid, fpath)
logger.debug('adding %s to local image store' % fpath)
shell.call(cmdstr)
logger.debug('%s added to local image store' % fpath)
name, imageid = self._get_image_reference(fpath)
rsp = kvmagent.AgentResponse()
rsp.backupStorageInstallPath = self._build_install_path(name, imageid)
rsp.size = linux.qcow2_size_and_actual_size(cmd.primaryStorageInstallPath)[0]
# we need to sum all the disk size within the chain ...
chain = linux.qcow2_get_file_chain(cmd.primaryStorageInstallPath)
rsp.actualSize = sum([ linux.qcow2_size_and_actual_size(f)[1] for f in chain ])
return jsonobject.dumps(rsp)
def get_backing_chain(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetBackingChainRsp()
abs_path = translate_absolute_path_from_install_path(cmd.installPath)
with lvm.RecursiveOperateLv(abs_path, shared=True, skip_deactivate_tags=[IMAGE_TAG], delete_when_exception=False):
rsp.backingChain = linux.qcow2_get_file_chain(abs_path)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
def get_backing_chain(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = GetBackingChainRsp()
abs_path = translate_absolute_path_from_install_path(cmd.installPath)
with lvm.RecursiveOperateLv(abs_path,
shared=True,
skip_deactivate_tags=[IMAGE_TAG],
delete_when_exception=False):
rsp.backingChain = linux.qcow2_get_file_chain(abs_path)
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
def create_template_from_volume(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = AgentRsp()
volume_abs_path = translate_absolute_path_from_install_path(
cmd.volumePath)
install_abs_path = translate_absolute_path_from_install_path(
cmd.installPath)
if cmd.sharedVolume:
lvm.do_active_lv(volume_abs_path, lvm.LvmlockdLockType.SHARE, True)
with lvm.RecursiveOperateLv(volume_abs_path,
shared=cmd.sharedVolume,
skip_deactivate_tags=[IMAGE_TAG]):
virtual_size = linux.qcow2_virtualsize(volume_abs_path)
total_size = 0
compress = False
for qcow2 in linux.qcow2_get_file_chain(volume_abs_path):
if bash.bash_r("qemu-img check %s | grep compressed" %
volume_abs_path) == 0:
compress = True
total_size += int(lvm.get_lv_size(qcow2))
if total_size > virtual_size:
total_size = virtual_size
if bash.bash_r("qemu-img info --backing-chain %s | grep compress" %
volume_abs_path) == 0:
compress = True
if not lvm.lv_exists(install_abs_path):
lvm.create_lv_from_absolute_path(
install_abs_path, total_size,
"%s::%s::%s" % (VOLUME_TAG, cmd.hostUuid, time.time()))
with lvm.OperateLv(install_abs_path,
shared=False,
delete_when_exception=True):
linux.create_template(volume_abs_path, install_abs_path,
compress)
logger.debug(
'successfully created template[%s] from volume[%s]' %
(cmd.installPath, cmd.volumePath))
if cmd.compareQcow2 is True:
logger.debug("comparing qcow2 between %s and %s")
bash.bash_errorout("time qemu-img compare %s %s" %
(volume_abs_path, install_abs_path))
logger.debug("confirmed qcow2 %s and %s are identical" %
(volume_abs_path, install_abs_path))
rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid)
return jsonobject.dumps(rsp)
def copy_bits_to_remote(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
if cmd.dstUsername != 'root':
raise Exception("cannot support migrate to non-root user host")
chain = sum([linux.qcow2_get_file_chain(p) for p in cmd.paths], [])
if cmd.sendCommandUrl:
Report.url = cmd.sendCommandUrl
report = Report(cmd.threadContext, cmd.threadContextStack)
report.processType = "LocalStorageMigrateVolume"
report.resourceUuid = cmd.volumeUuid
PFILE = shell.call('mktemp /tmp/tmp-XXXXXX').strip()
PASSWORD_FILE = linux.write_to_temp_file(cmd.dstPassword)
start = 10
end = 90
if cmd.stage:
start, end = get_scale(cmd.stage)
total = 0
for path in set(chain):
total = total + os.path.getsize(path)
written = 0
def _get_progress(synced):
logger.debug(
"getProgress in localstorage-agent, synced: %s, total: %s" %
(synced, total))
if not os.path.exists(PFILE):
return synced
fpread = open(PFILE, 'r')
lines = fpread.readlines()
if not lines:
fpread.close()
return synced
last = str(lines[-1]).strip().split('\r')[-1]
if not last or len(last.split()) < 1:
fpread.close()
return synced
line = last.split()[0]
if not line.isdigit():
return synced
if total > 0:
synced = long(line)
if synced < total:
percent = int(
round(
float(written + synced) / float(total) *
(end - start) + start))
report.progress_report(percent, "report")
synced = written
fpread.close()
return synced
for path in set(chain):
PATH = path
USER = cmd.dstUsername
IP = cmd.dstIp
PORT = (cmd.dstPort and cmd.dstPort or "22")
DIR = os.path.dirname(path)
_, _, err = bash_progress_1(
# Fixes ZSTAC-13430: handle extremely complex password like ~ ` !@#$%^&*()_+-=[]{}|?<>;:'"/ .
'rsync -av --progress --relative {{PATH}} --rsh="/usr/bin/sshpass -f{{PASSWORD_FILE}} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p {{PORT}} -l {{USER}}" {{IP}}:/ 1>{{PFILE}}',
_get_progress,
False)
if err:
linux.rm_file_force(PASSWORD_FILE)
linux.rm_file_force(PFILE)
raise Exception('fail to migrate vm to host, because %s' %
str(err))
written += os.path.getsize(path)
bash_errorout(
'/usr/bin/sshpass -f{{PASSWORD_FILE}} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p {{PORT}} {{USER}}@{{IP}} "/bin/sync {{PATH}}"'
)
percent = int(
round(float(written) / float(total) * (end - start) + start))
report.progress_report(percent, "report")
linux.rm_file_force(PASSWORD_FILE)
linux.rm_file_force(PFILE)
rsp = AgentResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.storagePath)
return jsonobject.dumps(rsp)
def copy_bits_to_remote(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
chain = sum([linux.qcow2_get_file_chain(p) for p in cmd.paths], [])
if cmd.sendCommandUrl:
Report.url = cmd.sendCommandUrl
report = Report(cmd.threadContext, cmd.threadContextStack)
report.processType = "LocalStorageMigrateVolume"
report.resourceUuid = cmd.uuid
PFILE = shell.call('mktemp /tmp/tmp-XXXXXX').strip()
start = 10
end = 90
if cmd.stage:
start, end = get_scale(cmd.stage)
total = 0
for path in set(chain):
total = total + os.path.getsize(path)
written = 0
def _get_progress(synced):
logger.debug("getProgress in localstorage-agent, synced: %s, total: %s" % (synced, total))
if not os.path.exists(PFILE):
return synced
fpread = open(PFILE, 'r')
lines = fpread.readlines()
if not lines:
fpread.close()
return synced
last = str(lines[-1]).strip().split('\r')[-1]
if not last or len(last.split()) < 1:
fpread.close()
return synced
line = last.split()[0]
if not line.isdigit():
return synced
if total > 0:
synced = long(line)
if synced < total:
percent = int(round(float(written + synced) / float(total) * (end - start) + start))
report.progress_report(percent, "report")
synced = written
fpread.close()
return synced
for path in set(chain):
PATH = path
PASSWORD = linux.shellquote(cmd.dstPassword)
USER = cmd.dstUsername
IP = cmd.dstIp
PORT = (cmd.dstPort and cmd.dstPort or "22")
DIR = os.path.dirname(path)
if cmd.dstUsername == 'root':
_, _, err = bash_progress_1(
'rsync -av --progress --relative {{PATH}} --rsh="/usr/bin/sshpass -p {{PASSWORD}} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p {{PORT}} -l {{USER}}" {{IP}}:/ 1>{{PFILE}}', _get_progress, False)
if err:
raise Exception('fail to migrate vm to host, because %s' % str(err))
else:
raise Exception("cannot support migrate to non-root user host")
written += os.path.getsize(path)
bash_errorout('/usr/bin/sshpass -p {{PASSWORD}} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p {{PORT}} {{USER}}@{{IP}} "/bin/sync {{PATH}}"')
percent = int(round(float(written) / float(total) * (end - start) + start))
report.progress_report(percent, "report")
if os.path.exists(PFILE):
os.remove(PFILE)
rsp = AgentResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.storagePath)
return jsonobject.dumps(rsp)