def sftp_download(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
hostname = cmd.hostname
prikey = cmd.sshKey
port = cmd.sshPort
pool, image_name = self._parse_install_path(cmd.primaryStorageInstallPath)
tmp_image_name = 'tmp-%s' % image_name
prikey_file = linux.write_to_temp_file(prikey)
@rollbackable
def _0():
tpath = "%s/%s" % (pool, tmp_image_name)
shell.call('rbd info %s > /dev/null && rbd rm %s' % (tpath, tpath))
_0()
try:
shell.call(
'set -o pipefail; ssh -p %d -o StrictHostKeyChecking=no -i %s root@%s "cat %s" | rbd import --image-format 2 - %s/%s' %
(port, prikey_file, hostname, cmd.backupStorageInstallPath, pool, tmp_image_name))
finally:
os.remove(prikey_file)
@rollbackable
def _1():
shell.call('rbd rm %s/%s' % (pool, tmp_image_name))
_1()
file_format = shell.call(
"set -o pipefail; qemu-img info rbd:%s/%s | grep 'file format' | cut -d ':' -f 2" % (pool, tmp_image_name))
file_format = file_format.strip()
if file_format not in ['qcow2', 'raw']:
raise Exception('unknown image format: %s' % file_format)
if file_format == 'qcow2':
conf_path = None
try:
with open('/etc/ceph/ceph.conf', 'r') as fd:
conf = fd.read()
conf = '%s\n%s\n' % (conf, 'rbd default format = 2')
conf_path = linux.write_to_temp_file(conf)
shell.call('qemu-img convert -f qcow2 -O rbd rbd:%s/%s rbd:%s/%s:conf=%s' % (
pool, tmp_image_name, pool, image_name, conf_path))
shell.call('rbd rm %s/%s' % (pool, tmp_image_name))
finally:
if conf_path:
os.remove(conf_path)
else:
shell.call('rbd mv %s/%s %s/%s' % (pool, tmp_image_name, pool, image_name))
rsp = AgentResponse()
self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
def sftp_download(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
hostname = cmd.hostname
prikey = cmd.sshKey
port = cmd.sshPort
pool, image_name = self._parse_install_path(cmd.primaryStorageInstallPath)
tmp_image_name = 'tmp-%s' % image_name
prikey_file = linux.write_to_temp_file(prikey)
@rollbackable
def _0():
tpath = "%s/%s" % (pool, tmp_image_name)
shell.call('rbd info %s > /dev/null && rbd rm %s' % (tpath, tpath))
_0()
try:
shell.call(
'set -o pipefail; ssh -p %d -o StrictHostKeyChecking=no -i %s root@%s "cat %s" | rbd import --image-format 2 - %s/%s' %
(port, prikey_file, hostname, cmd.backupStorageInstallPath, pool, tmp_image_name))
finally:
os.remove(prikey_file)
@rollbackable
def _1():
shell.call('rbd rm %s/%s' % (pool, tmp_image_name))
_1()
file_format = shell.call(
"set -o pipefail; qemu-img info rbd:%s/%s | grep 'file format' | cut -d ':' -f 2" % (pool, tmp_image_name))
file_format = file_format.strip()
if file_format not in ['qcow2', 'raw']:
raise Exception('unknown image format: %s' % file_format)
if file_format == 'qcow2':
conf_path = None
try:
with open('/etc/ceph/ceph.conf', 'r') as fd:
conf = fd.read()
conf = '%s\n%s\n' % (conf, 'rbd default format = 2')
conf_path = linux.write_to_temp_file(conf)
shell.call('qemu-img convert -f qcow2 -O rbd rbd:%s/%s rbd:%s/%s:conf=%s' % (
pool, tmp_image_name, pool, image_name, conf_path))
shell.call('rbd rm %s/%s' % (pool, tmp_image_name))
finally:
if conf_path:
os.remove(conf_path)
else:
shell.call('rbd mv %s/%s %s/%s' % (pool, tmp_image_name, pool, image_name))
rsp = AgentResponse()
self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
def do_sftp_download(self, cmd, pool, image_name):
hostname = cmd.hostname
prikey = cmd.sshKey
port = cmd.sshPort
if cmd.bandWidth is not None:
bandWidth = 'pv -q -L %s |' % cmd.bandWidth
else:
bandWidth = ''
tmp_image_name = 'tmp-%s' % image_name
prikey_file = linux.write_to_temp_file(prikey)
@rollbackable
def _0():
tpath = "%s/%s" % (pool, tmp_image_name)
shell.call('rbd info %s > /dev/null && rbd rm %s' % (tpath, tpath))
_0()
try:
shell.run('rbd rm %s/%s' % (pool, tmp_image_name))
shell.call('set -o pipefail; ssh -p %d -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s cat %s | %s rbd import --image-format 2 - %s/%s' % (port, prikey_file, hostname, remote_shell_quote(cmd.backupStorageInstallPath), bandWidth, pool, tmp_image_name))
finally:
os.remove(prikey_file)
@rollbackable
def _1():
shell.call('rbd rm %s/%s' % (pool, tmp_image_name))
_1()
file_format = shell.call(
"set -o pipefail; qemu-img info rbd:%s/%s | grep 'file format' | cut -d ':' -f 2" % (pool, tmp_image_name))
file_format = file_format.strip()
if file_format not in ['qcow2', 'raw']:
raise Exception('unknown image format: %s' % file_format)
shell.run('rbd rm %s/%s' % (pool, image_name))
if file_format == 'qcow2':
conf_path = None
try:
with open('/etc/ceph/ceph.conf', 'r') as fd:
conf = fd.read()
conf = '%s\n%s\n' % (conf, 'rbd default format = 2')
conf_path = linux.write_to_temp_file(conf)
shell.call('qemu-img convert -f qcow2 -O rbd rbd:%s/%s rbd:%s/%s:conf=%s' % (
pool, tmp_image_name, pool, image_name, conf_path))
shell.call('rbd rm %s/%s' % (pool, tmp_image_name))
finally:
if conf_path:
os.remove(conf_path)
else:
shell.call('rbd mv %s/%s %s/%s' % (pool, tmp_image_name, pool, image_name))
def apply_iptables_save_doc(content):
f = linux.write_to_temp_file(content)
try:
logger.debug('apply iptables:\n %s' % content)
shell.call('/sbin/iptables-restore < %s' % f)
finally:
os.remove(f)
def clean_ipsets(ipset_names):
destroy_cmds = ['destroy %s' % set_name for set_name in ipset_names]
tmp = linux.write_to_temp_file('\n'.join(destroy_cmds))
o = shell.ShellCmd('ipset restore -f %s' % tmp)
o(False)
if o.return_code != 0:
logger.warn('fail to cleanup ipsets, %s' % o.stderr)
else:
logger.debug('success cleanup ipsets')
os.remove(tmp)
def clean_ipsets(ipset_names):
destroy_cmds = ['destroy %s' % set_name for set_name in ipset_names]
tmp = linux.write_to_temp_file('\n'.join(destroy_cmds))
o = shell.ShellCmd('ipset restore -f %s' % tmp)
o(False)
if o.return_code != 0:
logger.warn('fail to cleanup ipsets, %s' % o.stderr)
else:
logger.debug('success cleanup ipsets')
os.remove(tmp)
def sftp_upload(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
src_path = self._normalize_install_path(cmd.primaryStorageInstallPath)
prikey_file = linux.write_to_temp_file(cmd.sshKey)
bs_folder = os.path.dirname(cmd.backupStorageInstallPath)
rsp = AgentResponse()
rsp.success = False
rsp.error = 'unsupported SimpleSftpBackupStorage, only supports fusionstor backupstorage'
return jsonobject.dumps(rsp)
def sftp_upload(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
src_path = self._normalize_install_path(cmd.primaryStorageInstallPath)
prikey_file = linux.write_to_temp_file(cmd.sshKey)
bs_folder = os.path.dirname(cmd.backupStorageInstallPath)
rsp = AgentResponse()
rsp.success = False
rsp.error = 'unsupported SimpleSftpBackupStorage, only supports fusionstor backupstorage'
return jsonobject.dumps(rsp)
def download(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
pool, image_name = self._parse_install_path(cmd.installPath)
tmp_image_name = 'tmp-%s' % image_name
if cmd.url.startswith('http://') or cmd.url.startswith('https://'):
shell.call('set -o pipefail; wget --no-check-certificate -q -O - %s | rbd import --image-format 2 - %s/%s'
% (cmd.url, pool, tmp_image_name))
actual_size = linux.get_file_size_by_http_head(cmd.url)
elif cmd.url.startswith('file://'):
src_path = cmd.url.lstrip('file:')
src_path = os.path.normpath(src_path)
if not os.path.isfile(src_path):
raise Exception('cannot find the file[%s]' % src_path)
shell.call("rbd import --image-format 2 %s %s/%s" % (src_path, pool, tmp_image_name))
actual_size = os.path.getsize(src_path)
else:
raise Exception('unknown url[%s]' % cmd.url)
@rollbackable
def _1():
shell.call('rbd rm %s/%s' % (pool, tmp_image_name))
_1()
file_format = shell.call("set -o pipefail; qemu-img info rbd:%s/%s | grep 'file format' | cut -d ':' -f 2" % (pool, tmp_image_name))
file_format = file_format.strip()
if file_format not in ['qcow2', 'raw']:
raise Exception('unknown image format: %s' % file_format)
if file_format == 'qcow2':
conf_path = None
try:
with open('/etc/ceph/ceph.conf', 'r') as fd:
conf = fd.read()
conf = '%s\n%s\n' % (conf, 'rbd default format = 2')
conf_path = linux.write_to_temp_file(conf)
shell.call('qemu-img convert -f qcow2 -O rbd rbd:%s/%s rbd:%s/%s:conf=%s' % (pool, tmp_image_name, pool, image_name, conf_path))
shell.call('rbd rm %s/%s' % (pool, tmp_image_name))
finally:
if conf_path:
os.remove(conf_path)
else:
shell.call('rbd mv %s/%s %s/%s' % (pool, tmp_image_name, pool, image_name))
o = shell.call('rbd --format json info %s/%s' % (pool, image_name))
image_stats = jsonobject.loads(o)
rsp = DownloadRsp()
rsp.size = long(image_stats.size_)
rsp.actualSize = actual_size
self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
def sftp_download(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
hostname = cmd.hostname
prikey = cmd.sshKey
pool, image_name = self._parse_install_path(cmd.primaryStorageInstallPath)
tmp_image_name = 'tmp-%s' % image_name
prikey_file = linux.write_to_temp_file(prikey)
rsp = AgentResponse()
rsp.success = False
rsp.error = 'unsupported SimpleSftpBackupStorage, only supports fusionstor backupstorage'
#self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
def sftp_download(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
hostname = cmd.hostname
prikey = cmd.sshKey
pool, image_name = self._parse_install_path(cmd.primaryStorageInstallPath)
tmp_image_name = 'tmp-%s' % image_name
prikey_file = linux.write_to_temp_file(prikey)
rsp = AgentResponse()
rsp.success = False
rsp.error = 'unsupported SimpleSftpBackupStorage, only supports fusionstor backupstorage'
#self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
def download(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
pool, image_name = self._parse_install_path(cmd.installPath)
tmp_image_name = 'tmp-%s' % image_name
shell.call(
'wget --no-check-certificate -q -O - %s | rbd import --image-format 2 - %s/%s'
% (cmd.url, pool, tmp_image_name))
@rollbackable
def _1():
shell.call('rbd rm %s/%s' % (pool, tmp_image_name))
_1()
file_format = shell.call(
"qemu-img info rbd:%s/%s | grep 'file format' | cut -d ':' -f 2" %
(pool, tmp_image_name))
file_format = file_format.strip()
if file_format not in ['qcow2', 'raw']:
raise Exception('unknown image format: %s' % file_format)
if file_format == 'qcow2':
conf_path = None
try:
with open('/etc/ceph/ceph.conf', 'r') as fd:
conf = fd.read()
conf = '%s\n%s\n' % (conf, 'rbd default format = 2')
conf_path = linux.write_to_temp_file(conf)
shell.call(
'qemu-img convert -f qcow2 -O rbd rbd:%s/%s rbd:%s/%s:conf=%s'
% (pool, tmp_image_name, pool, image_name, conf_path))
shell.call('rbd rm %s/%s' % (pool, tmp_image_name))
finally:
if conf_path:
os.remove(conf_path)
else:
shell.call('rbd mv %s/%s %s/%s' %
(pool, tmp_image_name, pool, image_name))
o = shell.call('rbd --format json info %s/%s' % (pool, image_name))
image_stats = jsonobject.loads(o)
rsp = DownloadRsp()
rsp.size = long(image_stats.size_)
self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
def iptable_restore(self, marshall_func=None, sort_nat_func=None, sort_filter_func=None, sort_mangle_func=None):
content = self._to_iptables_string(marshall_func, sort_nat_func, sort_filter_func, sort_mangle_func)
f = linux.write_to_temp_file(content)
try:
shell.call('/sbin/iptables-restore < %s' % f)
except Exception as e:
err ='''Failed to apply iptables rules:
shell error description:
%s
iptable rules:
%s
''' % (str(e), content)
raise IPTablesError(err)
finally:
os.remove(f)
def create_bootstrap_iso(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
isoinfo = jsonobject.dumps(cmd.isoInfo, True)
tmpfile = linux.write_to_temp_file(isoinfo)
isodir = tempfile.mkdtemp()
try:
dst = os.path.join(isodir, 'cmdline.json')
shell.ShellCmd('cp %s %s' % (tmpfile, dst))()
shell.ShellCmd('/usr/bin/mkisofs -quiet -r -o %s %s' % (cmd.isoPath, isodir))()
return jsonobject.dumps(CreateVritualRouterBootstrapIsoRsp())
finally:
if not isodir:
shutil.rmtree(isodir)
if not tmpfile:
os.remove(tmpfile)
def create_bootstrap_iso(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
isoinfo = jsonobject.dumps(cmd.isoInfo, True)
tmpfile = linux.write_to_temp_file(isoinfo)
isodir = tempfile.mkdtemp()
try:
dst = os.path.join(isodir, 'cmdline.json')
shell.ShellCmd('cp %s %s' % (tmpfile, dst))()
shell.ShellCmd('/usr/bin/mkisofs -quiet -r -o %s %s' % (cmd.isoPath, isodir))()
return jsonobject.dumps(CreateVritualRouterBootstrapIsoRsp())
finally:
if not isodir:
shutil.rmtree(isodir)
if not tmpfile:
os.remove(tmpfile)
def sftp_upload(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
src_path = self._normalize_install_path(cmd.primaryStorageInstallPath)
prikey_file = linux.write_to_temp_file(cmd.sshKey)
bs_folder = os.path.dirname(cmd.backupStorageInstallPath)
shell.call('ssh -o StrictHostKeyChecking=no -i %s root@%s "mkdir -p %s"' %
(prikey_file, cmd.hostname, bs_folder))
try:
shell.call("set -o pipefail; rbd export %s - | ssh -o StrictHostKeyChecking=no -i %s root@%s 'cat > %s'" %
(src_path, prikey_file, cmd.hostname, cmd.backupStorageInstallPath))
finally:
os.remove(prikey_file)
return jsonobject.dumps(AgentResponse())
def sftp_upload(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
src_path = self._normalize_install_path(cmd.primaryStorageInstallPath)
prikey_file = linux.write_to_temp_file(cmd.sshKey)
bs_folder = os.path.dirname(cmd.backupStorageInstallPath)
shell.call('ssh -p %d -o StrictHostKeyChecking=no -i %s root@%s "mkdir -p %s"' %
(cmd.sshPort, prikey_file, cmd.hostname, bs_folder))
try:
shell.call("set -o pipefail; rbd export %s - | ssh -o StrictHostKeyChecking=no -i %s root@%s 'cat > %s'" %
(src_path, prikey_file, cmd.hostname, cmd.backupStorageInstallPath))
finally:
os.remove(prikey_file)
return jsonobject.dumps(AgentResponse())
def iptable_restore(self, marshall_func=None, sort_nat_func=None, sort_filter_func=None, sort_mangle_func=None):
content = self._to_iptables_string(marshall_func, sort_nat_func, sort_filter_func, sort_mangle_func)
f = linux.write_to_temp_file(content)
try:
shell.call('/sbin/iptables-restore < %s' % f)
except Exception as e:
res = shell.call('lsof /run/xtables.lock')
err ='''Failed to apply iptables rules:
shell error description:
%s
result of lsof /run/xtables.lock
%s
iptable rules:
%s
''' % (str(e), str(res), content)
raise IPTablesError(err)
finally:
os.remove(f)
def listvm(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
rsp = ListVmRsp()
if cmd.sshPassword and not cmd.sshPrivKey:
target, port = getSshTargetAndPort(cmd.libvirtURI)
ssh_pswd_file = linux.write_to_temp_file(cmd.sshPassword)
if not os.path.exists(V2V_PRIV_KEY) or not os.path.exists(V2V_PUB_KEY):
shell.check_run("yes | ssh-keygen -t rsa -N '' -f {}".format(V2V_PRIV_KEY))
cmdstr = "HOME={4} timeout 30 sshpass -f {0} ssh-copy-id -i {5} -p {1} {2} {3}".format(
ssh_pswd_file,
port,
DEF_SSH_OPTS,
target,
os.path.expanduser("~"),
V2V_PUB_KEY)
shell.check_run(cmdstr)
linux.rm_file_force(ssh_pswd_file)
rsp.qemuVersion, rsp.libvirtVersion, rsp.vms, rsp.v2vCaps = listVirtualMachines(cmd.libvirtURI,
cmd.saslUser,
cmd.saslPass,
cmd.sshPrivKey)
return jsonobject.dumps(rsp)
def copy_bits_to_remote(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
if cmd.dstUsername != 'root':
raise Exception("cannot support migrate to non-root user host")
chain = sum([linux.qcow2_get_file_chain(p) for p in cmd.paths], [])
if cmd.sendCommandUrl:
Report.url = cmd.sendCommandUrl
report = Report(cmd.threadContext, cmd.threadContextStack)
report.processType = "LocalStorageMigrateVolume"
report.resourceUuid = cmd.volumeUuid
PFILE = shell.call('mktemp /tmp/tmp-XXXXXX').strip()
PASSWORD_FILE = linux.write_to_temp_file(cmd.dstPassword)
start = 10
end = 90
if cmd.stage:
start, end = get_scale(cmd.stage)
total = 0
for path in set(chain):
total = total + os.path.getsize(path)
written = 0
def _get_progress(synced):
logger.debug(
"getProgress in localstorage-agent, synced: %s, total: %s" %
(synced, total))
if not os.path.exists(PFILE):
return synced
fpread = open(PFILE, 'r')
lines = fpread.readlines()
if not lines:
fpread.close()
return synced
last = str(lines[-1]).strip().split('\r')[-1]
if not last or len(last.split()) < 1:
fpread.close()
return synced
line = last.split()[0]
if not line.isdigit():
return synced
if total > 0:
synced = long(line)
if synced < total:
percent = int(
round(
float(written + synced) / float(total) *
(end - start) + start))
report.progress_report(percent, "report")
synced = written
fpread.close()
return synced
for path in set(chain):
PATH = path
USER = cmd.dstUsername
IP = cmd.dstIp
PORT = (cmd.dstPort and cmd.dstPort or "22")
DIR = os.path.dirname(path)
_, _, err = bash_progress_1(
# Fixes ZSTAC-13430: handle extremely complex password like ~ ` !@#$%^&*()_+-=[]{}|?<>;:'"/ .
'rsync -av --progress --relative {{PATH}} --rsh="/usr/bin/sshpass -f{{PASSWORD_FILE}} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p {{PORT}} -l {{USER}}" {{IP}}:/ 1>{{PFILE}}',
_get_progress,
False)
if err:
linux.rm_file_force(PASSWORD_FILE)
linux.rm_file_force(PFILE)
raise Exception('fail to migrate vm to host, because %s' %
str(err))
written += os.path.getsize(path)
bash_errorout(
'/usr/bin/sshpass -f{{PASSWORD_FILE}} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p {{PORT}} {{USER}}@{{IP}} "/bin/sync {{PATH}}"'
)
percent = int(
round(float(written) / float(total) * (end - start) + start))
report.progress_report(percent, "report")
linux.rm_file_force(PASSWORD_FILE)
linux.rm_file_force(PFILE)
rsp = AgentResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(
cmd.storagePath)
return jsonobject.dumps(rsp)
def stream_body(task, fpath, entity, boundary):
def _progress_consumer(total):
task.downloadedSize = total
@thread.AsyncThread
def _do_import(task, fpath):
shell.check_run("cat %s | rbd import --image-format 2 - %s" % (fpath, task.tmpPath))
while True:
headers = cherrypy._cpreqbody.Part.read_headers(entity.fp)
p = CustomPart(entity.fp, headers, boundary, fpath, _progress_consumer)
if not p.filename:
continue
# start consumer
_do_import(task, fpath)
try:
p.process()
except Exception as e:
logger.warn('process image %s failed: %s' % (task.imageUuid, str(e)))
pass
finally:
if p.wfd is not None:
p.wfd.close()
break
if task.downloadedSize != task.expectedSize:
task.fail('incomplete upload, got %d, expect %d' % (task.downloadedSize, task.expectedSize))
shell.run('rbd rm %s' % task.tmpPath)
return
file_format = None
try:
file_format = linux.get_img_fmt('rbd:'+task.tmpPath)
except Exception as e:
task.fail('upload image %s failed: %s' % (task.imageUuid, str(e)))
return
if file_format == 'qcow2':
if linux.qcow2_get_backing_file('rbd:'+task.tmpPath):
task.fail('Qcow2 image %s has backing file' % task.imageUuid)
shell.run('rbd rm %s' % task.tmpPath)
return
conf_path = None
try:
with open('/etc/ceph/ceph.conf', 'r') as fd:
conf = fd.read()
conf = '%s\n%s\n' % (conf, 'rbd default format = 2')
conf_path = linux.write_to_temp_file(conf)
shell.check_run('qemu-img convert -f qcow2 -O rbd rbd:%s rbd:%s:conf=%s' % (task.tmpPath, task.dstPath, conf_path))
except Exception as e:
task.fail('cannot convert Qcow2 image %s to rbd' % task.imageUuid)
logger.warn('convert image %s failed: %s', (task.imageUuid, str(e)))
return
finally:
shell.run('rbd rm %s' % task.tmpPath)
if conf_path:
os.remove(conf_path)
else:
shell.check_run('rbd mv %s %s' % (task.tmpPath, task.dstPath))
task.success()
def download(self, req):
rsp = DownloadRsp()
def _get_origin_format(path):
qcow2_length = 0x9007
if path.startswith('http://') or path.startswith(
'https://') or path.startswith('ftp://'):
resp = urllib2.urlopen(path)
qhdr = resp.read(qcow2_length)
resp.close()
elif path.startswith('sftp://'):
fd, tmp_file = tempfile.mkstemp()
get_header_from_pipe_cmd = "timeout 60 head --bytes=%d %s > %s" % (
qcow2_length, pipe_path, tmp_file)
clean_cmd = "pkill -f %s" % pipe_path
shell.run(
'%s & %s && %s' %
(scp_to_pipe_cmd, get_header_from_pipe_cmd, clean_cmd))
qhdr = os.read(fd, qcow2_length)
if os.path.exists(tmp_file):
os.remove(tmp_file)
else:
resp = open(path)
qhdr = resp.read(qcow2_length)
resp.close()
if len(qhdr) < qcow2_length:
return "raw"
return get_image_format_from_buf(qhdr)
def get_origin_format(fpath, fail_if_has_backing_file=True):
image_format = _get_origin_format(fpath)
if image_format == "derivedQcow2" and fail_if_has_backing_file:
raise Exception('image has backing file or %s is not exist!' %
fpath)
return image_format
cmd = jsonobject.loads(req[http.REQUEST_BODY])
shell = traceable_shell.get_shell(cmd)
pool, image_name = self._parse_install_path(cmd.installPath)
tmp_image_name = 'tmp-%s' % image_name
@rollbackable
def _1():
shell.check_run('rbd rm %s/%s' % (pool, tmp_image_name))
def _getRealSize(length):
'''length looks like: 10245K'''
logger.debug(length)
if not length[-1].isalpha():
return length
units = {
"g": lambda x: x * 1024 * 1024 * 1024,
"m": lambda x: x * 1024 * 1024,
"k": lambda x: x * 1024,
}
try:
if not length[-1].isalpha():
return length
return units[length[-1].lower()](int(length[:-1]))
except:
logger.warn(linux.get_exception_stacktrace())
return length
# whether we have an upload request
if cmd.url.startswith(self.UPLOAD_PROTO):
self._prepare_upload(cmd)
rsp.size = 0
rsp.uploadPath = self._get_upload_path(req)
self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
if cmd.sendCommandUrl:
Report.url = cmd.sendCommandUrl
report = Report(cmd.threadContext, cmd.threadContextStack)
report.processType = "AddImage"
report.resourceUuid = cmd.imageUuid
report.progress_report("0", "start")
url = urlparse.urlparse(cmd.url)
if url.scheme in ('http', 'https', 'ftp'):
image_format = get_origin_format(cmd.url, True)
cmd.url = linux.shellquote(cmd.url)
# roll back tmp ceph file after import it
_1()
_, PFILE = tempfile.mkstemp()
content_length = shell.call(
"""curl -sLI %s|awk '/[cC]ontent-[lL]ength/{print $NF}'""" %
cmd.url).splitlines()[-1]
total = _getRealSize(content_length)
def _getProgress(synced):
last = linux.tail_1(PFILE).strip()
if not last or len(last.split(
)) < 1 or 'HTTP request sent, awaiting response' in last:
return synced
logger.debug("last synced: %s" % last)
written = _getRealSize(last.split()[0])
if total > 0 and synced < written:
synced = written
if synced < total:
percent = int(round(float(synced) / float(total) * 90))
report.progress_report(percent, "report")
return synced
logger.debug("content-length is: %s" % total)
_, _, err = shell.bash_progress_1(
'set -o pipefail;wget --no-check-certificate -O - %s 2>%s| rbd import --image-format 2 - %s/%s'
% (cmd.url, PFILE, pool, tmp_image_name), _getProgress)
if err:
raise err
actual_size = linux.get_file_size_by_http_head(cmd.url)
if os.path.exists(PFILE):
os.remove(PFILE)
elif url.scheme == 'sftp':
port = (url.port, 22)[url.port is None]
_, PFILE = tempfile.mkstemp()
ssh_pswd_file = None
pipe_path = PFILE + "fifo"
scp_to_pipe_cmd = "scp -P %d -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null %s@%s:%s %s" % (
port, url.username, url.hostname, url.path, pipe_path)
sftp_command = "sftp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=no -P %s -b /dev/stdin %s@%s" % (
port, url.username, url.hostname) + " <<EOF\n%s\nEOF\n"
if url.password is not None:
ssh_pswd_file = linux.write_to_temp_file(url.password)
scp_to_pipe_cmd = 'sshpass -f %s %s' % (ssh_pswd_file,
scp_to_pipe_cmd)
sftp_command = 'sshpass -f %s %s' % (ssh_pswd_file,
sftp_command)
actual_size = shell.call(
sftp_command %
("ls -l " + url.path)).splitlines()[1].strip().split()[4]
os.mkfifo(pipe_path)
image_format = get_origin_format(cmd.url, True)
cmd.url = linux.shellquote(cmd.url)
# roll back tmp ceph file after import it
_1()
def _get_progress(synced):
if not os.path.exists(PFILE):
return synced
last = linux.tail_1(PFILE).strip()
if not last or not last.isdigit():
return synced
report.progress_report(int(last) * 90 / 100, "report")
return synced
get_content_from_pipe_cmd = "pv -s %s -n %s 2>%s" % (
actual_size, pipe_path, PFILE)
import_from_pipe_cmd = "rbd import --image-format 2 - %s/%s" % (
pool, tmp_image_name)
_, _, err = shell.bash_progress_1(
'set -o pipefail; %s & %s | %s' %
(scp_to_pipe_cmd, get_content_from_pipe_cmd,
import_from_pipe_cmd), _get_progress)
if ssh_pswd_file:
linux.rm_file_force(ssh_pswd_file)
linux.rm_file_force(PFILE)
linux.rm_file_force(pipe_path)
if err:
raise err
elif url.scheme == 'file':
src_path = cmd.url.lstrip('file:')
src_path = os.path.normpath(src_path)
if not os.path.isfile(src_path):
raise Exception('cannot find the file[%s]' % src_path)
image_format = get_origin_format(src_path, True)
# roll back tmp ceph file after import it
_1()
shell.check_run("rbd import --image-format 2 %s %s/%s" %
(src_path, pool, tmp_image_name))
actual_size = os.path.getsize(src_path)
else:
raise Exception('unknown url[%s]' % cmd.url)
file_format = shell.call(
"set -o pipefail; %s rbd:%s/%s | grep 'file format' | cut -d ':' -f 2"
% (qemu_img.subcmd('info'), pool, tmp_image_name))
file_format = file_format.strip()
if file_format not in ['qcow2', 'raw']:
raise Exception('unknown image format: %s' % file_format)
if file_format == 'qcow2':
conf_path = None
try:
with open('/etc/ceph/ceph.conf', 'r') as fd:
conf = fd.read()
conf = '%s\n%s\n' % (conf, 'rbd default format = 2')
conf_path = linux.write_to_temp_file(conf)
shell.check_run(
'%s -f qcow2 -O rbd rbd:%s/%s rbd:%s/%s:conf=%s' %
(qemu_img.subcmd('convert'), pool, tmp_image_name, pool,
image_name, conf_path))
shell.check_run('rbd rm %s/%s' % (pool, tmp_image_name))
finally:
if conf_path:
os.remove(conf_path)
else:
shell.check_run('rbd mv %s/%s %s/%s' %
(pool, tmp_image_name, pool, image_name))
report.progress_report("100", "finish")
@rollbackable
def _2():
shell.check_run('rbd rm %s/%s' % (pool, image_name))
_2()
o = shell.call('rbd --format json info %s/%s' % (pool, image_name))
image_stats = jsonobject.loads(o)
rsp.size = long(image_stats.size_)
rsp.actualSize = actual_size
if image_format == "qcow2":
rsp.format = "raw"
else:
rsp.format = image_format
self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
def download(self, req):
rsp = DownloadRsp()
def _get_origin_format(path):
qcow2_length = 0x9007
if path.startswith('http://') or path.startswith('https://') or path.startswith('ftp://'):
resp = urllib2.urlopen(path)
qhdr = resp.read(qcow2_length)
resp.close()
elif path.startswith('sftp://'):
fd, tmp_file = tempfile.mkstemp()
get_header_from_pipe_cmd = "timeout 60 head --bytes=%d %s > %s" % (qcow2_length, pipe_path, tmp_file)
clean_cmd = "pkill -f %s" % pipe_path
shell.run('%s & %s && %s' % (scp_to_pipe_cmd, get_header_from_pipe_cmd, clean_cmd))
qhdr = os.read(fd, qcow2_length)
if os.path.exists(tmp_file):
os.remove(tmp_file)
else:
resp = open(path)
qhdr = resp.read(qcow2_length)
resp.close()
if len(qhdr) < qcow2_length:
return "raw"
return get_image_format_from_buf(qhdr)
def get_origin_format(fpath, fail_if_has_backing_file=True):
image_format = _get_origin_format(fpath)
if image_format == "derivedQcow2" and fail_if_has_backing_file:
raise Exception('image has backing file or %s is not exist!' % fpath)
return image_format
cmd = jsonobject.loads(req[http.REQUEST_BODY])
pool, image_name = self._parse_install_path(cmd.installPath)
tmp_image_name = 'tmp-%s' % image_name
@rollbackable
def _1():
shell.check_run('rbd rm %s/%s' % (pool, tmp_image_name))
def _getRealSize(length):
'''length looks like: 10245K'''
logger.debug(length)
if not length[-1].isalpha():
return length
units = {
"g": lambda x: x * 1024 * 1024 * 1024,
"m": lambda x: x * 1024 * 1024,
"k": lambda x: x * 1024,
}
try:
if not length[-1].isalpha():
return length
return units[length[-1].lower()](int(length[:-1]))
except:
logger.warn(linux.get_exception_stacktrace())
return length
# whether we have an upload request
if cmd.url.startswith(self.UPLOAD_PROTO):
self._prepare_upload(cmd)
rsp.size = 0
rsp.uploadPath = self._get_upload_path(req)
self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
if cmd.sendCommandUrl:
Report.url = cmd.sendCommandUrl
report = Report(cmd.threadContext, cmd.threadContextStack)
report.processType = "AddImage"
report.resourceUuid = cmd.imageUuid
report.progress_report("0", "start")
url = urlparse.urlparse(cmd.url)
if url.scheme in ('http', 'https', 'ftp'):
image_format = get_origin_format(cmd.url, True)
cmd.url = linux.shellquote(cmd.url)
# roll back tmp ceph file after import it
_1()
_, PFILE = tempfile.mkstemp()
content_length = shell.call("""curl -sLI %s|awk '/[cC]ontent-[lL]ength/{print $NF}'""" % cmd.url).splitlines()[-1]
total = _getRealSize(content_length)
def _getProgress(synced):
logger.debug("getProgress in ceph-bs-agent, synced: %s, total: %s" % (synced, total))
last = linux.tail_1(PFILE).strip()
if not last or len(last.split()) < 1 or 'HTTP request sent, awaiting response' in last:
return synced
logger.debug("last synced: %s" % last)
written = _getRealSize(last.split()[0])
if total > 0 and synced < written:
synced = written
if synced < total:
percent = int(round(float(synced) / float(total) * 90))
report.progress_report(percent, "report")
return synced
logger.debug("content-length is: %s" % total)
_, _, err = bash_progress_1('set -o pipefail;wget --no-check-certificate -O - %s 2>%s| rbd import --image-format 2 - %s/%s'
% (cmd.url, PFILE, pool, tmp_image_name), _getProgress)
if err:
raise err
actual_size = linux.get_file_size_by_http_head(cmd.url)
if os.path.exists(PFILE):
os.remove(PFILE)
elif url.scheme == 'sftp':
port = (url.port, 22)[url.port is None]
_, PFILE = tempfile.mkstemp()
pipe_path = PFILE + "fifo"
scp_to_pipe_cmd = "scp -P %d -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null %s@%s:%s %s" % (port, url.username, url.hostname, url.path, pipe_path)
sftp_command = "sftp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=no -P %s -b /dev/stdin %s@%s" % (port, url.username, url.hostname) + " <<EOF\n%s\nEOF\n"
if url.password is not None:
scp_to_pipe_cmd = 'sshpass -p %s %s' % (linux.shellquote(url.password), scp_to_pipe_cmd)
sftp_command = 'sshpass -p %s %s' % (linux.shellquote(url.password), sftp_command)
actual_size = shell.call(sftp_command % ("ls -l " + url.path)).splitlines()[1].strip().split()[4]
os.mkfifo(pipe_path)
image_format = get_origin_format(cmd.url, True)
cmd.url = linux.shellquote(cmd.url)
# roll back tmp ceph file after import it
_1()
def _get_progress(synced):
logger.debug("getProgress in add image")
if not os.path.exists(PFILE):
return synced
last = linux.tail_1(PFILE).strip()
if not last or not last.isdigit():
return synced
report.progress_report(int(last)*90/100, "report")
return synced
get_content_from_pipe_cmd = "pv -s %s -n %s 2>%s" % (actual_size, pipe_path, PFILE)
import_from_pipe_cmd = "rbd import --image-format 2 - %s/%s" % (pool, tmp_image_name)
_, _, err = bash_progress_1('set -o pipefail; %s & %s | %s' %
(scp_to_pipe_cmd, get_content_from_pipe_cmd, import_from_pipe_cmd), _get_progress)
if os.path.exists(PFILE):
os.remove(PFILE)
if os.path.exists(pipe_path):
os.remove(pipe_path)
if err:
raise err
elif url.scheme == 'file':
src_path = cmd.url.lstrip('file:')
src_path = os.path.normpath(src_path)
if not os.path.isfile(src_path):
raise Exception('cannot find the file[%s]' % src_path)
image_format = get_origin_format(src_path, True)
# roll back tmp ceph file after import it
_1()
shell.check_run("rbd import --image-format 2 %s %s/%s" % (src_path, pool, tmp_image_name))
actual_size = os.path.getsize(src_path)
else:
raise Exception('unknown url[%s]' % cmd.url)
file_format = shell.call(
"set -o pipefail; qemu-img info rbd:%s/%s | grep 'file format' | cut -d ':' -f 2" % (pool, tmp_image_name))
file_format = file_format.strip()
if file_format not in ['qcow2', 'raw']:
raise Exception('unknown image format: %s' % file_format)
if file_format == 'qcow2':
conf_path = None
try:
with open('/etc/ceph/ceph.conf', 'r') as fd:
conf = fd.read()
conf = '%s\n%s\n' % (conf, 'rbd default format = 2')
conf_path = linux.write_to_temp_file(conf)
shell.check_run('qemu-img convert -f qcow2 -O rbd rbd:%s/%s rbd:%s/%s:conf=%s' % (pool, tmp_image_name, pool, image_name, conf_path))
shell.check_run('rbd rm %s/%s' % (pool, tmp_image_name))
finally:
if conf_path:
os.remove(conf_path)
else:
shell.check_run('rbd mv %s/%s %s/%s' % (pool, tmp_image_name, pool, image_name))
report.progress_report("100", "finish")
@rollbackable
def _2():
shell.check_run('rbd rm %s/%s' % (pool, image_name))
_2()
o = shell.call('rbd --format json info %s/%s' % (pool, image_name))
image_stats = jsonobject.loads(o)
rsp.size = long(image_stats.size_)
rsp.actualSize = actual_size
if image_format == "qcow2":
rsp.format = "raw"
else:
rsp.format = image_format
self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
def download(self, req):
rsp = DownloadRsp()
def isDerivedQcow2Image(path):
if path.startswith('http://') or path.startswith('https://'):
resp = urllib2.urlopen(path)
qhdr = resp.read(72)
resp.close()
else:
resp = open(path)
qhdr = resp.read(72)
resp.close
if len(qhdr) != 72:
return False
if qhdr[:4] != 'QFI\xfb':
return False
return qhdr[16:20] != '\x00\x00\x00\00'
def fail_if_has_backing_file(fpath):
if isDerivedQcow2Image(fpath):
raise Exception('image has backing file or %s is not exist!' %
fpath)
cmd = jsonobject.loads(req[http.REQUEST_BODY])
pool, image_name = self._parse_install_path(cmd.installPath)
tmp_image_name = 'tmp-%s' % image_name
@rollbackable
def _1():
shell.call('rbd rm %s/%s' % (pool, tmp_image_name))
def _getRealSize(length):
'''length looks like: 10245K'''
logger.debug(length)
if not length[-1].isalpha():
return length
units = {
"g": lambda x: x * 1024 * 1024 * 1024,
"m": lambda x: x * 1024 * 1024,
"k": lambda x: x * 1024,
}
try:
if not length[-1].isalpha():
return length
return units[length[-1].lower()](int(length[:-1]))
except:
logger.warn(linux.get_exception_stacktrace())
return length
# whether we have an upload request
if cmd.url.startswith(self.UPLOAD_PROTO):
self._prepare_upload(cmd)
rsp.size = 0
rsp.uploadPath = self._get_upload_path(req)
self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
report = Report(cmd.threadContext, cmd.threadContextStack)
report.processType = "AddImage"
report.resourceUuid = cmd.imageUuid
report.progress_report("0", "start")
if cmd.url.startswith('http://') or cmd.url.startswith('https://'):
fail_if_has_backing_file(cmd.url)
cmd.url = linux.shellquote(cmd.url)
# roll back tmp ceph file after import it
_1()
if cmd.sendCommandUrl:
Report.url = cmd.sendCommandUrl
PFILE = shell.call('mktemp /tmp/tmp-XXXXXX').strip()
content_length = shell.call('curl -sI %s|grep Content-Length' %
cmd.url).strip().split()[1]
total = _getRealSize(content_length)
def _getProgress(synced):
logger.debug(
"getProgress in ceph-bs-agent, synced: %s, total: %s" %
(synced, total))
last = shell.call('tail -1 %s' % PFILE).strip()
if not last or len(last.split()) < 1:
return synced
logger.debug("last synced: %s" % last)
written = _getRealSize(last.split()[0])
if total > 0 and synced < written:
synced = written
if synced < total:
percent = int(round(float(synced) / float(total) * 90))
report.progress_report(percent, "report")
return synced
logger.debug("content-length is: %s" % total)
_, _, err = bash_progress_1(
'set -o pipefail;wget --no-check-certificate -O - %s 2>%s| rbd import --image-format 2 - %s/%s'
% (cmd.url, PFILE, pool, tmp_image_name), _getProgress)
if err:
raise err
actual_size = linux.get_file_size_by_http_head(cmd.url)
if os.path.exists(PFILE):
os.remove(PFILE)
elif cmd.url.startswith('file://'):
src_path = cmd.url.lstrip('file:')
src_path = os.path.normpath(src_path)
if not os.path.isfile(src_path):
raise Exception('cannot find the file[%s]' % src_path)
fail_if_has_backing_file(src_path)
# roll back tmp ceph file after import it
_1()
shell.call("rbd import --image-format 2 %s %s/%s" %
(src_path, pool, tmp_image_name))
actual_size = os.path.getsize(src_path)
else:
raise Exception('unknown url[%s]' % cmd.url)
file_format = shell.call(
"set -o pipefail; qemu-img info rbd:%s/%s | grep 'file format' | cut -d ':' -f 2"
% (pool, tmp_image_name))
file_format = file_format.strip()
if file_format not in ['qcow2', 'raw']:
raise Exception('unknown image format: %s' % file_format)
if file_format == 'qcow2':
conf_path = None
try:
with open('/etc/ceph/ceph.conf', 'r') as fd:
conf = fd.read()
conf = '%s\n%s\n' % (conf, 'rbd default format = 2')
conf_path = linux.write_to_temp_file(conf)
shell.call(
'qemu-img convert -f qcow2 -O rbd rbd:%s/%s rbd:%s/%s:conf=%s'
% (pool, tmp_image_name, pool, image_name, conf_path))
shell.call('rbd rm %s/%s' % (pool, tmp_image_name))
finally:
if conf_path:
os.remove(conf_path)
else:
shell.call('rbd mv %s/%s %s/%s' %
(pool, tmp_image_name, pool, image_name))
report.progress_report("100", "finish")
@rollbackable
def _2():
shell.call('rbd rm %s/%s' % (pool, image_name))
_2()
o = shell.call('rbd --format json info %s/%s' % (pool, image_name))
image_stats = jsonobject.loads(o)
rsp.size = long(image_stats.size_)
rsp.actualSize = actual_size
self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
def download(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
pool, image_name = self._parse_install_path(cmd.installPath)
tmp_image_name = 'tmp-%s' % image_name
if cmd.url.startswith('http://') or cmd.url.startswith('https://'):
shell.call(
'set -o pipefail; wget --no-check-certificate -q -O - %s | rbd import --image-format 2 - %s/%s'
% (cmd.url, pool, tmp_image_name))
actual_size = linux.get_file_size_by_http_head(cmd.url)
elif cmd.url.startswith('file://'):
src_path = cmd.url.lstrip('file:')
src_path = os.path.normpath(src_path)
if not os.path.isfile(src_path):
raise Exception('cannot find the file[%s]' % src_path)
shell.call("rbd import --image-format 2 %s %s/%s" %
(src_path, pool, tmp_image_name))
actual_size = os.path.getsize(src_path)
else:
raise Exception('unknown url[%s]' % cmd.url)
@rollbackable
def _1():
shell.call('rbd rm %s/%s' % (pool, tmp_image_name))
_1()
file_format = shell.call(
"set -o pipefail; qemu-img info rbd:%s/%s | grep 'file format' | cut -d ':' -f 2"
% (pool, tmp_image_name))
file_format = file_format.strip()
if file_format not in ['qcow2', 'raw']:
raise Exception('unknown image format: %s' % file_format)
if file_format == 'qcow2':
conf_path = None
try:
with open('/etc/ceph/ceph.conf', 'r') as fd:
conf = fd.read()
conf = '%s\n%s\n' % (conf, 'rbd default format = 2')
conf_path = linux.write_to_temp_file(conf)
shell.call(
'qemu-img convert -f qcow2 -O rbd rbd:%s/%s rbd:%s/%s:conf=%s'
% (pool, tmp_image_name, pool, image_name, conf_path))
shell.call('rbd rm %s/%s' % (pool, tmp_image_name))
finally:
if conf_path:
os.remove(conf_path)
else:
shell.call('rbd mv %s/%s %s/%s' %
(pool, tmp_image_name, pool, image_name))
o = shell.call('rbd --format json info %s/%s' % (pool, image_name))
image_stats = jsonobject.loads(o)
rsp = DownloadRsp()
rsp.size = long(image_stats.size_)
rsp.actualSize = actual_size
rsp.format = file_format
self._set_capacity_to_response(rsp)
return jsonobject.dumps(rsp)
def stream_body(task, fpath, entity, boundary):
def _progress_consumer(total):
task.downloadedSize = total
@thread.AsyncThread
def _do_import(task, fpath):
shell.check_run("cat %s | rbd import --image-format 2 - %s" %
(fpath, task.tmpPath))
while True:
headers = cherrypy._cpreqbody.Part.read_headers(entity.fp)
p = CustomPart(entity.fp, headers, boundary, fpath, _progress_consumer)
if not p.filename:
continue
# start consumer
_do_import(task, fpath)
try:
p.process()
except Exception as e:
logger.warn('process image %s failed: %s' %
(task.imageUuid, str(e)))
pass
finally:
if p.wfd is not None:
p.wfd.close()
break
if task.downloadedSize != task.expectedSize:
task.fail('incomplete upload, got %d, expect %d' %
(task.downloadedSize, task.expectedSize))
shell.run('rbd rm %s' % task.tmpPath)
return
file_format = None
try:
file_format = linux.get_img_fmt('rbd:' + task.tmpPath)
except Exception as e:
task.fail('upload image %s failed: %s' % (task.imageUuid, str(e)))
return
if file_format == 'qcow2':
if linux.qcow2_get_backing_file('rbd:' + task.tmpPath):
task.fail('Qcow2 image %s has backing file' % task.imageUuid)
shell.run('rbd rm %s' % task.tmpPath)
return
conf_path = None
try:
with open('/etc/ceph/ceph.conf', 'r') as fd:
conf = fd.read()
conf = '%s\n%s\n' % (conf, 'rbd default format = 2')
conf_path = linux.write_to_temp_file(conf)
shell.check_run('%s -f qcow2 -O rbd rbd:%s rbd:%s:conf=%s' %
(qemu_img.subcmd('convert'), task.tmpPath,
task.dstPath, conf_path))
except Exception as e:
task.fail('cannot convert Qcow2 image %s to rbd' % task.imageUuid)
logger.warn('convert image %s failed: %s',
(task.imageUuid, str(e)))
return
finally:
shell.run('rbd rm %s' % task.tmpPath)
if conf_path:
os.remove(conf_path)
else:
shell.check_run('rbd mv %s %s' % (task.tmpPath, task.dstPath))
task.success()
def copy_bits_to_remote(self, req):
cmd = jsonobject.loads(req[http.REQUEST_BODY])
if cmd.dstUsername != 'root':
raise Exception("cannot support migrate to non-root user host")
chain = sum([linux.qcow2_get_file_chain(p) for p in cmd.paths], [])
if cmd.sendCommandUrl:
Report.url = cmd.sendCommandUrl
report = Report(cmd.threadContext, cmd.threadContextStack)
report.processType = "LocalStorageMigrateVolume"
report.resourceUuid = cmd.volumeUuid
PFILE = shell.call('mktemp /tmp/tmp-XXXXXX').strip()
PASSWORD_FILE = linux.write_to_temp_file(cmd.dstPassword)
start = 10
end = 90
if cmd.stage:
start, end = get_scale(cmd.stage)
total = 0
for path in set(chain):
total = total + os.path.getsize(path)
written = 0
def _get_progress(synced):
logger.debug("getProgress in localstorage-agent, synced: %s, total: %s" % (synced, total))
if not os.path.exists(PFILE):
return synced
fpread = open(PFILE, 'r')
lines = fpread.readlines()
if not lines:
fpread.close()
return synced
last = str(lines[-1]).strip().split('\r')[-1]
if not last or len(last.split()) < 1:
fpread.close()
return synced
line = last.split()[0]
if not line.isdigit():
return synced
if total > 0:
synced = long(line)
if synced < total:
percent = int(round(float(written + synced) / float(total) * (end - start) + start))
report.progress_report(percent, "report")
synced = written
fpread.close()
return synced
for path in set(chain):
PATH = path
USER = cmd.dstUsername
IP = cmd.dstIp
PORT = (cmd.dstPort and cmd.dstPort or "22")
DIR = os.path.dirname(path)
_, _, err = bash_progress_1(
# Fixes ZSTAC-13430: handle extremely complex password like ~ ` !@#$%^&*()_+-=[]{}|?<>;:'"/ .
'rsync -av --progress --relative {{PATH}} --rsh="/usr/bin/sshpass -f{{PASSWORD_FILE}} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p {{PORT}} -l {{USER}}" {{IP}}:/ 1>{{PFILE}}', _get_progress, False)
if err:
linux.rm_file_force(PASSWORD_FILE)
linux.rm_file_force(PFILE)
raise Exception('fail to migrate vm to host, because %s' % str(err))
written += os.path.getsize(path)
bash_errorout('/usr/bin/sshpass -f{{PASSWORD_FILE}} ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -p {{PORT}} {{USER}}@{{IP}} "/bin/sync {{PATH}}"')
percent = int(round(float(written) / float(total) * (end - start) + start))
report.progress_report(percent, "report")
linux.rm_file_force(PASSWORD_FILE)
linux.rm_file_force(PFILE)
rsp = AgentResponse()
rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.storagePath)
return jsonobject.dumps(rsp)
def do_sftp_download(self, cmd, pool, image_name):
hostname = cmd.hostname
prikey = cmd.sshKey
port = cmd.sshPort
if cmd.bandWidth is not None:
bandWidth = 'pv -q -L %s |' % cmd.bandWidth
else:
bandWidth = ''
tmp_image_name = 'tmp-%s' % image_name
prikey_file = linux.write_to_temp_file(prikey)
@rollbackable
def _0():
tpath = "%s/%s" % (pool, tmp_image_name)
shell.call('rbd info %s > /dev/null && rbd rm %s' % (tpath, tpath))
_0()
def rbd_check_rm(pool, name):
if shell.run('rbd info %s/%s' % (pool, name)) == 0:
shell.check_run('rbd rm %s/%s' % (pool, name))
try:
rbd_check_rm(pool, tmp_image_name)
shell.call(
self._wrap_shareable_cmd(
cmd,
'set -o pipefail; ssh -p %d -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i %s root@%s cat %s | %s rbd import --image-format 2 - %s/%s'
% (port, prikey_file, hostname,
remote_shell_quote(cmd.backupStorageInstallPath),
bandWidth, pool, tmp_image_name)))
finally:
os.remove(prikey_file)
@rollbackable
def _1():
shell.call('rbd rm %s/%s' % (pool, tmp_image_name))
_1()
file_format = shell.call(
"set -o pipefail; %s rbd:%s/%s | grep 'file format' | cut -d ':' -f 2"
% (qemu_img.subcmd('info'), pool, tmp_image_name))
file_format = file_format.strip()
if file_format not in ['qcow2', 'raw']:
raise Exception('unknown image format: %s' % file_format)
rbd_check_rm(pool, image_name)
if file_format == 'qcow2':
conf_path = None
try:
with open('/etc/ceph/ceph.conf', 'r') as fd:
conf = fd.read()
conf = '%s\n%s\n' % (conf, 'rbd default format = 2')
conf_path = linux.write_to_temp_file(conf)
shell.call('%s -f qcow2 -O rbd rbd:%s/%s rbd:%s/%s:conf=%s' %
(qemu_img.subcmd('convert'), pool, tmp_image_name,
pool, image_name, conf_path))
shell.call('rbd rm %s/%s' % (pool, tmp_image_name))
finally:
if conf_path:
os.remove(conf_path)
else:
shell.call('rbd mv %s/%s %s/%s' %
(pool, tmp_image_name, pool, image_name))
