def runSshCmd(libvirtURI, keystr, cmdstr): target, port = getSshTargetAndPort(libvirtURI) ssh_opts = DEF_SSH_OPTS ssh_cmd = "ssh" if not os.path.exists(V2V_PRIV_KEY) else "ssh -i {}".format(V2V_PRIV_KEY) if not keystr: return shell.check_run( "{} {} -p {} {} {}".format( ssh_cmd, target, port, DEF_SSH_OPTS, linux.shellquote(cmdstr))) tmpkeyfile = None with tempfile.NamedTemporaryFile(delete=False) as f: f.write(keystr) tmpkeyfile = f.name try: return shell.check_run("ssh {} -p {} {} -i {} {}".format( target, port, DEF_SSH_OPTS, tmpkeyfile, linux.shellquote(cmdstr))) finally: os.remove(tmpkeyfile)
def qcow2_clone_with_option(src, dst, opt=""): # NOTE(weiw): qcow2 doesn't support specify backing file and preallocation at same time pattern = re.compile("\-o\ preallocation\=\w+ ") opt = re.sub(pattern, " ", opt) fmt = get_img_fmt(src) shell.check_run('/usr/bin/qemu-img create -F %s %s -b %s -f qcow2 %s' % (fmt, opt, src, dst)) shell.check_run('chmod 666 %s' % dst)
def upload_image(self, hostname, fpath): imf = self.commit_image(fpath) cmdstr = '%s -url %s:%s push %s' % (self.ZSTORE_CLI_PATH, hostname, self.ZSTORE_DEF_PORT, fpath) logger.debug('pushing %s to image store' % fpath) shell.check_run(cmdstr) logger.debug('%s pushed to image store' % fpath) return imf
def upload_image(self, hostname, fpath): imf = self.commit_image(fpath) cmdstr = '%s -url %s:%s push %s' % (self.ZSTORE_CLI_PATH, hostname, self.ZSTORE_DEF_PORT, fpath) logger.debug('pushing %s to image store' % fpath) shell.check_run(cmdstr) logger.debug('%s pushed to image store' % fpath) return imf
def add_interface_to_bridge(self, req): cmd = jsonobject.loads(req[http.REQUEST_BODY]) rsp = kvmagent.AgentResponse() oldbr = shell.call("""brctl show | awk '$4 == "%s" {print $1}'""" % cmd.physicalInterfaceName).strip() if oldbr == cmd.bridgeName: return jsonobject.dumps(rsp) if oldbr: shell.run("brctl delif %s %s" % (oldbr, cmd.physicalInterfaceName)) shell.check_run("brctl addif %s %s" % (cmd.bridgeName, cmd.physicalInterfaceName)) return jsonobject.dumps(rsp)
def init(self, req): cmd = jsonobject.loads(req[http.REQUEST_BODY]) o = shell.call('ceph mon_status') mon_status = jsonobject.loads(o) fsid = mon_status.monmap.fsid_ existing_pools = shell.call('ceph osd lspools') for pool in cmd.pools: if pool.predefined and pool.name not in existing_pools: raise Exception('cannot find pool[%s] in the ceph cluster, you must create it manually' % pool.name) elif pool.name not in existing_pools: shell.check_run('ceph osd pool create %s 128' % pool.name) rsp = InitRsp() rsp.fsid = fsid self._set_capacity_to_response(rsp) return jsonobject.dumps(rsp)
def init(self, req): cmd = jsonobject.loads(req[http.REQUEST_BODY]) o = shell.call('ceph mon_status') mon_status = jsonobject.loads(o) fsid = mon_status.monmap.fsid_ existing_pools = shell.call('ceph osd lspools') for pool in cmd.pools: if pool.predefined and pool.name not in existing_pools: raise Exception('cannot find pool[%s] in the ceph cluster, you must create it manually' % pool.name) elif pool.name not in existing_pools: shell.check_run('ceph osd pool create %s 128' % pool.name) rsp = InitRsp() rsp.fsid = fsid self._set_capacity_to_response(rsp) return jsonobject.dumps(rsp)
def init(self, req): rsp = AgentRsp() cmd = jsonobject.loads(req[http.REQUEST_BODY]) spath = None if cmd.storagePath: spath = getRealStoragePath(cmd.storagePath) linux.mkdir(spath) with open('/etc/exports.d/zs-v2v.exports', 'w') as f: f.write("{} *(rw,sync,no_root_squash)\n".format(spath)) shell.check_run('systemctl restart nfs-server') if spath is not None: fstype = shell.call("""stat -f -c '%T' {}""".format(spath)).strip() if fstype not in [ "xfs", "ext2", "ext3", "ext4", "jfs", "btrfs" ]: raise Exception("unexpected fstype '{}' on '{}'".format(fstype, cmd.storagePath)) return jsonobject.dumps(rsp)
def delete_bridge(bridge_name): vifs = get_all_bridge_interface(bridge_name) for vif in vifs: shell.check_run("brctl delif %s %s" % (bridge_name, vif)) shell.check_run("ip link set %s down" % bridge_name) shell.check_run("brctl delbr %s" % bridge_name)
def listvm(self, req): cmd = jsonobject.loads(req[http.REQUEST_BODY]) rsp = ListVmRsp() if cmd.sshPassword and not cmd.sshPrivKey: target, port = getSshTargetAndPort(cmd.libvirtURI) ssh_pswd_file = linux.write_to_temp_file(cmd.sshPassword) if not os.path.exists(V2V_PRIV_KEY) or not os.path.exists(V2V_PUB_KEY): shell.check_run("yes | ssh-keygen -t rsa -N '' -f {}".format(V2V_PRIV_KEY)) cmdstr = "HOME={4} timeout 30 sshpass -f {0} ssh-copy-id -i {5} -p {1} {2} {3}".format( ssh_pswd_file, port, DEF_SSH_OPTS, target, os.path.expanduser("~"), V2V_PUB_KEY) shell.check_run(cmdstr) linux.rm_file_force(ssh_pswd_file) rsp.qemuVersion, rsp.libvirtVersion, rsp.vms, rsp.v2vCaps = listVirtualMachines(cmd.libvirtURI, cmd.saslUser, cmd.saslPass, cmd.sshPrivKey) return jsonobject.dumps(rsp)
def delete_image(_): # in case image is deleted, we don't have to wait for timeout img = "%s/%s" % (pool, image_name) shell.check_run('rbd info %s && rbd rm %s' % (img, img)) return True
def qcow2_create_with_option(dst, size, opt=""): shell.check_run('/usr/bin/qemu-img create -f qcow2 %s %s %s' % (opt, dst, size)) shell.check_run('chmod 666 %s' % dst)
def activate_and_remove(f): active_lv(f, shared=False) backing = linux.qcow2_get_backing_file(f) shell.check_run("lvremove -y -Stags={%s} %s" % (tag, f)) return f
def stream_body(task, fpath, entity, boundary): def _progress_consumer(total): task.downloadedSize = total @thread.AsyncThread def _do_import(task, fpath): shell.check_run("cat %s | rbd import --image-format 2 - %s" % (fpath, task.tmpPath)) while True: headers = cherrypy._cpreqbody.Part.read_headers(entity.fp) p = CustomPart(entity.fp, headers, boundary, fpath, _progress_consumer) if not p.filename: continue # start consumer _do_import(task, fpath) try: p.process() except Exception as e: logger.warn('process image %s failed: %s' % (task.imageUuid, str(e))) pass finally: if p.wfd is not None: p.wfd.close() break if task.downloadedSize != task.expectedSize: task.fail('incomplete upload, got %d, expect %d' % (task.downloadedSize, task.expectedSize)) shell.run('rbd rm %s' % task.tmpPath) return file_format = None try: file_format = linux.get_img_fmt('rbd:'+task.tmpPath) except Exception as e: task.fail('upload image %s failed: %s' % (task.imageUuid, str(e))) return if file_format == 'qcow2': if linux.qcow2_get_backing_file('rbd:'+task.tmpPath): task.fail('Qcow2 image %s has backing file' % task.imageUuid) shell.run('rbd rm %s' % task.tmpPath) return conf_path = None try: with open('/etc/ceph/ceph.conf', 'r') as fd: conf = fd.read() conf = '%s\n%s\n' % (conf, 'rbd default format = 2') conf_path = linux.write_to_temp_file(conf) shell.check_run('qemu-img convert -f qcow2 -O rbd rbd:%s rbd:%s:conf=%s' % (task.tmpPath, task.dstPath, conf_path)) except Exception as e: task.fail('cannot convert Qcow2 image %s to rbd' % task.imageUuid) logger.warn('convert image %s failed: %s', (task.imageUuid, str(e))) return finally: shell.run('rbd rm %s' % task.tmpPath) if conf_path: os.remove(conf_path) else: shell.check_run('rbd mv %s %s' % (task.tmpPath, task.dstPath)) task.success()
def download(self, req): rsp = DownloadRsp() def _get_origin_format(path): qcow2_length = 0x9007 if path.startswith('http://') or path.startswith( 'https://') or path.startswith('ftp://'): resp = urllib2.urlopen(path) qhdr = resp.read(qcow2_length) resp.close() elif path.startswith('sftp://'): fd, tmp_file = tempfile.mkstemp() get_header_from_pipe_cmd = "timeout 60 head --bytes=%d %s > %s" % ( qcow2_length, pipe_path, tmp_file) clean_cmd = "pkill -f %s" % pipe_path shell.run( '%s & %s && %s' % (scp_to_pipe_cmd, get_header_from_pipe_cmd, clean_cmd)) qhdr = os.read(fd, qcow2_length) if os.path.exists(tmp_file): os.remove(tmp_file) else: resp = open(path) qhdr = resp.read(qcow2_length) resp.close() if len(qhdr) < qcow2_length: return "raw" return get_image_format_from_buf(qhdr) def get_origin_format(fpath, fail_if_has_backing_file=True): image_format = _get_origin_format(fpath) if image_format == "derivedQcow2" and fail_if_has_backing_file: raise Exception('image has backing file or %s is not exist!' % fpath) return image_format cmd = jsonobject.loads(req[http.REQUEST_BODY]) pool, image_name = self._parse_install_path(cmd.installPath) tmp_image_name = 'tmp-%s' % image_name @rollbackable def _1(): shell.check_run('rbd rm %s/%s' % (pool, tmp_image_name)) def _getRealSize(length): '''length looks like: 10245K''' logger.debug(length) if not length[-1].isalpha(): return length units = { "g": lambda x: x * 1024 * 1024 * 1024, "m": lambda x: x * 1024 * 1024, "k": lambda x: x * 1024, } try: if not length[-1].isalpha(): return length return units[length[-1].lower()](int(length[:-1])) except: logger.warn(linux.get_exception_stacktrace()) return length # whether we have an upload request if cmd.url.startswith(self.UPLOAD_PROTO): self._prepare_upload(cmd) rsp.size = 0 rsp.uploadPath = self._get_upload_path(req) self._set_capacity_to_response(rsp) return jsonobject.dumps(rsp) if cmd.sendCommandUrl: Report.url = cmd.sendCommandUrl report = Report(cmd.threadContext, cmd.threadContextStack) report.processType = "AddImage" report.resourceUuid = cmd.imageUuid report.progress_report("0", "start") url = urlparse.urlparse(cmd.url) if url.scheme in ('http', 'https', 'ftp'): image_format = get_origin_format(cmd.url, True) cmd.url = linux.shellquote(cmd.url) # roll back tmp ceph file after import it _1() _, PFILE = tempfile.mkstemp() content_length = shell.call('curl -sI %s|grep Content-Length' % cmd.url).strip().split()[1] total = _getRealSize(content_length) def _getProgress(synced): logger.debug( "getProgress in ceph-bs-agent, synced: %s, total: %s" % (synced, total)) last = shell.call('tail -1 %s' % PFILE).strip() if not last or len(last.split()) < 1: return synced logger.debug("last synced: %s" % last) written = _getRealSize(last.split()[0]) if total > 0 and synced < written: synced = written if synced < total: percent = int(round(float(synced) / float(total) * 90)) report.progress_report(percent, "report") return synced logger.debug("content-length is: %s" % total) _, _, err = bash_progress_1( 'set -o pipefail;wget --no-check-certificate -O - %s 2>%s| rbd import --image-format 2 - %s/%s' % (cmd.url, PFILE, pool, tmp_image_name), _getProgress) if err: raise err actual_size = linux.get_file_size_by_http_head(cmd.url) if os.path.exists(PFILE): os.remove(PFILE) elif url.scheme == 'sftp': port = (url.port, 22)[url.port is None] _, PFILE = tempfile.mkstemp() pipe_path = PFILE + "fifo" scp_to_pipe_cmd = "scp -P %d -o StrictHostKeyChecking=no %s@%s:%s %s" % ( port, url.username, url.hostname, url.path, pipe_path) sftp_command = "sftp -o StrictHostKeyChecking=no -o BatchMode=no -P %s -b /dev/stdin %s@%s" % ( port, url.username, url.hostname) + " <<EOF\n%s\nEOF\n" if url.password is not None: scp_to_pipe_cmd = 'sshpass -p "%s" %s' % (url.password, scp_to_pipe_cmd) sftp_command = 'sshpass -p "%s" %s' % (url.password, sftp_command) actual_size = shell.call( sftp_command % ("ls -l " + url.path)).splitlines()[1].strip().split()[4] os.mkfifo(pipe_path) image_format = get_origin_format(cmd.url, True) cmd.url = linux.shellquote(cmd.url) # roll back tmp ceph file after import it _1() def _get_progress(synced): logger.debug("getProgress in add image") if not os.path.exists(PFILE): return synced last = shell.call('tail -1 %s' % PFILE).strip() if not last or not last.isdigit(): return synced report.progress_report(int(last) * 90 / 100, "report") return synced get_content_from_pipe_cmd = "pv -s %s -n %s 2>%s" % ( actual_size, pipe_path, PFILE) import_from_pipe_cmd = "rbd import --image-format 2 - %s/%s" % ( pool, tmp_image_name) _, _, err = bash_progress_1( 'set -o pipefail; %s & %s | %s' % (scp_to_pipe_cmd, get_content_from_pipe_cmd, import_from_pipe_cmd), _get_progress) if os.path.exists(PFILE): os.remove(PFILE) if os.path.exists(pipe_path): os.remove(pipe_path) if err: raise err elif url.scheme == 'file': src_path = cmd.url.lstrip('file:') src_path = os.path.normpath(src_path) if not os.path.isfile(src_path): raise Exception('cannot find the file[%s]' % src_path) image_format = get_origin_format(src_path, True) # roll back tmp ceph file after import it _1() shell.check_run("rbd import --image-format 2 %s %s/%s" % (src_path, pool, tmp_image_name)) actual_size = os.path.getsize(src_path) else: raise Exception('unknown url[%s]' % cmd.url) file_format = shell.call( "set -o pipefail; qemu-img info rbd:%s/%s | grep 'file format' | cut -d ':' -f 2" % (pool, tmp_image_name)) file_format = file_format.strip() if file_format not in ['qcow2', 'raw']: raise Exception('unknown image format: %s' % file_format) if file_format == 'qcow2': conf_path = None try: with open('/etc/ceph/ceph.conf', 'r') as fd: conf = fd.read() conf = '%s\n%s\n' % (conf, 'rbd default format = 2') conf_path = linux.write_to_temp_file(conf) shell.check_run( 'qemu-img convert -f qcow2 -O rbd rbd:%s/%s rbd:%s/%s:conf=%s' % (pool, tmp_image_name, pool, image_name, conf_path)) shell.check_run('rbd rm %s/%s' % (pool, tmp_image_name)) finally: if conf_path: os.remove(conf_path) else: shell.check_run('rbd mv %s/%s %s/%s' % (pool, tmp_image_name, pool, image_name)) report.progress_report("100", "finish") @rollbackable def _2(): shell.check_run('rbd rm %s/%s' % (pool, image_name)) _2() o = shell.call('rbd --format json info %s/%s' % (pool, image_name)) image_stats = jsonobject.loads(o) rsp.size = long(image_stats.size_) rsp.actualSize = actual_size if image_format == "qcow2": rsp.format = "raw" else: rsp.format = image_format self._set_capacity_to_response(rsp) return jsonobject.dumps(rsp)
def _2(): shell.check_run('rbd rm %s/%s' % (pool, image_name))
def rbd_check_rm(pool, name): if shell.run('rbd info %s/%s' % (pool, name)) == 0: shell.check_run('rbd rm %s/%s' % (pool, name))
def download(self, req): rsp = DownloadRsp() def _get_origin_format(path): qcow2_length = 0x9007 if path.startswith('http://') or path.startswith('https://') or path.startswith('ftp://'): resp = urllib2.urlopen(path) qhdr = resp.read(qcow2_length) resp.close() elif path.startswith('sftp://'): fd, tmp_file = tempfile.mkstemp() get_header_from_pipe_cmd = "timeout 60 head --bytes=%d %s > %s" % (qcow2_length, pipe_path, tmp_file) clean_cmd = "pkill -f %s" % pipe_path shell.run('%s & %s && %s' % (scp_to_pipe_cmd, get_header_from_pipe_cmd, clean_cmd)) qhdr = os.read(fd, qcow2_length) if os.path.exists(tmp_file): os.remove(tmp_file) else: resp = open(path) qhdr = resp.read(qcow2_length) resp.close() if len(qhdr) < qcow2_length: return "raw" return get_image_format_from_buf(qhdr) def get_origin_format(fpath, fail_if_has_backing_file=True): image_format = _get_origin_format(fpath) if image_format == "derivedQcow2" and fail_if_has_backing_file: raise Exception('image has backing file or %s is not exist!' % fpath) return image_format cmd = jsonobject.loads(req[http.REQUEST_BODY]) pool, image_name = self._parse_install_path(cmd.installPath) tmp_image_name = 'tmp-%s' % image_name @rollbackable def _1(): shell.check_run('rbd rm %s/%s' % (pool, tmp_image_name)) def _getRealSize(length): '''length looks like: 10245K''' logger.debug(length) if not length[-1].isalpha(): return length units = { "g": lambda x: x * 1024 * 1024 * 1024, "m": lambda x: x * 1024 * 1024, "k": lambda x: x * 1024, } try: if not length[-1].isalpha(): return length return units[length[-1].lower()](int(length[:-1])) except: logger.warn(linux.get_exception_stacktrace()) return length # whether we have an upload request if cmd.url.startswith(self.UPLOAD_PROTO): self._prepare_upload(cmd) rsp.size = 0 rsp.uploadPath = self._get_upload_path(req) self._set_capacity_to_response(rsp) return jsonobject.dumps(rsp) if cmd.sendCommandUrl: Report.url = cmd.sendCommandUrl report = Report(cmd.threadContext, cmd.threadContextStack) report.processType = "AddImage" report.resourceUuid = cmd.imageUuid report.progress_report("0", "start") url = urlparse.urlparse(cmd.url) if url.scheme in ('http', 'https', 'ftp'): image_format = get_origin_format(cmd.url, True) cmd.url = linux.shellquote(cmd.url) # roll back tmp ceph file after import it _1() _, PFILE = tempfile.mkstemp() content_length = shell.call("""curl -sLI %s|awk '/[cC]ontent-[lL]ength/{print $NF}'""" % cmd.url).splitlines()[-1] total = _getRealSize(content_length) def _getProgress(synced): logger.debug("getProgress in ceph-bs-agent, synced: %s, total: %s" % (synced, total)) last = linux.tail_1(PFILE).strip() if not last or len(last.split()) < 1 or 'HTTP request sent, awaiting response' in last: return synced logger.debug("last synced: %s" % last) written = _getRealSize(last.split()[0]) if total > 0 and synced < written: synced = written if synced < total: percent = int(round(float(synced) / float(total) * 90)) report.progress_report(percent, "report") return synced logger.debug("content-length is: %s" % total) _, _, err = bash_progress_1('set -o pipefail;wget --no-check-certificate -O - %s 2>%s| rbd import --image-format 2 - %s/%s' % (cmd.url, PFILE, pool, tmp_image_name), _getProgress) if err: raise err actual_size = linux.get_file_size_by_http_head(cmd.url) if os.path.exists(PFILE): os.remove(PFILE) elif url.scheme == 'sftp': port = (url.port, 22)[url.port is None] _, PFILE = tempfile.mkstemp() pipe_path = PFILE + "fifo" scp_to_pipe_cmd = "scp -P %d -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null %s@%s:%s %s" % (port, url.username, url.hostname, url.path, pipe_path) sftp_command = "sftp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o BatchMode=no -P %s -b /dev/stdin %s@%s" % (port, url.username, url.hostname) + " <<EOF\n%s\nEOF\n" if url.password is not None: scp_to_pipe_cmd = 'sshpass -p %s %s' % (linux.shellquote(url.password), scp_to_pipe_cmd) sftp_command = 'sshpass -p %s %s' % (linux.shellquote(url.password), sftp_command) actual_size = shell.call(sftp_command % ("ls -l " + url.path)).splitlines()[1].strip().split()[4] os.mkfifo(pipe_path) image_format = get_origin_format(cmd.url, True) cmd.url = linux.shellquote(cmd.url) # roll back tmp ceph file after import it _1() def _get_progress(synced): logger.debug("getProgress in add image") if not os.path.exists(PFILE): return synced last = linux.tail_1(PFILE).strip() if not last or not last.isdigit(): return synced report.progress_report(int(last)*90/100, "report") return synced get_content_from_pipe_cmd = "pv -s %s -n %s 2>%s" % (actual_size, pipe_path, PFILE) import_from_pipe_cmd = "rbd import --image-format 2 - %s/%s" % (pool, tmp_image_name) _, _, err = bash_progress_1('set -o pipefail; %s & %s | %s' % (scp_to_pipe_cmd, get_content_from_pipe_cmd, import_from_pipe_cmd), _get_progress) if os.path.exists(PFILE): os.remove(PFILE) if os.path.exists(pipe_path): os.remove(pipe_path) if err: raise err elif url.scheme == 'file': src_path = cmd.url.lstrip('file:') src_path = os.path.normpath(src_path) if not os.path.isfile(src_path): raise Exception('cannot find the file[%s]' % src_path) image_format = get_origin_format(src_path, True) # roll back tmp ceph file after import it _1() shell.check_run("rbd import --image-format 2 %s %s/%s" % (src_path, pool, tmp_image_name)) actual_size = os.path.getsize(src_path) else: raise Exception('unknown url[%s]' % cmd.url) file_format = shell.call( "set -o pipefail; qemu-img info rbd:%s/%s | grep 'file format' | cut -d ':' -f 2" % (pool, tmp_image_name)) file_format = file_format.strip() if file_format not in ['qcow2', 'raw']: raise Exception('unknown image format: %s' % file_format) if file_format == 'qcow2': conf_path = None try: with open('/etc/ceph/ceph.conf', 'r') as fd: conf = fd.read() conf = '%s\n%s\n' % (conf, 'rbd default format = 2') conf_path = linux.write_to_temp_file(conf) shell.check_run('qemu-img convert -f qcow2 -O rbd rbd:%s/%s rbd:%s/%s:conf=%s' % (pool, tmp_image_name, pool, image_name, conf_path)) shell.check_run('rbd rm %s/%s' % (pool, tmp_image_name)) finally: if conf_path: os.remove(conf_path) else: shell.check_run('rbd mv %s/%s %s/%s' % (pool, tmp_image_name, pool, image_name)) report.progress_report("100", "finish") @rollbackable def _2(): shell.check_run('rbd rm %s/%s' % (pool, image_name)) _2() o = shell.call('rbd --format json info %s/%s' % (pool, image_name)) image_stats = jsonobject.loads(o) rsp.size = long(image_stats.size_) rsp.actualSize = actual_size if image_format == "qcow2": rsp.format = "raw" else: rsp.format = image_format self._set_capacity_to_response(rsp) return jsonobject.dumps(rsp)
def _2(): shell.check_run('rbd rm %s/%s' % (pool, image_name))
def _1(): shell.check_run('rbd rm %s/%s' % (pool, tmp_image_name))
def raw_create(dst, size): shell.check_run('/usr/bin/qemu-img create -f raw %s %s' % (dst, size)) shell.check_run('chmod 666 %s' % dst)
def raw_clone(src, dst): shell.check_run('/usr/bin/qemu-img create -b %s -f raw %s' % (src, dst)) shell.check_run('chmod 666 %s' % dst)
def qcow2_clone(src, dst): fmt = get_img_fmt(src) shell.check_run('/usr/bin/qemu-img create -F %s -b %s -f qcow2 %s' % (fmt, src, dst)) shell.check_run('chmod 666 %s' % dst)
def stream_body(task, fpath, entity, boundary): def _progress_consumer(total): task.downloadedSize = total @thread.AsyncThread def _do_import(task, fpath): shell.check_run("cat %s | rbd import --image-format 2 - %s" % (fpath, task.tmpPath)) while True: headers = cherrypy._cpreqbody.Part.read_headers(entity.fp) p = CustomPart(entity.fp, headers, boundary, fpath, _progress_consumer) if not p.filename: continue # start consumer _do_import(task, fpath) try: p.process() except Exception as e: logger.warn('process image %s failed: %s' % (task.imageUuid, str(e))) pass finally: if p.wfd is not None: p.wfd.close() break if task.downloadedSize != task.expectedSize: task.fail('incomplete upload, got %d, expect %d' % (task.downloadedSize, task.expectedSize)) shell.run('rbd rm %s' % task.tmpPath) return file_format = None qcow2_length = 0x9007 image_format = "raw" image_header_file = "tmp/%s_header" % task.imageUuid shell.check_run("rbd export %s - | head -%d > %s" % (task.tmpPath, qcow2_length, image_header_file)) fd = open(image_header_file) qhdr = fd.read(qcow2_length) fd.close() linux.rm_file_force(image_header_file) if len(qhdr) >= qcow2_length: image_format = get_image_format_from_buf(qhdr) try: file_format = linux.get_img_fmt('rbd:' + task.tmpPath) except Exception as e: task.fail('upload image %s failed: %s' % (task.imageUuid, str(e))) return if file_format == 'qcow2': if linux.qcow2_get_backing_file('rbd:' + task.tmpPath): task.fail('Qcow2 image %s has backing file' % task.imageUuid) shell.run('rbd rm %s' % task.tmpPath) return conf_path = None try: with open('/etc/ceph/ceph.conf', 'r') as fd: conf = fd.read() conf = '%s\n%s\n' % (conf, 'rbd default format = 2') conf_path = linux.write_to_temp_file(conf) shell.check_run( 'qemu-img convert -f qcow2 -O rbd rbd:%s rbd:%s:conf=%s' % (task.tmpPath, task.dstPath, conf_path)) shell.check_run('rbd rm %s' % task.tmpPath) finally: if conf_path: os.remove(conf_path) image_format = "raw" else: shell.check_run('rbd mv %s %s' % (task.tmpPath, task.dstPath)) task.image_format = image_format task.success()
def download(self, req): rsp = DownloadRsp() def isDerivedQcow2Image(path): return getOriginalFormat(path) == "derivedQcow2" def getOriginalFormat(path): if path.startswith('http://') or path.startswith('https://'): resp = urllib2.urlopen(path) qhdr = resp.read(0x9007) resp.close() else: resp = open(path) qhdr = resp.read(0x9007) resp.close() if len(qhdr) < 0x9007: return "raw" if qhdr[:4] == 'QFI\xfb': if qhdr[16:20] == '\x00\x00\x00\00': return "qcow2" else: return "derivedQcow2" if qhdr[0x8001:0x8006] == 'CD001': return 'iso' if qhdr[0x8801:0x8806] == 'CD001': return 'iso' if qhdr[0x9001:0x9006] == 'CD001': return 'iso' return "raw" def fail_if_has_backing_file(fpath): if isDerivedQcow2Image(fpath): raise Exception('image has backing file or %s is not exist!' % fpath) cmd = jsonobject.loads(req[http.REQUEST_BODY]) pool, image_name = self._parse_install_path(cmd.installPath) tmp_image_name = 'tmp-%s' % image_name @rollbackable def _1(): shell.check_run('rbd rm %s/%s' % (pool, tmp_image_name)) def _getRealSize(length): '''length looks like: 10245K''' logger.debug(length) if not length[-1].isalpha(): return length units = { "g": lambda x: x * 1024 * 1024 * 1024, "m": lambda x: x * 1024 * 1024, "k": lambda x: x * 1024, } try: if not length[-1].isalpha(): return length return units[length[-1].lower()](int(length[:-1])) except: logger.warn(linux.get_exception_stacktrace()) return length # whether we have an upload request if cmd.url.startswith(self.UPLOAD_PROTO): self._prepare_upload(cmd) rsp.size = 0 rsp.uploadPath = self._get_upload_path(req) self._set_capacity_to_response(rsp) return jsonobject.dumps(rsp) if cmd.sendCommandUrl: Report.url = cmd.sendCommandUrl report = Report(cmd.threadContext, cmd.threadContextStack) report.processType = "AddImage" report.resourceUuid = cmd.imageUuid report.progress_report("0", "start") if cmd.url.startswith('http://') or cmd.url.startswith('https://'): fail_if_has_backing_file(cmd.url) image_format = getOriginalFormat(cmd.url) cmd.url = linux.shellquote(cmd.url) # roll back tmp ceph file after import it _1() PFILE = shell.call('mktemp /tmp/tmp-XXXXXX').strip() content_length = shell.call('curl -sI %s|grep Content-Length' % cmd.url).strip().split()[1] total = _getRealSize(content_length) def _getProgress(synced): logger.debug("getProgress in ceph-bs-agent, synced: %s, total: %s" % (synced, total)) last = shell.call('tail -1 %s' % PFILE).strip() if not last or len(last.split()) < 1: return synced logger.debug("last synced: %s" % last) written = _getRealSize(last.split()[0]) if total > 0 and synced < written: synced = written if synced < total: percent = int(round(float(synced) / float(total) * 90)) report.progress_report(percent, "report") return synced logger.debug("content-length is: %s" % total) _, _, err = bash_progress_1('set -o pipefail;wget --no-check-certificate -O - %s 2>%s| rbd import --image-format 2 - %s/%s' % (cmd.url, PFILE, pool, tmp_image_name), _getProgress) if err: raise err actual_size = linux.get_file_size_by_http_head(cmd.url) if os.path.exists(PFILE): os.remove(PFILE) elif cmd.url.startswith('file://'): src_path = cmd.url.lstrip('file:') src_path = os.path.normpath(src_path) if not os.path.isfile(src_path): raise Exception('cannot find the file[%s]' % src_path) fail_if_has_backing_file(src_path) # roll back tmp ceph file after import it _1() image_format = getOriginalFormat(src_path) shell.check_run("rbd import --image-format 2 %s %s/%s" % (src_path, pool, tmp_image_name)) actual_size = os.path.getsize(src_path) else: raise Exception('unknown url[%s]' % cmd.url) file_format = shell.call( "set -o pipefail; qemu-img info rbd:%s/%s | grep 'file format' | cut -d ':' -f 2" % (pool, tmp_image_name)) file_format = file_format.strip() if file_format not in ['qcow2', 'raw']: raise Exception('unknown image format: %s' % file_format) if file_format == 'qcow2': conf_path = None try: with open('/etc/ceph/ceph.conf', 'r') as fd: conf = fd.read() conf = '%s\n%s\n' % (conf, 'rbd default format = 2') conf_path = linux.write_to_temp_file(conf) shell.check_run('qemu-img convert -f qcow2 -O rbd rbd:%s/%s rbd:%s/%s:conf=%s' % (pool, tmp_image_name, pool, image_name, conf_path)) shell.check_run('rbd rm %s/%s' % (pool, tmp_image_name)) finally: if conf_path: os.remove(conf_path) else: shell.check_run('rbd mv %s/%s %s/%s' % (pool, tmp_image_name, pool, image_name)) report.progress_report("100", "finish") @rollbackable def _2(): shell.check_run('rbd rm %s/%s' % (pool, image_name)) _2() o = shell.call('rbd --format json info %s/%s' % (pool, image_name)) image_stats = jsonobject.loads(o) rsp.size = long(image_stats.size_) rsp.actualSize = actual_size if image_format == "qcow2": rsp.format = "raw" else: rsp.format = image_format self._set_capacity_to_response(rsp) return jsonobject.dumps(rsp)
def _do_import(task, fpath): shell.check_run("cat %s | rbd import --image-format 2 - %s" % (fpath, task.tmpPath))
def _do_import(task, fpath): shell.check_run("cat %s | rbd import --image-format 2 - %s" % (fpath, task.tmpPath))
def _1(): shell.check_run('rbd rm %s/%s' % (pool, tmp_image_name))
def delete_image(_): # in case image is deleted, we don't have to wait for timeout img = "%s/%s" % (pool, image_name) shell.check_run('rbd info %s && rbd rm %s' % (img, img)) return True
def activate_and_remove(f): active_lv(f, shared=False) backing = linux.qcow2_get_backing_file(f) shell.check_run("lvremove -y -Stags={%s} %s" % (tag, f)) return f
def qcow2_clone_with_option(src, dst, opt=""): fmt = get_img_fmt(src) shell.check_run('/usr/bin/qemu-img create -F %s %s -b %s -f qcow2 %s' % (fmt, opt, src, dst)) shell.check_run('chmod 666 %s' % dst)