def wait_iscsi_mknode(iscsiServerIp, iscsiServerPort, iscsiIqn, e=None): disks_by_dev = bash.bash_o( "ls /dev/disk/by-path | grep %s:%s | grep %s" % (iscsiServerIp, iscsiServerPort, iscsiIqn)).strip().splitlines() sid = bash.bash_o( "iscsiadm -m session | grep %s:%s | grep %s | awk '{print $2}'" % (iscsiServerIp, iscsiServerPort, iscsiIqn)).strip("[]\n ") if sid == "" or sid is None: err = "sid not found, this may because chap authentication failed" if e != None and e != "": err += " ,error: %s" % e raise RetryException(e) #Get the host_Number of iqn, Will match the HTCL attribute of iscsi according to Host_number host_Number = bash.bash_o( "iscsiadm -m session -P 3 --sid=%s | grep 'Host Number:' | awk '{print $3}'" % sid).strip() #Use HCTL, IQN, "-" to match the number of unmounted Luns according to lsscsi --transport disks_by_no_mapping_lun = bash.bash_o( "lsscsi --transport | grep -w %s | awk '{print $1,$NF}' | grep -E '\<%s\>:[[:digit:]]*:[[:digit:]]*:[[:digit:]]*' | awk '{print $NF}' | grep -x '-'" % (iscsiIqn, host_Number)).strip().splitlines() disks_by_iscsi = bash.bash_o( "iscsiadm -m session -P 3 --sid=%s | grep Lun" % sid).strip().splitlines() if len(disks_by_dev) < (len(disks_by_iscsi) - len(disks_by_no_mapping_lun)): raise RetryException( "iscsiadm says there are [%s] disks but only found [%s] disks on /dev/disk[%s], so not all disks loged in, and you can check the iscsi mounted disk by lsscsi --transport" "it may recover after a while so check and login again" % ((len(disks_by_iscsi) - len(disks_by_no_mapping_lun)), len(disks_by_dev), disks_by_dev))
def get_fc_luns(self): o = bash.bash_o("ls -1c /sys/bus/scsi/devices/target*/fc_transport | grep ^target | awk -F 'target' '{print $2}'") fc_targets = o.strip().splitlines() if len(fc_targets) == 0 or (len(fc_targets) == 1 and fc_targets[0] == ""): logger.debug("not find any fc targets") return [] o = bash.bash_o("lsscsi | grep '\/dev\/'").strip().splitlines() if len(o) == 0 or (len(o) == 1 and o[0] == ""): logger.debug("not find any usable fc disks") return [] luns = [] for fc_target in fc_targets: t = filter(lambda x: "[%s" % fc_target in x, o) luns.extend(map(lambda x: self.get_device_info(x.split("/dev/")[1]), t)) luns_info = {} for lun in luns: # type: FiberChannelLunStruct if lun.storageWwnn not in luns_info or len(luns_info[lun.storageWwnn])==0: luns_info[lun.storageWwnn] = [] luns_info[lun.storageWwnn].append(lun) elif lun.wwids[0] not in map(lambda x:x.wwids[0], luns_info[lun.storageWwnn]): luns_info[lun.storageWwnn].append(lun) result = [] for i in luns_info.values(): result.extend(i) return result
def clean_iscsi_cache_configuration(path,iscsiServerIp,iscsiServerPort): #clean cache configuration file:/var/lib/iscsi/nodes/iqnxxx/ip,port results = bash.bash_o(("ls %s/*/ | grep %s | grep %s" % (path, iscsiServerIp, iscsiServerPort))).strip().splitlines() if results is None or len(results) == 0: return for result in results: dpaths = bash.bash_o("dirname %s/*/%s" % (path, result)).strip().splitlines() if dpaths is None or len(dpaths) == 0: continue for dpath in dpaths: linux.rm_dir_force("%s/%s" % (dpath, result))
def create_vg_if_not_found(vgUuid, diskPaths, hostUuid, forceWipe=False): @linux.retry(times=5, sleep_time=random.uniform(0.1, 3)) def find_vg(vgUuid, raise_exception=True): cmd = shell.ShellCmd( "timeout 5 vgscan --ignorelockingfailure; vgs --nolocking %s -otags | grep %s" % (vgUuid, INIT_TAG)) cmd(is_exception=False) if cmd.return_code != 0 and raise_exception: raise RetryException("can not find vg %s with tag %s" % (vgUuid, INIT_TAG)) elif cmd.return_code != 0: return False return True try: find_vg(vgUuid) except RetryException as e: if forceWipe is True: running_vm = bash.bash_o( "virsh list | grep running | awk '{print $2}'").strip( ).split() if running_vm != [] and running_vm[0] != "": for vm in running_vm: bash.bash_r("virsh destroy %s" % vm) r = bash.bash_r("drbdadm down all") if r == 0: bash.bash_r("mkdir -p %s" % BACKUP_DIR) bash.bash_r("mv /etc/drbd.d/*.res %s" % BACKUP_DIR) lvm.wipe_fs(diskPaths, vgUuid) cmd = shell.ShellCmd( "vgcreate -qq --addtag '%s::%s::%s::%s' --metadatasize %s %s %s" % (INIT_TAG, hostUuid, time.time(), bash.bash_o("hostname").strip(), DEFAULT_VG_METADATA_SIZE, vgUuid, " ".join(diskPaths))) cmd(is_exception=False) logger.debug("created vg %s, ret: %s, stdout: %s, stderr: %s" % (vgUuid, cmd.return_code, cmd.stdout, cmd.stderr)) if cmd.return_code == 0 and find_vg(vgUuid, False) is True: return True try: if find_vg(vgUuid) is True: return True except RetryException as ee: raise Exception( "can not find vg %s with disks: %s and create vg return: %s %s %s " % (vgUuid, diskPaths, cmd.return_code, cmd.stdout, cmd.stderr)) except Exception as ee: raise ee except Exception as e: raise e return False
def wait_iscsi_mknode(iscsiServerIp, iscsiServerPort, iscsiIqn, e = None): disks_by_dev = bash.bash_o("ls /dev/disk/by-path | grep %s:%s | grep %s" % (iscsiServerIp, iscsiServerPort, iscsiIqn)).strip().splitlines() sid = bash.bash_o("iscsiadm -m session | grep %s:%s | grep %s | awk '{print $2}'" % (iscsiServerIp, iscsiServerPort, iscsiIqn)).strip("[]\n ") if sid == "" or sid is None: err = "sid not found, this may because chap authentication failed" if e != None and e != "": err += " ,error: %s" % e raise RetryException(e) disks_by_iscsi = bash.bash_o("iscsiadm -m session -P 3 --sid=%s | grep Lun" % sid).strip().splitlines() if len(disks_by_dev) != len(disks_by_iscsi): raise RetryException("disks number by /dev/disk not equal to iscsiadm")
def wait_iscsi_mknode(iscsiServerIp, iscsiServerPort, iscsiIqn, e=None): disks_by_dev = bash.bash_o("ls /dev/disk/by-path | grep %s:%s | grep %s" % (iscsiServerIp, iscsiServerPort, iscsiIqn)).strip().splitlines() sid = bash.bash_o("iscsiadm -m session | grep %s:%s | grep %s | awk '{print $2}'" % (iscsiServerIp, iscsiServerPort, iscsiIqn)).strip("[]\n ") if sid == "" or sid is None: err = "sid not found, this may because chap authentication failed" if e != None and e != "": err += " ,error: %s" % e raise RetryException(e) disks_by_iscsi = bash.bash_o("iscsiadm -m session -P 3 --sid=%s | grep Lun" % sid).strip().splitlines() if len(disks_by_dev) < len(disks_by_iscsi): raise RetryException("iscsiadm says there are [%s] disks[%s] but only found [%s] disks on /dev/disk[%s], so not all disks loged in, " "it may recover after a while so check and login again" %(len(disks_by_iscsi), disks_by_iscsi, len(disks_by_dev), disks_by_dev))
def check_lv_on_pv_valid(vgUuid, pvUuid, lv_path=None): pv_name = bash.bash_o( "timeout -s SIGKILL 10 pvs --noheading --nolocking -oname -Spv_uuid=%s" % pvUuid).strip() one_active_lv = lv_path if lv_path is not None else bash.bash_o( "timeout -s SIGKILL 10 lvs --noheading --nolocking -opath,devices,tags " + "-Sactive=active %s | grep %s | grep %s | awk '{print $1}' | head -n1" % (vgUuid, pv_name, VOLUME_TAG)).strip() if one_active_lv == "": return True r = bash.bash_r("qemu-img info %s" % one_active_lv) if r != 0: return False return True
def wait_iscsi_mknode(iscsiServerIp, iscsiServerPort, iqn): disks_by_dev = bash.bash_o( "ls /dev/disk/by-path | grep %s:%s | grep %s" % (iscsiServerIp, iscsiServerPort, iqn)).strip().splitlines() sid = bash.bash_o( "iscsiadm -m session | grep %s:%s | grep %s | awk '{print $2}'" % (iscsiServerIp, iscsiServerPort, iqn)).strip("[]\n ") disks_by_iscsi = bash.bash_o( "iscsiadm -m session -P 3 --sid=%s | grep Lun" % sid).strip().splitlines() if len(disks_by_dev) != len(disks_by_iscsi): raise RetryException( "disks number by /dev/disk not equal to iscsiadm")
def clean_iscsi_cache_configuration(path, iscsiServerIp, iscsiServerPort): #clean cache configuration file:/var/lib/iscsi/nodes/iqnxxx/ip,port results = bash.bash_o( ("ls %s/*/ | grep %s | grep %s" % (path, iscsiServerIp, iscsiServerPort))).strip().splitlines() if results is None or len(results) == 0: return for result in results: dpaths = bash.bash_o("dirname %s/*/%s" % (path, result)).strip().splitlines() if dpaths is None or len(dpaths) == 0: continue for dpath in dpaths: linux.rm_dir_force("%s/%s" % (dpath, result))
def is_volume_on_pvs(volume_path, pvUuids, includingMissing=True): files = linux.qcow2_get_file_chain(volume_path) if len(files) == 0: # could not read qcow2 logger.debug("can not read volume %s, return true" % volume_path) return True pv_names = [] for p in pvUuids: name = get_pv_name_by_uuid(p) if name != "": pv_names.append(get_pv_name_by_uuid(p) + "(") if includingMissing: pv_names.append("unknown") for f in files: o = bash.bash_o( "timeout -s SIGKILL 10 lvs --noheading --nolocking %s -odevices" % f).strip().lower() # type: str logger.debug("volume %s is on pv %s" % (volume_path, o)) if len(filter(lambda n: o.find(n.lower()) > 0, pv_names)) > 0: logger.debug("lv %s on pv %s(%s), return true" % (volume_path, pvUuids, pv_names)) return True if o == "" and includingMissing: logger.debug("pv of lv %s is missing, return true") return True return False
def install_drbd(): mod_installed = bash.bash_r("lsmod | grep drbd") == 0 mod_exists = bash.bash_r("modinfo drbd") == 0 utils_installed = bash.bash_r( "rpm -ql drbd-utils || rpm -ql drbd84-utils") == 0 basearch = platform.machine() releasever = bash.bash_o("awk '{print $3}' /etc/zstack-release") utils_exists, o = bash.bash_ro( "ls /opt/zstack-dvd/{}/{}/Packages/drbd-utils*".format( basearch, releasever)) if mod_installed and utils_exists: return if not mod_installed: if mod_exists: bash.bash_errorout("modprobe drbd") else: raise Exception("drbd mod not installed and not exists!") if not utils_installed: if utils_exists == 0: bash.bash_errorout("rpm -ivh %s" % o) else: raise Exception("drbd utils not installed and not exists!")
def wipe_fs(disks, expected_vg=None): for disk in disks: exists_vg = None r = bash.bash_r("pvdisplay %s | grep %s" % (disk, expected_vg)) if r == 0: continue r, o = bash.bash_ro("pvs --nolocking --noheading -o vg_name %s" % disk) if r == 0 and o.strip() != "": exists_vg = o.strip() backup = backup_super_block(disk) if bash.bash_r("grep %s %s" % (expected_vg, backup)) == 0: raise Exception("found vg uuid in superblock backup while not found in lvm command!") need_flush_mpath = False bash.bash_roe("partprobe -s %s" % disk) cmd_type = bash.bash_o("lsblk %s -oTYPE | grep mpath" % disk) if cmd_type.strip() != "": need_flush_mpath = True bash.bash_roe("wipefs -af %s" % disk) if need_flush_mpath: bash.bash_roe("multipath -f %s && systemctl restart multipathd.service && sleep 1" % disk) if exists_vg is not None: logger.debug("found vg %s exists on this pv %s, start wipe" % (exists_vg, disk)) try: drop_vg_lock(exists_vg) remove_device_map_for_vg(exists_vg) finally: pass
def remove_device_map_for_vg(vgUuid): o = bash.bash_o("dmsetup ls | grep %s | awk '{print $1}'" % vgUuid).strip().splitlines() if len(o) == 0: return for dm in o: bash.bash_roe("dmsetup remove %s" % dm.strip())
def config_lvm(host_id, enableLvmetad=False): lvm.backup_lvm_config() lvm.reset_lvm_conf_default() lvm.config_lvm_by_sed("use_lvmlockd", "use_lvmlockd=1", ["lvm.conf", "lvmlocal.conf"]) if enableLvmetad: lvm.config_lvm_by_sed("use_lvmetad", "use_lvmetad=1", ["lvm.conf", "lvmlocal.conf"]) else: lvm.config_lvm_by_sed("use_lvmetad", "use_lvmetad=0", ["lvm.conf", "lvmlocal.conf"]) lvm.config_lvm_by_sed("host_id", "host_id=%s" % host_id, ["lvm.conf", "lvmlocal.conf"]) lvm.config_lvm_by_sed("sanlock_lv_extend", "sanlock_lv_extend=%s" % DEFAULT_SANLOCK_LV_SIZE, ["lvm.conf", "lvmlocal.conf"]) lvm.config_lvm_by_sed("lvmlockd_lock_retries", "lvmlockd_lock_retries=6", ["lvm.conf", "lvmlocal.conf"]) lvm.config_lvm_by_sed("issue_discards", "issue_discards=1", ["lvm.conf", "lvmlocal.conf"]) lvm.config_lvm_by_sed("reserved_stack", "reserved_stack=256", ["lvm.conf", "lvmlocal.conf"]) lvm.config_lvm_by_sed("reserved_memory", "reserved_memory=131072", ["lvm.conf", "lvmlocal.conf"]) lvm.config_lvm_filter(["lvm.conf", "lvmlocal.conf"]) lvm.config_sanlock_by_sed("sh_retries", "sh_retries=20") lvm.config_sanlock_by_sed("logfile_priority", "logfile_priority=7") lvm.config_sanlock_by_sed("renewal_read_extend_sec", "renewal_read_extend_sec=24") lvm.config_sanlock_by_sed("debug_renew", "debug_renew=1") lvm.config_sanlock_by_sed("use_watchdog", "use_watchdog=0") sanlock_hostname = "%s-%s-%s" % (cmd.vgUuid[:8], cmd.hostUuid[:8], bash.bash_o("hostname").strip()[:20]) lvm.config_sanlock_by_sed("our_host_name", "our_host_name=%s" % sanlock_hostname)
def get_used_qmp_file(): t = bash.bash_o("ps aux | grep -Eo -- '-qmp unix:%s/\w*\.sock'" % QMP_SOCKET_PATH).splitlines() qmp = [] for i in t: qmp.append(i.split("/")[-1]) return qmp
def clear_stalled_qmp_socket(): def get_used_qmp_file(): t = bash.bash_o("ps aux | grep -Eo -- '-qmp unix:%s/\w*\.sock'" % QMP_SOCKET_PATH).splitlines() qmp = [] for i in t: qmp.append(i.split("/")[-1]) return qmp exists_qmp_files = set( bash.bash_o("ls %s" % QMP_SOCKET_PATH).splitlines()) if len(exists_qmp_files) == 0: return running_qmp_files = set(get_used_qmp_file()) if len(running_qmp_files) == 0: bash.bash_roe("/bin/rm %s/*" % QMP_SOCKET_PATH) return need_delete_qmp_files = exists_qmp_files.difference(running_qmp_files) if len(need_delete_qmp_files) == 0: return for f in need_delete_qmp_files: bash.bash_roe("/bin/rm %s/%s" % (QMP_SOCKET_PATH, f))
def get_lv_locking_type(path): if not lv_is_active(path): return LvmlockdLockType.NULL output = bash.bash_o( "lvmlockctl -i | grep %s | head -n1 | awk '{print $3}'" % lv_uuid(path)) return LvmlockdLockType.from_abbr(output.strip())
def convert_volume_provisioning(self, req): cmd = jsonobject.loads(req[http.REQUEST_BODY]) rsp = ConvertVolumeProvisioningRsp() if cmd.provisioningStrategy != "ThinProvisioning": raise NotImplementedError abs_path = translate_absolute_path_from_install_path(cmd.installPath) with lvm.RecursiveOperateLv(abs_path, shared=False): image_offest = long( bash.bash_o( "qemu-img check %s | grep 'Image end offset' | awk -F ': ' '{print $2}'" % abs_path).strip()) current_size = long(lvm.get_lv_size(abs_path)) virtual_size = linux.qcow2_virtualsize(abs_path) size = image_offest + cmd.addons[ lvm.thinProvisioningInitializeSize] if size > current_size: size = current_size if size > virtual_size: size = virtual_size lvm.resize_lv(abs_path, size, True) rsp.actualSize = size rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid) return jsonobject.dumps(rsp)
def remove_partial_lv_dm(vgUuid): o = bash.bash_o("lvs --noheading --nolocking %s -opath,tags -Slv_health_status=partial | grep %s" % (vgUuid, COMMON_TAG)).strip().splitlines() if len(o) == 0: return for volume in o: bash.bash_roe("dmsetup remove %s" % volume.strip().split(" ")[0])
def get_fc_luns(self, rescan): o = bash.bash_o( "ls -1c /sys/bus/scsi/devices/target*/fc_transport | grep ^target | awk -F 'target' '{print $2}'" ) fc_targets = o.strip().splitlines() if len(fc_targets) == 0 or (len(fc_targets) == 1 and fc_targets[0] == ""): logger.debug("not find any fc targets") return [] o = bash.bash_o("lsscsi | grep '\/dev\/'").strip().splitlines() if len(o) == 0 or (len(o) == 1 and o[0] == ""): logger.debug("not find any usable fc disks") return [] luns = [None] * len(fc_targets) def get_lun_info(fc_target, i): t = filter(lambda x: "[%s" % fc_target in x, o) mapped_t = map( lambda x: self.get_device_info(x.split("/dev/")[1], rescan), t) luns[i] = filter(lambda x: x is not None, mapped_t) threads = [] for idx, fc_target in enumerate(fc_targets, start=0): threads.append( thread.ThreadFacade.run_in_thread(get_lun_info, [fc_target, idx])) for t in threads: t.join() luns_info = {} for lun_list in luns: for lun in lun_list: # type: FiberChannelLunStruct if lun.storageWwnn not in luns_info or len( luns_info[lun.storageWwnn]) == 0: luns_info[lun.storageWwnn] = [] luns_info[lun.storageWwnn].append(lun) elif lun.wwids[0] not in map(lambda x: x.wwids[0], luns_info[lun.storageWwnn]): luns_info[lun.storageWwnn].append(lun) result = [] for i in luns_info.values(): result.extend(i) return result
def list_local_up_drbd(vgUuid): if bash.bash_r("drbd-overview | grep -v %s" % DrbdNetState.Unconfigured) == 1: return [] names = bash.bash_o( "drbd-overview | grep -v %s | awk -F ':' '{print $2}' | awk '{print $1}'" % DrbdNetState.Unconfigured).strip().splitlines() return [DrbdResource(name) for name in names]
def do_find_qcow2(vgUuid): paths = [] raw_paths = bash.bash_o('lvs --nolocking --noheading -Slv_name=~".*%s" -Stags={%s} -opath %s' % (QCOW2_SUFFIX, DONE_TAG, vgUuid)).strip().splitlines() for raw_path in raw_paths: paths.append(raw_path.strip()) sys.stdout.write(",".join(paths)) sys.stdout.flush() return paths
def up_all_resouces(): all_names = bash.bash_o( "ls /etc/drbd.d/ | grep -v global_common.conf").strip().splitlines() for name in all_names: try: DrbdResource(name.split(".")[0]) except Exception as e: logger.warn("up resource %s failed: %s" % (name, e.message))
def get_config_path_from_name(name): if bash.bash_r("drbdadm dump %s" % name) == 0: return bash.bash_o( "drbdadm dump %s | grep 'defined at' | awk '{print $4}'" % name).split(":")[0] if bash.bash_r("ls /etc/drbd.d/%s.res" % name) == 0: return "/etc/drbd.d/%s.res" % name raise Exception("can not find drbd resource %s" % name)
def get_running_vm_root_volume_on_pv(vgUuid, pvUuids, checkIo=True): # 1. get "-drive ... -device ... bootindex=1, # 2. get "-boot order=dc ... -drive id=drive-virtio-disk" # 3. make sure io has error # 4. filter for pv out = bash.bash_o("pgrep -a qemu-kvm | grep %s" % vgUuid).strip().split("\n") if len(out) == 0: return [] vms = [] for o in out: vm = VmStruct() vm.pid = o.split(" ")[0] vm.cmdline = o.split(" ", 3)[-1] vm.uuid = o.split(" -uuid ")[-1].split(" ")[0] if "bootindex=1" in vm.cmdline: vm.root_volume = vm.cmdline.split("bootindex=1")[0].split( " -drive file=")[-1].split(",")[0] elif " -boot order=dc" in vm.cmdline: # TODO(weiw): maybe support scsi volume as boot volume one day vm.root_volume = vm.cmdline.split("id=drive-virtio-disk0")[ 0].split(" -drive file=")[-1].split(",")[0] else: logger.warn( "found strange vm[pid: %s, cmdline: %s], can not find boot volume" % (vm.pid, vm.cmdline)) continue r = bash.bash_r("qemu-img info --backing-chain %s" % vm.root_volume) if checkIo is True and r == 0: logger.debug("volume %s for vm %s io success, skiped" % (vm.root_volume, vm.uuid)) continue out = bash.bash_o("virsh dumpxml %s | grep \"source file='/dev/\"" % vm.uuid).strip().splitlines() if len(out) != 0: for file in out: vm.volumes.append(file.strip().split("'")[1]) if is_volume_on_pvs(vm.root_volume, pvUuids, True): vms.append(vm) return vms
def get_lockspace(vgUuid): @linux.retry(times=3, sleep_time=0.5) def _do_get_lockspace(vgUuid): o = bash.bash_o("sanlock client gets | awk '{print $2}' | grep %s" % vgUuid).strip() if o == "": raise RetryException return o out = bash.bash_o("sanlock client gets | awk '{print $2}' | grep %s" % vgUuid).strip() if out != "": return out try: logger.debug("retrying get lockspace for vg %s" % vgUuid) out = _do_get_lockspace(vgUuid) except Exception as e: out = bash.bash_o("sanlock client gets | awk '{print $2}' | grep %s" % vgUuid).strip() return out
def get_device_info(self, dev_name): # type: (str) -> FiberChannelLunStruct s = FiberChannelLunStruct() o = shell.call( "lsblk --pair -b -p -o NAME,VENDOR,MODEL,WWN,SERIAL,HCTL,TYPE,SIZE /dev/%s" % dev_name).strip().split("\n")[0] if o == "": raise Exception("can not get device information from %s" % dev_name) def get_data(e): return e.split("=")[1].strip().strip('"') def get_wwids(dev): return shell.call( "udevadm info -n %s | grep 'by-id' | grep -v DEVLINKS | awk -F 'by-id/' '{print $2}'" % dev).strip().split() def get_path(dev): return shell.call( "udevadm info -n %s | grep 'by-path' | grep -v DEVLINKS | head -n1 | awk -F 'by-path/' '{print $2}'" % dev).strip() def get_storage_wwnn(hctl): o = shell.call( "systool -c fc_transport -A node_name | grep '\"target%s\"' -B2 | grep node_name | awk '{print $NF}'" % ":".join(hctl.split(":")[0:3])) return o.strip().strip('"') for entry in o.split('" '): # type: str if entry.startswith("VENDOR"): s.vendor = get_data(entry) elif entry.startswith("MODEL"): s.model = get_data(entry) elif entry.startswith("WWN"): s.wwn = get_data(entry) elif entry.startswith("SERIAL"): s.serial = get_data(entry) elif entry.startswith('HCTL'): s.hctl = get_data(entry) elif entry.startswith('SIZE'): s.size = get_data(entry) elif entry.startswith('TYPE'): s.type = get_data(entry) s.wwids = get_wwids(dev_name) s.wwids.sort() s.path = get_path(dev_name) if lvm.is_slave_of_multipath("/dev/%s" % dev_name): s.type = "mpath" wwid = bash.bash_o( "multipath -l /dev/%s | head -n1 | awk '{print $2}'" % dev_name).strip().strip("()") s.wwids = [wwid] if wwid != "" else s.wwids s.storageWwnn = get_storage_wwnn(s.hctl) return s
def remove_partial_lv_dm(vgUuid): o = bash.bash_o( "lvs --noheading --nolocking --readonly %s -opath,tags -Slv_health_status=partial | grep %s" % (vgUuid, COMMON_TAG)).strip().splitlines() if len(o) == 0: return for volume in o: bash.bash_roe("dmsetup remove %s" % volume.strip().split(" ")[0])
def test_network_ok_to_peer(peer_address, via_dev=None): if not via_dev: via_dev = bash.bash_o("ip -o r get %s | awk '{print $3}'" % peer_address).strip() for i in range(5): recv = bash.bash_r("timeout 2 arping -w 1 -b %s -I %s -c 1" % (peer_address, via_dev)) if recv == 0: return True return False
def get_disk_info_by_path(path): # type: (str) -> IscsiLunStruct abs_path = bash.bash_o("readlink -e /dev/disk/by-path/%s" % path).strip() candidate_struct = lvm.get_device_info(abs_path.split("/")[-1]) lun_struct = IscsiLunStruct() lun_struct.path = path lun_struct.size = candidate_struct.size lun_struct.hctl = candidate_struct.hctl lun_struct.serial = candidate_struct.serial lun_struct.model = candidate_struct.model lun_struct.vendor = candidate_struct.vendor lun_struct.type = candidate_struct.type lun_struct.wwn = candidate_struct.wwn lun_struct.wwids = candidate_struct.wwids if lvm.is_slave_of_multipath(abs_path): lun_struct.type = "mpath" mpath_wwid = bash.bash_o("multipath -l %s | head -n1 | awk '{print $2}'" % abs_path).strip("() \n") lun_struct.wwids = [mpath_wwid] return lun_struct
def get_name_from_config_path(config_path): """ :type config_path: str """ if bash.bash_r("head -n 1 %s" % config_path) == 0: return bash.bash_o("head -n 1 %s | awk '{print $2}'" % config_path).strip() else: return config_path.split("/")[-1].split(".")[0]
def make_ctx(self): ctx = {} for k, v in self.__dict__.items(): if isinstance(v, str): ctx[k] = v elif isinstance(v, DrbdStruct): for m, n in v.__dict__.items(): ctx["%s_%s" % (k, m)] = n ctx["local_host_hostname"] = bash.bash_o("hostname").strip() return ctx
def do_find_qcow2(vgUuid): paths = [] raw_paths = bash.bash_o( 'lvs --nolocking --noheading -Slv_name=~".*%s" -Stags={%s} -opath %s' % (QCOW2_SUFFIX, DONE_TAG, vgUuid)).strip().splitlines() for raw_path in raw_paths: paths.append(raw_path.strip()) sys.stdout.write(",".join(paths)) sys.stdout.flush() return paths
def fix_global_lock(): if not ENABLE_DUP_GLOBAL_CHECK: return vg_names = bash.bash_o("lvmlockctl -i | grep lock_type=sanlock | awk '{print $2}'").strip().splitlines() # type: list vg_names.sort() if len(vg_names) < 2: return for vg_name in vg_names[1:]: bash.bash_roe("lvmlockctl --gl-disable %s" % vg_name) bash.bash_roe("lvmlockctl --gl-enable %s" % vg_names[0])
def up_all_resouces(): all_names = bash.bash_o( "ls /etc/drbd.d/ | grep -v global_common.conf").strip().splitlines() for name in all_names: try: r = DrbdResource(name.split(".")[0]) if r.config.local_host.minor is not None and linux.linux_lsof( r.config.local_host.get_drbd_device()).strip() == "": r.demote() except Exception as e: logger.warn("up resource %s failed: %s" % (name, e.message))
def config_lvm_filter(files): if not os.path.exists(LVM_CONFIG_PATH): raise Exception("can not find lvm config path: %s, config lvm failed" % LVM_CONFIG_PATH) vgs = bash.bash_o("vgs --nolocking -oname --noheading").splitlines() filter_str = 'filter=["r|\\/dev\\/cdrom|"' for vg in vgs: filter_str += ', "r\\/dev\\/mapper\\/%s.*\\/"' % vg.strip() filter_str += ']' for file in files: bash.bash_r("sed -i 's/.*\\b%s.*/%s/g' %s/%s" % ("filter", filter_str, LVM_CONFIG_PATH, file))
def delete_lv(path, raise_exception=True): logger.debug("deleting lv %s" % path) # remove meta-lv if any if lv_exists(get_meta_lv_path(path)): shell.run("lvremove -y %s" % get_meta_lv_path(path)) if not lv_exists(path): return if raise_exception: o = bash.bash_errorout("lvremove -y %s" % path) else: o = bash.bash_o("lvremove -y %s" % path) return o
def fix_global_lock(): if not ENABLE_DUP_GLOBAL_CHECK: return vg_names = bash.bash_o( "lvmlockctl -i | grep lock_type=sanlock | awk '{print $2}'").strip( ).splitlines() # type: list vg_names.sort() if len(vg_names) < 2: return for vg_name in vg_names[1:]: bash.bash_roe("lvmlockctl --gl-disable %s" % vg_name) bash.bash_roe("lvmlockctl --gl-enable %s" % vg_names[0])
def check_pv_status(vgUuid, timeout): r, o, e = bash.bash_roe( "timeout -s SIGKILL %s pvs --noheading --nolocking -Svg_name=%s -oname,missing" % (timeout, vgUuid)) if len(o) == 0 or r != 0: logger.warn( "can not find shared block in shared block group %s, detail: [return_code: %s, stdout: %s, stderr: %s]" % (vgUuid, r, o, e)) return True, "" for pvs_out in o: if "unknown" in pvs_out: s = "disk in shared block group %s missing" % vgUuid logger.warn("%s, details: %s" % (s, o)) return False, s if "missing" in pvs_out: s = "disk %s in shared block group %s exists but state is missing" % ( pvs_out.strip().split(" ")[0], vgUuid) logger.warn("%s, details: %s" % (s, o)) return False, s health, o, e = bash.bash_roe('timeout -s SIGKILL %s vgck %s' % (10 if timeout < 10 else timeout, vgUuid)) if health != 0: s = "vgck %s failed, details: %s" % (vgUuid, e) logger.warn(s) return False, s health = bash.bash_o( 'timeout -s SIGKILL %s vgs -oattr --nolocking --readonly --noheadings --shared %s ' % (10 if timeout < 10 else timeout, vgUuid)).strip() if health == "": logger.warn("can not get proper attr of vg, return false") return False, "primary storage %s attr get error, expect 'wz--ns' got %s" % ( vgUuid, health) if health[0] != "w": return False, "primary storage %s permission error, expect 'w' but now is %s, deatils: %s" % ( vgUuid, health.stdout.strip()[0], health) if health[1] != "z": return False, "primary storage %s resizeable error, expect 'z' but now is %s, deatils: %s" % ( vgUuid, health.stdout.strip()[1], health) if health[3] != "-": return False, "primary storage %s partial error, expect '-' but now is %s, deatils: %s" % ( vgUuid, health.stdout.strip()[3], health) if health[5] != "s": return False, "primary storage %s shared mode error, expect 's' but now is %s, deatils: %s" % ( vgUuid, health.stdout.strip()[5], health) return True, ""
def get_multipath_dmname(dev_name): # if is multipath dev, return; # if is one of multipath paths, return multipath dev(dm-xxx); # else return None slaves = shell.call("ls /sys/class/block/%s/slaves/" % dev_name).strip().splitlines() if slaves is not None and len(slaves) > 0 and slaves[0].strip() != "": return dev_name r = bash.bash_r("multipath /dev/%s -l | grep policy" % dev_name) if r != 0: return None o = bash.bash_o("multipath -l /dev/%s | head -n1 | awk -F 'dm' '{print $2}' | awk '{print $1}'" % dev_name).strip() return "dm%s" % o
def get_running_vm_root_volume_on_pv(vgUuid, pvUuids, checkIo=True): # 1. get "-drive ... -device ... bootindex=1, # 2. get "-boot order=dc ... -drive id=drive-virtio-disk" # 3. make sure io has error # 4. filter for pv out = bash.bash_o("pgrep -a qemu-kvm | grep %s" % vgUuid).strip().split("\n") if len(out) == 0: return [] vms = [] for o in out: vm = VmStruct() vm.pid = o.split(" ")[0] vm.cmdline = o.split(" ", 3)[-1] vm.uuid = o.split(" -uuid ")[-1].split(" ")[0] if "bootindex=1" in vm.cmdline: vm.root_volume = vm.cmdline.split("bootindex=1")[0].split(" -drive file=")[-1].split(",")[0] elif " -boot order=dc" in vm.cmdline: # TODO(weiw): maybe support scsi volume as boot volume one day vm.root_volume = vm.cmdline.split("id=drive-virtio-disk0")[0].split(" -drive file=")[-1].split(",")[0] else: logger.warn("found strange vm[pid: %s, cmdline: %s], can not find boot volume" % (vm.pid, vm.cmdline)) continue r = bash.bash_r("qemu-img info --backing-chain %s" % vm.root_volume) if checkIo is True and r == 0: logger.debug("volume %s for vm %s io success, skiped" % (vm.root_volume, vm.uuid)) continue out = bash.bash_o("virsh dumpxml %s | grep \"source file='/dev/\"" % vm.uuid).strip().splitlines() if len(out) != 0: for file in out: vm.volumes.append(file.strip().split("'")[1]) if is_volume_on_pvs(vm.root_volume, pvUuids, True): vms.append(vm) return vms
def get_device_info(self, dev_name): # type: (str) -> FiberChannelLunStruct s = FiberChannelLunStruct() o = shell.call( "lsblk --pair -b -p -o NAME,VENDOR,MODEL,WWN,SERIAL,HCTL,TYPE,SIZE /dev/%s" % dev_name).strip().split("\n")[0] if o == "": raise Exception("can not get device information from %s" % dev_name) def get_data(e): return e.split("=")[1].strip().strip('"') def get_wwids(dev): return shell.call( "udevadm info -n %s | grep 'by-id' | grep -v DEVLINKS | awk -F 'by-id/' '{print $2}'" % dev).strip().split() def get_path(dev): return shell.call( "udevadm info -n %s | grep 'by-path' | grep -v DEVLINKS | head -n1 | awk -F 'by-path/' '{print $2}'" % dev).strip() def get_storage_wwnn(hctl): o = shell.call( "systool -c fc_transport -A node_name | grep '\"target%s\"' -B2 | grep node_name | awk '{print $NF}'" % ":".join(hctl.split(":")[0:3])) return o.strip().strip('"') for entry in o.split('" '): # type: str if entry.startswith("VENDOR"): s.vendor = get_data(entry) elif entry.startswith("MODEL"): s.model = get_data(entry) elif entry.startswith("WWN"): s.wwn = get_data(entry) elif entry.startswith("SERIAL"): s.serial = get_data(entry) elif entry.startswith('HCTL'): s.hctl = get_data(entry) elif entry.startswith('SIZE'): s.size = get_data(entry) elif entry.startswith('TYPE'): s.type = get_data(entry) s.wwids = get_wwids(dev_name) s.wwids.sort() s.path = get_path(dev_name) if lvm.is_slave_of_multipath("/dev/%s" % dev_name): s.type = "mpath" wwid = bash.bash_o("multipath -l /dev/%s | head -n1 | awk '{print $2}'" % dev_name).strip().strip("()") s.wwids = [wwid] if wwid != "" else s.wwids s.storageWwnn = get_storage_wwnn(s.hctl) return s
def get_invalid_pv_uuids(vgUuid, checkIo = False): invalid_pv_uuids = [] pvs_outs = bash.bash_o( "timeout -s SIGKILL 10 pvs --noheading --nolocking -Svg_name=%s -ouuid,name,missing" % vgUuid).strip().split("\n") if len(pvs_outs) == 0: return for pvs_out in pvs_outs: pv_uuid = pvs_out.strip().split(" ")[0] if "unknown" in pvs_out: invalid_pv_uuids.append(pv_uuid) elif "missing" in pvs_out: invalid_pv_uuids.append(pv_uuid) elif checkIo is True and check_lv_on_pv_valid(vgUuid, pv_uuid) is False: invalid_pv_uuids.append(pv_uuid) return invalid_pv_uuids
def get_slave_path(self, multipath_path): def get_wwids(dev_name): result = [] wwids = shell.call( "udevadm info -n %s | grep 'by-id' | grep -v DEVLINKS | awk -F 'by-id/' '{print $2}'" % dev_name).strip().split() wwids.sort() for wwid in wwids: if "lvm-pv" not in wwid: result.append(wwid) if len(result) == 0: return wwids dm = bash.bash_o("realpath %s | grep -E -o 'dm-.*'" % multipath_path) slaves = shell.call("ls -1 /sys/class/block/%s/slaves/" % dm).strip().split("\n") if slaves is None or len(slaves) == 0: raise "can not find any slave from multpath device: %s" % multipath_path return "/dev/disk/by-id/%s" % get_wwids(slaves[0])[0]
def get_lv_locking_type(path): @linux.retry(times=5, sleep_time=random.uniform(0.1, 3)) def _get_lv_locking_type(path): output = bash.bash_o("lvmlockctl -i | grep %s | head -n1 | awk '{print $3}'" % lv_uuid(path)) return LvmlockdLockType.from_abbr(output.strip(), raise_exception=True) locking_type = LvmlockdLockType.NULL with lock.FileLock(path.split("/")[-1]): try: if not lv_is_active(path): return locking_type locking_type = _get_lv_locking_type(path) except Exception as e: output = bash.bash_o("lvmlockctl -i | grep %s | head -n1 | awk '{print $3}'" % lv_uuid(path)) locking_type = LvmlockdLockType.from_abbr(output.strip(), raise_exception=False) return locking_type
def check_pv_status(vgUuid, timeout): r, o , e = bash.bash_roe("timeout -s SIGKILL %s pvs --noheading --nolocking -Svg_name=%s -oname,missing" % (timeout, vgUuid)) if len(o) == 0 or r != 0: s = "can not find shared block in shared block group %s, detail: [return_code: %s, stdout: %s, stderr: %s]" % (vgUuid, r, o, e) logger.warn(s) return False, s for pvs_out in o: if "unknown" in pvs_out: s = "disk in shared block group %s missing" % vgUuid logger.warn("%s, details: %s" % (s, o)) return False, s if "missing" in pvs_out: s = "disk %s in shared block group %s exists but state is missing" % (pvs_out.strip().split(" ")[0], vgUuid) logger.warn("%s, details: %s" % (s, o)) return False, s # r, s = lvm_vgck(vgUuid, timeout) # if r is False: # return r, s health = bash.bash_o('timeout -s SIGKILL %s vgs -oattr --nolocking --noheadings --shared %s ' % (10 if timeout < 10 else timeout, vgUuid)).strip() if health == "": logger.warn("can not get proper attr of vg, return false") return False, "primary storage %s attr get error, expect 'wz--ns' got %s" % (vgUuid, health) if health[0] != "w": return False, "primary storage %s permission error, expect 'w' but now is %s, deatils: %s" % (vgUuid, health.stdout.strip()[0], health) if health[1] != "z": return False, "primary storage %s resizeable error, expect 'z' but now is %s, deatils: %s" % (vgUuid, health.stdout.strip()[1], health) if health[3] != "-": return False, "primary storage %s partial error, expect '-' but now is %s, deatils: %s" % (vgUuid, health.stdout.strip()[3], health) if health[5] != "s": return False, "primary storage %s shared mode error, expect 's' but now is %s, deatils: %s" % (vgUuid, health.stdout.strip()[5], health) return True, ""
def clear_stalled_qmp_socket(): def get_used_qmp_file(): t = bash.bash_o("ps aux | grep -Eo -- '-qmp unix:%s/\w*\.sock'" % QMP_SOCKET_PATH).splitlines() qmp = [] for i in t: qmp.append(i.split("/")[-1]) return qmp exists_qmp_files = set(bash.bash_o("ls %s" % QMP_SOCKET_PATH).splitlines()) if len(exists_qmp_files) == 0: return running_qmp_files = set(get_used_qmp_file()) if len(running_qmp_files) == 0: bash.bash_roe("/bin/rm %s/*" % QMP_SOCKET_PATH) return need_delete_qmp_files = exists_qmp_files.difference(running_qmp_files) if len(need_delete_qmp_files) == 0: return for f in need_delete_qmp_files: bash.bash_roe("/bin/rm %s/%s" % (QMP_SOCKET_PATH, f))
def convert_volume_provisioning(self, req): cmd = jsonobject.loads(req[http.REQUEST_BODY]) rsp = ConvertVolumeProvisioningRsp() if cmd.provisioningStrategy != "ThinProvisioning": raise NotImplementedError abs_path = translate_absolute_path_from_install_path(cmd.installPath) with lvm.RecursiveOperateLv(abs_path, shared=False): image_offest = long( bash.bash_o("qemu-img check %s | grep 'Image end offset' | awk -F ': ' '{print $2}'" % abs_path).strip()) current_size = long(lvm.get_lv_size(abs_path)) virtual_size = linux.qcow2_virtualsize(abs_path) size = image_offest + cmd.addons[lvm.thinProvisioningInitializeSize] if size > current_size: size = current_size if size > virtual_size: size = virtual_size lvm.resize_lv(abs_path, size, True) rsp.actualSize = size rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size(cmd.vgUuid) return jsonobject.dumps(rsp)
def create_vg_if_not_found(vgUuid, diskPaths, hostUuid, forceWipe=False): @linux.retry(times=5, sleep_time=random.uniform(0.1, 3)) def find_vg(vgUuid, raise_exception = True): cmd = shell.ShellCmd("timeout 5 vgscan --ignorelockingfailure; vgs --nolocking %s -otags | grep %s" % (vgUuid, INIT_TAG)) cmd(is_exception=False) if cmd.return_code != 0 and raise_exception: raise RetryException("can not find vg %s with tag %s" % (vgUuid, INIT_TAG)) elif cmd.return_code != 0: return False return True try: find_vg(vgUuid) except RetryException as e: if forceWipe is True: lvm.wipe_fs(diskPaths, vgUuid) cmd = shell.ShellCmd("vgcreate -qq --shared --addtag '%s::%s::%s::%s' --metadatasize %s %s %s" % (INIT_TAG, hostUuid, time.time(), bash.bash_o("hostname").strip(), DEFAULT_VG_METADATA_SIZE, vgUuid, " ".join(diskPaths))) cmd(is_exception=False) logger.debug("created vg %s, ret: %s, stdout: %s, stderr: %s" % (vgUuid, cmd.return_code, cmd.stdout, cmd.stderr)) if cmd.return_code == 0 and find_vg(vgUuid, False) is True: return True try: if find_vg(vgUuid) is True: return True except RetryException as ee: raise Exception("can not find vg %s with disks: %s and create vg return: %s %s %s " % (vgUuid, diskPaths, cmd.return_code, cmd.stdout, cmd.stderr)) except Exception as ee: raise ee except Exception as e: raise e return False
def _do_get_lockspace(vgUuid): o = bash.bash_o("sanlock client gets | awk '{print $2}' | grep %s" % vgUuid).strip() if o == "": raise RetryException return o
def _get_lv_locking_type(path): output = bash.bash_o("lvmlockctl -i | grep %s | head -n1 | awk '{print $3}'" % lv_uuid(path)) return LvmlockdLockType.from_abbr(output.strip(), raise_exception=True)
def get_vg_lvm_uuid(vgUuid): return bash.bash_o("vgs --nolocking --noheading -ouuid %s" % vgUuid).strip()
def get_pv_uuid_by_path(pvPath): return bash.bash_o( "timeout -s SIGKILL 10 pvs --noheading --nolocking -ouuid %s" % pvPath).strip()
def get_multipath_name(dev_name): return bash.bash_o("multipath /dev/%s -l -v1" % dev_name).strip()