def scan_sg_devices(self, req): #1. find fc devices #2. distinct by device wwid and storage wwn rsp = FcSanScanRsp() bash.bash_roe("timeout 120 /usr/bin/rescan-scsi-bus.sh -r") rsp.fiberChannelLunStructs = self.get_fc_luns() linux.set_fail_if_no_path() return jsonobject.dumps(rsp)
def enable_multipath(self, req): rsp = AgentRsp() lvm.enable_multipath() r = bash.bash_r("grep '^[[:space:]]*alias' /etc/multipath.conf") if r == 0: bash.bash_roe( "sed -i 's/^[[:space:]]*alias/#alias/g' /etc/multipath.conf") bash.bash_roe("systemctl reload multipathd") linux.set_fail_if_no_path() return jsonobject.dumps(rsp)
def check_disks(self, req): cmd = jsonobject.loads(req[http.REQUEST_BODY]) rsp = AgentRsp() if cmd.failIfNoPath: linux.set_fail_if_no_path() for diskUuid in cmd.sharedBlockUuids: disk = CheckDisk(diskUuid) path = disk.get_path() if cmd.rescan: disk.rescan(path.split("/")[-1]) if cmd.vgUuid is not None and lvm.vg_exists(cmd.vgUuid): rsp.totalCapacity, rsp.availableCapacity = lvm.get_vg_size( cmd.vgUuid, False) return jsonobject.dumps(rsp)
def enable_multipath(self, req): rsp = AgentRsp() lvm.enable_multipath() bash.bash_roe( "sed -i 's/^[[:space:]]*alias/#alias/g' /etc/multipath.conf") current_t = time.time() bash.bash_roe( "mv /etc/multipath/bindings /etc/multipath/bindings.%s " % current_t + "&& md5sum /etc/multipath/bindings.* | awk 'p[$1]++ { printf \"rm %s\\n\",$2;}' | bash" ) bash.bash_roe( "mv /etc/multipath/wwids /etc/multipath/wwids.%s " % current_t + "&& md5sum /etc/multipath/wwids.* | awk 'p[$1]++ { printf \"rm %s\\n\",$2;}' | bash" ) bash.bash_roe("multipath -F; systemctl restart multipathd.service") linux.set_fail_if_no_path() return jsonobject.dumps(rsp)
def iscsi_login(self, req): cmd = jsonobject.loads(req[http.REQUEST_BODY]) rsp = IscsiLoginRsp() @linux.retry(times=5, sleep_time=1) def discovery_iscsi(iscsiServerIp, iscsiServerPort): r, o, e = bash.bash_roe( "timeout 10 iscsiadm -m discovery --type sendtargets --portal %s:%s" % (iscsiServerIp, iscsiServerPort)) if r != 0: raise RetryException( "can not discovery iscsi portal %s:%s, cause %s" % (iscsiServerIp, iscsiServerPort, e)) iqns = [] for i in o.splitlines(): if i.startswith("%s:%s," % (iscsiServerIp, iscsiServerPort)): iqns.append(i.strip().split(" ")[-1]) return iqns @linux.retry(times=20, sleep_time=1) def wait_iscsi_mknode(iscsiServerIp, iscsiServerPort, iscsiIqn, e=None): disks_by_dev = bash.bash_o( "ls /dev/disk/by-path | grep %s:%s | grep %s" % (iscsiServerIp, iscsiServerPort, iscsiIqn)).strip().splitlines() sid = bash.bash_o( "iscsiadm -m session | grep %s:%s | grep %s | awk '{print $2}'" % (iscsiServerIp, iscsiServerPort, iscsiIqn)).strip("[]\n ") if sid == "" or sid is None: err = "sid not found, this may because chap authentication failed" if e != None and e != "": err += " ,error: %s" % e raise RetryException(e) bash.bash_o("iscsiadm -m session -r %s --rescan" % sid) #Get the host_Number of iqn, Will match the HTCL attribute of iscsi according to Host_number host_Number = bash.bash_o( "iscsiadm -m session -P 3 --sid=%s | grep 'Host Number:' | awk '{print $3}'" % sid).strip() #Use HCTL, IQN, "-" to match the number of unmounted Luns according to lsscsi --transport disks_by_no_mapping_lun = bash.bash_o( "lsscsi --transport | grep -w %s | awk '{print $1,$NF}' | grep -E '\<%s\>:[[:digit:]]*:[[:digit:]]*:[[:digit:]]*' | awk '{print $NF}' | grep -x '-'" % (iscsiIqn, host_Number)).strip().splitlines() disks_by_iscsi = bash.bash_o( "iscsiadm -m session -P 3 --sid=%s | grep Lun" % sid).strip().splitlines() if len(disks_by_dev) < (len(disks_by_iscsi) - len(disks_by_no_mapping_lun)): raise RetryException( "iscsiadm says there are [%s] disks but only found [%s] disks on /dev/disk[%s], so not all disks loged in, and you can check the iscsi mounted disk by lsscsi --transport" "it may recover after a while so check and login again" % ((len(disks_by_iscsi) - len(disks_by_no_mapping_lun)), len(disks_by_dev), disks_by_dev)) def check_iscsi_conf(): shell.call( "sed -i 's/.*iscsid.startup.*=.*/iscsid.startup = \/bin\/systemctl start iscsid.socket iscsiuio.soccket/' /etc/iscsi/iscsid.conf", exception=False) check_iscsi_conf() path = "/var/lib/iscsi/nodes" self.clean_iscsi_cache_configuration(path, cmd.iscsiServerIp, cmd.iscsiServerPort) iqns = cmd.iscsiTargets if iqns is None or len(iqns) == 0: try: iqns = discovery_iscsi(cmd.iscsiServerIp, cmd.iscsiServerPort) except Exception as e: current_hostname = shell.call('hostname') current_hostname = current_hostname.strip(' \t\n\r') rsp.error = "login iscsi server %s:%s on host %s failed, because %s" % \ (cmd.iscsiServerIp, cmd.iscsiServerPort, current_hostname, e.message) rsp.success = False return jsonobject.dumps(rsp) if iqns is None or len(iqns) == 0: rsp.iscsiTargetStructList = [] return jsonobject.dumps(rsp) for iqn in iqns: t = IscsiTargetStruct() t.iqn = iqn try: if cmd.iscsiChapUserName and cmd.iscsiChapUserPassword: bash.bash_o( 'iscsiadm --mode node --targetname "%s" -p %s:%s --op=update --name node.session.auth.authmethod --value=CHAP' % (iqn, cmd.iscsiServerIp, cmd.iscsiServerPort)) bash.bash_o( 'iscsiadm --mode node --targetname "%s" -p %s:%s --op=update --name node.session.auth.username --value=%s' % (iqn, cmd.iscsiServerIp, cmd.iscsiServerPort, cmd.iscsiChapUserName)) bash.bash_o( 'iscsiadm --mode node --targetname "%s" -p %s:%s --op=update --name node.session.auth.password --value=%s' % (iqn, cmd.iscsiServerIp, cmd.iscsiServerPort, linux.shellquote(cmd.iscsiChapUserPassword))) r, o, e = bash.bash_roe( 'iscsiadm --mode node --targetname "%s" -p %s:%s --login' % (iqn, cmd.iscsiServerIp, cmd.iscsiServerPort)) wait_iscsi_mknode(cmd.iscsiServerIp, cmd.iscsiServerPort, iqn, e) finally: if bash.bash_r( "ls /dev/disk/by-path | grep %s:%s | grep %s" % (cmd.iscsiServerIp, cmd.iscsiServerPort, iqn)) != 0: rsp.iscsiTargetStructList.append(t) else: disks = bash.bash_o( "ls /dev/disk/by-path | grep %s:%s | grep %s" % (cmd.iscsiServerIp, cmd.iscsiServerPort, iqn)).strip().splitlines() for d in disks: lun_struct = self.get_disk_info_by_path(d.strip()) if lun_struct is not None: t.iscsiLunStructList.append(lun_struct) rsp.iscsiTargetStructList.append(t) linux.set_fail_if_no_path() return jsonobject.dumps(rsp)
def heartbeat_on_sharedblock(): failure = 0 while self.run_fencer(cmd.vgUuid, created_time): try: time.sleep(cmd.interval) global last_multipath_run if cmd.fail_if_no_path and time.time( ) - last_multipath_run > 4: last_multipath_run = time.time() linux.set_fail_if_no_path() health = lvm.check_vg_status(cmd.vgUuid, cmd.storageCheckerTimeout, check_pv=False) logger.debug( "sharedblock group primary storage %s fencer run result: %s" % (cmd.vgUuid, health)) if health[0] is True: failure = 0 continue failure += 1 if failure < cmd.maxAttempts: continue try: logger.warn("shared block storage %s fencer fired!" % cmd.vgUuid) self.report_storage_status([cmd.vgUuid], 'Disconnected', health[1]) # we will check one qcow2 per pv to determine volumes on pv should be kill invalid_pv_uuids = lvm.get_invalid_pv_uuids( cmd.vgUuid, cmd.checkIo) vms = lvm.get_running_vm_root_volume_on_pv( cmd.vgUuid, invalid_pv_uuids, True) killed_vm_uuids = [] for vm in vms: kill = shell.ShellCmd('kill -9 %s' % vm.pid) kill(False) if kill.return_code == 0: logger.warn( 'kill the vm[uuid:%s, pid:%s] because we lost connection to the storage.' 'failed to run health check %s times' % (vm.uuid, vm.pid, cmd.maxAttempts)) killed_vm_uuids.append(vm.uuid) else: logger.warn( 'failed to kill the vm[uuid:%s, pid:%s] %s' % (vm.uuid, vm.pid, kill.stderr)) for volume in vm.volumes: used_process = linux.linux_lsof(volume) if len(used_process) == 0: try: lvm.deactive_lv(volume, False) except Exception as e: logger.debug( "deactivate volume %s for vm %s failed, %s" % (volume, vm.uuid, e.message)) content = traceback.format_exc() logger.warn("traceback: %s" % content) else: logger.debug( "volume %s still used: %s, skip to deactivate" % (volume, used_process)) if len(killed_vm_uuids) != 0: self.report_self_fencer_triggered( [cmd.vgUuid], ','.join(killed_vm_uuids)) lvm.remove_partial_lv_dm(cmd.vgUuid) if lvm.check_vg_status(cmd.vgUuid, cmd.storageCheckerTimeout, True)[0] is False: lvm.drop_vg_lock(cmd.vgUuid) lvm.remove_device_map_for_vg(cmd.vgUuid) # reset the failure count failure = 0 except Exception as e: logger.warn("kill vm failed, %s" % e.message) content = traceback.format_exc() logger.warn("traceback: %s" % content) except Exception as e: logger.debug( 'self-fencer on sharedblock primary storage %s stopped abnormally, try again soon...' % cmd.vgUuid) content = traceback.format_exc() logger.warn(content) if not self.run_fencer(cmd.vgUuid, created_time): logger.debug( 'stop self-fencer on sharedblock primary storage %s for judger failed' % cmd.vgUuid) else: logger.warn( 'stop self-fencer on sharedblock primary storage %s' % cmd.vgUuid)