def lv_is_active(path):
# NOTE(weiw): use readonly to get active may return 'unknown'
cmd = shell.ShellCmd("lvs --nolocking --noheadings %s -oactive | grep -w active" % path)
cmd(is_exception=False)
return cmd.return_code == 0
def is_ip_existing(ip):
ip_str = ' %s/' % ip
cmd = shell.ShellCmd('ip a|grep inet|grep "%s"' % ip_str)
cmd(is_exception=False)
return cmd.return_code == 0
def is_interface_bridge(bridge_name):
cmd = shell.ShellCmd("brctl show |sed -n '2,$p'|cut -f 1")
cmd(is_exception=False)
bridges = cmd.stdout.split('\n')
if bridge_name in bridges:
return True
def set_device_uuid_alias(interf, l2NetworkUuid):
cmd = shell.ShellCmd("ip link set dev %s alias \"uuid: %s\"" % (interf, l2NetworkUuid))
cmd(is_exception=False)
def flush_device_ip(dev):
if is_network_device_existing(dev):
cmd = shell.ShellCmd("ip addr flush dev %s" % dev)
cmd(is_exception=False)
return cmd.stdout
def raw_create(dst, size):
shell.ShellCmd('/usr/bin/qemu-img create -f raw %s %s' % (dst, size))()
shell.ShellCmd('chmod 666 %s' % dst)()
def get_pid_by_process_name(name):
cmd = shell.ShellCmd('ps -ae | grep -w %s' % name)
output = cmd(False)
if cmd.return_code != 0:
return None
return output.split()[0]
def kill_progresses_using_mount_path(mount_path):
o = shell.ShellCmd("pkill -9 -e -f '%s'" % mount_path)
o(False)
logger.warn('kill the progresses with mount path: %s, killed process: %s' % (mount_path, o.stdout))
def linux_lsof(file):
cmd = shell.ShellCmd("lsof %s | grep -v '^COMMAND'" % file)
cmd(is_exception=False)
return cmd.stdout
def touch_heartbeat_file():
touch = shell.ShellCmd('timeout %s touch %s' % (cmd.storageCheckerTimeout, heartbeat_file_path))
touch(False)
if touch.return_code != 0:
logger.warn('unable to touch %s, %s %s' % (heartbeat_file_path, touch.stderr, touch.stdout))
return touch.return_code == 0
def kill_and_umount(mount_path, is_nfs):
do_kill_and_umount(mount_path, is_nfs)
if is_nfs:
shell.ShellCmd("systemctl start nfs-client.target")(False)
def heartbeat_on_sharedblock():
failure = 0
while self.run_sharedblock_fencer:
try:
time.sleep(cmd.interval)
health = lvm.check_vg_status(cmd.vgUuid, cmd.storageCheckerTimeout)
logger.debug("sharedblock group primary storage %s fencer run result: %s" % (cmd.vgUuid, health))
if health[0] is True:
failure = 0
continue
failure += 1
if failure < cmd.maxAttempts:
continue
try:
logger.warn("shared block storage %s fencer fired!" % cmd.vgUuid)
# we will check one qcow2 per pv to determine volumes on pv should be kill
invalid_pv_uuids = lvm.get_invalid_pv_uuids(cmd.vgUuid, cmd.checkIo)
vms = lvm.get_running_vm_root_volume_on_pv(cmd.vgUuid, invalid_pv_uuids, cmd.checkIo)
for vm in vms:
kill = shell.ShellCmd('kill -9 %s' % vm.pid)
kill(False)
if kill.return_code == 0:
logger.warn(
'kill the vm[uuid:%s, pid:%s] because we lost connection to the storage.'
'failed to run health check %s times' % (vm.uuid, vm.pid, cmd.maxAttempts))
else:
logger.warn(
'failed to kill the vm[uuid:%s, pid:%s] %s' % (vm.uuid, vm.pid, kill.stderr))
for volume in vm.volumes:
used_process = linux.linux_lsof(volume)
if len(used_process) == 0:
try:
lvm.deactive_lv(volume, False)
except Exception as e:
logger.debug("deactivate volume %s for vm %s failed, %s" % (volume, vm.uuid, e.message))
content = traceback.format_exc()
logger.warn("traceback: %s" % content)
else:
logger.debug("volume %s still used: %s, skip to deactivate" % (volume, used_process))
lvm.remove_partial_lv_dm(cmd.vgUuid)
if lvm.check_vg_status(cmd.vgUuid, cmd.storageCheckerTimeout, False)[0] is False:
lvm.drop_vg_lock(cmd.vgUuid)
lvm.remove_device_map_for_vg(cmd.vgUuid)
# reset the failure count
failure = 0
except Exception as e:
logger.warn("kill vm failed, %s" % e.message)
content = traceback.format_exc()
logger.warn("traceback: %s" % content)
finally:
self.report_storage_status([cmd.vgUuid], 'Disconnected', health[1])
except Exception as e:
logger.debug('self-fencer on sharedblock primary storage %s stopped abnormally' % cmd.vgUuid)
content = traceback.format_exc()
logger.warn(content)
logger.debug('stop self-fencer on sharedblock primary storage %s' % cmd.vgUuid)
def clean_duplicate_configs():
cmd = shell.ShellCmd("md5sum %s/* " % LVM_CONFIG_BACKUP_PATH +
" | awk 'p[$1]++ { printf \"rm %s\\n\",$2;}' | bash")
cmd(is_exception=False)
def get_lv_locking_type(path):
if not lv_is_active(path):
return LvmlockdLockType.NULL
cmd = shell.ShellCmd("lvmlockctl -i | grep %s | awk '{print $3}'" % lv_uuid(path))
cmd(is_exception=True)
return LvmlockdLockType.from_abbr(cmd.stdout.strip())
def qcow2_clone(src, dst):
fmt = get_img_fmt(src)
shell.ShellCmd('/usr/bin/qemu-img create -F %s -b %s -f qcow2 %s' %
(fmt, src, dst))()
shell.ShellCmd('chmod 666 %s' % dst)()
def qcow2_fill(seek, length, path, raise_excpetion=False):
cmd = shell.ShellCmd("qemu-io -c 'write %s %s' %s" % (seek, length, path))
cmd(raise_excpetion)
def raw_clone(src, dst):
shell.ShellCmd('/usr/bin/qemu-img create -b %s -f raw %s' % (src, dst))()
shell.ShellCmd('chmod 666 %s' % dst)()
def __call_shellcmd(cmd, exception=False, workdir=None):
shellcmd = shell.ShellCmd(cmd, workdir)
shellcmd(exception)
return shellcmd
def get_cpu_num():
out = shell.ShellCmd("cat /proc/cpuinfo | grep 'processor' | wc -l")()
return int(out)
def set_node_ip(self, node_ip):
self.node_ip = node_ip
#need to manually set nodeName
shell.ShellCmd("echo 'management.server.ip=%s' >> %s" %
(self.node_ip, self.node_zstack_properties))()
def check(_):
cmd = shell.ShellCmd('ps %s > /dev/null' % pid)
cmd(False)
return cmd.return_code != 0
def delete_heartbeat_file():
delete = shell.ShellCmd("timeout %s rbd rm --id zstack %s -m %s" %
(cmd.storageCheckerTimeout, cmd.heartbeatImagePath, mon_url))
delete(False)
def umount(path, is_exception=True):
cmd = shell.ShellCmd('umount -f -l %s' % path)
cmd(is_exception=is_exception)
return cmd.return_code == 0
def vm_match_storage_type(vmUuid, isFileSystem):
o = shell.ShellCmd("virsh dumpxml %s | grep \"disk type='file'\"" % vmUuid)
o(False)
if (o.return_code == 0 and isFileSystem) or (o.return_code != 0 and not isFileSystem):
return True
return False
def get_device_ip(dev):
cmd = shell.ShellCmd("ip addr show dev %s|grep inet|grep -v inet6|awk -F'inet' '{print $2}'|awk '{print $1}'|awk -F'/24' '{print $1}'" % dev)
cmd(is_exception=False)
return cmd.stdout if cmd.stdout != "" else None
def _get_memory(word):
out = shell.ShellCmd("cat /proc/meminfo | grep '%s'" % word)()
(name, capacity) = out.split(':')
capacity = re.sub('[k|K][b|B]', '', capacity).strip()
#capacity = capacity.rstrip('kB').rstrip('KB').rstrip('kb').strip()
return sizeunit.KiloByte.toByte(long(capacity))
def is_network_device_existing(dev):
cmd = shell.ShellCmd('ip link show %s' % dev)
cmd(is_exception=False)
return cmd.return_code == 0
def get_used_disk_apparent_size(dir_path):
output = shell.ShellCmd('du --apparent-size --max-depth=1 %s | tail -1' %
dir_path)()
return long(output.split()[0])
def get_all_bridge_interface(bridge_name):
cmd = shell.ShellCmd("brctl show %s|sed -n '2,$p'|cut -f 6-10" % bridge_name)
cmd(is_exception=False)
vifs = cmd.stdout.split('\n')
return [v.strip(" \t\r\n") for v in vifs]
def lv_uuid(path):
cmd = shell.ShellCmd("lvs --nolocking --readonly --noheadings %s -ouuid" % path)
cmd(is_exception=False)
return cmd.stdout.strip()
