def ping(self, req): cmd = jsonobject.loads(req[http.REQUEST_BODY]) if cmd.uuid not in self.mount_path.keys(): self.mount_path[cmd.uuid] = cmd.mountPath mount_path = self.mount_path[cmd.uuid] # if nfs service stop, os.path.isdir will hung if not linux.timeout_isdir(mount_path) or not linux.is_mounted( path=mount_path): raise Exception( 'the mount path[%s] of the nfs primary storage[uuid:%s] is not existing' % (mount_path, cmd.uuid)) test_file = os.path.join(mount_path, '%s-ping-test-file' % uuidhelper.uuid()) touch = shell.ShellCmd('timeout 60 touch %s' % test_file) touch(False) if touch.return_code == 124: raise Exception( 'unable to access the mount path[%s] of the nfs primary storage[uuid:%s] in 60s, timeout' % (mount_path, cmd.uuid)) elif touch.return_code != 0: touch.raise_error() linux.rm_file_force(test_file) return jsonobject.dumps(NfsResponse())
def connect(self, req): none_shared_mount_fs_type = [ 'xfs', 'ext2', 'ext3', 'ext4', 'vfat', 'tmpfs', 'btrfs' ] cmd = jsonobject.loads(req[http.REQUEST_BODY]) if not linux.timeout_isdir(cmd.mountPoint): raise kvmagent.KvmError( '%s is not a directory, the mount point seems not setup' % cmd.mountPoint) folder_fs_type = shell.call("df -T %s|tail -1|awk '{print $2}'" % cmd.mountPoint).strip() if folder_fs_type in none_shared_mount_fs_type: raise kvmagent.KvmError( '%s filesystem is %s, which is not a shared mount point type.' % (cmd.mountPoint, folder_fs_type)) id_dir = os.path.join(cmd.mountPoint, "zstack_smp_id_file") shell.call("mkdir -p %s" % id_dir) lock_file = os.path.join(id_dir, "uuid.lock") @lock.file_lock(lock_file, locker=lock.Flock()) def check_other_smp_and_set_id_file(uuid, existUuids): o = shell.ShellCmd('''\ ls %s | grep -v %s | grep -o "[0-9a-f]\{8\}[0-9a-f]\{4\}[1-5][0-9a-f]\{3\}[89ab][0-9a-f]\{3\}[0-9a-f]\{12\}"\ ''' % (id_dir, uuid)) o(False) if o.return_code != 0: file_uuids = [] else: file_uuids = o.stdout.splitlines() for file_uuid in file_uuids: if file_uuid in existUuids: raise Exception( "the mount point [%s] has been occupied by other SMP[uuid:%s], Please attach this directly" % (cmd.mountPoint, file_uuid)) logger.debug("existing id files: %s" % file_uuids) self.id_files[uuid] = os.path.join(id_dir, uuid) if not os.path.exists(self.id_files[uuid]): # check if hosts in the same cluster mount the same path but different storages. rsp.isFirst = True for file_uuid in file_uuids: linux.rm_file_force(os.path.join(id_dir, file_uuid)) linux.touch_file(self.id_files[uuid]) linux.sync() rsp = ConnectRsp() check_other_smp_and_set_id_file(cmd.uuid, cmd.existUuids) rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity( cmd.mountPoint) return jsonobject.dumps(rsp)
def setup_self_fencer(self, req): cmd = jsonobject.loads(req[http.REQUEST_BODY]) self.run_filesystem_fencer = True @thread.AsyncThread def heartbeat_file_fencer(mount_path, ps_uuid): heartbeat_file_path = os.path.join( mount_path, 'heartbeat-file-kvm-host-%s.hb' % cmd.hostUuid) try: failure = 0 while self.run_filesystem_fencer: time.sleep(cmd.interval) touch = shell.ShellCmd( 'timeout %s touch %s; exit $?' % (cmd.storageCheckerTimeout, heartbeat_file_path)) touch(False) if touch.return_code == 0: failure = 0 continue logger.warn( 'unable to touch %s, %s %s' % (heartbeat_file_path, touch.stderr, touch.stdout)) failure += 1 if failure == cmd.maxAttempts: logger.warn( 'failed to touch the heartbeat file[%s] %s times, we lost the connection to the storage,' 'shutdown ourselves' % (heartbeat_file_path, cmd.maxAttempts)) self.report_storage_status([ps_uuid], 'Disconnected') kill_vm(cmd.maxAttempts, [mount_path], True) break logger.debug('stop heartbeat[%s] for filesystem self-fencer' % heartbeat_file_path) except: content = traceback.format_exc() logger.warn(content) for mount_point, uuid in zip(cmd.mountPoints, cmd.uuids): if not linux.timeout_isdir(mount_point): raise Exception('the mount point[%s] is not a directory' % mount_point) heartbeat_file_fencer(mount_point, uuid) return jsonobject.dumps(AgentRsp())
def connect(self, req): none_shared_mount_fs_type = ['xfs', 'ext2', 'ext3', 'ext4', 'vfat', 'tmpfs', 'btrfs'] cmd = jsonobject.loads(req[http.REQUEST_BODY]) if not linux.timeout_isdir(cmd.mountPoint): raise kvmagent.KvmError('%s is not a directory, the mount point seems not setup' % cmd.mountPoint) folder_fs_type = shell.call("df -T %s|tail -1|awk '{print $2}'" % cmd.mountPoint).strip() if folder_fs_type in none_shared_mount_fs_type: raise kvmagent.KvmError( '%s filesystem is %s, which is not a shared mount point type.' % (cmd.mountPoint, folder_fs_type)) id_dir = os.path.join(cmd.mountPoint, "zstack_smp_id_file") shell.call("mkdir -p %s" % id_dir) lock_file = os.path.join(id_dir, "uuid.lock") @lock.file_lock(lock_file, locker=lock.Flock()) def check_other_smp_and_set_id_file(uuid, existUuids): o = shell.ShellCmd('''\ ls %s | grep -v %s | grep -o "[0-9a-f]\{8\}[0-9a-f]\{4\}[1-5][0-9a-f]\{3\}[89ab][0-9a-f]\{3\}[0-9a-f]\{12\}"\ ''' % (id_dir, uuid)) o(False) if o.return_code != 0: file_uuids = [] else: file_uuids = o.stdout.splitlines() for file_uuid in file_uuids: if file_uuid in existUuids: raise Exception( "the mount point [%s] has been occupied by other SMP[uuid:%s], Please attach this directly" % (cmd.mountPoint, file_uuid)) logger.debug("existing id files: %s" % file_uuids) self.id_files[uuid] = os.path.join(id_dir, uuid) if not os.path.exists(self.id_files[uuid]): # check if hosts in the same cluster mount the same path but different storages. rsp.isFirst = True for file_uuid in file_uuids: linux.rm_file_force(os.path.join(id_dir, file_uuid)) linux.touch_file(self.id_files[uuid]) linux.sync() rsp = ConnectRsp() check_other_smp_and_set_id_file(cmd.uuid, cmd.existUuids) rsp.totalCapacity, rsp.availableCapacity = self._get_disk_capacity(cmd.mountPoint) return jsonobject.dumps(rsp)
def ping(self, req): cmd = jsonobject.loads(req[http.REQUEST_BODY]) mount_path = self.mount_path[cmd.uuid] # if nfs service stop, os.path.isdir will hung if not linux.timeout_isdir(mount_path) or not linux.is_mounted(path=mount_path): raise Exception('the mount path[%s] of the nfs primary storage[uuid:%s] is not existing' % (mount_path, cmd.uuid)) test_file = os.path.join(mount_path, '%s-ping-test-file' % uuidhelper.uuid()) touch = shell.ShellCmd('timeout 60 touch %s' % test_file) touch(False) if touch.return_code == 124: raise Exception('unable to access the mount path[%s] of the nfs primary storage[uuid:%s] in 60s, timeout' % (mount_path, cmd.uuid)) elif touch.return_code != 0: touch.raise_error() linux.rm_file_force(test_file) return jsonobject.dumps(NfsResponse())
def setup_self_fencer(self, req): cmd = jsonobject.loads(req[http.REQUEST_BODY]) @thread.AsyncThread def heartbeat_file_fencer(mount_path, ps_uuid, mounted_by_zstack): def try_remount_fs(): if mount_path_is_nfs(mount_path): shell.run("systemctl start nfs-client.target") while self.run_filesystem_fencer(ps_uuid, created_time): if linux.is_mounted( path=mount_path) and touch_heartbeat_file(): self.report_storage_status([ps_uuid], 'Connected') logger.debug( "fs[uuid:%s] is reachable again, report to management" % ps_uuid) break try: logger.debug( 'fs[uuid:%s] is unreachable, it will be remounted after 180s' % ps_uuid) time.sleep(180) if not self.run_filesystem_fencer( ps_uuid, created_time): break linux.remount(url, mount_path, options) self.report_storage_status([ps_uuid], 'Connected') logger.debug( "remount fs[uuid:%s] success, report to management" % ps_uuid) break except: logger.warn( 'remount fs[uuid:%s] fail, try again soon' % ps_uuid) kill_progresses_using_mount_path(mount_path) logger.debug('stop remount fs[uuid:%s]' % ps_uuid) def after_kill_vm(): if not killed_vm_pids or not mounted_by_zstack: return try: kill_and_umount(mount_path, mount_path_is_nfs(mount_path)) except UmountException: if shell.run('ps -p %s' % ' '.join(killed_vm_pids)) == 0: virsh_list = shell.call( "timeout 10 virsh list --all || echo 'cannot obtain virsh list'" ) logger.debug("virsh_list:\n" + virsh_list) logger.error( 'kill vm[pids:%s] failed because of unavailable fs[mountPath:%s].' ' please retry "umount -f %s"' % (killed_vm_pids, mount_path, mount_path)) return try_remount_fs() def touch_heartbeat_file(): touch = shell.ShellCmd( 'timeout %s touch %s' % (cmd.storageCheckerTimeout, heartbeat_file_path)) touch(False) if touch.return_code != 0: logger.warn( 'unable to touch %s, %s %s' % (heartbeat_file_path, touch.stderr, touch.stdout)) return touch.return_code == 0 heartbeat_file_path = os.path.join( mount_path, 'heartbeat-file-kvm-host-%s.hb' % cmd.hostUuid) created_time = time.time() with self.fencer_lock: self.run_filesystem_fencer_timestamp[ps_uuid] = created_time try: failure = 0 url = shell.call("mount | grep -e '%s' | awk '{print $1}'" % mount_path).strip() options = shell.call( "mount | grep -e '%s' | awk -F '[()]' '{print $2}'" % mount_path).strip() while self.run_filesystem_fencer(ps_uuid, created_time): time.sleep(cmd.interval) if touch_heartbeat_file(): failure = 0 continue failure += 1 if failure == cmd.maxAttempts: logger.warn( 'failed to touch the heartbeat file[%s] %s times, we lost the connection to the storage,' 'shutdown ourselves' % (heartbeat_file_path, cmd.maxAttempts)) self.report_storage_status([ps_uuid], 'Disconnected') killed_vm_pids = kill_vm(cmd.maxAttempts, [mount_path], True) after_kill_vm() logger.debug('stop heartbeat[%s] for filesystem self-fencer' % heartbeat_file_path) except: content = traceback.format_exc() logger.warn(content) for mount_path, uuid, mounted_by_zstack in zip(cmd.mountPaths, cmd.uuids, cmd.mountedByZStack): if not linux.timeout_isdir(mount_path): raise Exception('the mount path[%s] is not a directory' % mount_path) heartbeat_file_fencer(mount_path, uuid, mounted_by_zstack) return jsonobject.dumps(AgentRsp())
def setup_self_fencer(self, req): cmd = jsonobject.loads(req[http.REQUEST_BODY]) self.run_filesystem_fencer = True @thread.AsyncThread def heartbeat_file_fencer(heartbeat_file_path, ps_uuid): try: failure = 0 while self.run_filesystem_fencer: time.sleep(cmd.interval) touch = shell.ShellCmd( 'timeout %s touch %s; exit $?' % (cmd.storageCheckerTimeout, heartbeat_file_path)) touch(False) if touch.return_code == 0: failure = 0 continue logger.warn( 'unable to touch %s, %s %s' % (heartbeat_file_path, touch.stderr, touch.stdout)) failure += 1 if failure == cmd.maxAttempts: logger.warn( 'failed to touch the heartbeat file[%s] %s times, we lost the connection to the storage,' 'shutdown ourselves' % (heartbeat_file_path, cmd.maxAttempts)) mountPath = (os.path.split(heartbeat_file_path))[0] self.report_storage_status([ps_uuid], 'Disconnected') kill_vm(cmd.maxAttempts, mountPath, True) logger.debug('stop heartbeat[%s] for filesystem self-fencer' % heartbeat_file_path) except: content = traceback.format_exc() logger.warn(content) gateway = cmd.storageGateway if not gateway: gateway = linux.get_gateway_by_default_route() @thread.AsyncThread def storage_gateway_fencer(gw): failure = 0 try: while self.run_filesystem_fencer: time.sleep(cmd.interval) ping = shell.ShellCmd( "nmap -sP -PI %s | grep 'Host is up'" % gw) ping(False) if ping.return_code == 0: failure = 0 continue logger.warn( 'unable to ping the storage gateway[%s], %s %s' % (gw, ping.stderr, ping.stdout)) failure += 1 if failure == cmd.maxAttempts: logger.warn( 'failed to ping storage gateway[%s] %s times, we lost connection to the storage,' 'shutdown ourselves' % (gw, cmd.maxAttempts)) self.report_storage_status(cmd.psUuids, 'Disconnected') kill_vm(cmd.maxAttempts) logger.debug( 'stop gateway[%s] fencer for filesystem self-fencer' % gw) except: content = traceback.format_exc() logger.warn(content) for mount_point, uuid in zip(cmd.mountPoints, cmd.uuids): if not linux.timeout_isdir(mount_point): raise Exception('the mount point[%s] is not a directory' % mount_point) hb_file = os.path.join( mount_point, 'heartbeat-file-kvm-host-%s.hb' % cmd.hostUuid) heartbeat_file_fencer(hb_file, uuid) if gateway: storage_gateway_fencer(gateway) else: logger.warn( 'cannot find storage gateway, unable to setup storage gateway fencer' ) return jsonobject.dumps(AgentRsp())
def setup_self_fencer(self, req): cmd = jsonobject.loads(req[http.REQUEST_BODY]) @thread.AsyncThread def heartbeat_file_fencer(mount_path, ps_uuid, mounted_by_zstack, url, options): def try_remount_fs(): if mount_path_is_nfs(mount_path): shell.run("systemctl start nfs-client.target") while self.run_fencer(ps_uuid, created_time): if linux.is_mounted( path=mount_path) and touch_heartbeat_file(): self.report_storage_status([ps_uuid], 'Connected') logger.debug( "fs[uuid:%s] is reachable again, report to management" % ps_uuid) break try: logger.debug( 'fs[uuid:%s] is unreachable, it will be remounted after 180s' % ps_uuid) time.sleep(180) if not self.run_fencer(ps_uuid, created_time): break linux.remount(url, mount_path, options) self.report_storage_status([ps_uuid], 'Connected') logger.debug( "remount fs[uuid:%s] success, report to management" % ps_uuid) break except: logger.warn( 'remount fs[uuid:%s] fail, try again soon' % ps_uuid) kill_progresses_using_mount_path(mount_path) logger.debug('stop remount fs[uuid:%s]' % ps_uuid) def after_kill_vm(): if not killed_vm_pids or not mounted_by_zstack: return try: kill_and_umount(mount_path, mount_path_is_nfs(mount_path)) except UmountException: if shell.run('ps -p %s' % ' '.join(killed_vm_pids)) == 0: virsh_list = shell.call( "timeout 10 virsh list --all || echo 'cannot obtain virsh list'" ) logger.debug("virsh_list:\n" + virsh_list) logger.error( 'kill vm[pids:%s] failed because of unavailable fs[mountPath:%s].' ' please retry "umount -f %s"' % (killed_vm_pids, mount_path, mount_path)) return def touch_heartbeat_file(): touch = shell.ShellCmd( 'timeout %s touch %s' % (cmd.storageCheckerTimeout, heartbeat_file_path)) touch(False) if touch.return_code != 0: logger.warn( 'unable to touch %s, %s %s' % (heartbeat_file_path, touch.stderr, touch.stdout)) return touch.return_code == 0 def prepare_heartbeat_dir(): heartbeat_dir = os.path.join(mount_path, "zs-heartbeat") if not mounted_by_zstack or linux.is_mounted(mount_path): if not os.path.exists(heartbeat_dir): os.makedirs(heartbeat_dir, 0755) else: if os.path.exists(heartbeat_dir): linux.rm_dir_force(heartbeat_dir) return heartbeat_dir heartbeat_file_dir = prepare_heartbeat_dir() heartbeat_file_path = os.path.join( heartbeat_file_dir, 'heartbeat-file-kvm-host-%s.hb' % cmd.hostUuid) created_time = time.time() self.setup_fencer(ps_uuid, created_time) try: failure = 0 while self.run_fencer(ps_uuid, created_time): time.sleep(cmd.interval) if touch_heartbeat_file(): failure = 0 continue failure += 1 if failure == cmd.maxAttempts: logger.warn( 'failed to touch the heartbeat file[%s] %s times, we lost the connection to the storage,' 'shutdown ourselves' % (heartbeat_file_path, cmd.maxAttempts)) self.report_storage_status([ps_uuid], 'Disconnected') if cmd.strategy == 'Permissive': continue killed_vms = kill_vm(cmd.maxAttempts, [mount_path], True) if len(killed_vms) != 0: self.report_self_fencer_triggered( [ps_uuid], ','.join(killed_vms.keys())) clean_network_config(killed_vms.keys()) killed_vm_pids = killed_vms.values() after_kill_vm() if mounted_by_zstack and not linux.is_mounted( mount_path): try_remount_fs() prepare_heartbeat_dir() logger.debug('stop heartbeat[%s] for filesystem self-fencer' % heartbeat_file_path) except: content = traceback.format_exc() logger.warn(content) for mount_path, uuid, mounted_by_zstack, url, options in zip( cmd.mountPaths, cmd.uuids, cmd.mountedByZStack, cmd.urls, cmd.mountOptions): if not linux.timeout_isdir(mount_path): raise Exception('the mount path[%s] is not a directory' % mount_path) heartbeat_file_fencer(mount_path, uuid, mounted_by_zstack, url, options) return jsonobject.dumps(AgentRsp())
def setup_self_fencer(self, req): cmd = jsonobject.loads(req[http.REQUEST_BODY]) @thread.AsyncThread def heartbeat_file_fencer(mount_path, ps_uuid, mounted_by_zstack): def try_remount_fs(): if mount_path_is_nfs(mount_path): shell.run("systemctl start nfs-client.target") while self.run_fencer(ps_uuid, created_time): if linux.is_mounted(path=mount_path) and touch_heartbeat_file(): self.report_storage_status([ps_uuid], 'Connected') logger.debug("fs[uuid:%s] is reachable again, report to management" % ps_uuid) break try: logger.debug('fs[uuid:%s] is unreachable, it will be remounted after 180s' % ps_uuid) time.sleep(180) if not self.run_fencer(ps_uuid, created_time): break linux.remount(url, mount_path, options) self.report_storage_status([ps_uuid], 'Connected') logger.debug("remount fs[uuid:%s] success, report to management" % ps_uuid) break except: logger.warn('remount fs[uuid:%s] fail, try again soon' % ps_uuid) kill_progresses_using_mount_path(mount_path) logger.debug('stop remount fs[uuid:%s]' % ps_uuid) def after_kill_vm(): if not killed_vm_pids or not mounted_by_zstack: return try: kill_and_umount(mount_path, mount_path_is_nfs(mount_path)) except UmountException: if shell.run('ps -p %s' % ' '.join(killed_vm_pids)) == 0: virsh_list = shell.call("timeout 10 virsh list --all || echo 'cannot obtain virsh list'") logger.debug("virsh_list:\n" + virsh_list) logger.error('kill vm[pids:%s] failed because of unavailable fs[mountPath:%s].' ' please retry "umount -f %s"' % (killed_vm_pids, mount_path, mount_path)) return try_remount_fs() def touch_heartbeat_file(): touch = shell.ShellCmd('timeout %s touch %s' % (cmd.storageCheckerTimeout, heartbeat_file_path)) touch(False) if touch.return_code != 0: logger.warn('unable to touch %s, %s %s' % (heartbeat_file_path, touch.stderr, touch.stdout)) return touch.return_code == 0 heartbeat_file_path = os.path.join(mount_path, 'heartbeat-file-kvm-host-%s.hb' % cmd.hostUuid) created_time = time.time() self.setup_fencer(ps_uuid, created_time) try: failure = 0 url = shell.call("mount | grep -e '%s' | awk '{print $1}'" % mount_path).strip() options = shell.call("mount | grep -e '%s' | awk -F '[()]' '{print $2}'" % mount_path).strip() while self.run_fencer(ps_uuid, created_time): time.sleep(cmd.interval) if touch_heartbeat_file(): failure = 0 continue failure += 1 if failure == cmd.maxAttempts: logger.warn('failed to touch the heartbeat file[%s] %s times, we lost the connection to the storage,' 'shutdown ourselves' % (heartbeat_file_path, cmd.maxAttempts)) self.report_storage_status([ps_uuid], 'Disconnected') killed_vms = kill_vm(cmd.maxAttempts, [mount_path], True) if len(killed_vms) != 0: self.report_self_fencer_triggered([ps_uuid], ','.join(killed_vms.keys())) killed_vm_pids = killed_vms.values() after_kill_vm() logger.debug('stop heartbeat[%s] for filesystem self-fencer' % heartbeat_file_path) except: content = traceback.format_exc() logger.warn(content) for mount_path, uuid, mounted_by_zstack in zip(cmd.mountPaths, cmd.uuids, cmd.mountedByZStack): if not linux.timeout_isdir(mount_path): raise Exception('the mount path[%s] is not a directory' % mount_path) heartbeat_file_fencer(mount_path, uuid, mounted_by_zstack) return jsonobject.dumps(AgentRsp())
def setup_self_fencer(self, req): cmd = jsonobject.loads(req[http.REQUEST_BODY]) self.run_filesystem_fencer = True @thread.AsyncThread def heartbeat_file_fencer(mount_path, ps_uuid): heartbeat_file_path = os.path.join( mount_path, 'heartbeat-file-kvm-host-%s.hb' % cmd.hostUuid) try: failure = 0 while self.run_filesystem_fencer: time.sleep(cmd.interval) touch = shell.ShellCmd( 'timeout %s touch %s; exit $?' % (cmd.storageCheckerTimeout, heartbeat_file_path)) touch(False) if touch.return_code == 0: failure = 0 continue logger.warn( 'unable to touch %s, %s %s' % (heartbeat_file_path, touch.stderr, touch.stdout)) failure += 1 if failure == cmd.maxAttempts: logger.warn( 'failed to touch the heartbeat file[%s] %s times, we lost the connection to the storage,' 'shutdown ourselves' % (heartbeat_file_path, cmd.maxAttempts)) self.report_storage_status([ps_uuid], 'Disconnected') kill_vm(cmd.maxAttempts, [mount_path], True) break logger.debug('stop heartbeat[%s] for filesystem self-fencer' % heartbeat_file_path) except: content = traceback.format_exc() logger.warn(content) gateway = cmd.storageGateway if not gateway: gateway = linux.get_gateway_by_default_route() @thread.AsyncThread def storage_gateway_fencer(gw): failure = 0 try: while self.run_filesystem_fencer: time.sleep(cmd.interval) ping = shell.ShellCmd( "nmap -sP -PI %s | grep 'Host is up'" % gw) ping(False) if ping.return_code == 0: failure = 0 continue logger.warn( 'unable to ping the storage gateway[%s], %s %s' % (gw, ping.stderr, ping.stdout)) failure += 1 if failure == cmd.maxAttempts: logger.warn( 'failed to ping storage gateway[%s] %s times, we lost connection to the storage,' 'shutdown ourselves' % (gw, cmd.maxAttempts)) # there might be a race condition with very small probabilities because report action is async, # cause vm ha slow because starting vm might select this broken host # due to ps-report arriving later than vm-report. # a sure card is to modify it to sync action with try-except # but if http post fail to retry too many times, kill vm will be too late. # so for ha efficiency, that's it.. self.report_storage_status(cmd.psUuids, 'Disconnected') kill_vm(cmd.maxAttempts, cmd.mountPoints, True) logger.debug( 'stop gateway[%s] fencer for filesystem self-fencer' % gw) except: content = traceback.format_exc() logger.warn(content) for mount_point, uuid in zip(cmd.mountPoints, cmd.uuids): if not linux.timeout_isdir(mount_point): raise Exception('the mount point[%s] is not a directory' % mount_point) heartbeat_file_fencer(mount_point, uuid) if gateway: storage_gateway_fencer(gateway) else: logger.warn( 'cannot find storage gateway, unable to setup storage gateway fencer' ) return jsonobject.dumps(AgentRsp())