def _cleanup(self): self.logger.info('Cleaning up temporary files') du = DshUtils() root_dir = os.sep dirlist = set([os.path.join(root_dir, 'tmp'), os.path.join(root_dir, 'var', 'tmp')]) # get tmp dir from the environment for envname in 'TMPDIR', 'TEMP', 'TMP': dirname = os.getenv(envname) if dirname: dirlist.add(dirname) p = re.compile(r'^pbs\.\d+') for tmpdir in dirlist: # list the contents of each tmp dir and # get the file list to be deleted self.logger.info('Cleaning up ' + tmpdir + ' dir') ftd = [] files = du.listdir(path=tmpdir) bn = os.path.basename ftd.extend([f for f in files if bn(f).startswith('PtlPbs')]) ftd.extend([f for f in files if bn(f).startswith('STDIN')]) ftd.extend([f for f in files if bn(f).startswith('pbsscrpt')]) ftd.extend([f for f in files if bn(f).startswith('pbs.conf.')]) ftd.extend([f for f in files if p.match(bn(f))]) for f in ftd: du.rm(path=f, sudo=True, recursive=True, force=True, level=logging.DEBUG) tmpdir = tempfile.gettempdir() os.chdir(tmpdir) tmppath = os.path.join(tmpdir, 'dejagnutemp%s' % os.getpid()) if du.isdir(path=tmppath): du.rm(path=tmppath, recursive=True, sudo=True, force=True, level=logging.DEBUG)
def _cleanup(self): self.logger.info('Cleaning up temporary files') du = DshUtils() tmpdir = tempfile.gettempdir() ftd = [] if tmpdir: files = du.listdir(path=tmpdir) bn = os.path.basename ftd.extend([f for f in files if bn(f).startswith('PtlPbs')]) ftd.extend([f for f in files if bn(f).startswith('STDIN')]) for f in ftd: du.rm(path=f, sudo=True, recursive=True, force=True, level=logging.DEBUG) os.chdir(tmpdir) tmppath = os.path.join(tmpdir, 'dejagnutemp%s' % os.getpid()) if du.isdir(path=tmppath): du.rm(path=tmppath, recursive=True, sudo=True, force=True, level=logging.DEBUG)
def check_hardware_status_and_core_files(self, test): """ function checks hardware status and core files every 5 minutes """ du = DshUtils() systems = list(self.param_dict['servers']) systems.extend(self.param_dict['moms']) systems.extend(self.param_dict['comms']) systems = list(set(systems)) if hasattr(test, 'test'): _test = test.test elif hasattr(test, 'context'): _test = test.context else: return None for name in ['servers', 'moms', 'comms', 'clients']: mlist = None if (hasattr(_test, name) and (getattr(_test, name, None) is not None)): mlist = getattr(_test, name).values() if mlist: for mc in mlist: platform = mc.platform if ((platform not in ['linux', 'shasta', 'cray']) and (mc.hostname in systems)): systems.remove(mc.hostname) self.hardware_report_timer = Timer( 300, self.check_hardware_status_and_core_files, args=(test, )) self.hardware_report_timer.start() for hostname in systems: hr = SystemInfo() hr.get_system_info(hostname) # monitors disk used_disk_percent = getattr(hr, 'system_disk_used_percent', None) if used_disk_percent is None: _msg = hostname _msg += ": unable to get disk info" self.hardware_report_timer.cancel() raise SkipTest(_msg) elif 70 <= used_disk_percent < 95: _msg = hostname + ": disk usage is at " _msg += str(used_disk_percent) + "%" _msg += ", disk cleanup is recommended." self.logger.warning(_msg) elif used_disk_percent >= 95: _msg = hostname + ":disk usage > 95%, skipping the test(s)" self.hardware_report_timer.cancel() raise SkipTest(_msg) # checks for core files pbs_conf = du.parse_pbs_config(hostname) mom_priv_path = os.path.join(pbs_conf["PBS_HOME"], "mom_priv") if du.isdir(hostname=hostname, path=mom_priv_path): mom_priv_files = du.listdir(hostname=hostname, path=mom_priv_path, sudo=True, fullpath=False) if fnmatch.filter(mom_priv_files, "core*"): _msg = hostname + ": core files found in " _msg += mom_priv_path self.logger.warning(_msg) server_priv_path = os.path.join(pbs_conf["PBS_HOME"], "server_priv") if du.isdir(hostname=hostname, path=server_priv_path): server_priv_files = du.listdir(hostname=hostname, path=server_priv_path, sudo=True, fullpath=False) if fnmatch.filter(server_priv_files, "core*"): _msg = hostname + ": core files found in " _msg += server_priv_path self.logger.warning(_msg) sched_priv_path = os.path.join(pbs_conf["PBS_HOME"], "sched_priv") if du.isdir(hostname=hostname, path=sched_priv_path): sched_priv_files = du.listdir(hostname=hostname, path=sched_priv_path, sudo=True, fullpath=False) if fnmatch.filter(sched_priv_files, "core*"): _msg = hostname + ": core files found in " _msg += sched_priv_path self.logger.warning(_msg) for u in PBS_ALL_USERS: user_home_files = du.listdir(hostname=hostname, path=u.home, sudo=True, fullpath=False, runas=u.name) if user_home_files and fnmatch.filter(user_home_files, "core*"): _msg = hostname + ": user-" + str(u) _msg += ": core files found in " self.logger.warning(_msg + u.home)
def check_hardware_status_and_core_files(self): """ function checks hardware status and core files every 5 minutes """ du = DshUtils() self.hardware_report_timer = Timer( 300, self.check_hardware_status_and_core_files) self.hardware_report_timer.start() systems = list(self.param_dict['servers']) systems.extend(self.param_dict['moms']) systems.extend(self.param_dict['comms']) systems = list(set(systems)) for hostname in systems: hr = SystemInfo() hr.get_system_info(hostname) # monitors disk used_disk_percent = getattr(hr, 'system_disk_used_percent', None) if used_disk_percent is None: _msg = hostname _msg += ": unable to get disk info" self.hardware_report_timer.cancel() raise SkipTest(_msg) elif 70 <= used_disk_percent < 95: _msg = hostname + ": disk usage is at " _msg += str(used_disk_percent) + "%" _msg += ", disk cleanup is recommended." self.logger.warning(_msg) elif used_disk_percent >= 95: _msg = hostname + ":disk usage > 95%, skipping the test(s)" self.hardware_report_timer.cancel() raise SkipTest(_msg) # checks for core files pbs_conf = du.parse_pbs_config(hostname) mom_priv_path = os.path.join(pbs_conf["PBS_HOME"], "mom_priv") if du.isdir(hostname=hostname, path=mom_priv_path): mom_priv_files = du.listdir( hostname=hostname, path=mom_priv_path, sudo=True, fullpath=False) if fnmatch.filter(mom_priv_files, "core*"): _msg = hostname + ": core files found in " _msg += mom_priv_path self.logger.warning(_msg) server_priv_path = os.path.join( pbs_conf["PBS_HOME"], "server_priv") if du.isdir(hostname=hostname, path=server_priv_path): server_priv_files = du.listdir( hostname=hostname, path=server_priv_path, sudo=True, fullpath=False) if fnmatch.filter(server_priv_files, "core*"): _msg = hostname + ": core files found in " _msg += server_priv_path self.logger.warning(_msg) sched_priv_path = os.path.join(pbs_conf["PBS_HOME"], "sched_priv") if du.isdir(hostname=hostname, path=sched_priv_path): sched_priv_files = du.listdir( hostname=hostname, path=sched_priv_path, sudo=True, fullpath=False) if fnmatch.filter(sched_priv_files, "core*"): _msg = hostname + ": core files found in " _msg += sched_priv_path self.logger.warning(_msg) for u in PBS_ALL_USERS: user_home_files = du.listdir(hostname=hostname, path=u.home, sudo=True, fullpath=False, runas=u.name) if user_home_files and fnmatch.filter( user_home_files, "core*"): _msg = hostname + ": user-" + str(u) _msg += ": core files found in " self.logger.warning(_msg + u.home)
def _cleanup(self): self.logger.info('Cleaning up temporary files') du = DshUtils() hosts = set(self.param_dict['moms']).union( set(self.param_dict['servers'])) for user in PBS_USERS: self.logger.debug('Cleaning %s\'s home directory' % (str(user))) runas = PbsUser.get_user(user) for host in hosts: ret = du.run_cmd(host, cmd=['echo', '$HOME'], sudo=True, runas=runas, logerr=False, as_script=True, level=logging.DEBUG) if ret['rc'] == 0: path = ret['out'][0].strip() else: return None ftd = [] files = du.listdir(host, path=path, runas=user, level=logging.DEBUG) bn = os.path.basename ftd.extend([f for f in files if bn(f).startswith('PtlPbs')]) ftd.extend([f for f in files if bn(f).startswith('STDIN')]) if len(ftd) > 1000: for i in range(0, len(ftd), 1000): j = i + 1000 du.rm(host, path=ftd[i:j], runas=user, force=True, level=logging.DEBUG) root_dir = os.sep dirlist = set([os.path.join(root_dir, 'tmp'), os.path.join(root_dir, 'var', 'tmp')]) # get tmp dir from the environment for envname in 'TMPDIR', 'TEMP', 'TMP': dirname = os.getenv(envname) if dirname: dirlist.add(dirname) p = re.compile(r'^pbs\.\d+') for tmpdir in dirlist: # list the contents of each tmp dir and # get the file list to be deleted self.logger.info('Cleaning up ' + tmpdir + ' dir') ftd = [] files = du.listdir(path=tmpdir) bn = os.path.basename ftd.extend([f for f in files if bn(f).startswith('PtlPbs')]) ftd.extend([f for f in files if bn(f).startswith('STDIN')]) ftd.extend([f for f in files if bn(f).startswith('pbsscrpt')]) ftd.extend([f for f in files if bn(f).startswith('pbs.conf.')]) ftd.extend([f for f in files if p.match(bn(f))]) for f in ftd: du.rm(path=f, sudo=True, recursive=True, force=True, level=logging.DEBUG) for f in du.tmpfilelist: du.rm(path=f, sudo=True, force=True, level=logging.DEBUG) del du.tmpfilelist[:] tmpdir = tempfile.gettempdir() os.chdir(tmpdir) tmppath = os.path.join(tmpdir, 'dejagnutemp%s' % os.getpid()) if du.isdir(path=tmppath): du.rm(path=tmppath, recursive=True, sudo=True, force=True, level=logging.DEBUG)