class Job(ResourceResv): """ PBS Job. Attributes and Resources :param username: Job username :type username: str or None :param attrs: Job attributes :type attrs: Dictionary :param jobname: Name of the PBS job :type jobname: str or None """ dflt_attributes = { ATTR_N: 'STDIN', ATTR_j: 'n', ATTR_m: 'a', ATTR_p: '0', ATTR_r: 'y', ATTR_k: 'oe', } runtime = 100 du = DshUtils() def __init__(self, username=TEST_USER, attrs={}, jobname=None): self.platform = self.du.get_platform() self.server = {} self.script = None self.script_body = None if username is not None: self.username = str(username) else: self.username = None self.du = None self.interactive_handle = None if self.platform == 'cray' or self.platform == 'craysim': if 'Resource_List.select' in attrs: select = attrs['Resource_List.select'] attrs['Resource_List.select'] = self.add_cray_vntype(select) elif 'Resource_List.vntype' not in attrs: attrs['Resource_List.vntype'] = 'cray_compute' PBSObject.__init__(self, None, attrs, self.dflt_attributes) if jobname is not None: self.custom_attrs[ATTR_N] = jobname self.attributes[ATTR_N] = jobname self.set_variable_list(self.username) self.set_sleep_time(100) def __del__(self): del self.__dict__ def add_cray_vntype(self, select=None): """ Cray specific function to add vntype as ``cray_compute`` to each select chunk :param select: PBS select statement :type select: str or None """ ra = [] r = select.split('+') for i in r: select = PbsTypeSelect(i) novntype = 'vntype' not in select.resources nohost = 'host' not in select.resources novnode = 'vnode' not in select.resources if novntype and nohost and novnode: i = i + ":vntype=cray_compute" ra.append(i) select_str = '' for l in ra: select_str = select_str + "+" + l select_str = select_str[1:] return select_str def set_attributes(self, a={}): """ set attributes and custom attributes on this job. custom attributes are used when converting attributes to CLI. In case of Cray platform if 'Resource_List.vntype' is set already then remove it and add vntype value to each chunk of a select statement. :param a: Attribute dictionary :type a: Dictionary """ if isinstance(a, list): a = OrderedDict(a) self.attributes = OrderedDict(list(self.dflt_attributes.items()) + list(self.attributes.items()) + list(a.items())) if self.platform == 'cray' or self.platform == 'craysim': s = 'Resource_List.select' in a v = 'Resource_List.vntype' in self.custom_attrs if s and v: del self.custom_attrs['Resource_List.vntype'] select = a['Resource_List.select'] a['Resource_List.select'] = self.add_cray_vntype(select) self.custom_attrs = OrderedDict(list(self.custom_attrs.items()) + list(a.items())) def set_variable_list(self, user=None, workdir=None): """ Customize the ``Variable_List`` job attribute to ``<user>`` """ if user is None: userinfo = pwd.getpwuid(os.getuid()) user = userinfo[0] homedir = userinfo[5] else: try: homedir = pwd.getpwnam(user)[5] except Exception: homedir = "" self.username = user s = ['PBS_O_HOME=' + homedir] s += ['PBS_O_LANG=en_US.UTF-8'] s += ['PBS_O_LOGNAME=' + user] s += ['PBS_O_PATH=/usr/bin:/bin:/usr/bin:/usr/local/bin'] s += ['PBS_O_MAIL=/var/spool/mail/' + user] s += ['PBS_O_SHELL=/bin/bash'] s += ['PBS_O_SYSTEM=Linux'] if workdir is not None: wd = workdir else: wd = os.getcwd() s += ['PBS_O_WORKDIR=' + str(wd)] self.attributes[ATTR_v] = ",".join(s) self.set_attributes() def set_sleep_time(self, duration): """ Set the sleep duration for this job. :param duration: The duration, in seconds, to sleep :type duration: int """ self.set_execargs('/bin/sleep', duration) def set_execargs(self, executable, arguments=None): """ Set the executable and arguments to use for this job :param executable: path to an executable. No checks are made. :type executable: str :param arguments: arguments to executable. :type arguments: str or list or int """ msg = ['job: executable set to ' + str(executable)] if arguments is not None: msg += [' with arguments: ' + str(arguments)] self.logger.info("".join(msg)) self.attributes[ATTR_executable] = executable if arguments is not None: args = '' xml_beginargs = '<jsdl-hpcpa:Argument>' xml_endargs = '</jsdl-hpcpa:Argument>' if isinstance(arguments, list): for a in arguments: args += xml_beginargs + str(a) + xml_endargs elif isinstance(arguments, str): args = xml_beginargs + arguments + xml_endargs elif isinstance(arguments, int): args = xml_beginargs + str(arguments) + xml_endargs self.attributes[ATTR_Arglist] = args else: self.unset_attributes([ATTR_Arglist]) self.set_attributes() def create_script(self, body=None, asuser=None, hostname=None): """ Create a job script from a given body of text into a temporary location :param body: the body of the script :type body: str or None :param asuser: Optionally the user to own this script, defaults ot current user :type asuser: str or None :param hostname: The host on which the job script is to be created :type hostname: str or None """ if body is None: return None if isinstance(body, list): body = '\n'.join(body) if self.platform == 'cray' or self.platform == 'craysim': body = body.split("\n") for i, line in enumerate(body): if line.startswith("#PBS") and "select=" in line: if 'Resource_List.vntype' in self.attributes: self.unset_attributes(['Resource_List.vntype']) line_arr = line.split(" ") for j, element in enumerate(line_arr): select = element.startswith("select=") lselect = element.startswith("-lselect=") if select or lselect: if lselect: sel_str = element[9:] else: sel_str = element[7:] sel_str = self.add_cray_vntype(select=sel_str) if lselect: line_arr[j] = "-lselect=" + sel_str else: line_arr[j] = "select=" + sel_str body[i] = " ".join(line_arr) body = '\n'.join(body) # If the user has a userhost, the job will run from there # so the script should be made there if self.username: user = PbsUser.get_user(self.username) if user.host: hostname = user.host asuser = user.name self.script_body = body if self.du is None: self.du = DshUtils() # First create the temporary file as current user and only change # its mode once the current user has written to it fn = self.du.create_temp_file(hostname, prefix='PtlPbsJobScript', asuser=asuser, body=body) self.du.chmod(hostname, fn, mode=0o755) self.script = fn return fn def create_subjob_id(self, job_array_id, subjob_index): """ insert subjob index into the square brackets of job array id :param job_array_id: PBS parent array job id :type job_array_id: str :param subjob_index: index of subjob :type subjob_index: int :returns: subjob id string """ idx = job_array_id.find('[]') return job_array_id[:idx + 1] + str(subjob_index) + \ job_array_id[idx + 1:] def create_eatcpu_job(self, duration=None, hostname=None): """ Create a job that eats cpu indefinitely or for the given duration of time :param duration: The duration, in seconds, to sleep :type duration: int :param hostname: hostname on which to execute the job :type hostname: str or None """ if self.du is None: self.du = DshUtils() shebang_line = '#!' + self.du.which(hostname, exe='python3') body = """ import signal import sys x = 0 def receive_alarm(signum, stack): sys.exit() signal.signal(signal.SIGALRM, receive_alarm) if (len(sys.argv) > 1): input_time = sys.argv[1] print('Terminating after %s seconds' % input_time) signal.alarm(int(input_time)) else: print('Running indefinitely') while True: x += 1 """ script_body = shebang_line + body script_path = self.du.create_temp_file(hostname=hostname, body=script_body, suffix='.py') pbs_conf = self.du.parse_pbs_config(hostname) shell_path = os.path.join(pbs_conf['PBS_EXEC'], 'bin', 'pbs_python') a = {ATTR_S: shell_path} self.set_attributes(a) mode = 0o755 if not self.du.chmod(hostname=hostname, path=script_path, mode=mode, sudo=True): raise AssertionError("Failed to set permissions for file %s" " to %s" % (script_path, oct(mode))) self.set_execargs(script_path, duration) return script_path
class PTLTestData(Plugin): """ Save post analysis data on test cases failure or error """ name = 'PTLTestData' score = sys.maxint - 3 logger = logging.getLogger(__name__) def __init__(self): self.sharedpath = None self.du = DshUtils() self.__syncth = None self.__queue = Queue.Queue() def options(self, parser, env): """ Register command line options """ pass def set_data(self, sharedpath): self.sharedpath = sharedpath def configure(self, options, config): """ Configure the plugin and system, based on selected options """ self.config = config self.enabled = True def __get_sntnbi_name(self, test): if hasattr(test, 'test'): _test = test.test sn = _test.__class__.__name__ elif hasattr(test, 'context'): _test = test.context sn = _test.__name__ else: return ('unknown', 'unknown', 'unknown') tn = getattr(_test, '_testMethodName', 'unknown') if (hasattr(_test, 'server') and (getattr(_test, 'server', None) is not None)): bi = _test.server.attributes['pbs_version'] else: bi = 'unknown' return (sn, tn, bi) def __save_home(self, test, status): if hasattr(test, 'test'): _test = test.test elif hasattr(test, 'context'): _test = test.context else: # test does not have any PBS Objects, so just return return if not hasattr(_test, 'server'): # test does not have any PBS Objects, so just return return st = getattr(test, 'start_time', None) if st is not None: st = time.mktime(st.timetuple()) else: st = time.time() st -= 180 # starttime - 3 min et = getattr(test, 'end_time', None) if et is not None: et = time.mktime(et.timetuple()) else: et = time.time() hostname = socket.gethostname().split('.')[0] lp = os.environ.get('PBS_JOBID', time.strftime("%Y%b%d_%H_%m_%S", time.localtime())) sn, tn, bi = self.__get_sntnbi_name(test) if getattr(_test, 'servers', None) is not None: shosts = map(lambda x: x.split('.')[0], _test.servers.host_keys()) else: shosts = [] if getattr(_test, 'schedulers', None) is not None: schosts = map(lambda x: x.split('.')[0], _test.schedulers.host_keys()) else: schosts = [] if getattr(_test, 'moms', None) is not None: mhosts = map(lambda x: x.split('.')[0], _test.moms.host_keys()) else: mhosts = [] hosts = [] hosts.extend(shosts) hosts.extend(schosts) hosts.extend(mhosts) hosts.append(hostname) hosts = sorted(set(hosts)) for host in hosts: confpath = self.du.get_pbs_conf_file(host) tmpdir = self.du.get_tempdir(host) datadir = os.path.join(tmpdir, bi, sn, hostname, tn, lp) _s = ['#!/bin/bash'] _s += ['. %s' % (confpath)] _s += ['mkdir -p %s' % (datadir)] _s += ['chmod -R 0755 %s' % (datadir)] if host == _test.server.shortname: _l = '${PBS_EXEC}/bin/qstat -tf > %s/qstat_tf &' % (datadir) _s += [_l] _l = '${PBS_EXEC}/bin/pbsnodes -av > %s/pbsnodes &' % (datadir) _s += [_l] _l = '${PBS_EXEC}/bin/qmgr -c "p s"' _l += ' > %s/print_server &' % (datadir) _s += [_l] _s += ['echo "%s" >> %s/uptime' % ('*' * 80, datadir)] _s += ['echo "On host : %s" >> %s/uptime' % (host, datadir)] _s += ['uptime >> %s/uptime' % (datadir)] _s += ['echo "" >> %s/uptime' % (datadir)] _s += ['echo "%s" >> %s/netstat' % ('*' * 80, datadir)] _s += ['echo "On host : %s" >> %s/netstat' % (host, datadir)] _cmd = self.du.which(host, 'netstat') if _cmd == 'netstat': _cmd = 'ss' if sys.platform.startswith('linux'): _cmd += ' -ap' else: _cmd += ' -an' _s += ['%s >> %s/netstat' % (_cmd, datadir)] _s += ['echo "" >> %s/netstat' % (datadir)] _s += ['echo "%s" >> %s/ps' % ('*' * 80, datadir)] _s += ['echo "On host : %s" >> %s/ps' % (host, datadir)] _s += ['ps -ef | grep pbs_ >> %s/ps' % (datadir)] _s += ['echo "" >> %s/ps' % (datadir)] _s += ['echo "%s" >> %s/df' % ('*' * 80, datadir)] _s += ['echo "On host : %s" >> %s/df' % (host, datadir)] _s += ['df -h >> %s/df' % (datadir)] _s += ['echo "" >> %s/df' % (datadir)] _s += ['echo "%s" >> %s/vmstat' % ('*' * 80, datadir)] _s += ['echo "On host : %s" >> %s/vmstat' % (host, datadir)] _s += ['vmstat >> %s/vmstat' % (datadir)] _s += ['echo "" >> %s/vmstat' % (datadir)] _dst = os.path.join(datadir, 'PBS_' + host) _s += ['cp -rp ${PBS_HOME} %s' % (_dst)] _s += ['tar -cf %s/datastore.tar %s/datastore' % (_dst, _dst)] _s += ['gzip -rf %s/datastore.tar' % (_dst)] _s += ['rm -rf %s/datastore' % (_dst)] _s += ['rm -rf %s/*_logs' % (_dst)] _s += ['rm -rf %s/server_priv/accounting' % (_dst)] _s += ['cp %s %s/pbs.conf.%s' % (confpath, _dst, host)] if host == hostname: _s += ['cat > %s/logfile_%s <<EOF' % (datadir, status)] _s += ['%s' % (getattr(test, 'err_in_string', ''))] _s += [''] _s += ['EOF'] _s += ['wait'] fd, fn = self.du.mkstemp(hostname, mode=0755, body='\n'.join(_s)) os.close(fd) self.du.run_cmd(hostname, cmd=fn, sudo=True, logerr=False) self.du.rm(hostname, fn, force=True, sudo=True) svr = _test.servers[host] if svr is not None: self.__save_logs(svr, _dst, 'server_logs', st, et) _adst = os.path.join(_dst, 'server_priv') self.__save_logs(svr, _adst, 'accounting', st, et) if getattr(_test, 'moms', None) is not None: self.__save_logs(_test.moms[host], _dst, 'mom_logs', st, et) if getattr(_test, 'schedulers', None) is not None: self.__save_logs(_test.schedulers[host], _dst, 'sched_logs', st, et) if ((self.sharedpath is not None) and (self.__syncth is not None)): self.__queue.put((host, datadir, bi, sn, hostname, tn, lp)) def __save_logs(self, obj, dst, name, st, et, jid=None): if name == 'accounting': logs = obj.log_lines('accounting', n='ALL', starttime=st, endtime=et) logs = map(lambda x: x + '\n', logs) elif name == 'tracejob': logs = obj.log_lines('tracejob', id=jid, n='ALL') name += '_' + jid else: logs = obj.log_lines(obj, n='ALL', starttime=st, endtime=et) f = open(os.path.join(dst, name), 'w+') f.writelines(logs) f.close() def begin(self): if self.sharedpath is not None: self.__syncth = SyncData(self.sharedpath, self.__queue) self.__syncth.daemon = True self.__syncth.start() def addError(self, test, err): self.__save_home(test, 'ERROR') def addFailure(self, test, err): self.__save_home(test, 'FAIL') def finalize(self, result): if ((self.sharedpath is not None) and (self.__syncth is not None)): while not self.__queue.empty(): pass self.__syncth.stop() self.__syncth.join()