class Job(ResourceResv):
"""
PBS Job. Attributes and Resources
:param username: Job username
:type username: str or None
:param attrs: Job attributes
:type attrs: Dictionary
:param jobname: Name of the PBS job
:type jobname: str or None
"""
dflt_attributes = {
ATTR_N: 'STDIN',
ATTR_j: 'n',
ATTR_m: 'a',
ATTR_p: '0',
ATTR_r: 'y',
ATTR_k: 'oe',
}
runtime = 100
du = DshUtils()
def __init__(self, username=TEST_USER, attrs={}, jobname=None):
self.platform = self.du.get_platform()
self.server = {}
self.script = None
self.script_body = None
if username is not None:
self.username = str(username)
else:
self.username = None
self.du = None
self.interactive_handle = None
if self.platform == 'cray' or self.platform == 'craysim':
if 'Resource_List.select' in attrs:
select = attrs['Resource_List.select']
attrs['Resource_List.select'] = self.add_cray_vntype(select)
elif 'Resource_List.vntype' not in attrs:
attrs['Resource_List.vntype'] = 'cray_compute'
PBSObject.__init__(self, None, attrs, self.dflt_attributes)
if jobname is not None:
self.custom_attrs[ATTR_N] = jobname
self.attributes[ATTR_N] = jobname
self.set_variable_list(self.username)
self.set_sleep_time(100)
def __del__(self):
del self.__dict__
def add_cray_vntype(self, select=None):
"""
Cray specific function to add vntype as ``cray_compute`` to each
select chunk
:param select: PBS select statement
:type select: str or None
"""
ra = []
r = select.split('+')
for i in r:
select = PbsTypeSelect(i)
novntype = 'vntype' not in select.resources
nohost = 'host' not in select.resources
novnode = 'vnode' not in select.resources
if novntype and nohost and novnode:
i = i + ":vntype=cray_compute"
ra.append(i)
select_str = ''
for l in ra:
select_str = select_str + "+" + l
select_str = select_str[1:]
return select_str
def set_attributes(self, a={}):
"""
set attributes and custom attributes on this job.
custom attributes are used when converting attributes to CLI.
In case of Cray platform if 'Resource_List.vntype' is set
already then remove it and add vntype value to each chunk of a
select statement.
:param a: Attribute dictionary
:type a: Dictionary
"""
if isinstance(a, list):
a = OrderedDict(a)
self.attributes = OrderedDict(list(self.dflt_attributes.items()) +
list(self.attributes.items()) +
list(a.items()))
if self.platform == 'cray' or self.platform == 'craysim':
s = 'Resource_List.select' in a
v = 'Resource_List.vntype' in self.custom_attrs
if s and v:
del self.custom_attrs['Resource_List.vntype']
select = a['Resource_List.select']
a['Resource_List.select'] = self.add_cray_vntype(select)
self.custom_attrs = OrderedDict(list(self.custom_attrs.items()) +
list(a.items()))
def set_variable_list(self, user=None, workdir=None):
"""
Customize the ``Variable_List`` job attribute to ``<user>``
"""
if user is None:
userinfo = pwd.getpwuid(os.getuid())
user = userinfo[0]
homedir = userinfo[5]
else:
try:
homedir = pwd.getpwnam(user)[5]
except Exception:
homedir = ""
self.username = user
s = ['PBS_O_HOME=' + homedir]
s += ['PBS_O_LANG=en_US.UTF-8']
s += ['PBS_O_LOGNAME=' + user]
s += ['PBS_O_PATH=/usr/bin:/bin:/usr/bin:/usr/local/bin']
s += ['PBS_O_MAIL=/var/spool/mail/' + user]
s += ['PBS_O_SHELL=/bin/bash']
s += ['PBS_O_SYSTEM=Linux']
if workdir is not None:
wd = workdir
else:
wd = os.getcwd()
s += ['PBS_O_WORKDIR=' + str(wd)]
self.attributes[ATTR_v] = ",".join(s)
self.set_attributes()
def set_sleep_time(self, duration):
"""
Set the sleep duration for this job.
:param duration: The duration, in seconds, to sleep
:type duration: int
"""
self.set_execargs('/bin/sleep', duration)
def set_execargs(self, executable, arguments=None):
"""
Set the executable and arguments to use for this job
:param executable: path to an executable. No checks are made.
:type executable: str
:param arguments: arguments to executable.
:type arguments: str or list or int
"""
msg = ['job: executable set to ' + str(executable)]
if arguments is not None:
msg += [' with arguments: ' + str(arguments)]
self.logger.info("".join(msg))
self.attributes[ATTR_executable] = executable
if arguments is not None:
args = ''
xml_beginargs = '<jsdl-hpcpa:Argument>'
xml_endargs = '</jsdl-hpcpa:Argument>'
if isinstance(arguments, list):
for a in arguments:
args += xml_beginargs + str(a) + xml_endargs
elif isinstance(arguments, str):
args = xml_beginargs + arguments + xml_endargs
elif isinstance(arguments, int):
args = xml_beginargs + str(arguments) + xml_endargs
self.attributes[ATTR_Arglist] = args
else:
self.unset_attributes([ATTR_Arglist])
self.set_attributes()
def create_script(self, body=None, asuser=None, hostname=None):
"""
Create a job script from a given body of text into a
temporary location
:param body: the body of the script
:type body: str or None
:param asuser: Optionally the user to own this script,
defaults ot current user
:type asuser: str or None
:param hostname: The host on which the job script is to
be created
:type hostname: str or None
"""
if body is None:
return None
if isinstance(body, list):
body = '\n'.join(body)
if self.platform == 'cray' or self.platform == 'craysim':
body = body.split("\n")
for i, line in enumerate(body):
if line.startswith("#PBS") and "select=" in line:
if 'Resource_List.vntype' in self.attributes:
self.unset_attributes(['Resource_List.vntype'])
line_arr = line.split(" ")
for j, element in enumerate(line_arr):
select = element.startswith("select=")
lselect = element.startswith("-lselect=")
if select or lselect:
if lselect:
sel_str = element[9:]
else:
sel_str = element[7:]
sel_str = self.add_cray_vntype(select=sel_str)
if lselect:
line_arr[j] = "-lselect=" + sel_str
else:
line_arr[j] = "select=" + sel_str
body[i] = " ".join(line_arr)
body = '\n'.join(body)
# If the user has a userhost, the job will run from there
# so the script should be made there
if self.username:
user = PbsUser.get_user(self.username)
if user.host:
hostname = user.host
asuser = user.name
self.script_body = body
if self.du is None:
self.du = DshUtils()
# First create the temporary file as current user and only change
# its mode once the current user has written to it
fn = self.du.create_temp_file(hostname, prefix='PtlPbsJobScript',
asuser=asuser, body=body)
self.du.chmod(hostname, fn, mode=0o755)
self.script = fn
return fn
def create_subjob_id(self, job_array_id, subjob_index):
"""
insert subjob index into the square brackets of job array id
:param job_array_id: PBS parent array job id
:type job_array_id: str
:param subjob_index: index of subjob
:type subjob_index: int
:returns: subjob id string
"""
idx = job_array_id.find('[]')
return job_array_id[:idx + 1] + str(subjob_index) + \
job_array_id[idx + 1:]
def create_eatcpu_job(self, duration=None, hostname=None):
"""
Create a job that eats cpu indefinitely or for the given
duration of time
:param duration: The duration, in seconds, to sleep
:type duration: int
:param hostname: hostname on which to execute the job
:type hostname: str or None
"""
if self.du is None:
self.du = DshUtils()
shebang_line = '#!' + self.du.which(hostname, exe='python3')
body = """
import signal
import sys
x = 0
def receive_alarm(signum, stack):
sys.exit()
signal.signal(signal.SIGALRM, receive_alarm)
if (len(sys.argv) > 1):
input_time = sys.argv[1]
print('Terminating after %s seconds' % input_time)
signal.alarm(int(input_time))
else:
print('Running indefinitely')
while True:
x += 1
"""
script_body = shebang_line + body
script_path = self.du.create_temp_file(hostname=hostname,
body=script_body,
suffix='.py')
pbs_conf = self.du.parse_pbs_config(hostname)
shell_path = os.path.join(pbs_conf['PBS_EXEC'],
'bin', 'pbs_python')
a = {ATTR_S: shell_path}
self.set_attributes(a)
mode = 0o755
if not self.du.chmod(hostname=hostname, path=script_path, mode=mode,
sudo=True):
raise AssertionError("Failed to set permissions for file %s"
" to %s" % (script_path, oct(mode)))
self.set_execargs(script_path, duration)
return script_path
class PTLTestData(Plugin):
"""
Save post analysis data on test cases failure or error
"""
name = 'PTLTestData'
score = sys.maxint - 3
logger = logging.getLogger(__name__)
def __init__(self):
self.sharedpath = None
self.du = DshUtils()
self.__syncth = None
self.__queue = Queue.Queue()
def options(self, parser, env):
"""
Register command line options
"""
pass
def set_data(self, sharedpath):
self.sharedpath = sharedpath
def configure(self, options, config):
"""
Configure the plugin and system, based on selected options
"""
self.config = config
self.enabled = True
def __get_sntnbi_name(self, test):
if hasattr(test, 'test'):
_test = test.test
sn = _test.__class__.__name__
elif hasattr(test, 'context'):
_test = test.context
sn = _test.__name__
else:
return ('unknown', 'unknown', 'unknown')
tn = getattr(_test, '_testMethodName', 'unknown')
if (hasattr(_test, 'server')
and (getattr(_test, 'server', None) is not None)):
bi = _test.server.attributes['pbs_version']
else:
bi = 'unknown'
return (sn, tn, bi)
def __save_home(self, test, status):
if hasattr(test, 'test'):
_test = test.test
elif hasattr(test, 'context'):
_test = test.context
else:
# test does not have any PBS Objects, so just return
return
if not hasattr(_test, 'server'):
# test does not have any PBS Objects, so just return
return
st = getattr(test, 'start_time', None)
if st is not None:
st = time.mktime(st.timetuple())
else:
st = time.time()
st -= 180 # starttime - 3 min
et = getattr(test, 'end_time', None)
if et is not None:
et = time.mktime(et.timetuple())
else:
et = time.time()
hostname = socket.gethostname().split('.')[0]
lp = os.environ.get('PBS_JOBID',
time.strftime("%Y%b%d_%H_%m_%S", time.localtime()))
sn, tn, bi = self.__get_sntnbi_name(test)
if getattr(_test, 'servers', None) is not None:
shosts = map(lambda x: x.split('.')[0], _test.servers.host_keys())
else:
shosts = []
if getattr(_test, 'schedulers', None) is not None:
schosts = map(lambda x: x.split('.')[0],
_test.schedulers.host_keys())
else:
schosts = []
if getattr(_test, 'moms', None) is not None:
mhosts = map(lambda x: x.split('.')[0], _test.moms.host_keys())
else:
mhosts = []
hosts = []
hosts.extend(shosts)
hosts.extend(schosts)
hosts.extend(mhosts)
hosts.append(hostname)
hosts = sorted(set(hosts))
for host in hosts:
confpath = self.du.get_pbs_conf_file(host)
tmpdir = self.du.get_tempdir(host)
datadir = os.path.join(tmpdir, bi, sn, hostname, tn, lp)
_s = ['#!/bin/bash']
_s += ['. %s' % (confpath)]
_s += ['mkdir -p %s' % (datadir)]
_s += ['chmod -R 0755 %s' % (datadir)]
if host == _test.server.shortname:
_l = '${PBS_EXEC}/bin/qstat -tf > %s/qstat_tf &' % (datadir)
_s += [_l]
_l = '${PBS_EXEC}/bin/pbsnodes -av > %s/pbsnodes &' % (datadir)
_s += [_l]
_l = '${PBS_EXEC}/bin/qmgr -c "p s"'
_l += ' > %s/print_server &' % (datadir)
_s += [_l]
_s += ['echo "%s" >> %s/uptime' % ('*' * 80, datadir)]
_s += ['echo "On host : %s" >> %s/uptime' % (host, datadir)]
_s += ['uptime >> %s/uptime' % (datadir)]
_s += ['echo "" >> %s/uptime' % (datadir)]
_s += ['echo "%s" >> %s/netstat' % ('*' * 80, datadir)]
_s += ['echo "On host : %s" >> %s/netstat' % (host, datadir)]
_cmd = self.du.which(host, 'netstat')
if _cmd == 'netstat':
_cmd = 'ss'
if sys.platform.startswith('linux'):
_cmd += ' -ap'
else:
_cmd += ' -an'
_s += ['%s >> %s/netstat' % (_cmd, datadir)]
_s += ['echo "" >> %s/netstat' % (datadir)]
_s += ['echo "%s" >> %s/ps' % ('*' * 80, datadir)]
_s += ['echo "On host : %s" >> %s/ps' % (host, datadir)]
_s += ['ps -ef | grep pbs_ >> %s/ps' % (datadir)]
_s += ['echo "" >> %s/ps' % (datadir)]
_s += ['echo "%s" >> %s/df' % ('*' * 80, datadir)]
_s += ['echo "On host : %s" >> %s/df' % (host, datadir)]
_s += ['df -h >> %s/df' % (datadir)]
_s += ['echo "" >> %s/df' % (datadir)]
_s += ['echo "%s" >> %s/vmstat' % ('*' * 80, datadir)]
_s += ['echo "On host : %s" >> %s/vmstat' % (host, datadir)]
_s += ['vmstat >> %s/vmstat' % (datadir)]
_s += ['echo "" >> %s/vmstat' % (datadir)]
_dst = os.path.join(datadir, 'PBS_' + host)
_s += ['cp -rp ${PBS_HOME} %s' % (_dst)]
_s += ['tar -cf %s/datastore.tar %s/datastore' % (_dst, _dst)]
_s += ['gzip -rf %s/datastore.tar' % (_dst)]
_s += ['rm -rf %s/datastore' % (_dst)]
_s += ['rm -rf %s/*_logs' % (_dst)]
_s += ['rm -rf %s/server_priv/accounting' % (_dst)]
_s += ['cp %s %s/pbs.conf.%s' % (confpath, _dst, host)]
if host == hostname:
_s += ['cat > %s/logfile_%s <<EOF' % (datadir, status)]
_s += ['%s' % (getattr(test, 'err_in_string', ''))]
_s += ['']
_s += ['EOF']
_s += ['wait']
fd, fn = self.du.mkstemp(hostname, mode=0755, body='\n'.join(_s))
os.close(fd)
self.du.run_cmd(hostname, cmd=fn, sudo=True, logerr=False)
self.du.rm(hostname, fn, force=True, sudo=True)
svr = _test.servers[host]
if svr is not None:
self.__save_logs(svr, _dst, 'server_logs', st, et)
_adst = os.path.join(_dst, 'server_priv')
self.__save_logs(svr, _adst, 'accounting', st, et)
if getattr(_test, 'moms', None) is not None:
self.__save_logs(_test.moms[host], _dst, 'mom_logs', st, et)
if getattr(_test, 'schedulers', None) is not None:
self.__save_logs(_test.schedulers[host], _dst, 'sched_logs',
st, et)
if ((self.sharedpath is not None) and (self.__syncth is not None)):
self.__queue.put((host, datadir, bi, sn, hostname, tn, lp))
def __save_logs(self, obj, dst, name, st, et, jid=None):
if name == 'accounting':
logs = obj.log_lines('accounting',
n='ALL',
starttime=st,
endtime=et)
logs = map(lambda x: x + '\n', logs)
elif name == 'tracejob':
logs = obj.log_lines('tracejob', id=jid, n='ALL')
name += '_' + jid
else:
logs = obj.log_lines(obj, n='ALL', starttime=st, endtime=et)
f = open(os.path.join(dst, name), 'w+')
f.writelines(logs)
f.close()
def begin(self):
if self.sharedpath is not None:
self.__syncth = SyncData(self.sharedpath, self.__queue)
self.__syncth.daemon = True
self.__syncth.start()
def addError(self, test, err):
self.__save_home(test, 'ERROR')
def addFailure(self, test, err):
self.__save_home(test, 'FAIL')
def finalize(self, result):
if ((self.sharedpath is not None) and (self.__syncth is not None)):
while not self.__queue.empty():
pass
self.__syncth.stop()
self.__syncth.join()
class PTLTestData(Plugin):
"""
Save post analysis data on test cases failure or error
"""
name = 'PTLTestData'
score = sys.maxint - 3
logger = logging.getLogger(__name__)
def __init__(self):
self.sharedpath = None
self.du = DshUtils()
self.__syncth = None
self.__queue = Queue.Queue()
def options(self, parser, env):
"""
Register command line options
"""
pass
def set_data(self, sharedpath):
self.sharedpath = sharedpath
def configure(self, options, config):
"""
Configure the plugin and system, based on selected options
"""
self.config = config
self.enabled = True
def __get_sntnbi_name(self, test):
if hasattr(test, 'test'):
_test = test.test
sn = _test.__class__.__name__
elif hasattr(test, 'context'):
_test = test.context
sn = _test.__name__
else:
return ('unknown', 'unknown', 'unknown')
tn = getattr(_test, '_testMethodName', 'unknown')
if (hasattr(_test, 'server') and
(getattr(_test, 'server', None) is not None)):
bi = _test.server.attributes['pbs_version']
else:
bi = 'unknown'
return (sn, tn, bi)
def __save_home(self, test, status):
if hasattr(test, 'test'):
_test = test.test
elif hasattr(test, 'context'):
_test = test.context
else:
# test does not have any PBS Objects, so just return
return
if not hasattr(_test, 'server'):
# test does not have any PBS Objects, so just return
return
st = getattr(test, 'start_time', None)
if st is not None:
st = time.mktime(st.timetuple())
else:
st = time.time()
st -= 180 # starttime - 3 min
et = getattr(test, 'end_time', None)
if et is not None:
et = time.mktime(et.timetuple())
else:
et = time.time()
hostname = socket.gethostname().split('.')[0]
lp = os.environ.get('PBS_JOBID', time.strftime("%Y%b%d_%H_%m_%S",
time.localtime()))
sn, tn, bi = self.__get_sntnbi_name(test)
if getattr(_test, 'servers', None) is not None:
shosts = map(lambda x: x.split('.')[0], _test.servers.host_keys())
else:
shosts = []
if getattr(_test, 'schedulers', None) is not None:
schosts = map(lambda x: x.split('.')[0],
_test.schedulers.host_keys())
else:
schosts = []
if getattr(_test, 'moms', None) is not None:
mhosts = map(lambda x: x.split('.')[0], _test.moms.host_keys())
else:
mhosts = []
hosts = []
hosts.extend(shosts)
hosts.extend(schosts)
hosts.extend(mhosts)
hosts.append(hostname)
hosts = sorted(set(hosts))
for host in hosts:
confpath = self.du.get_pbs_conf_file(host)
tmpdir = self.du.get_tempdir(host)
datadir = os.path.join(tmpdir, bi, sn, hostname, tn, lp)
_s = ['#!/bin/bash']
_s += ['. %s' % (confpath)]
_s += ['mkdir -p %s' % (datadir)]
_s += ['chmod -R 0755 %s' % (datadir)]
if host == _test.server.shortname:
_l = '${PBS_EXEC}/bin/qstat -tf > %s/qstat_tf &' % (datadir)
_s += [_l]
_l = '${PBS_EXEC}/bin/pbsnodes -av > %s/pbsnodes &' % (datadir)
_s += [_l]
_l = '${PBS_EXEC}/bin/qmgr -c "p s"'
_l += ' > %s/print_server &' % (datadir)
_s += [_l]
_s += ['echo "%s" >> %s/uptime' % ('*' * 80, datadir)]
_s += ['echo "On host : %s" >> %s/uptime' % (host, datadir)]
_s += ['uptime >> %s/uptime' % (datadir)]
_s += ['echo "" >> %s/uptime' % (datadir)]
_s += ['echo "%s" >> %s/netstat' % ('*' * 80, datadir)]
_s += ['echo "On host : %s" >> %s/netstat' % (host, datadir)]
_cmd = self.du.which(host, 'netstat')
if _cmd == 'netstat':
_cmd = 'ss'
if sys.platform.startswith('linux'):
_cmd += ' -ap'
else:
_cmd += ' -an'
_s += ['%s >> %s/netstat' % (_cmd, datadir)]
_s += ['echo "" >> %s/netstat' % (datadir)]
_s += ['echo "%s" >> %s/ps' % ('*' * 80, datadir)]
_s += ['echo "On host : %s" >> %s/ps' % (host, datadir)]
_s += ['ps -ef | grep pbs_ >> %s/ps' % (datadir)]
_s += ['echo "" >> %s/ps' % (datadir)]
_s += ['echo "%s" >> %s/df' % ('*' * 80, datadir)]
_s += ['echo "On host : %s" >> %s/df' % (host, datadir)]
_s += ['df -h >> %s/df' % (datadir)]
_s += ['echo "" >> %s/df' % (datadir)]
_s += ['echo "%s" >> %s/vmstat' % ('*' * 80, datadir)]
_s += ['echo "On host : %s" >> %s/vmstat' % (host, datadir)]
_s += ['vmstat >> %s/vmstat' % (datadir)]
_s += ['echo "" >> %s/vmstat' % (datadir)]
_dst = os.path.join(datadir, 'PBS_' + host)
_s += ['cp -rp ${PBS_HOME} %s' % (_dst)]
_s += ['tar -cf %s/datastore.tar %s/datastore' % (_dst, _dst)]
_s += ['gzip -rf %s/datastore.tar' % (_dst)]
_s += ['rm -rf %s/datastore' % (_dst)]
_s += ['rm -rf %s/*_logs' % (_dst)]
_s += ['rm -rf %s/server_priv/accounting' % (_dst)]
_s += ['cp %s %s/pbs.conf.%s' % (confpath, _dst, host)]
if host == hostname:
_s += ['cat > %s/logfile_%s <<EOF' % (datadir, status)]
_s += ['%s' % (getattr(test, 'err_in_string', ''))]
_s += ['']
_s += ['EOF']
_s += ['wait']
fd, fn = self.du.mkstemp(hostname, mode=0755, body='\n'.join(_s))
os.close(fd)
self.du.run_cmd(hostname, cmd=fn, sudo=True, logerr=False)
self.du.rm(hostname, fn, force=True, sudo=True)
svr = _test.servers[host]
if svr is not None:
self.__save_logs(svr, _dst, 'server_logs', st, et)
_adst = os.path.join(_dst, 'server_priv')
self.__save_logs(svr, _adst, 'accounting', st, et)
if getattr(_test, 'moms', None) is not None:
self.__save_logs(_test.moms[host], _dst, 'mom_logs', st, et)
if getattr(_test, 'schedulers', None) is not None:
self.__save_logs(_test.schedulers[host], _dst, 'sched_logs',
st, et)
if ((self.sharedpath is not None) and (self.__syncth is not None)):
self.__queue.put((host, datadir, bi, sn, hostname, tn, lp))
def __save_logs(self, obj, dst, name, st, et, jid=None):
if name == 'accounting':
logs = obj.log_lines('accounting', n='ALL', starttime=st,
endtime=et)
logs = map(lambda x: x + '\n', logs)
elif name == 'tracejob':
logs = obj.log_lines('tracejob', id=jid, n='ALL')
name += '_' + jid
else:
logs = obj.log_lines(obj, n='ALL', starttime=st, endtime=et)
f = open(os.path.join(dst, name), 'w+')
f.writelines(logs)
f.close()
def begin(self):
if self.sharedpath is not None:
self.__syncth = SyncData(self.sharedpath, self.__queue)
self.__syncth.daemon = True
self.__syncth.start()
def addError(self, test, err):
self.__save_home(test, 'ERROR')
def addFailure(self, test, err):
self.__save_home(test, 'FAIL')
def finalize(self, result):
if ((self.sharedpath is not None) and (self.__syncth is not None)):
while not self.__queue.empty():
pass
self.__syncth.stop()
self.__syncth.join()
