def getInitialAVs(resIdx, attrs, exeName, ptds):
""" Takes attrs, which is a list of name, value pairs, and builds
a build Resource object and Execution object
"""
executableAVs = []
buildAVs = []
buildOSAVs = []
AppName = ""
BuildMachType = ""
BuildMachine = ""
BuildOSName = ""
BuildOSVersion = ""
BuildOSType = ""
# regular expressions for parsing
for name, value, type in attrs:
if name == "BuildDataBegin":
pass
elif name == "BuildDataEnd":
break
# Get executable specific data
elif name == "ApplicationName":
AppName = value
elif name == "ApplicationExeTrialName" or name == "ApplicationExeUserComment":
AppTrialName = value
elif name.startswith("Application"):
if name == "ApplicationLanguages":
value = fixListFormat(value)
executableAVs.append(("Languages", value, "string"))
elif name == "ApplicationParadigms":
value = fixListFormat(value)
executableAVs.append(("Concurrency", value, "string"))
elif name == "ApplicationExeName":
executableAVs.append(("Executable Name", value, "string"))
elif name == "ApplicationExeSize":
executableAVs.append(("Executable Size", value, "string"))
elif name == "ApplicationExePerms":
executableAVs.append(("Executable Permissions", value, "string"))
elif name == "ApplicationExeUID":
executableAVs.append(("Executable UID", value, "string"))
elif name == "ApplicationExeGID":
executableAVs.append(("Executable GID", value, "string"))
elif name == "ApplicationExeTimestamp":
executableAVs.append(("Executable Timestamp", value, "string"))
elif name == "ApplicationExeUsername":
executableAVs.append(("Username", value, "string"))
# get Build specific data
elif name.startswith("Build") and not (name.find("BuildOS") >= 0):
if name == "BuildEnv":
BuildEnv = [] # a list of attr value pairs
# where attr = "BuildEnv"+ (enviroment variable name)
# and value is the environment variable's value
# e.g. for PATH=/usr/bin:/usr/local/bin
# attr = BuildEnvPATH
# value = /usr/bin:/usr/local/bin
BuildEnv = StringToList("Env_", "@@@", value)
for (n, v) in BuildEnv:
if v != "":
buildAVs.append((n, v, "string"))
else:
if name == "BuildNode" or name == "BuildMachine":
BuildMachine = value
if name == "BuildNode":
BuildMachType = "node"
buildAVs.append(("Node Name", value, "string"))
elif name == "BuildMachine":
BuildMachType = "machine"
buildAVs.append(("Machine Name", value, "string"))
if name == "BuildDateTime":
buildAVs.append(("DateTime", value, "string"))
# get Build operating system specific data
elif name.startswith("BuildOS"):
if name == "BuildOSName":
BuildOSName = value
elif name == "BuildOSReleaseVersion":
BuildOSVersion = value
elif name == "BuildOSReleaseType":
BuildOSType = value
if exeName == "" or exeName == None:
raise PTexception("missing execution name in Build.getInitialAVs")
exeNewName = ptds.getNewResourceName(exeName)
# ATTR CHANGE TODO
# executableAVs should be attrs of build, not execution
exe = Execution(exeNewName, executableAVs)
resIdx.addResource(exe)
newBuildName = ptds.getNewResourceName("build")
build = Resource(newBuildName, "build", buildAVs)
resIdx.addResource(build)
if BuildMachine == "" or BuildMachine == None:
raise PTexception("missing build machine name in build data file for " " execution:%s" % exeName)
if BuildMachType == "node":
fullname, type = getMachineName(ptds, BuildMachine, BuildMachType)
# print fullname
buildMachine = Resource(fullname, type)
else: # machine
fullname, type = getMachineName(ptds, BuildMachine, BuildMachType)
buildMachine = Resource(fullname, type)
if BuildOSName == "" or BuildOSName == None:
raise PTexception("missing build operating system name for execution" "execution:%s" % exeName)
if BuildOSVersion == "" or BuildOSVersion == None or BuildOSType == "" or BuildOSType == None:
raise PTexception("missing build operating system details for execution" "execution:%s" % exeName)
buildOSNewName = ptds.getNewResourceName(BuildOSName + " " + BuildOSVersion + " " + BuildOSType)
buildOS = Resource(buildOSNewName, "operatingSystem", buildOSAVs)
resIdx.addResources([buildOS, buildMachine])
build.addAttribute("Node Name", buildMachine.name, "resource")
build.addAttribute("OS Name", buildOS.name, "resource")
return (build, exe)
def getInitialAVs(resIdx, attrs, exe, runMachineArg, ptds):
"""Parses the attribute value pairs and extracts the needed information.
"""
runAVs = []
runOSAVs = []
runOSName = ""
runOSVersion = ""
runOSType = ""
runMachine = ""
numberOfProcesses = -1
threadsPerProcess = -1
for name, value, type in attrs:
#print '(%s,%s)' % (name,value)
if name == "RunDataBegin":
pass
elif (name == "LibraryBegin") or (name == "InputDeckBegin") \
or (name == "RunDataEnd") or (name == "SubmissionBegin"):
break
#Get run specific data
elif name == "RunMachine":
runMachine = value
elif name == "PageSize":
pageSize = value
runAVs.append((name, value, "string"))
elif name == "NumberOfProcesses":
numberOfProcesses = value
runAVs.append((name, value, "string"))
elif name == "ProcessesPerNode":
processesPerNode = value
runAVs.append((name, value, "string"))
elif name == "ThreadsPerProcess":
threadsPerProcess = value
runAVs.append((name, value, "string"))
elif name.startswith("Run") and not (name.find("RunOS") >= 0):
if name == "RunEnv":
RunEnv = [] # a list of attr value pairs
#where attr = "RunEnv"+ (enviroment variable name)
#and value is the environment variable's value
#e.g. for PATH=/usr/bin:/usr/local/bin
# attr = RunEnvPATH
# value = /usr/bin:/usr/local/bin
RunEnv = StringToList("Env_", "@@@", value)
for (n, v) in RunEnv:
if (v != ""):
runAVs.append((n, v, "string"))
else:
runAVs.append((name, value, "string"))
elif name.startswith("RunOS"):
if name == "RunOSName":
runOSName = value
elif name == "RunOSReleaseVersion":
runOSVersion = value
elif name == "RunOSReleaseType":
runOSType = value
elif (value != ""):
runOSAVs.append((name, value, "string"))
else:
runAVs.append((name, value, "string"))
if runOSName == "" or runOSName == None:
raise PTexception("no operating system name given in run data file for"\
" execution:%s" % exe.name )
if runOSVersion == "" or runOSVersion == None or \
runOSType == "" or runOSType == None:
raise PTexception("no operating system details given in run data file "\
"for execution:%s" % exe.name )
if runMachine == "" or runMachine == None:
raise PTexception("no run machine name given in data file "\
"for execution:%s" % exe.name )
runOSNewName = ptds.getNewResourceName(runOSName+" "+runOSVersion+" "+\
runOSType)
runOS = Resource(runOSNewName, "operatingSystem", runOSAVs)
resIdx.addResource(runOS)
# if the run data doesn't know the machine name, then see if the user
# sent it in as an argument to PTdFgen
if runMachine == "" or runMachine == None:
runMachine = runMachineArg
# if they didn't do that, then we still don't know the machine
if runMachine == "" or runMachine == None:
runMach = None
else:
fullname, type = getMachineName(ptds, runMachine, "machine")
#print "machine is %s " % runMachine
runMach = Resource(fullname, type)
resIdx.addResource(runMach)
exe.addAttributes(runAVs)
if threadsPerProcess < 0:
raise PTexception("ThreadsPerProcess attribute not given in run data "\
"file for execution:%s" % exe.name )
if numberOfProcesses < 0:
raise PTexception("NumberOfProcesses attribute not given in run data "\
"file for execution:%s" % exe.name )
return (runMach, runOS, exe, numberOfProcesses, \
threadsPerProcess)
def getInitialAVs(resIdx, attrs, exeName, ptds):
""" Takes attrs, which is a list of name, value pairs, and builds
a build Resource object and Execution object
"""
executableAVs = []
buildAVs = []
buildOSAVs = []
AppName = ""
BuildMachType = ""
BuildMachine = ""
BuildOSName = ""
BuildOSVersion = ""
BuildOSType = ""
# regular expressions for parsing
for name,value,type in attrs:
if name == "BuildDataBegin":
pass
elif name == "BuildDataEnd":
break
#Get executable specific data
elif name == "ApplicationName":
AppName = value
elif name == "ApplicationExeTrialName" or \
name == "ApplicationExeUserComment":
AppTrialName = value
elif name.startswith("Application"):
if name == "ApplicationLanguages":
value = fixListFormat(value)
executableAVs.append(("Languages", value, "string"))
elif name == "ApplicationParadigms":
value = fixListFormat(value)
executableAVs.append(("Concurrency", value, "string"))
elif name == "ApplicationExeName":
executableAVs.append(("Executable Name", value, "string"))
elif name == "ApplicationExeSize":
executableAVs.append(("Executable Size", value, "string"))
elif name == "ApplicationExePerms":
executableAVs.append(("Executable Permissions", value, "string"))
elif name == "ApplicationExeUID":
executableAVs.append(("Executable UID", value, "string"))
elif name == "ApplicationExeGID":
executableAVs.append(("Executable GID", value, "string"))
elif name == "ApplicationExeTimestamp":
executableAVs.append(("Executable Timestamp", value, "string"))
elif name == "ApplicationExeUsername":
executableAVs.append(("Username", value, "string"))
# get Build specific data
elif name.startswith("Build") and not (name.find("BuildOS") >= 0):
if name == "BuildEnv":
BuildEnv = [] # a list of attr value pairs
#where attr = "BuildEnv"+ (enviroment variable name)
#and value is the environment variable's value
#e.g. for PATH=/usr/bin:/usr/local/bin
# attr = BuildEnvPATH
# value = /usr/bin:/usr/local/bin
BuildEnv = StringToList("Env_","@@@", value)
for (n,v) in BuildEnv:
if(v != ""):
buildAVs.append((n,v,"string"))
else:
if name == "BuildNode" or name == "BuildMachine":
BuildMachine = value
if name == "BuildNode":
BuildMachType = 'node'
buildAVs.append(("Node Name",value,"string"))
elif name == "BuildMachine":
BuildMachType = 'machine'
buildAVs.append(("Machine Name",value,"string"))
if name == "BuildDateTime":
buildAVs.append(("DateTime",value,"string"))
#get Build operating system specific data
elif name.startswith("BuildOS"):
if name == "BuildOSName":
BuildOSName = value
elif name == "BuildOSReleaseVersion":
BuildOSVersion = value
elif name == "BuildOSReleaseType":
BuildOSType = value
if exeName == "" or exeName == None:
raise PTexception("missing execution name in Build.getInitialAVs")
exeNewName = ptds.getNewResourceName(exeName)
# ATTR CHANGE TODO
# executableAVs should be attrs of build, not execution
exe = Execution(exeNewName, executableAVs)
resIdx.addResource(exe)
newBuildName = ptds.getNewResourceName("build")
build = Resource(newBuildName, "build", buildAVs)
resIdx.addResource(build)
if BuildMachine == "" or BuildMachine == None:
raise PTexception("missing build machine name in build data file for "
" execution:%s" % exeName)
if BuildMachType == "node":
fullname,type = getMachineName(ptds, BuildMachine, BuildMachType)
#print fullname
buildMachine = Resource(fullname,type)
else: # machine
fullname,type = getMachineName(ptds, BuildMachine, BuildMachType)
buildMachine = Resource(fullname,type)
if BuildOSName == "" or BuildOSName == None:
raise PTexception("missing build operating system name for execution"\
"execution:%s" % exeName)
if BuildOSVersion == "" or BuildOSVersion == None or \
BuildOSType == "" or BuildOSType == None:
raise PTexception("missing build operating system details for execution"\
"execution:%s" % exeName)
buildOSNewName = ptds.getNewResourceName(BuildOSName+" "+BuildOSVersion+\
" "+BuildOSType)
buildOS = Resource(buildOSNewName, "operatingSystem", buildOSAVs)
resIdx.addResources([buildOS,buildMachine])
build.addAttribute("Node Name", buildMachine.name, "resource")
build.addAttribute("OS Name", buildOS.name, "resource")
return(build,exe)
def parsePBSoutput(resIdx, runfilename, ptds):
# parse the stdout and stderr of PBS to get additional run information
# we expect there to be a files called jobprefix.pbs.out and
# jobprefix.pbs.err where jobprefix
# is the same prefix as the .run file parsed in process()
fileprefix = runfilename.rstrip('.run')
pbsstdout = fileprefix + ".pbs.out"
pbsstderr = fileprefix + ".pbs.err"
try:
f = open(pbsstdout, 'r')
except:
raise PTexception(
"Run.parsePBSoutput could not open pbs stdout file: %s" %
pbsstdout)
[exe] = resIdx.findResourcesByType("execution")
lines = f.readlines()
for line in lines:
if line.startswith("Job startup at"):
jobStartTime = line.strip("Job startup at").strip()
if jobStartTime != "":
exe.addAttribute("jobStartTime", jobStartTime, "string")
elif line.startswith("Jobs exit status"):
jobExitStatus = line.strip("Jobs exit status code is ").strip()
if jobExitStatus != "":
exe.addAttribute("jobExitStatus", jobExitStatus, "string")
elif line.find("completed ") >= 0:
jobCompletionTime = ""
for w in line.split()[3:]:
jobCompletionTime += "%s " % w
if jobCompletionTime.strip() != "":
exe.addAttribute("jobCompletionTime", jobCompletionTime,
"string")
elif line.startswith("Nodes used"):
jobNodes = line.strip("Nodes used: ").strip()
if jobNodes != "":
exe.addAttribute("jobNodes", jobNodes, "string")
nodes = jobNodes.split()
count = 1
for node in nodes:
fullname, type = getMachineName(ptds, node, "node")
if fullname != "":
exe.addAttribute("run node_%d" % count, fullname,
"resource")
count += 1
elif line.startswith("Job Resources used"):
jobResources = line.strip("Job Resources used: ").strip()
if jobResources != "":
exe.addAttribute("jobResourcesUsed", jobResources, "string")
f.close()
# look though stderr file
try:
f = open(pbsstderr, 'r')
except:
raise PTexception(
"Run.parsePBSoutput could not open pbs stderr file: %s" %
pbsstderr)
lines = f.readlines()
count = 1
for line in lines:
runError = line.strip()
if runError != "":
exe.addAttribute("RunErrorMsg_%d" % count, runError, "string")
count += 1
f.close()
def getInitialAVs(resIdx, attrs, exe, runMachineArg, ptds):
"""Parses the attribute value pairs and extracts the needed information.
"""
runAVs = []
runOSAVs = []
runOSName = ""
runOSVersion = ""
runOSType = ""
runMachine = ""
numberOfProcesses = -1
threadsPerProcess = -1
for name, value, type in attrs:
# print '(%s,%s)' % (name,value)
if name == "RunDataBegin":
pass
elif (
(name == "LibraryBegin")
or (name == "InputDeckBegin")
or (name == "RunDataEnd")
or (name == "SubmissionBegin")
):
break
# Get run specific data
elif name == "RunMachine":
runMachine = value
elif name == "PageSize":
pageSize = value
runAVs.append((name, value, "string"))
elif name == "NumberOfProcesses":
numberOfProcesses = value
runAVs.append((name, value, "string"))
elif name == "ProcessesPerNode":
processesPerNode = value
runAVs.append((name, value, "string"))
elif name == "ThreadsPerProcess":
threadsPerProcess = value
runAVs.append((name, value, "string"))
elif name.startswith("Run") and not (name.find("RunOS") >= 0):
if name == "RunEnv":
RunEnv = [] # a list of attr value pairs
# where attr = "RunEnv"+ (enviroment variable name)
# and value is the environment variable's value
# e.g. for PATH=/usr/bin:/usr/local/bin
# attr = RunEnvPATH
# value = /usr/bin:/usr/local/bin
RunEnv = StringToList("Env_", "@@@", value)
for (n, v) in RunEnv:
if v != "":
runAVs.append((n, v, "string"))
else:
runAVs.append((name, value, "string"))
elif name.startswith("RunOS"):
if name == "RunOSName":
runOSName = value
elif name == "RunOSReleaseVersion":
runOSVersion = value
elif name == "RunOSReleaseType":
runOSType = value
elif value != "":
runOSAVs.append((name, value, "string"))
else:
runAVs.append((name, value, "string"))
if runOSName == "" or runOSName == None:
raise PTexception("no operating system name given in run data file for" " execution:%s" % exe.name)
if runOSVersion == "" or runOSVersion == None or runOSType == "" or runOSType == None:
raise PTexception("no operating system details given in run data file " "for execution:%s" % exe.name)
if runMachine == "" or runMachine == None:
raise PTexception("no run machine name given in data file " "for execution:%s" % exe.name)
runOSNewName = ptds.getNewResourceName(runOSName + " " + runOSVersion + " " + runOSType)
runOS = Resource(runOSNewName, "operatingSystem", runOSAVs)
resIdx.addResource(runOS)
# if the run data doesn't know the machine name, then see if the user
# sent it in as an argument to PTdFgen
if runMachine == "" or runMachine == None:
runMachine = runMachineArg
# if they didn't do that, then we still don't know the machine
if runMachine == "" or runMachine == None:
runMach = None
else:
fullname, type = getMachineName(ptds, runMachine, "machine")
# print "machine is %s " % runMachine
runMach = Resource(fullname, type)
resIdx.addResource(runMach)
exe.addAttributes(runAVs)
if threadsPerProcess < 0:
raise PTexception("ThreadsPerProcess attribute not given in run data " "file for execution:%s" % exe.name)
if numberOfProcesses < 0:
raise PTexception("NumberOfProcesses attribute not given in run data " "file for execution:%s" % exe.name)
return (runMach, runOS, exe, numberOfProcesses, threadsPerProcess)
def parsePBSoutput(resIdx, runfilename, ptds):
# parse the stdout and stderr of PBS to get additional run information
# we expect there to be a files called jobprefix.pbs.out and
# jobprefix.pbs.err where jobprefix
# is the same prefix as the .run file parsed in process()
fileprefix = runfilename.rstrip(".run")
pbsstdout = fileprefix + ".pbs.out"
pbsstderr = fileprefix + ".pbs.err"
try:
f = open(pbsstdout, "r")
except:
raise PTexception("Run.parsePBSoutput could not open pbs stdout file: %s" % pbsstdout)
[exe] = resIdx.findResourcesByType("execution")
lines = f.readlines()
for line in lines:
if line.startswith("Job startup at"):
jobStartTime = line.strip("Job startup at").strip()
if jobStartTime != "":
exe.addAttribute("jobStartTime", jobStartTime, "string")
elif line.startswith("Jobs exit status"):
jobExitStatus = line.strip("Jobs exit status code is ").strip()
if jobExitStatus != "":
exe.addAttribute("jobExitStatus", jobExitStatus, "string")
elif line.find("completed ") >= 0:
jobCompletionTime = ""
for w in line.split()[3:]:
jobCompletionTime += "%s " % w
if jobCompletionTime.strip() != "":
exe.addAttribute("jobCompletionTime", jobCompletionTime, "string")
elif line.startswith("Nodes used"):
jobNodes = line.strip("Nodes used: ").strip()
if jobNodes != "":
exe.addAttribute("jobNodes", jobNodes, "string")
nodes = jobNodes.split()
count = 1
for node in nodes:
fullname, type = getMachineName(ptds, node, "node")
if fullname != "":
exe.addAttribute("run node_%d" % count, fullname, "resource")
count += 1
elif line.startswith("Job Resources used"):
jobResources = line.strip("Job Resources used: ").strip()
if jobResources != "":
exe.addAttribute("jobResourcesUsed", jobResources, "string")
f.close()
# look though stderr file
try:
f = open(pbsstderr, "r")
except:
raise PTexception("Run.parsePBSoutput could not open pbs stderr file: %s" % pbsstderr)
lines = f.readlines()
count = 1
for line in lines:
runError = line.strip()
if runError != "":
exe.addAttribute("RunErrorMsg_%d" % count, runError, "string")
count += 1
f.close()
except PTexception, a:
raise
if testMode:
print a
raise PTexception(a)
else:
print a.value
return -1
if sInfo:
ptds = None
try:
resIdx = ResourceIndex()
ptds = connectToDB(testMode, opt)
## set the machine name from command line arg:
fullname,type = getMachineName(ptds, sInfo.machineName, "machine")
mach = Resource(fullname, type)
resIdx.addResource(mach)
parsePerf.getSysPerfInfo(resIdx, sInfo.dataDir, sInfo.perfTools, ptds)
ptdfname = sInfo.dataDir + "/sys.ptdf"
f = open(ptdfname,'w')
# ResourceIndex.PTdF returns a list of strings to write
writeLst = resIdx.PTdF()
for w in writeLst:
f.write(w)
f.close()
print "PTDF system data generation complete."
except PTexception, a:
if testMode:
print a.value
except PTexception, a:
raise
if testMode:
print a
raise PTexception(a)
else:
print a.value
return -1
if sInfo:
ptds = None
try:
resIdx = ResourceIndex()
ptds = connectToDB(testMode, opt)
## set the machine name from command line arg:
fullname, type = getMachineName(ptds, sInfo.machineName, "machine")
mach = Resource(fullname, type)
resIdx.addResource(mach)
parsePerf.getSysPerfInfo(resIdx, sInfo.dataDir, sInfo.perfTools,
ptds)
ptdfname = sInfo.dataDir + "/sys.ptdf"
f = open(ptdfname, 'w')
# ResourceIndex.PTdF returns a list of strings to write
writeLst = resIdx.PTdF()
for w in writeLst:
f.write(w)
f.close()
print "PTDF system data generation complete."
except PTexception, a:
if testMode: