Пример #1
0
def getInitialAVs(resIdx, attrs, exeName, ptds):
    """ Takes attrs, which is a list of name, value pairs, and builds
        a build Resource object and Execution object
    """

    executableAVs = []
    buildAVs = []
    buildOSAVs = []
    AppName = ""
    BuildMachType = ""
    BuildMachine = ""
    BuildOSName = ""
    BuildOSVersion = ""
    BuildOSType = ""

    # regular expressions for parsing

    for name, value, type in attrs:
        if name == "BuildDataBegin":
            pass
        elif name == "BuildDataEnd":
            break
        # Get executable specific data
        elif name == "ApplicationName":
            AppName = value
        elif name == "ApplicationExeTrialName" or name == "ApplicationExeUserComment":
            AppTrialName = value
        elif name.startswith("Application"):
            if name == "ApplicationLanguages":
                value = fixListFormat(value)
                executableAVs.append(("Languages", value, "string"))
            elif name == "ApplicationParadigms":
                value = fixListFormat(value)
                executableAVs.append(("Concurrency", value, "string"))
            elif name == "ApplicationExeName":
                executableAVs.append(("Executable Name", value, "string"))
            elif name == "ApplicationExeSize":
                executableAVs.append(("Executable Size", value, "string"))
            elif name == "ApplicationExePerms":
                executableAVs.append(("Executable Permissions", value, "string"))
            elif name == "ApplicationExeUID":
                executableAVs.append(("Executable UID", value, "string"))
            elif name == "ApplicationExeGID":
                executableAVs.append(("Executable GID", value, "string"))
            elif name == "ApplicationExeTimestamp":
                executableAVs.append(("Executable Timestamp", value, "string"))
            elif name == "ApplicationExeUsername":
                executableAVs.append(("Username", value, "string"))
        # get Build specific data
        elif name.startswith("Build") and not (name.find("BuildOS") >= 0):
            if name == "BuildEnv":
                BuildEnv = []  # a list of attr value pairs
                # where attr = "BuildEnv"+ (enviroment variable name)
                # and value is the environment variable's value
                # e.g.  for PATH=/usr/bin:/usr/local/bin
                # attr = BuildEnvPATH
                # value = /usr/bin:/usr/local/bin
                BuildEnv = StringToList("Env_", "@@@", value)
                for (n, v) in BuildEnv:
                    if v != "":
                        buildAVs.append((n, v, "string"))
            else:
                if name == "BuildNode" or name == "BuildMachine":
                    BuildMachine = value
                if name == "BuildNode":
                    BuildMachType = "node"
                    buildAVs.append(("Node Name", value, "string"))
                elif name == "BuildMachine":
                    BuildMachType = "machine"
                    buildAVs.append(("Machine Name", value, "string"))
                if name == "BuildDateTime":
                    buildAVs.append(("DateTime", value, "string"))
                # get Build operating system  specific data
        elif name.startswith("BuildOS"):
            if name == "BuildOSName":
                BuildOSName = value
            elif name == "BuildOSReleaseVersion":
                BuildOSVersion = value
            elif name == "BuildOSReleaseType":
                BuildOSType = value

    if exeName == "" or exeName == None:
        raise PTexception("missing execution name in Build.getInitialAVs")

    exeNewName = ptds.getNewResourceName(exeName)
    # ATTR CHANGE TODO
    # executableAVs should be attrs of build, not execution
    exe = Execution(exeNewName, executableAVs)
    resIdx.addResource(exe)

    newBuildName = ptds.getNewResourceName("build")
    build = Resource(newBuildName, "build", buildAVs)
    resIdx.addResource(build)

    if BuildMachine == "" or BuildMachine == None:
        raise PTexception("missing build machine name in build data file for " " execution:%s" % exeName)

    if BuildMachType == "node":
        fullname, type = getMachineName(ptds, BuildMachine, BuildMachType)
        # print fullname
        buildMachine = Resource(fullname, type)
    else:  # machine
        fullname, type = getMachineName(ptds, BuildMachine, BuildMachType)
        buildMachine = Resource(fullname, type)

    if BuildOSName == "" or BuildOSName == None:
        raise PTexception("missing build operating system name for execution" "execution:%s" % exeName)
    if BuildOSVersion == "" or BuildOSVersion == None or BuildOSType == "" or BuildOSType == None:
        raise PTexception("missing build operating system details for execution" "execution:%s" % exeName)
    buildOSNewName = ptds.getNewResourceName(BuildOSName + " " + BuildOSVersion + " " + BuildOSType)
    buildOS = Resource(buildOSNewName, "operatingSystem", buildOSAVs)
    resIdx.addResources([buildOS, buildMachine])
    build.addAttribute("Node Name", buildMachine.name, "resource")
    build.addAttribute("OS Name", buildOS.name, "resource")

    return (build, exe)
Пример #2
0
def getInitialAVs(resIdx, attrs, exe, runMachineArg, ptds):
    """Parses the attribute value pairs and extracts the needed information.
    """
    runAVs = []
    runOSAVs = []
    runOSName = ""
    runOSVersion = ""
    runOSType = ""
    runMachine = ""
    numberOfProcesses = -1
    threadsPerProcess = -1

    for name, value, type in attrs:
        #print '(%s,%s)' % (name,value)
        if name == "RunDataBegin":
            pass
        elif (name == "LibraryBegin") or (name == "InputDeckBegin") \
             or (name == "RunDataEnd") or (name == "SubmissionBegin"):
            break
        #Get run specific data
        elif name == "RunMachine":
            runMachine = value
        elif name == "PageSize":
            pageSize = value
            runAVs.append((name, value, "string"))
        elif name == "NumberOfProcesses":
            numberOfProcesses = value
            runAVs.append((name, value, "string"))
        elif name == "ProcessesPerNode":
            processesPerNode = value
            runAVs.append((name, value, "string"))
        elif name == "ThreadsPerProcess":
            threadsPerProcess = value
            runAVs.append((name, value, "string"))
        elif name.startswith("Run") and not (name.find("RunOS") >= 0):
            if name == "RunEnv":
                RunEnv = []  # a list of attr value pairs
                #where attr = "RunEnv"+ (enviroment variable name)
                #and value is the environment variable's value
                #e.g.  for PATH=/usr/bin:/usr/local/bin
                # attr = RunEnvPATH
                # value = /usr/bin:/usr/local/bin
                RunEnv = StringToList("Env_", "@@@", value)
                for (n, v) in RunEnv:
                    if (v != ""):
                        runAVs.append((n, v, "string"))
            else:
                runAVs.append((name, value, "string"))
        elif name.startswith("RunOS"):
            if name == "RunOSName":
                runOSName = value
            elif name == "RunOSReleaseVersion":
                runOSVersion = value
            elif name == "RunOSReleaseType":
                runOSType = value
            elif (value != ""):
                runOSAVs.append((name, value, "string"))

        else:
            runAVs.append((name, value, "string"))

    if runOSName == "" or runOSName == None:
        raise PTexception("no operating system name given in run data file for"\
                          " execution:%s" % exe.name )
    if runOSVersion == "" or runOSVersion == None or \
       runOSType == "" or runOSType == None:
        raise PTexception("no operating system details given in run data file "\
                          "for execution:%s" % exe.name )

    if runMachine == "" or runMachine == None:
        raise PTexception("no run machine name given in data file "\
                         "for execution:%s" % exe.name )

    runOSNewName = ptds.getNewResourceName(runOSName+" "+runOSVersion+" "+\
                             runOSType)
    runOS = Resource(runOSNewName, "operatingSystem", runOSAVs)
    resIdx.addResource(runOS)

    # if the run data doesn't know the machine name, then see if the user
    # sent it in as an argument to PTdFgen
    if runMachine == "" or runMachine == None:
        runMachine = runMachineArg
    # if they didn't do that, then we still don't know the machine
    if runMachine == "" or runMachine == None:
        runMach = None
    else:
        fullname, type = getMachineName(ptds, runMachine, "machine")
        #print "machine is %s " % runMachine
        runMach = Resource(fullname, type)
        resIdx.addResource(runMach)
    exe.addAttributes(runAVs)

    if threadsPerProcess < 0:
        raise PTexception("ThreadsPerProcess attribute not given in run data "\
                              "file for execution:%s" % exe.name )

    if numberOfProcesses < 0:
        raise PTexception("NumberOfProcesses attribute not given in run data "\
                              "file for execution:%s" % exe.name )
    return  (runMach, runOS, exe, numberOfProcesses, \
            threadsPerProcess)
Пример #3
0
def getInitialAVs(resIdx, attrs, exeName, ptds):
    """ Takes attrs, which is a list of name, value pairs, and builds
        a build Resource object and Execution object
    """

    executableAVs = []
    buildAVs = []
    buildOSAVs = []
    AppName = ""
    BuildMachType = ""
    BuildMachine = ""
    BuildOSName = ""
    BuildOSVersion = ""
    BuildOSType = ""

    # regular expressions for parsing


    for name,value,type in attrs:
        if name == "BuildDataBegin":
           pass
        elif name == "BuildDataEnd":
           break
	#Get executable specific data
        elif name == "ApplicationName":
           AppName = value 
        elif name == "ApplicationExeTrialName" or \
	     name == "ApplicationExeUserComment":
	   AppTrialName = value
        elif name.startswith("Application"):
           if name  == "ApplicationLanguages":
              value = fixListFormat(value)
              executableAVs.append(("Languages", value, "string"))
           elif name == "ApplicationParadigms":
              value = fixListFormat(value)
              executableAVs.append(("Concurrency", value, "string"))
           elif name == "ApplicationExeName":
              executableAVs.append(("Executable Name", value, "string"))
           elif name == "ApplicationExeSize":
              executableAVs.append(("Executable Size", value, "string"))
           elif name == "ApplicationExePerms":
              executableAVs.append(("Executable Permissions", value, "string"))
           elif name == "ApplicationExeUID":
              executableAVs.append(("Executable UID", value, "string"))
           elif name == "ApplicationExeGID":
              executableAVs.append(("Executable GID", value, "string"))
           elif name == "ApplicationExeTimestamp":
              executableAVs.append(("Executable Timestamp", value, "string"))
           elif name == "ApplicationExeUsername":
              executableAVs.append(("Username", value, "string"))
        # get Build specific data
        elif name.startswith("Build") and not (name.find("BuildOS") >= 0):
           if name == "BuildEnv":
              BuildEnv = [] # a list of attr value pairs
	           #where attr = "BuildEnv"+ (enviroment variable name)
	           #and value is the environment variable's value
	           #e.g.  for PATH=/usr/bin:/usr/local/bin
	           # attr = BuildEnvPATH
	           # value = /usr/bin:/usr/local/bin
              BuildEnv = StringToList("Env_","@@@", value)
	      for (n,v) in BuildEnv:
                  if(v != ""):
	             buildAVs.append((n,v,"string"))
           else:
	      if name == "BuildNode" or name == "BuildMachine":
                 BuildMachine = value
              if name == "BuildNode":
		 BuildMachType = 'node'
                 buildAVs.append(("Node Name",value,"string"))
              elif name == "BuildMachine":
		 BuildMachType = 'machine'
                 buildAVs.append(("Machine Name",value,"string"))
              if name == "BuildDateTime":
                 buildAVs.append(("DateTime",value,"string"))
        #get Build operating system  specific data
        elif name.startswith("BuildOS"):
	       if name == "BuildOSName":
	          BuildOSName = value
               elif name == "BuildOSReleaseVersion":
                  BuildOSVersion = value
               elif name == "BuildOSReleaseType":
                  BuildOSType = value

    if exeName == "" or exeName == None:
       raise PTexception("missing execution name in Build.getInitialAVs") 

    exeNewName = ptds.getNewResourceName(exeName)
    # ATTR CHANGE TODO
    # executableAVs should be attrs of build, not execution
    exe = Execution(exeNewName, executableAVs)
    resIdx.addResource(exe)

  
    newBuildName = ptds.getNewResourceName("build") 
    build = Resource(newBuildName, "build", buildAVs)
    resIdx.addResource(build)
    
    if BuildMachine == "" or BuildMachine == None:
       raise PTexception("missing build machine name in build data file for "
                         " execution:%s" % exeName)

    if BuildMachType == "node":
        fullname,type = getMachineName(ptds, BuildMachine, BuildMachType)
        #print fullname
        buildMachine = Resource(fullname,type)
    else: # machine
        fullname,type = getMachineName(ptds, BuildMachine, BuildMachType)
        buildMachine = Resource(fullname,type)

    if BuildOSName == "" or BuildOSName == None:
       raise PTexception("missing build operating system name for execution"\
                         "execution:%s" % exeName)
    if BuildOSVersion == "" or BuildOSVersion == None or \
       BuildOSType == "" or BuildOSType == None: 
       raise PTexception("missing build operating system details for execution"\
                         "execution:%s" % exeName)
    buildOSNewName = ptds.getNewResourceName(BuildOSName+" "+BuildOSVersion+\
                              " "+BuildOSType)
    buildOS = Resource(buildOSNewName, "operatingSystem", buildOSAVs)
    resIdx.addResources([buildOS,buildMachine])
    build.addAttribute("Node Name", buildMachine.name, "resource")
    build.addAttribute("OS Name", buildOS.name, "resource")


    return(build,exe)
Пример #4
0
def parsePBSoutput(resIdx, runfilename, ptds):
    # parse the stdout and stderr of PBS to get additional run information
    # we expect there to be a files called jobprefix.pbs.out and
    # jobprefix.pbs.err where jobprefix
    # is the same prefix as the .run file parsed in process()
    fileprefix = runfilename.rstrip('.run')
    pbsstdout = fileprefix + ".pbs.out"
    pbsstderr = fileprefix + ".pbs.err"
    try:
        f = open(pbsstdout, 'r')
    except:
        raise PTexception(
            "Run.parsePBSoutput could not open pbs stdout file: %s" %
            pbsstdout)

    [exe] = resIdx.findResourcesByType("execution")

    lines = f.readlines()

    for line in lines:
        if line.startswith("Job startup at"):
            jobStartTime = line.strip("Job startup at").strip()
            if jobStartTime != "":
                exe.addAttribute("jobStartTime", jobStartTime, "string")
        elif line.startswith("Jobs exit status"):
            jobExitStatus = line.strip("Jobs exit status code is ").strip()
            if jobExitStatus != "":
                exe.addAttribute("jobExitStatus", jobExitStatus, "string")
        elif line.find("completed ") >= 0:
            jobCompletionTime = ""
            for w in line.split()[3:]:
                jobCompletionTime += "%s " % w
            if jobCompletionTime.strip() != "":
                exe.addAttribute("jobCompletionTime", jobCompletionTime,
                                 "string")
        elif line.startswith("Nodes used"):
            jobNodes = line.strip("Nodes used: ").strip()
            if jobNodes != "":
                exe.addAttribute("jobNodes", jobNodes, "string")
                nodes = jobNodes.split()
                count = 1
                for node in nodes:
                    fullname, type = getMachineName(ptds, node, "node")
                    if fullname != "":
                        exe.addAttribute("run node_%d" % count, fullname,
                                         "resource")
                    count += 1
        elif line.startswith("Job Resources used"):
            jobResources = line.strip("Job Resources used: ").strip()
            if jobResources != "":
                exe.addAttribute("jobResourcesUsed", jobResources, "string")
    f.close()

    # look though stderr file
    try:
        f = open(pbsstderr, 'r')
    except:
        raise PTexception(
            "Run.parsePBSoutput could not open pbs stderr file: %s" %
            pbsstderr)
    lines = f.readlines()
    count = 1
    for line in lines:
        runError = line.strip()
        if runError != "":
            exe.addAttribute("RunErrorMsg_%d" % count, runError, "string")
        count += 1

    f.close()
Пример #5
0
def getInitialAVs(resIdx, attrs, exe, runMachineArg, ptds):
    """Parses the attribute value pairs and extracts the needed information.
    """
    runAVs = []
    runOSAVs = []
    runOSName = ""
    runOSVersion = ""
    runOSType = ""
    runMachine = ""
    numberOfProcesses = -1
    threadsPerProcess = -1

    for name, value, type in attrs:
        # print '(%s,%s)' % (name,value)
        if name == "RunDataBegin":
            pass
        elif (
            (name == "LibraryBegin")
            or (name == "InputDeckBegin")
            or (name == "RunDataEnd")
            or (name == "SubmissionBegin")
        ):
            break
        # Get run specific data
        elif name == "RunMachine":
            runMachine = value
        elif name == "PageSize":
            pageSize = value
            runAVs.append((name, value, "string"))
        elif name == "NumberOfProcesses":
            numberOfProcesses = value
            runAVs.append((name, value, "string"))
        elif name == "ProcessesPerNode":
            processesPerNode = value
            runAVs.append((name, value, "string"))
        elif name == "ThreadsPerProcess":
            threadsPerProcess = value
            runAVs.append((name, value, "string"))
        elif name.startswith("Run") and not (name.find("RunOS") >= 0):
            if name == "RunEnv":
                RunEnv = []  # a list of attr value pairs
                # where attr = "RunEnv"+ (enviroment variable name)
                # and value is the environment variable's value
                # e.g.  for PATH=/usr/bin:/usr/local/bin
                # attr = RunEnvPATH
                # value = /usr/bin:/usr/local/bin
                RunEnv = StringToList("Env_", "@@@", value)
                for (n, v) in RunEnv:
                    if v != "":
                        runAVs.append((n, v, "string"))
            else:
                runAVs.append((name, value, "string"))
        elif name.startswith("RunOS"):
            if name == "RunOSName":
                runOSName = value
            elif name == "RunOSReleaseVersion":
                runOSVersion = value
            elif name == "RunOSReleaseType":
                runOSType = value
            elif value != "":
                runOSAVs.append((name, value, "string"))

        else:
            runAVs.append((name, value, "string"))

    if runOSName == "" or runOSName == None:
        raise PTexception("no operating system name given in run data file for" " execution:%s" % exe.name)
    if runOSVersion == "" or runOSVersion == None or runOSType == "" or runOSType == None:
        raise PTexception("no operating system details given in run data file " "for execution:%s" % exe.name)

    if runMachine == "" or runMachine == None:
        raise PTexception("no run machine name given in data file " "for execution:%s" % exe.name)

    runOSNewName = ptds.getNewResourceName(runOSName + " " + runOSVersion + " " + runOSType)
    runOS = Resource(runOSNewName, "operatingSystem", runOSAVs)
    resIdx.addResource(runOS)

    # if the run data doesn't know the machine name, then see if the user
    # sent it in as an argument to PTdFgen
    if runMachine == "" or runMachine == None:
        runMachine = runMachineArg
    # if they didn't do that, then we still don't know the machine
    if runMachine == "" or runMachine == None:
        runMach = None
    else:
        fullname, type = getMachineName(ptds, runMachine, "machine")
        # print "machine is %s " % runMachine
        runMach = Resource(fullname, type)
        resIdx.addResource(runMach)
    exe.addAttributes(runAVs)

    if threadsPerProcess < 0:
        raise PTexception("ThreadsPerProcess attribute not given in run data " "file for execution:%s" % exe.name)

    if numberOfProcesses < 0:
        raise PTexception("NumberOfProcesses attribute not given in run data " "file for execution:%s" % exe.name)
    return (runMach, runOS, exe, numberOfProcesses, threadsPerProcess)
Пример #6
0
def parsePBSoutput(resIdx, runfilename, ptds):
    # parse the stdout and stderr of PBS to get additional run information
    # we expect there to be a files called jobprefix.pbs.out and
    # jobprefix.pbs.err where jobprefix
    # is the same prefix as the .run file parsed in process()
    fileprefix = runfilename.rstrip(".run")
    pbsstdout = fileprefix + ".pbs.out"
    pbsstderr = fileprefix + ".pbs.err"
    try:
        f = open(pbsstdout, "r")
    except:
        raise PTexception("Run.parsePBSoutput could not open pbs stdout file: %s" % pbsstdout)

    [exe] = resIdx.findResourcesByType("execution")

    lines = f.readlines()

    for line in lines:
        if line.startswith("Job startup at"):
            jobStartTime = line.strip("Job startup at").strip()
            if jobStartTime != "":
                exe.addAttribute("jobStartTime", jobStartTime, "string")
        elif line.startswith("Jobs exit status"):
            jobExitStatus = line.strip("Jobs exit status code is ").strip()
            if jobExitStatus != "":
                exe.addAttribute("jobExitStatus", jobExitStatus, "string")
        elif line.find("completed ") >= 0:
            jobCompletionTime = ""
            for w in line.split()[3:]:
                jobCompletionTime += "%s " % w
            if jobCompletionTime.strip() != "":
                exe.addAttribute("jobCompletionTime", jobCompletionTime, "string")
        elif line.startswith("Nodes used"):
            jobNodes = line.strip("Nodes used: ").strip()
            if jobNodes != "":
                exe.addAttribute("jobNodes", jobNodes, "string")
                nodes = jobNodes.split()
                count = 1
                for node in nodes:
                    fullname, type = getMachineName(ptds, node, "node")
                    if fullname != "":
                        exe.addAttribute("run node_%d" % count, fullname, "resource")
                    count += 1
        elif line.startswith("Job Resources used"):
            jobResources = line.strip("Job Resources used: ").strip()
            if jobResources != "":
                exe.addAttribute("jobResourcesUsed", jobResources, "string")
    f.close()

    # look though stderr file
    try:
        f = open(pbsstderr, "r")
    except:
        raise PTexception("Run.parsePBSoutput could not open pbs stderr file: %s" % pbsstderr)
    lines = f.readlines()
    count = 1
    for line in lines:
        runError = line.strip()
        if runError != "":
            exe.addAttribute("RunErrorMsg_%d" % count, runError, "string")
        count += 1

    f.close()
Пример #7
0
       except PTexception, a:
           raise
           if testMode:
              print a
              raise PTexception(a)
           else:
              print a.value
              return -1

    if sInfo:
       ptds = None
       try:
          resIdx = ResourceIndex()
          ptds = connectToDB(testMode, opt)
          ## set the machine name from command line arg:
          fullname,type = getMachineName(ptds, sInfo.machineName, "machine")
          mach = Resource(fullname, type)
          resIdx.addResource(mach)

          parsePerf.getSysPerfInfo(resIdx, sInfo.dataDir, sInfo.perfTools, ptds)
          ptdfname = sInfo.dataDir + "/sys.ptdf"
          f = open(ptdfname,'w')
          # ResourceIndex.PTdF returns a list of strings to write
          writeLst = resIdx.PTdF()
          for w in writeLst:
             f.write(w)
          f.close()
          print "PTDF system data generation complete."
       except PTexception, a:
          if testMode:
             print a.value
Пример #8
0
        except PTexception, a:
            raise
            if testMode:
                print a
                raise PTexception(a)
            else:
                print a.value
                return -1

    if sInfo:
        ptds = None
        try:
            resIdx = ResourceIndex()
            ptds = connectToDB(testMode, opt)
            ## set the machine name from command line arg:
            fullname, type = getMachineName(ptds, sInfo.machineName, "machine")
            mach = Resource(fullname, type)
            resIdx.addResource(mach)

            parsePerf.getSysPerfInfo(resIdx, sInfo.dataDir, sInfo.perfTools,
                                     ptds)
            ptdfname = sInfo.dataDir + "/sys.ptdf"
            f = open(ptdfname, 'w')
            # ResourceIndex.PTdF returns a list of strings to write
            writeLst = resIdx.PTdF()
            for w in writeLst:
                f.write(w)
            f.close()
            print "PTDF system data generation complete."
        except PTexception, a:
            if testMode: