Exemple #1
0
def main(argv):                  

    try:                                
        opts, args = getopt.getopt(argv, "ht:", ["help", "threads=", "version", "nosubmit", "nobackground", "onefile"])
    except getopt.GetoptError:
        print "Error while converting options: unknown option(s) encountered.\n\n"
        usage(os.path.basename(sys.argv[0]))
        sys.exit(2)
    
    for opt, arg in opts:
        if opt in ["-h", "--help"]:
            usage(os.path.basename(sys.argv[0]))
            sys.exit()
        elif opt in ["--version"]:
            version()
            sys.exit()
        else:
            print "Unknown parameter", opt
    
    if len(args) < 1:
        print "Error: No workload file given.\n\n"
        usage(os.path.basename(sys.argv[0]))
        sys.exit(3)
        
    WorkloadFileName = args[0];
    if not os.path.isfile( WorkloadFileName ):
        print "Error: %s is not a valid file.\n\n" % WorkloadFileName
        usage(os.path.basename(sys.argv[0]))
        sys.exit(1)
        
    handlerXML = WLDocHandlers.readWorkloadSubmitFile( WorkloadFileName )
    printWorkload( handlerXML )
def main(argv):                  

    global DirSSER
    global NRandomTests
           
    try:                                
        opts, args = getopt.getopt(argv, "ht:", ["help", "threads=", "version", "nosubmit", "nobackground", "onefile"])
    except getopt.GetoptError:
        print "Error while converting options: unknown option(s) encountered.\n\n"
        usage(os.path.basename(sys.argv[0]))
        sys.exit(2)
    
    NThreads = 5
    NoSubmit = 0
    OneFile = 0
    NoBackground = 0
    
    for opt, arg in opts:
        if opt in ["-h", "--help"]:
            usage(os.path.basename(sys.argv[0]))
            sys.exit()
        elif opt in ["--version"]:
            version()
            sys.exit()
        elif opt in ["-t", "--threads"]: 
            try:
                NThreads = int(arg.strip()) 
            except ValueError:
                NThreads = 5
        elif opt in ["--nosubmit"]:
            NoSubmit = 1
        elif opt in ["--nobackground"]:
            NoBackground = 1
        elif opt in ["--onefile"]:
            OneFile = 1
        else:
            print "Unknown parameter", opt
    
    if len(args) < 1:
        print "Error: No workload file given.\n\n"
        usage(os.path.basename(sys.argv[0]))
        sys.exit(3)
        
    WorkloadFileName = args[0];
    if not os.path.isfile( WorkloadFileName ):
        print "Error: %s is not a valid file.\n\n" % WorkloadFileName
        usage(os.path.basename(sys.argv[0]))
        sys.exit(1)
        
    print "%s Parsing workload file %s" % \
          ( time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time())), WorkloadFileName )
    handlerXML = WLDocHandlers.readWorkloadSubmitFile( WorkloadFileName )
    print "%s Workload file processed, proceeding to submission"  % \
          ( time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time())) ) 
        
    parseWL( os.path.dirname(WorkloadFileName), handlerXML )
    
    print "%s All done" % time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time()))
def main(argv):                  

    try:                                
        opts, args = getopt.getopt(argv, "h", ["help", "version"])
    except getopt.GetoptError:
        print "Error while converting options: unknown option(s) encountered.\n\n"
        usage(os.path.basename(sys.argv[0]))
        sys.exit(2)
    
    for opt, arg in opts:
        if opt in ("-h", "--help"):
            usage(os.path.basename(sys.argv[0]))
            sys.exit()
        elif opt in ["--version"]:
            version()
            sys.exit()
        else:
            print "Unknown parameter", opt
    
    if len(args) < 1:
        #print "Error: No sites file given.\n\n"
        #usage(os.path.basename(sys.argv[0]))
        #sys.exit(3)
        SitesFileName = "grid-sites.xml"
    else:
        SitesFileName = args[0];
        
    if not os.path.isfile( SitesFileName ):
        if os.path.exists( SitesFileName ):
            print "\n\n****\nError: %s is not a file!\n****\n" % SitesFileName
        else:
            print "\n\n****\nError: %s does not exist!\n****\n" % SitesFileName
        usage(os.path.basename(sys.argv[0]))
        sys.exit(1)
        
    #---Read a sites file
    #print "%s Parsing sites file %s" % \
    #      ( time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time())), SitesFileName )
    handlerXML = WLDocHandlers.readSiteFile(SitesFileName)
    #print "%s Sites file processed, proceeding to submission"  % \
    #      time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time()))  
    
    DictionaryOfSites = handlerXML.getDictionaryOfSites()
    ListOfSites = AIStorageUtils.dict_sortbykey( DictionaryOfSites, AIStorageUtils.SORT_DESCENDING )
    for (id, Site) in ListOfSites: 
        print id, Site['location'], Site['machines']
    print
Exemple #4
0
def main(argv):                  

    global DirSSER
    global NRandomTests
           
    try:                                
        opts, args = getopt.getopt(argv[1:], "ht:d", ["help", "threads=", "version", "nosubmit", "nobackground", "onefile", "testid=", "projectid=", "testerid=", "starttime=","dedicated"])
    except getopt.GetoptError:
        print "Error while converting options: unknown option(s) encountered.\n\n"
        usage(os.path.basename(sys.argv[0]))
        sys.exit(2)

    NThreads = 5
    NoSubmit = 0
    OneFile = 0
    NoBackground = 0
    testid = 0
    testerid = 0
    starttime = time.time()
    projectid = "default"
    dedicatedMode=False

    #print opts
    #print args    

    for opt, arg in opts:
        #print "x", opt, arg

        if opt in ["-h", "--help"]:
            usage(os.path.basename(sys.argv[0]))
            sys.exit()
        elif opt in ["--version"]:
            version()
            sys.exit()
        elif opt in ["-t", "--threads"]: 
            try:
                NThreads = int(arg.strip()) 
            except ValueError:
                NThreads = 5
        elif opt in ["--nosubmit"]:
            NoSubmit = 1
        elif opt in ["--nobackground"]:
            NoBackground = 1
        elif opt in ["--onefile"]:
            OneFile = 1
        elif opt in ["--testid"]:
	    print "#### wl-submit arg: %s \n" % arg
            testid = int(arg.strip())
            #print testid
        elif opt in ["--testerid"]:
            testerid = int(arg.strip())
            #print testerid
        elif opt in ["--starttime"]:
            starttime = float(arg.strip())
            #print starttime
        elif opt in ["--projectid"]:
            projectid = str(arg.strip())
        elif opt in ["-d","--dedicated"]:
            dedicatedMode = True
        else:
            print "Unknown parameter", opt

    #sys.exit(0)
    
    if len(argv) < 1:
        print "Error: No workload file given.\n\n"
        usage(os.path.basename(sys.argv[0]))
        sys.exit(3)
        
    WorkloadFileName = argv[0];
    if not os.path.isfile( WorkloadFileName ):
        print "Error: %s is not a valid file.\n\n" % WorkloadFileName
        usage(os.path.basename(sys.argv[0]))
        sys.exit(1)
        
    print "%s Parsing workload file %s" % \
          ( time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time())), WorkloadFileName )
    handlerXML = WLDocHandlers.readWorkloadSubmitFile( WorkloadFileName, dedicatedMode )
    print "%s Workload file processed, proceeding to submission"  % \
          ( time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time())) ) 
        
    WorkloadDir = os.path.join(os.path.dirname(WorkloadFileName),"run")
    StartSubmissionTime, EndSubmissionTime, NTotalJobs, NTotalJobsInQueue = \
        runWL( WorkloadDir, handlerXML, NThreads, NoSubmit, NoBackground, OneFile, testid, projectid, testerid, starttime - time.time() )
    print "%s All done" % time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time()))
    
    tstart = time.strftime('%H:%M:%S', time.gmtime(StartSubmissionTime))
    tend = time.strftime('%H:%M:%S', time.gmtime(EndSubmissionTime))
    
    
    print "------ SUBMISSION SUMMARY ---------"
    print "Started submission at %s" % tstart
    print "Ended submission at %s" % tend
    print "Submission total time/Total Jobs/submitted/Jobs per second"
#          0123456789012345678901234567890123456789012345678901234567890123456789
#          0         1         2         3         4         5         6  
    DeltaTime = float(EndSubmissionTime - StartSubmissionTime)
    if DeltaTime > 0.e-8:
        Avg = float(NTotalJobs - NTotalJobsInQueue) / DeltaTime
    else:
        Avg = 0.0
def main(argv):

    global DirSSER
    global NRandomTests

    try:
        opts, args = getopt.getopt(argv, "ht:", ["help", "threads=", "version", "nosubmit", "background", "onefile"])
    except getopt.GetoptError:
        print "Error while converting options: unknown option(s) encountered.\n\n"
        usage(os.path.basename(sys.argv[0]))
        sys.exit(2)

    NThreads = 30
    NoSubmit = 0
    OneFile = 0
    Background = 0

    for opt, arg in opts:
        if opt in ["-h", "--help"]:
            usage(os.path.basename(sys.argv[0]))
            sys.exit()
        elif opt in ["--version"]:
            version()
            sys.exit()
        elif opt in ["-t", "--threads"]:
            try:
                NThreads = int(arg.strip())
                print "[wl-exec-dagman.py] Number of threads set to %s " % NThreads
            except ValueError:
                NThreads = 30
        elif opt in ["--nosubmit"]:
            NoSubmit = 1
        elif opt in ["--background"]:
            Background = 1
        elif opt in ["--onefile"]:
            OneFile = 1
        else:
            print "Unknown parameter", opt

    if len(args) < 1:
        print "Error: No workload file given.\n\n"
        usage(os.path.basename(sys.argv[0]))
        sys.exit(3)

    WorkloadFileName = args[0]
    if not os.path.isfile(WorkloadFileName):
        print "Error: %s is not a valid file.\n\n" % WorkloadFileName
        usage(os.path.basename(sys.argv[0]))
        sys.exit(1)

    print "%s Parsing workload file %s" % (
        time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time())),
        WorkloadFileName,
    )
    handlerXML = WLDocHandlers.readWorkloadSubmitFile(WorkloadFileName)
    print "%s Workload file processed, proceeding to submission" % (
        time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time()))
    )

    WorkloadDir = os.path.join(os.path.dirname(WorkloadFileName), "run")
    print "Workload directory", WorkloadDir
    StartSubmissionTime, EndSubmissionTime, NTotalJobs, NTotalJobsInQueue = runWL(
        WorkloadDir, handlerXML, NThreads, NoSubmit, Background, OneFile
    )
    print "%s All done" % time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time()))

    tstart = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(StartSubmissionTime))
    tend = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(EndSubmissionTime))
    print "------ SUBMISSION SUMMARY ---------"
    print "Started submission at %s" % tstart
    print "Ended submission at %s" % tend
    print "Submission total time/Total Jobs/submitted/Jobs per second"
    #      0123456789012345678901234567890123456789012345678901234567890123456789
    #      0         1         2         3         4         5         6
    DeltaTime = float(EndSubmissionTime - StartSubmissionTime)
    if DeltaTime > 0.0e-8:
        Avg = float(NTotalJobs - NTotalJobsInQueue) / DeltaTime
    else:
        Avg = 0.0
    print "   %15.3f   /  %6d  / %6d  /  %.3f" % (DeltaTime, NTotalJobs, NTotalJobs - NTotalJobsInQueue, Avg)
Exemple #6
0
                 print "Cannot find JDF generator", "'"+JDFGeneratorNames[ii]+"'"
                 raise NameError, CurrentJDFGeneratorName
             JDFGeneratorFuncs[ii]( SubJobJDFFileName, JobComponentsList )
             ii = ii + 1
         #-- print job to its JDF file
         # CurrentJDFGeneratorFunc( SubJobJDFFileName, JobComponentsList )
         
     #--- print composite job - to multiple files if necessary
     for ii in range (0, NEngines):
         #Composite_JDFFileName = JobInfo['default_jdf']
         if (FileFormatsArray[ii] == 'default'):
             Composite_JDFFileName = JobInfo['jdf'][ii]
             print "STATUS! %d Writing default composite file to: %s " % (ii, Composite_JDFFileName)
             Composite_Name = JobInfo['id'] 
             Composite_Type = 'composite:' + WLUnit['CompositionType'] 
             WLDocHandlers.writeWorkloadSubmitFile( Composite_JDFFileName, Composite_Name, Composite_JobsInfoList, JDFGeneratorNames[ii], Composite_Type, GeneratorName ) 
     
 else:
     #-- get list of components for this unitary job
     JobComponentsList = WLUnit['unit']['jobs'][index]
     #--- statistics for this unit
     CurrentNComponents = len(JobComponentsList)
     NComponents = NComponents + CurrentNComponents
     if (MinComponents is None) or (MinComponents > CurrentNComponents): MinComponents = CurrentNComponents
     if (MaxComponents is None) or (MaxComponents < CurrentNComponents): MaxComponents = CurrentNComponents
     CurrentTotalJobNCPUs = 0
     for JobComponent in JobComponentsList: 
         if 'count' in JobComponent:
             CurrentNCPUs = JobComponent['count']
             # per job stats
             CurrentTotalJobNCPUs = CurrentTotalJobNCPUs + CurrentNCPUs
Exemple #7
0
             print "Submit duration set to default value (%d).\n" % SubmitDurationMS
             
     elif opt in ["-j", "--jdfgen"]: 
         JDFGenNames = arg.strip() 
         JDFGenNamesList = JDFGenNames.split(',')
         FileOrExit( os.path.join("jdfprint", "%s.py" % JDFGenName), 
                     "JDF generator (%s) " % JDFGenName )
         print "JDF Generator set to %s.\n" % JDFGenName
     elif opt in ["-s", "--sitesfile"]: 
         SitesFileName = arg.strip()
         FileOrExit( SitesFileName, "site description" )
     elif opt in ["-w", "--workload"]: 
         WorkloadDescriptionFileName = arg.strip() 
         FileOrExit( WorkloadDescriptionFileName, "workload description" )
     elif opt in ["--arrival"]:
         arrivalDistr = WLDocHandlers.getArrivalTimeDistribution(arg)
     else:
         print "Unknown parameter", opt
 
 #--- Load workload generators
 print "STATUS! Load workload generators."
 AWLGenLoader = WLLoader.GeneratorsLoader( ModulesPath = "apps", 
                                               AppGenSig = "WLGenerator", 
                                               AppGenFuncName = "generateWorkload",
                                               ErrPrefix = "Workload Generator:"  )
 try:
     AWLGenLoader.loadGenerators()
 except WLLoader.GeneratorLoaderError, e:
     print e
     sys.exit(4)