def main(argv): try: opts, args = getopt.getopt(argv, "ht:", ["help", "threads=", "version", "nosubmit", "nobackground", "onefile"]) except getopt.GetoptError: print "Error while converting options: unknown option(s) encountered.\n\n" usage(os.path.basename(sys.argv[0])) sys.exit(2) for opt, arg in opts: if opt in ["-h", "--help"]: usage(os.path.basename(sys.argv[0])) sys.exit() elif opt in ["--version"]: version() sys.exit() else: print "Unknown parameter", opt if len(args) < 1: print "Error: No workload file given.\n\n" usage(os.path.basename(sys.argv[0])) sys.exit(3) WorkloadFileName = args[0]; if not os.path.isfile( WorkloadFileName ): print "Error: %s is not a valid file.\n\n" % WorkloadFileName usage(os.path.basename(sys.argv[0])) sys.exit(1) handlerXML = WLDocHandlers.readWorkloadSubmitFile( WorkloadFileName ) printWorkload( handlerXML )
def main(argv): global DirSSER global NRandomTests try: opts, args = getopt.getopt(argv, "ht:", ["help", "threads=", "version", "nosubmit", "nobackground", "onefile"]) except getopt.GetoptError: print "Error while converting options: unknown option(s) encountered.\n\n" usage(os.path.basename(sys.argv[0])) sys.exit(2) NThreads = 5 NoSubmit = 0 OneFile = 0 NoBackground = 0 for opt, arg in opts: if opt in ["-h", "--help"]: usage(os.path.basename(sys.argv[0])) sys.exit() elif opt in ["--version"]: version() sys.exit() elif opt in ["-t", "--threads"]: try: NThreads = int(arg.strip()) except ValueError: NThreads = 5 elif opt in ["--nosubmit"]: NoSubmit = 1 elif opt in ["--nobackground"]: NoBackground = 1 elif opt in ["--onefile"]: OneFile = 1 else: print "Unknown parameter", opt if len(args) < 1: print "Error: No workload file given.\n\n" usage(os.path.basename(sys.argv[0])) sys.exit(3) WorkloadFileName = args[0]; if not os.path.isfile( WorkloadFileName ): print "Error: %s is not a valid file.\n\n" % WorkloadFileName usage(os.path.basename(sys.argv[0])) sys.exit(1) print "%s Parsing workload file %s" % \ ( time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time())), WorkloadFileName ) handlerXML = WLDocHandlers.readWorkloadSubmitFile( WorkloadFileName ) print "%s Workload file processed, proceeding to submission" % \ ( time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time())) ) parseWL( os.path.dirname(WorkloadFileName), handlerXML ) print "%s All done" % time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time()))
def main(argv): try: opts, args = getopt.getopt(argv, "h", ["help", "version"]) except getopt.GetoptError: print "Error while converting options: unknown option(s) encountered.\n\n" usage(os.path.basename(sys.argv[0])) sys.exit(2) for opt, arg in opts: if opt in ("-h", "--help"): usage(os.path.basename(sys.argv[0])) sys.exit() elif opt in ["--version"]: version() sys.exit() else: print "Unknown parameter", opt if len(args) < 1: #print "Error: No sites file given.\n\n" #usage(os.path.basename(sys.argv[0])) #sys.exit(3) SitesFileName = "grid-sites.xml" else: SitesFileName = args[0]; if not os.path.isfile( SitesFileName ): if os.path.exists( SitesFileName ): print "\n\n****\nError: %s is not a file!\n****\n" % SitesFileName else: print "\n\n****\nError: %s does not exist!\n****\n" % SitesFileName usage(os.path.basename(sys.argv[0])) sys.exit(1) #---Read a sites file #print "%s Parsing sites file %s" % \ # ( time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time())), SitesFileName ) handlerXML = WLDocHandlers.readSiteFile(SitesFileName) #print "%s Sites file processed, proceeding to submission" % \ # time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time())) DictionaryOfSites = handlerXML.getDictionaryOfSites() ListOfSites = AIStorageUtils.dict_sortbykey( DictionaryOfSites, AIStorageUtils.SORT_DESCENDING ) for (id, Site) in ListOfSites: print id, Site['location'], Site['machines'] print
def main(argv): global DirSSER global NRandomTests try: opts, args = getopt.getopt(argv[1:], "ht:d", ["help", "threads=", "version", "nosubmit", "nobackground", "onefile", "testid=", "projectid=", "testerid=", "starttime=","dedicated"]) except getopt.GetoptError: print "Error while converting options: unknown option(s) encountered.\n\n" usage(os.path.basename(sys.argv[0])) sys.exit(2) NThreads = 5 NoSubmit = 0 OneFile = 0 NoBackground = 0 testid = 0 testerid = 0 starttime = time.time() projectid = "default" dedicatedMode=False #print opts #print args for opt, arg in opts: #print "x", opt, arg if opt in ["-h", "--help"]: usage(os.path.basename(sys.argv[0])) sys.exit() elif opt in ["--version"]: version() sys.exit() elif opt in ["-t", "--threads"]: try: NThreads = int(arg.strip()) except ValueError: NThreads = 5 elif opt in ["--nosubmit"]: NoSubmit = 1 elif opt in ["--nobackground"]: NoBackground = 1 elif opt in ["--onefile"]: OneFile = 1 elif opt in ["--testid"]: print "#### wl-submit arg: %s \n" % arg testid = int(arg.strip()) #print testid elif opt in ["--testerid"]: testerid = int(arg.strip()) #print testerid elif opt in ["--starttime"]: starttime = float(arg.strip()) #print starttime elif opt in ["--projectid"]: projectid = str(arg.strip()) elif opt in ["-d","--dedicated"]: dedicatedMode = True else: print "Unknown parameter", opt #sys.exit(0) if len(argv) < 1: print "Error: No workload file given.\n\n" usage(os.path.basename(sys.argv[0])) sys.exit(3) WorkloadFileName = argv[0]; if not os.path.isfile( WorkloadFileName ): print "Error: %s is not a valid file.\n\n" % WorkloadFileName usage(os.path.basename(sys.argv[0])) sys.exit(1) print "%s Parsing workload file %s" % \ ( time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time())), WorkloadFileName ) handlerXML = WLDocHandlers.readWorkloadSubmitFile( WorkloadFileName, dedicatedMode ) print "%s Workload file processed, proceeding to submission" % \ ( time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time())) ) WorkloadDir = os.path.join(os.path.dirname(WorkloadFileName),"run") StartSubmissionTime, EndSubmissionTime, NTotalJobs, NTotalJobsInQueue = \ runWL( WorkloadDir, handlerXML, NThreads, NoSubmit, NoBackground, OneFile, testid, projectid, testerid, starttime - time.time() ) print "%s All done" % time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(time.time())) tstart = time.strftime('%H:%M:%S', time.gmtime(StartSubmissionTime)) tend = time.strftime('%H:%M:%S', time.gmtime(EndSubmissionTime)) print "------ SUBMISSION SUMMARY ---------" print "Started submission at %s" % tstart print "Ended submission at %s" % tend print "Submission total time/Total Jobs/submitted/Jobs per second" # 0123456789012345678901234567890123456789012345678901234567890123456789 # 0 1 2 3 4 5 6 DeltaTime = float(EndSubmissionTime - StartSubmissionTime) if DeltaTime > 0.e-8: Avg = float(NTotalJobs - NTotalJobsInQueue) / DeltaTime else: Avg = 0.0
def main(argv): global DirSSER global NRandomTests try: opts, args = getopt.getopt(argv, "ht:", ["help", "threads=", "version", "nosubmit", "background", "onefile"]) except getopt.GetoptError: print "Error while converting options: unknown option(s) encountered.\n\n" usage(os.path.basename(sys.argv[0])) sys.exit(2) NThreads = 30 NoSubmit = 0 OneFile = 0 Background = 0 for opt, arg in opts: if opt in ["-h", "--help"]: usage(os.path.basename(sys.argv[0])) sys.exit() elif opt in ["--version"]: version() sys.exit() elif opt in ["-t", "--threads"]: try: NThreads = int(arg.strip()) print "[wl-exec-dagman.py] Number of threads set to %s " % NThreads except ValueError: NThreads = 30 elif opt in ["--nosubmit"]: NoSubmit = 1 elif opt in ["--background"]: Background = 1 elif opt in ["--onefile"]: OneFile = 1 else: print "Unknown parameter", opt if len(args) < 1: print "Error: No workload file given.\n\n" usage(os.path.basename(sys.argv[0])) sys.exit(3) WorkloadFileName = args[0] if not os.path.isfile(WorkloadFileName): print "Error: %s is not a valid file.\n\n" % WorkloadFileName usage(os.path.basename(sys.argv[0])) sys.exit(1) print "%s Parsing workload file %s" % ( time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time())), WorkloadFileName, ) handlerXML = WLDocHandlers.readWorkloadSubmitFile(WorkloadFileName) print "%s Workload file processed, proceeding to submission" % ( time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time())) ) WorkloadDir = os.path.join(os.path.dirname(WorkloadFileName), "run") print "Workload directory", WorkloadDir StartSubmissionTime, EndSubmissionTime, NTotalJobs, NTotalJobsInQueue = runWL( WorkloadDir, handlerXML, NThreads, NoSubmit, Background, OneFile ) print "%s All done" % time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(time.time())) tstart = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(StartSubmissionTime)) tend = time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(EndSubmissionTime)) print "------ SUBMISSION SUMMARY ---------" print "Started submission at %s" % tstart print "Ended submission at %s" % tend print "Submission total time/Total Jobs/submitted/Jobs per second" # 0123456789012345678901234567890123456789012345678901234567890123456789 # 0 1 2 3 4 5 6 DeltaTime = float(EndSubmissionTime - StartSubmissionTime) if DeltaTime > 0.0e-8: Avg = float(NTotalJobs - NTotalJobsInQueue) / DeltaTime else: Avg = 0.0 print " %15.3f / %6d / %6d / %.3f" % (DeltaTime, NTotalJobs, NTotalJobs - NTotalJobsInQueue, Avg)
print "Cannot find JDF generator", "'"+JDFGeneratorNames[ii]+"'" raise NameError, CurrentJDFGeneratorName JDFGeneratorFuncs[ii]( SubJobJDFFileName, JobComponentsList ) ii = ii + 1 #-- print job to its JDF file # CurrentJDFGeneratorFunc( SubJobJDFFileName, JobComponentsList ) #--- print composite job - to multiple files if necessary for ii in range (0, NEngines): #Composite_JDFFileName = JobInfo['default_jdf'] if (FileFormatsArray[ii] == 'default'): Composite_JDFFileName = JobInfo['jdf'][ii] print "STATUS! %d Writing default composite file to: %s " % (ii, Composite_JDFFileName) Composite_Name = JobInfo['id'] Composite_Type = 'composite:' + WLUnit['CompositionType'] WLDocHandlers.writeWorkloadSubmitFile( Composite_JDFFileName, Composite_Name, Composite_JobsInfoList, JDFGeneratorNames[ii], Composite_Type, GeneratorName ) else: #-- get list of components for this unitary job JobComponentsList = WLUnit['unit']['jobs'][index] #--- statistics for this unit CurrentNComponents = len(JobComponentsList) NComponents = NComponents + CurrentNComponents if (MinComponents is None) or (MinComponents > CurrentNComponents): MinComponents = CurrentNComponents if (MaxComponents is None) or (MaxComponents < CurrentNComponents): MaxComponents = CurrentNComponents CurrentTotalJobNCPUs = 0 for JobComponent in JobComponentsList: if 'count' in JobComponent: CurrentNCPUs = JobComponent['count'] # per job stats CurrentTotalJobNCPUs = CurrentTotalJobNCPUs + CurrentNCPUs
print "Submit duration set to default value (%d).\n" % SubmitDurationMS elif opt in ["-j", "--jdfgen"]: JDFGenNames = arg.strip() JDFGenNamesList = JDFGenNames.split(',') FileOrExit( os.path.join("jdfprint", "%s.py" % JDFGenName), "JDF generator (%s) " % JDFGenName ) print "JDF Generator set to %s.\n" % JDFGenName elif opt in ["-s", "--sitesfile"]: SitesFileName = arg.strip() FileOrExit( SitesFileName, "site description" ) elif opt in ["-w", "--workload"]: WorkloadDescriptionFileName = arg.strip() FileOrExit( WorkloadDescriptionFileName, "workload description" ) elif opt in ["--arrival"]: arrivalDistr = WLDocHandlers.getArrivalTimeDistribution(arg) else: print "Unknown parameter", opt #--- Load workload generators print "STATUS! Load workload generators." AWLGenLoader = WLLoader.GeneratorsLoader( ModulesPath = "apps", AppGenSig = "WLGenerator", AppGenFuncName = "generateWorkload", ErrPrefix = "Workload Generator:" ) try: AWLGenLoader.loadGenerators() except WLLoader.GeneratorLoaderError, e: print e sys.exit(4)