def main(opts, args): taskDirs = multicrab.getTaskDirectories(opts) multicrab.checkCrabInPath() allJobs = [] for task in taskDirs: if not os.path.exists(task): print "%s: Task directory missing" % task continue jobs = multicrab.crabStatusToJobs(task) if not "Created" in jobs: print "%s: no 'Created' jobs to submit" % task continue allJobs.extend(filter(lambda j: isInRange(opts, j), jobs["Created"])) maxJobs = len(allJobs) if opts.maxJobs >= 0 and int(opts.maxJobs) < int(maxJobs): maxJobs = opts.maxJobs njobsSubmitted = 0 while njobsSubmitted < maxJobs: njobsToSubmit = min(opts.jobs, maxJobs - njobsSubmitted, len(allJobs)) njobsSubmitted += njobsToSubmit jobsToSubmit = {} for n in xrange(0, njobsToSubmit): job = allJobs.pop(0) multicrab._addToDictList(jobsToSubmit, job.task, job.id) for task, jobs in jobsToSubmit.iteritems(): pretty = multicrab.prettyJobnums(jobs) command = ["crab", "-c", task, "-submit", pretty] + args print "Submitting %d jobs from task %s" % (len(jobs), task) print "Command", " ".join(command) if not opts.test: ret = subprocess.call(command) if ret != 0: message = "Command '%s' failed with exit code %d" % ( " ".join(command), ret) if opts.allowFails: print message else: raise Exception() if njobsSubmitted < maxJobs: print "Submitted, sleeping %f seconds" % opts.sleep time.sleep(opts.sleep) else: print "Submitted" return 0
def main(opts, args): taskDirs = multicrab.getTaskDirectories(opts) multicrab.checkCrabInPath() allJobs = [] for task in taskDirs: if not os.path.exists(task): print "%s: Task directory missing" % task continue jobs = multicrab.crabStatusToJobs(task) if not "Created" in jobs: print "%s: no 'Created' jobs to submit" % task continue allJobs.extend(filter(lambda j: isInRange(opts, j), jobs["Created"])) maxJobs = len(allJobs) if opts.maxJobs >= 0 and int(opts.maxJobs) < int(maxJobs): maxJobs = opts.maxJobs njobsSubmitted = 0 while njobsSubmitted < maxJobs: njobsToSubmit = min(opts.jobs, maxJobs-njobsSubmitted, len(allJobs)) njobsSubmitted += njobsToSubmit jobsToSubmit = {} for n in xrange(0, njobsToSubmit): job = allJobs.pop(0) multicrab._addToDictList(jobsToSubmit, job.task, job.id) for task, jobs in jobsToSubmit.iteritems(): pretty = multicrab.prettyJobnums(jobs) command = ["crab", "-c", task, "-submit", pretty] + args print "Submitting %d jobs from task %s" % (len(jobs), task) print "Command", " ".join(command) if not opts.test: ret = subprocess.call(command) if ret != 0: message = "Command '%s' failed with exit code %d" % (" ".join(command), ret) if opts.allowFails: print message else: raise Exception() if njobsSubmitted < maxJobs: print "Submitted, sleeping %f seconds" % opts.sleep time.sleep(opts.sleep) else: print "Submitted" return 0
def main(opts): taskDirs = multicrab.getTaskDirectories(opts) multicrab.checkCrabInPath() for task in taskDirs: if not os.path.exists(task): print "%s: Task directory missing" % task continue jobsList = [] if opts.getoutput is None: jobs = multicrab.crabStatusToJobs(task, printCrab=False) for key in jobs.keys(): if "Done" in key: jobsList.extend([j.id for j in jobs[key]]) else: jobsList.extend(multicrab.prettyToJobList(opts.getoutput)) if len(jobsList) == 0: print "%s: no jobs to retrieve" % task continue # Getoutput loop maxJobs = len(jobsList) if opts.jobs > 0: maxJobs = opts.jobs for i in xrange(0, int(math.ceil(float(len(jobsList))/maxJobs))): jobsToGet = jobsList[i*maxJobs:(i+1)*maxJobs] jobsStr = ",".join([str(j) for j in jobsToGet]) command = ["crab", "-c", task, "-getoutput", jobsStr] print "Getting %d jobs from task %s" % (len(jobsToGet), task) print "Command", " ".join(command) ret = subprocess.call(command) if ret != 0: print "Command '%s' failed with exit code %s" % (" ".join(command), ret) if not opts.allowFails: return 1 return 0
def main(opts, args): if opts.report: multicrab.checkCrabInPath() cell = "\|\s+(?P<%s>\S+)\s+" lumi_re = re.compile((cell % "deliveredls")+ (cell % "delivered")+ (cell % "selectedls")+ (cell % "recorded")+"\|") #lumi_re = re.compile("\|\s(?P<recorded>\S+)\s") unit_re = re.compile("Recorded\(/(?P<unit>.*)\)") if not opts.truncate and os.path.exists(opts.output): f = open(opts.output, "r") data = json.load(f) f.close() files = [] # only if no explicit files, or some directories explicitly given if len(opts.files) == 0 or len(opts.dirs) > 0: crabdirs = multicrab.getTaskDirectories(opts) for d in crabdirs: if isMCTask(d): print " Ignoring task directory '%s', it looks like MC" % d continue if opts.report: cmd = ["crab", "-report", "-c", d] if opts.verbose: print " ".join(cmd) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = p.communicate()[0] ret = p.returncode if ret != 0: print "Call to 'crab -report -d %s' failed with return value %d" % (d, ret) print output return 1 if opts.verbose: print output files.append((d, os.path.join(d, "res", "lumiSummary.json"))) files.extend([(None, f) for f in opts.files]) data = {} for task, jsonfile in files: #print #print "================================================================================" #print "Dataset %s:" % d cmd = ["lumiCalc2.py", "-i", jsonfile, "--nowarning", "overview", "-b", "stable"] if opts.lumicalc1: cmd = ["lumiCalc.py", "-i", jsonfile, "--with-correction", "--nowarning", "overview", "-b", "stable"] #cmd = ["lumiCalc.py", "-c", "frontier://LumiCalc/CMS_LUMI_PROD", "-r", "132440", "--nowarning", "overview"] #ret = subprocess.call(cmd) if opts.verbose: print " ".join(cmd) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = p.communicate()[0] ret = p.returncode if ret != 0: print "Call to lumiCalc.py failed with return value %d with command" % ret print " ".join(cmd) print output return 1 if opts.verbose: print output lines = output.split("\n") lines.reverse() lumi = -1.0 unit = None for line in lines: m = unit_re.search(line) if m: unit = m.group("unit") break m = lumi_re.search(line) if m: lumi = float(m.group("recorded")) # lumiCalc2.py returns pb^-1 # if opts.lumicalc1: # lumi = lumi/1e6 # ub^-1 -> pb^-1, lumiCalc.py returns ub^-1 continue if unit == None: raise Exception("Didn't find unit information from lumiCalc output, command was %s" % " ".join(cmd)) lumi = convertLumi(lumi, unit) if task == None: print "File %s recorded luminosity %f pb^-1" % (jsonfile, lumi) else: print "Task %s recorded luminosity %f pb^-1" % (task, lumi) data[task] = lumi # Save the json file after each data task in case of future errors if len(data) > 0: f = open(opts.output, "wb") json.dump(data, f, sort_keys=True, indent=2) f.close() if len(data) > 0: f = open(opts.output, "wb") json.dump(data, f, sort_keys=True, indent=2) f.close() return 0
def main(opts): mc_conf_file = opts.cfgfile crab_conf_file = None py_conf_file = None json_files = [] if not os.path.exists(mc_conf_file): print "Multicrab configuration file %s doesn't exist!" % mc_conf_file return 1 mc_parser = ConfigParser.ConfigParser() mc_parser.read(mc_conf_file) crab_conf_file = mc_parser.get("MULTICRAB", "cfg") try: py_conf_file = mc_parser.get("COMMON", "CMSSW.pset") except ConfigParser.NoOptionError: pass for s in mc_parser.sections(): try: json_files.append(mc_parser.get(s, "CMSSW.lumi_mask")) except ConfigParser.NoOptionError: pass crab_parser = ConfigParser.ConfigParser() crab_parser.read(crab_conf_file) if py_conf_file == None: py_conf_file = crab_parser.get("CMSSW", "pset") try: json_files.append(crab_parser.get("CMSSW", "lumi_mask")) except ConfigParser.NoOptionError: pass # Unique list of json files keys = {} for f in json_files: keys[f] = 1 json_files = keys.keys() if crab_conf_file == None: print "Did not find crab configuration file" return 1 if py_conf_file == None: print "Did not find CMSSW python configuration file" return 1 # Check crab environment multicrab.checkCrabInPath() dirname = multicrab.createTaskDir(prefix=opts.prefix) shutil.copy(mc_conf_file, os.path.join(dirname, "multicrab.cfg")) flist = [crab_conf_file, py_conf_file] if len(json_files) > 0: flist.extend(json_files) for f in flist: shutil.copy(f, dirname) print "Copied %s to %s" % (", ".join(flist), dirname) print "Creating multicrab task" print print "############################################################" print os.chdir(dirname) subprocess.call(["multicrab", "-create"]) print print "############################################################" print print "Created multicrab task to subdirectory "+dirname print print "Jobs can be submitted by e.g. 'cd %s; multicrab -submit" % dirname print return 0
def main(opts): taskDirs = multicrab.getTaskDirectories(opts) multicrab.checkCrabInPath() resubmitMode = (len(opts.resubmit) > 0) resubmitIdListMode = (opts.resubmit not in ["failed", "aborted"]) if resubmitMode and resubmitIdListMode and len(taskDirs) != 1: print "Option '--resubmit job_id_list' can be used with only one task, trying to use with %d tasks" % len(taskDirs) return 1 if resubmitMode and resubmitIdListMode: resubmitJobList = multicrab.prettyToJobList(opts.resubmit) # Obtain all jobs to be (re)submitted allJobs = [] seBlackLists = {} for task in taskDirs: if not os.path.exists(task): print "%s: Task directory missing" % task continue cfgparser = ConfigParser.ConfigParser() cfgparser.read(os.path.join(task, "share", "crab.cfg")) if cfgparser.has_section("GRID"): availableOptions = cfgparser.options("GRID") blacklist = None for ao in availableOptions: if ao.lower() == "se_black_list": blacklist = cfgparser.get("GRID", ao) break seBlackLists[task] = blacklist jobs = multicrab.crabStatusToJobs(task, printCrab=False) if not resubmitMode: # normal submission if not "Created" in jobs: print "%s: no 'Created' jobs to submit" % task continue allJobs.extend(filter(lambda j: isInRange(opts, j), jobs["Created"])) elif not resubmitIdListMode: # resubmit all failed jobs status = "all" if opts.resubmit == "aborted": status = "aborted" for joblist in jobs.itervalues(): for job in joblist: if job.failed(status): allJobs.append(job) else: # resubmit explicit list of jobs for joblist in jobs.itervalues(): for job in joblist: if job.id in resubmitJobList: allJobs.append(job) resubmitJobList.remove(job.id) # Set the number of maximum jobs to submit maxJobs = len(allJobs) if opts.maxJobs >= 0 and int(opts.maxJobs) < int(maxJobs): maxJobs = opts.maxJobs submitCommand = "-submit" if len(opts.resubmit) > 0: submitCommand = "-resubmit" sites = [] siteSubmitIndex = 0 if len(opts.toSites) > 0: sites = opts.toSites.split(",") # Submission loop njobsSubmitted = 0 while njobsSubmitted < maxJobs: # Construct list of jobs per task to submit njobsToSubmit = min(opts.jobs, maxJobs-njobsSubmitted, len(allJobs)) njobsSubmitted += njobsToSubmit jobsToSubmit = {} for n in xrange(0, njobsToSubmit): job = allJobs.pop(0) aux.addToDictList(jobsToSubmit, job.task, job.id) # If explicit list of sites to submit was given, get the site to submit this time crabOptions = [] if len(sites) > 0: site = sites[siteSubmitIndex] siteSubmitIndex = (siteSubmitIndex+1) % len(sites) crabOptions.append("-GRID.se_black_list= -GRID.se_white_list="+site) # Actual submission for task, jobs in jobsToSubmit.iteritems(): pretty = multicrab.prettyJobnums(jobs) if len(jobs) == 1: pretty += "," # CRAB thinks one number is number of jobs, the comma translates it to job ID command = ["crab", "-c", task, submitCommand, pretty] if opts.crabArgs != "": command.extend(opts.crabArgs.split(" ")) if len(crabOptions) > 0: command.extend(crabOptions) if opts.addSeBlackList != "": lst = seBlackLists[task] if lst is None: lst = opts.addSeBlackList else: lst += ","+opts.addSeBlackList command.extend(["-GRID.se_black_list="+lst]) print "Submitting %d jobs from task %s" % (len(jobs), task) print "Command", " ".join(command) if not opts.test: timesLeft = 1 if opts.tryAgainTimes > 0: timesLeft = opts.tryAgainTimes while timesLeft > 0: ret = subprocess.call(command) if ret == 0: break else: timesLeft -= 1 message = "Command '%s' failed with exit code %d" % (" ".join(command), ret) if opts.allowFails: print message if opts.tryAgainTimes > 0: print message if timesLeft > 0: print "Trying again after %d seconds (%d trials left)" % (opts.tryAgainSeconds, timesLeft) time.sleep(opts.tryAgainSeconds) else: print "No trials left, continuing with next job block" else: raise Exception() # Sleep between submissions if njobsSubmitted < maxJobs: print "Submitted, sleeping %f seconds" % opts.sleep time.sleep(opts.sleep) else: print "Submitted" return 0
def main(opts): taskDirs = multicrab.getTaskDirectories(opts) multicrab.checkCrabInPath() resubmitJobs = {} failedJobs = {} stats = {} allJobs = 0 for task in taskDirs: if not os.path.exists(task): if opts.showMissing: print "%s: Task directory missing" % task continue jobs = multicrab.crabStatusToJobs(task) jobSummaries = {} njobs = 0 for key, item in jobs.iteritems(): hosts = {} for job in item: if job.host != None: hosts[job.host] = 1 l = len(item) jobSummaries[key] = JobSummary(item, hosts) njobs += l allJobs += l if key in stats: stats[key] += l else: stats[key] = l # First the succesfully done line = "%s (%d jobs):" % (task, njobs) for s in order_done: if s in jobSummaries: line = formatSummaries(opts, line, s, jobSummaries[s]) del jobSummaries[s] # Then the aborted-submitted to the end of the line line_end = "" for s in order_run: if s in jobSummaries: line_end = formatSummaries(opts, line_end, s, jobSummaries[s]) del jobSummaries[s] # Then the failed ones to the middle keys = jobSummaries.keys() keys.sort() for key in keys: line = formatSummaries(opts, line, key, jobSummaries[key]) line += line_end if line[-1] == ",": line = line[0:-1] print line # Infer the jobs to be resubmitted failed = [] for key, joblist in jobs.iteritems(): for job in joblist: if job.failed(opts.resubmit): failed.append( (job.id, job.jobExitCode) ) if len(failed) > 0: failed.sort() pretty = multicrab.prettyJobnums([x[0] for x in failed]) resubmitJobs[task] = pretty for jobId, jobCode in failed: multicrab._addToDictList(failedJobs, jobCode, "%s/res/CMSSW_%d.stdout" % (task, jobId)) print "----------------------------------------" print "Summary for %d task(s), total %d job(s):" % (len(taskDirs), allJobs) for s in order_done: if s in stats: print status_format % (s+":", stats[s]) del stats[s] b = [] for s in order_run: if s in stats: b.append(status_format % (s+":", stats[s])) del stats[s] keys = stats.keys() keys.sort() for key in keys: print status_format % (key+":", stats[key]) for line in b: print line print "----------------------------------------" if len(resubmitJobs) == 0: print "No failed/aborted jobs to resubmit" else: print "Following jobs failed/aborted, and can be resubmitted" print for task in taskDirs: if task in resubmitJobs: print "crab -c %s -resubmit %s" % (task, resubmitJobs[task]) print if opts.failedLogs: print "----------------------------------------" print "Log files of failed jobs" keys = failedJobs.keys() keys.sort() for code in keys: print print "Job exit code %d:" % code print "\n".join(failedJobs[code]) return 0
def main(opts): taskDirs = multicrab.getTaskDirectories(opts) multicrab.checkCrabInPath() resubmitMode = (len(opts.resubmit) > 0) resubmitIdListMode = (opts.resubmit not in ["failed", "aborted"]) if resubmitMode and resubmitIdListMode and len(taskDirs) != 1: print "Option '--resubmit job_id_list' can be used with only one task, trying to use with %d tasks" % len( taskDirs) return 1 if resubmitMode and resubmitIdListMode: resubmitJobList = multicrab.prettyToJobList(opts.resubmit) # Obtain all jobs to be (re)submitted allJobs = [] seBlackLists = {} for task in taskDirs: if not os.path.exists(task): print "%s: Task directory missing" % task continue cfgparser = ConfigParser.ConfigParser() cfgparser.read(os.path.join(task, "share", "crab.cfg")) if cfgparser.has_section("GRID"): availableOptions = cfgparser.options("GRID") blacklist = None for ao in availableOptions: if ao.lower() == "se_black_list": blacklist = cfgparser.get("GRID", ao) break seBlackLists[task] = blacklist jobs = multicrab.crabStatusToJobs(task, printCrab=False) if not resubmitMode: # normal submission if not "Created" in jobs: print "%s: no 'Created' jobs to submit" % task continue allJobs.extend( filter(lambda j: isInRange(opts, j), jobs["Created"])) elif not resubmitIdListMode: # resubmit all failed jobs status = "all" if opts.resubmit == "aborted": status = "aborted" for joblist in jobs.itervalues(): for job in joblist: if job.failed(status): allJobs.append(job) else: # resubmit explicit list of jobs for joblist in jobs.itervalues(): for job in joblist: if job.id in resubmitJobList: allJobs.append(job) resubmitJobList.remove(job.id) # Set the number of maximum jobs to submit maxJobs = len(allJobs) if opts.maxJobs >= 0 and int(opts.maxJobs) < int(maxJobs): maxJobs = opts.maxJobs submitCommand = "-submit" if len(opts.resubmit) > 0: submitCommand = "-resubmit" sites = [] siteSubmitIndex = 0 if len(opts.toSites) > 0: sites = opts.toSites.split(",") # Submission loop njobsSubmitted = 0 while njobsSubmitted < maxJobs: # Construct list of jobs per task to submit njobsToSubmit = min(opts.jobs, maxJobs - njobsSubmitted, len(allJobs)) njobsSubmitted += njobsToSubmit jobsToSubmit = {} for n in xrange(0, njobsToSubmit): job = allJobs.pop(0) aux.addToDictList(jobsToSubmit, job.task, job.id) # If explicit list of sites to submit was given, get the site to submit this time crabOptions = [] if len(sites) > 0: site = sites[siteSubmitIndex] siteSubmitIndex = (siteSubmitIndex + 1) % len(sites) crabOptions.append("-GRID.se_black_list= -GRID.se_white_list=" + site) # Actual submission for task, jobs in jobsToSubmit.iteritems(): pretty = multicrab.prettyJobnums(jobs) if len(jobs) == 1: pretty += "," # CRAB thinks one number is number of jobs, the comma translates it to job ID command = ["crab", "-c", task, submitCommand, pretty] if opts.crabArgs != "": command.extend(opts.crabArgs.split(" ")) if len(crabOptions) > 0: command.extend(crabOptions) if opts.addSeBlackList != "": lst = seBlackLists[task] if lst is None: lst = opts.addSeBlackList else: lst += "," + opts.addSeBlackList command.extend(["-GRID.se_black_list=" + lst]) print "Submitting %d jobs from task %s" % (len(jobs), task) print "Command", " ".join(command) if not opts.test: timesLeft = 1 if opts.tryAgainTimes > 0: timesLeft = opts.tryAgainTimes while timesLeft > 0: ret = subprocess.call(command) if ret == 0: break else: timesLeft -= 1 message = "Command '%s' failed with exit code %d" % ( " ".join(command), ret) if opts.allowFails: print message if opts.tryAgainTimes > 0: print message if timesLeft > 0: print "Trying again after %d seconds (%d trials left)" % ( opts.tryAgainSeconds, timesLeft) time.sleep(opts.tryAgainSeconds) else: print "No trials left, continuing with next job block" else: raise Exception() # Sleep between submissions if njobsSubmitted < maxJobs: print "Submitted, sleeping %f seconds" % opts.sleep time.sleep(opts.sleep) else: print "Submitted" return 0
def main(opts, args): workdir = os.getcwd() crabdirs = multicrab.getTaskDirectories(opts) for d in crabdirs: # Run crab -report if opts.report: multicrab.checkCrabInPath() cmd = ["crab", "-report", "-c", d] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = p.communicate()[0] ret = p.returncode if ret != 0: print "Call to 'crab -report -d %s' failed with return value %d" % ( d, ret) print output return 1 if os.path.exists(os.path.join(d, "task.tar.gz")): print "Skipping %s, task.tar.gz already exists" % d continue # Go to task directory (in order to get the paths in the archive correctly easily) os.chdir(d) # Remove default.tgz tmp = os.path.join("share", "default.tgz") if os.path.exists(tmp): os.remove(tmp) # Create the tar archive tar = tarfile.open("task.tar.gz", mode="w:gz") files = [] for f in [ "CMSSW_*.std*", "crab_fjr_*.xml", "Submission_*", "Watchdog_*.log*" ]: files.extend(glob.glob(os.path.join("res", f))) files.extend(["job", "log", "share"]) #print "\n".join(files) for f in files: if os.path.exists(f): tar.add(f) tar.close() # Keep share/crab.cfg files.remove("share") sharefiles = glob.glob(os.path.join("share", "*")) sharefiles = filter(lambda x: not "crab.cfg" in x, sharefiles) files.extend(sharefiles) # Delete the files just added to the archive for f in files: if os.path.isfile(f): #print "rm "+f try: os.remove(f) except OSError, e: print "Warning: failed to remove %s: %s" % (f, str(e)) elif os.path.isdir(f): #print "rm -fR "+f try: shutil.rmtree(f) except OSError, e: print "Warning: failed to remove %s: %s" % (f, str(e))
def main(opts, args): if opts.report: multicrab.checkCrabInPath() cell = "\|\s+(?P<%s>\S+)\s+" lumi_re = re.compile((cell % "deliveredls") + (cell % "delivered") + (cell % "selectedls") + (cell % "recorded") + "\|") #lumi_re = re.compile("\|\s(?P<recorded>\S+)\s") unit_re = re.compile("Recorded\(/(?P<unit>.*)\)") if not opts.truncate and os.path.exists(opts.output): f = open(opts.output, "r") data = json.load(f) f.close() files = [] # only if no explicit files, or some directories explicitly given if len(opts.files) == 0 or len(opts.dirs) > 0: crabdirs = multicrab.getTaskDirectories(opts) for d in crabdirs: if isMCTask(d): print " Ignoring task directory '%s', it looks like MC" % d continue if opts.report: cmd = ["crab", "-report", "-c", d] if opts.verbose: print " ".join(cmd) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = p.communicate()[0] ret = p.returncode if ret != 0: print "Call to 'crab -report -d %s' failed with return value %d" % ( d, ret) print output return 1 if opts.verbose: print output files.append((d, os.path.join(d, "res", "lumiSummary.json"))) files.extend([(None, f) for f in opts.files]) data = {} for task, jsonfile in files: #print #print "================================================================================" #print "Dataset %s:" % d cmd = [ "lumiCalc2.py", "-i", jsonfile, "--nowarning", "overview", "-b", "stable" ] if opts.lumicalc1: cmd = [ "lumiCalc.py", "-i", jsonfile, "--with-correction", "--nowarning", "overview", "-b", "stable" ] #cmd = ["lumiCalc.py", "-c", "frontier://LumiCalc/CMS_LUMI_PROD", "-r", "132440", "--nowarning", "overview"] #ret = subprocess.call(cmd) if opts.verbose: print " ".join(cmd) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = p.communicate()[0] ret = p.returncode if ret != 0: print "Call to lumiCalc.py failed with return value %d with command" % ret print " ".join(cmd) print output return 1 if opts.verbose: print output lines = output.split("\n") lines.reverse() lumi = -1.0 unit = None for line in lines: m = unit_re.search(line) if m: unit = m.group("unit") break m = lumi_re.search(line) if m: lumi = float(m.group("recorded")) # lumiCalc2.py returns pb^-1 # if opts.lumicalc1: # lumi = lumi/1e6 # ub^-1 -> pb^-1, lumiCalc.py returns ub^-1 continue if unit == None: raise Exception( "Didn't find unit information from lumiCalc output, command was %s" % " ".join(cmd)) lumi = convertLumi(lumi, unit) if task == None: print "File %s recorded luminosity %f pb^-1" % (jsonfile, lumi) else: print "Task %s recorded luminosity %f pb^-1" % (task, lumi) data[task] = lumi # Save the json file after each data task in case of future errors if len(data) > 0: f = open(opts.output, "wb") json.dump(data, f, sort_keys=True, indent=2) f.close() if len(data) > 0: f = open(opts.output, "wb") json.dump(data, f, sort_keys=True, indent=2) f.close() return 0
def main(opts): taskDirs = multicrab.getTaskDirectories(opts) multicrab.checkCrabInPath() resubmitJobs = {} failedJobs = {} stats = {} allJobs = 0 for task in taskDirs: if not os.path.exists(task): if opts.showMissing: print "%s: Task directory missing" % task continue jobs = multicrab.crabStatusToJobs(task) jobSummaries = {} njobs = 0 for key, item in jobs.iteritems(): hosts = {} for job in item: if job.host != None: hosts[job.host] = 1 l = len(item) jobSummaries[key] = JobSummary(item, hosts) njobs += l allJobs += l if key in stats: stats[key] += l else: stats[key] = l # First the succesfully done line = "%s (%d jobs):" % (task, njobs) for s in order_done: if s in jobSummaries: line = formatSummaries(opts, line, s, jobSummaries[s]) del jobSummaries[s] # Then the aborted-submitted to the end of the line line_end = "" for s in order_run: if s in jobSummaries: line_end = formatSummaries(opts, line_end, s, jobSummaries[s]) del jobSummaries[s] # Then the failed ones to the middle keys = jobSummaries.keys() keys.sort() for key in keys: line = formatSummaries(opts, line, key, jobSummaries[key]) line += line_end if line[-1] == ",": line = line[0:-1] print line # Infer the jobs to be resubmitted failed = [] for key, joblist in jobs.iteritems(): for job in joblist: if job.failed(opts.resubmit): failed.append((job.id, job.jobExitCode)) if len(failed) > 0: failed.sort() pretty = multicrab.prettyJobnums([x[0] for x in failed]) resubmitJobs[task] = pretty for jobId, jobCode in failed: multicrab._addToDictList( failedJobs, jobCode, "%s/res/CMSSW_%d.stdout" % (task, jobId)) print "----------------------------------------" print "Summary for %d task(s), total %d job(s):" % (len(taskDirs), allJobs) for s in order_done: if s in stats: print status_format % (s + ":", stats[s]) del stats[s] b = [] for s in order_run: if s in stats: b.append(status_format % (s + ":", stats[s])) del stats[s] keys = stats.keys() keys.sort() for key in keys: print status_format % (key + ":", stats[key]) for line in b: print line print "----------------------------------------" if len(resubmitJobs) == 0: print "No failed/aborted jobs to resubmit" else: print "Following jobs failed/aborted, and can be resubmitted" print for task in taskDirs: if task in resubmitJobs: print "crab -c %s -resubmit %s" % (task, resubmitJobs[task]) print if opts.failedLogs: print "----------------------------------------" print "Log files of failed jobs" keys = failedJobs.keys() keys.sort() for code in keys: print print "Job exit code %d:" % code print "\n".join(failedJobs[code]) return 0
def main(opts, args): workdir = os.getcwd() crabdirs = multicrab.getTaskDirectories(opts) for d in crabdirs: # Run crab -report if opts.report: multicrab.checkCrabInPath() cmd = ["crab", "-report", "-c", d] p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) output = p.communicate()[0] ret = p.returncode if ret != 0: print "Call to 'crab -report -d %s' failed with return value %d" % (d, ret) print output return 1 if os.path.exists(os.path.join(d, "task.tar.gz")): print "Skipping %s, task.tar.gz already exists" % d continue # Go to task directory (in order to get the paths in the archive correctly easily) os.chdir(d) # Remove default.tgz tmp = os.path.join("share", "default.tgz") if os.path.exists(tmp): os.remove(tmp) # Create the tar archive tar = tarfile.open("task.tar.gz", mode="w:gz") files = [] for f in ["CMSSW_*.std*", "crab_fjr_*.xml", "Submission_*", "Watchdog_*.log*"]: files.extend(glob.glob(os.path.join("res", f))) files.extend(["job", "log", "share"]) #print "\n".join(files) for f in files: if os.path.exists(f): tar.add(f) tar.close() # Keep share/crab.cfg files.remove("share") sharefiles = glob.glob(os.path.join("share", "*")) sharefiles = filter(lambda x: not "crab.cfg" in x, sharefiles) files.extend(sharefiles) # Delete the files just added to the archive for f in files: if os.path.isfile(f): #print "rm "+f try: os.remove(f) except OSError, e: print "Warning: failed to remove %s: %s" % (f, str(e)) elif os.path.isdir(f): #print "rm -fR "+f try: shutil.rmtree(f) except OSError, e: print "Warning: failed to remove %s: %s" % (f, str(e))
def main(opts): mc_conf_file = opts.cfgfile crab_conf_file = None py_conf_file = None json_files = [] if not os.path.exists(mc_conf_file): print "Multicrab configuration file %s doesn't exist!" % mc_conf_file return 1 mc_parser = ConfigParser.ConfigParser() mc_parser.read(mc_conf_file) crab_conf_file = mc_parser.get("MULTICRAB", "cfg") try: py_conf_file = mc_parser.get("COMMON", "CMSSW.pset") except ConfigParser.NoOptionError: pass for s in mc_parser.sections(): try: json_files.append(mc_parser.get(s, "CMSSW.lumi_mask")) except ConfigParser.NoOptionError: pass crab_parser = ConfigParser.ConfigParser() crab_parser.read(crab_conf_file) if py_conf_file == None: py_conf_file = crab_parser.get("CMSSW", "pset") try: json_files.append(crab_parser.get("CMSSW", "lumi_mask")) except ConfigParser.NoOptionError: pass # Unique list of json files keys = {} for f in json_files: keys[f] = 1 json_files = keys.keys() if crab_conf_file == None: print "Did not find crab configuration file" return 1 if py_conf_file == None: print "Did not find CMSSW python configuration file" return 1 dirname = multicrab.createTaskDir(prefix=opts.prefix) shutil.copy(mc_conf_file, os.path.join(dirname, "multicrab.cfg")) flist = [crab_conf_file, py_conf_file] if len(json_files) > 0: flist.extend(json_files) for f in flist: shutil.copy(f, dirname) if opts.codeVersion: multicrab.writeGitVersion(dirname) print "Copied %s to %s" % (", ".join(flist), dirname) if not opts.create: print print "Now (since you explicitly asked with --noCreate), you have to create the crab tasks yourself, e.g." print "cd %s; multicrab -create" % dirname print "Then you can submit them e.g. by multicrab -submit" return 0 # Check crab environment multicrab.checkCrabInPath() print "Creating multicrab task" print print "############################################################" print os.chdir(dirname) subprocess.call(["multicrab", "-create"]) print print "############################################################" print print "Created multicrab task to subdirectory "+dirname print print "Jobs can be submitted by e.g. 'cd %s; multicrab -submit" % dirname print return 0
def main(opts): taskDirs = multicrab.getTaskDirectories(opts) multicrab.checkCrabInPath() resubmitJobs = {} failedJobs = {} stats = {} allJobs = 0 if opts.byHost: global status_format status_format = status_format.replace("18s", "40s") if opts.save: out = open(opts.saveFile, "w") for task in taskDirs: if not os.path.exists(task): if opts.showMissing: print >> sys.stderr, "%s: Task directory missing" % task continue try: jobs = multicrab.crabStatusToJobs(task, opts.printCrab) except Exception: if not opts.allowFails: raise print "%s: crab -status failed" % task continue jobSummaries = {} njobs = 0 for key, item in jobs.iteritems(): hosts = {} for job in item: if job.host != None: aux.addToDictList(hosts, job.host, job) if opts.byHost: for host, joblist in hosts.iteritems(): jobSummaries[key + " " + host] = JobSummary( joblist, [host]) else: jobSummaries[key] = JobSummary(item, hosts) l = len(item) njobs += l allJobs += l if key in stats: stats[key] += l else: stats[key] = l # First the succesfully done line = "%s (%d jobs):" % (task, njobs) for s in order_done: if s in jobSummaries: line = formatSummaries(opts, line, s, jobSummaries[s]) del jobSummaries[s] # Then the aborted-submitted to the end of the line line_end = "" for s in order_run: if s in jobSummaries: line_end = formatSummaries(opts, line_end, s, jobSummaries[s]) del jobSummaries[s] # Then the failed ones to the middle keys = jobSummaries.keys() keys.sort() for key in keys: line = formatSummaries(opts, line, key, jobSummaries[key]) line += line_end if line[-1] == ",": line = line[0:-1] if opts.save: out.write(line) out.write("\n") print line # Infer the jobs to be resubmitted failed = [] for key, joblist in jobs.iteritems(): for job in joblist: if job.failed(opts.resubmit): failed.append((job.id, job.jobExitCode)) if len(failed) > 0: failed.sort() pretty = multicrab.prettyJobnums([x[0] for x in failed]) resubmitJobs[task] = pretty for jobId, jobCode in failed: aux.addToDictList(failedJobs, jobCode, "%s/res/CMSSW_%d.stdout" % (task, jobId)) summary = StringIO.StringIO() summary.write("----------------------------------------\n") print "Summary for %d task(s), total %d job(s):" % (len(taskDirs), allJobs) for s in order_done: if s in stats: summary.write(status_format % (s + ":", stats[s])) summary.write("\n") del stats[s] b = [] for s in order_run: if s in stats: b.append(status_format % (s + ":", stats[s])) del stats[s] keys = stats.keys() keys.sort() for key in keys: summary.write(status_format % (key + ":", stats[key])) summary.write("\n") for line in b: summary.write(line) summary.write("\n") summary.write("----------------------------------------\n") if len(resubmitJobs) == 0: summary.write("No failed/aborted jobs to resubmit\n") else: summary.write( "Following jobs failed/aborted, and can be resubmitted\n\n") for task in taskDirs: if task in resubmitJobs: summary.write("crab -c %s -resubmit %s\n" % (task, resubmitJobs[task])) summary.write("\n") if opts.failedLogs: summary.write("----------------------------------------\n") summary.write("Log files of failed jobs\n") keys = failedJobs.keys() keys.sort() for code in keys: summary.write("\nJob exit code %d:\n" % code) summary.write("\n".join(failedJobs[code])) summary.write("\n") if opts.save: out.write(summary.getvalue()) out.close() print summary.getvalue() return 0