def main() : parser = optparse.OptionParser() parser.add_option('--email', default='*****@*****.**') parser.add_option('--message', default='all jobs done, bam!') parser.add_option('--polleverysec', default=60*5, type='int') parser.add_option('--subject', default='All jobs done') parser.add_option('--user', default=os.environ['USER']) (opts, args) = parser.parse_args() receiver = opts.email username = opts.user message = opts.message subject = opts.subject everyNsec = opts.polleverysec nChecksDone = 0 countJobsCmd="qstat -u %(user)s | grep %(user)s | wc -l"% {'user':username} nJobsRunning = int(getCommandOutput(countJobsCmd)['stdout']) while nJobsRunning > 0 : sleep(everyNsec) nJobsRunning = int(getCommandOutput(countJobsCmd)['stdout']) nChecksDone += 1 message = message+("\n(after %(nchk)d checks every %(nsec)d sec)" %{'nchk' : nChecksDone, 'nsec' : everyNsec}) mailCmd = ("echo \"%(msg)s\" | mail -s \"%(sbj)s\" %(dest)s" % {'dest' : receiver, 'msg' : message, 'sbj':subject}) getCommandOutput(mailCmd)
def main(): parser = optparse.OptionParser() parser.add_option('--email', default='*****@*****.**') parser.add_option('--message', default='all jobs done, bam!') parser.add_option('--polleverysec', default=60 * 5, type='int') parser.add_option('--subject', default='All jobs done') parser.add_option('--user', default=os.environ['USER']) (opts, args) = parser.parse_args() receiver = opts.email username = opts.user message = opts.message subject = opts.subject everyNsec = opts.polleverysec nChecksDone = 0 countJobsCmd = "qstat -u %(user)s | grep %(user)s | wc -l" % { 'user': username } nJobsRunning = int(getCommandOutput(countJobsCmd)['stdout']) while nJobsRunning > 0: sleep(everyNsec) nJobsRunning = int(getCommandOutput(countJobsCmd)['stdout']) nChecksDone += 1 message = message + ("\n(after %(nchk)d checks every %(nsec)d sec)" % { 'nchk': nChecksDone, 'nsec': everyNsec }) mailCmd = ("echo \"%(msg)s\" | mail -s \"%(sbj)s\" %(dest)s" % { 'dest': receiver, 'msg': message, 'sbj': subject }) getCommandOutput(mailCmd)
def git_info(path): cmd = 'git describe --tags --dirty' out = getCommandOutput(cmd, cwd=path) tag_info = out['stdout'].strip() cmd = 'git status' modified_files = getCommandOutput(cmd, cwd=path)['stdout'].splitlines() modified_files = [l for l in modified_files if 'modified:' in l] return "{0} {1}".format(tag_info, ('\n'.join(['']+modified_files) if modified_files else ''))
def git_info(path): cmd = 'git describe --tags --dirty' out = getCommandOutput(cmd, cwd=path) tag_info = out['stdout'].strip() cmd = 'git status' modified_files = getCommandOutput(cmd, cwd=path)['stdout'].splitlines() modified_files = [l for l in modified_files if 'modified:' in l] return "{0} {1}".format( tag_info, ('\n'.join([''] + modified_files) if modified_files else ''))
def svn_info(path): # get tag or last rev cmd = 'svn info '+path out = getCommandOutput(cmd) url_origin = first([l for l in out['stdout'].splitlines() if 'URL: ' in l]) url_origin = url_origin.replace('/', ' ').split() if url_origin else [] tag = url_origin[url_origin.index('tags')+1] if 'tags' in url_origin else None last_rev = first(first([l for l in out['stdout'].splitlines() if 'Last Changed Rev:' in l]).split()[::-1]) # get list of modified files cmd = 'svn status '+path out = getCommandOutput(cmd) modified_files = [l for l in out['stdout'].splitlines() if l.startswith('M ')] return "{0} {1}".format(tag if tag else last_rev, ('\n'.join(['']+modified_files) if modified_files else ''))
def main(): options = parse_options() exe = options.executable inputdf = options.input include = options.include_regexp exclude = options.exclude_regexp tag = options.tag verbose = options.verbose submit = options.submit datasets = dataset.build_all_datasets_from_dir_or_file(inputdf) datasets = utils.filterWithRegexp (datasets, include, lambda _: _.name) if include else datasets datasets = utils.excludeWithRegexp(datasets, exclude, lambda _: _.name) if exclude else datasets for dset in datasets : if not get_filelist(dset.name): print "# skipping (missing filelist) {0}".format(dset.name) continue script = get_batch_script(dset, options) if not script: if verbose : print "skipping (do-not-overwrite) {0}".format(dset.name) continue cmd = "sbatch %s" % script if verbose : print cmd if submit : out = utils.getCommandOutput(cmd) if verbose : print out['stdout'] if not submit : print "This was a dry run; use '--submit' to actually submit the jobs"
def main(): options = parse_options() exe = options.executable inputdf = options.input include = options.include_regexp exclude = options.exclude_regexp tag = options.tag verbose = options.verbose submit = options.submit datasets = dataset.build_all_datasets_from_dir_or_file(inputdf) datasets = utils.filterWithRegexp( datasets, include, lambda _: _.name) if include else datasets datasets = utils.excludeWithRegexp( datasets, exclude, lambda _: _.name) if exclude else datasets for dset in datasets: if not get_filelist(dset.name): print "# skipping (missing filelist) {0}".format(dset.name) continue script = get_batch_script(dset, options) if not script: if verbose: print "skipping (do-not-overwrite) {0}".format(dset.name) continue cmd = "sbatch %s" % script if verbose: print cmd if submit: out = utils.getCommandOutput(cmd) if verbose: print out['stdout'] if not submit: print "This was a dry run; use '--submit' to actually submit the jobs"
def submit_batch_fill_job_per_group(group, opts): options_dict = vars(opts) group_name = group.name if hasattr(group, 'name') else group systematic = opts.syst if hasattr(opts, 'syst') and opts.syst else None verbose = opts.verbose options_dict['group'] = group_name options_with_value = dict((k,v) for k,v in options_dict.iteritems() if v and v is not True) # note to self: the line below assumes that the argument-less options have a default=False options_with_toggle = dict((k,v) for k,v in options_dict.iteritems() if v and v is True and k!="batch") def escape_regex(v) : return v if v!='.*' else "'.*'" def back_to_dash(v) : return v.replace('_','-') cmd_line_options = ' '.join(["--%s %s"%(back_to_dash(k), escape_regex(str(v))) for k,v in options_with_value.iteritems()] +["--%s"%back_to_dash(k) for k in options_with_toggle.keys()]) template = 'batch/templates/plot_emu.sh' default_log_dir = opts.output_dir.replace('out/', 'log/') if default_log_dir.count('/histos')==1: default_log_dir = default_log_dir.replace('/histos','') log_dir = mkdirIfNeeded(opts.log_dir if opts.log_dir else default_log_dir) script_dir = mkdirIfNeeded('batch/plot_emu') script_name = os.path.join(script_dir, group_name+("_{0}".format(systematic) if systematic else '')+'.sh') log_name = log_dir+'/'+group_name+("_{0}".format(systematic) if systematic else '')+'.log' script_file = open(script_name, 'w') script_file.write(open(template).read() .replace('%(opt)s', cmd_line_options) .replace('%(logfile)s', log_name) .replace('%(jobname)s', group_name) .replace('%(queue)s', opts.queue)) script_file.close() cmd = "sbatch %s"%script_name if verbose : print cmd out = getCommandOutput(cmd) if verbose : print out['stdout'] if out['stderr'] : print out['stderr']
def GenerateVersionFile(): # TODO: fix it. if HOST_OS == 'win': return versionInfo = utils.getCommandOutput( os.path.join('..', '..', 'dartium_tools', 'print_dart_version.sh')) file = open(VERSION_FILE, 'w') file.write(versionInfo) file.close()
def GenerateVersionFile(): # TODO: fix it. if HOST_OS == 'win': return versionInfo = utils.getCommandOutput(os.path.join('..', '..', 'dartium_tools', 'print_dart_version.sh')) file = open(VERSION_FILE, 'w') file.write(versionInfo) file.close()
def main(): parser = optparse.OptionParser() parser.add_option('--from-address', default='*****@*****.**') parser.add_option('--to-address', default='*****@*****.**') parser.add_option('--message', default='all jobs done, bam!') parser.add_option('--polleverysec', default=60 * 5, type='int') parser.add_option('--subject', default='jobs done from `pwd`') parser.add_option('--user', default=os.environ['USER']) parser.add_option('--exec-cmd', help='on completion, execute this command') (opts, args) = parser.parse_args() receiver = opts.to_address username = opts.user message = opts.message subject = opts.subject everyNsec = opts.polleverysec nChecksDone = 0 countJobsCmd = "qstat -u %(user)s | grep %(user)s | egrep '(Q|R)' | wc -l" % { 'user': username } nJobsRunning = int(getCommandOutput(countJobsCmd)['stdout']) while nJobsRunning > 0: sleep(everyNsec) nJobsRunning = int(getCommandOutput(countJobsCmd)['stdout']) nChecksDone += 1 preamble = "From: %s \n" % opts.from_address message = preamble + message message += ("\n(after %(nchk)d checks every %(nsec)d sec)" % { 'nchk': nChecksDone, 'nsec': everyNsec }) subject = opts.subject mailCmd = ("echo -e \"%(msg)s\" | mail -s \"%(sbj)s\" %(dest)s" % { 'dest': receiver, 'msg': message, 'sbj': subject }) print mailCmd getCommandOutput(mailCmd) if opts.exec_cmd: print "now executing {}".format(opts.exec_cmd) out = getCommandOutput(opts.exec_cmd) print out['stdout'] print out['stderr']
def merge(input_files=[], output_filename=''): "hadd a list of input files; return name merged file" input_files = [f for f in input_files if f is not output_filename] cmd = "hadd %s %s" % (output_filename, ' '.join(input_files)) out = utils.getCommandOutput(cmd) log.info(cmd) if out['returncode']!=0: log.info(out['stdout']) log.error(out['stderr']) return output_filename if out['returncode']==0 else None
def merge(input_files=[], output_filename=''): "hadd a list of input files; return name merged file" input_files = [f for f in input_files if f is not output_filename] cmd = "hadd %s %s" % (output_filename, ' '.join(input_files)) out = utils.getCommandOutput(cmd) log.info(cmd) if out['returncode'] != 0: log.info(out['stdout']) log.error(out['stderr']) return output_filename if out['returncode'] == 0 else None
def checkFileExists(file): if isRemote(file): if isSRAID(file): file = utils.translateToSRAURL(settings, file) result = utils.getCommandOutput("curl -L -I %s && echo $?"%(file), False) if result == "": return False return True else: return os.path.exists(file)
def checkFileExists(file): if isRemote(file): if isSRAID(file): file = utils.translateToSRAURL(settings, file) result = utils.getCommandOutput("curl -L -I %s && echo $?" % (file), False) if result == "": return False return True else: return os.path.exists(file)
def svn_info(path): # get tag or last rev cmd = 'svn info ' + path out = getCommandOutput(cmd) url_origin = first([l for l in out['stdout'].splitlines() if 'URL: ' in l]) url_origin = url_origin.replace('/', ' ').split() if url_origin else [] tag = url_origin[url_origin.index('tags') + 1] if 'tags' in url_origin else None last_rev = first( first([ l for l in out['stdout'].splitlines() if 'Last Changed Rev:' in l ]).split()[::-1]) # get list of modified files cmd = 'svn status ' + path out = getCommandOutput(cmd) modified_files = [ l for l in out['stdout'].splitlines() if l.startswith('M ') ] return "{0} {1}".format( tag if tag else last_rev, ('\n'.join([''] + modified_files) if modified_files else ''))
def getTagOrRev(dir) : "tag if available, otherwise rev" cmd = 'svn info '+dir res = getCommandOutput(cmd) lines = res['stdout'].split('\n') urlLine = next((l for l in lines if l.startswith('URL:')), None) revLine = next((l for l in lines if l.startswith('Revision:')), None) if not urlLine or not revLine : return pkg = extractPkg(urlLine) tag = extractTag(urlLine) rev = extractRev(revLine) assert tag or rev, "cannot extract info for %s"%dir return "%s @ %s"%(pkg, tag if tag else rev)
def submit_batch_fill_job_per_group_per_selection(group=None, selection='', opts=None): "if we are processing cached selections, we can submit one job per selection" options_dict = vars(opts) group_name = group.name if hasattr(group, 'name') else group systematic = opts.syst if hasattr(opts, 'syst') and opts.syst else None verbose = opts.verbose options_dict['group'] = group_name options_dict['region'] = selection options_dict['regions'] = None options_with_value = dict( (k, v) for k, v in options_dict.iteritems() if v and v is not True) # note to self: the line below assumes that the argument-less options have a default=False options_with_toggle = dict((k, v) for k, v in options_dict.iteritems() if v and v is True and k != "batch") def escape_regex(v): return v if v != '.*' else "'.*'" def back_to_dash(v): return v.replace('_', '-') cmd_line_options = ' '.join([ "--%s %s" % (back_to_dash(k), escape_regex(str(v))) for k, v in options_with_value.iteritems() ] + ["--%s" % back_to_dash(k) for k in options_with_toggle.keys()]) template = 'batch/templates/plot_emu.sh' default_log_dir = opts.output_dir.replace('out/', 'log/') if default_log_dir.count('/histos') == 1: default_log_dir = default_log_dir.replace('/histos', '') log_dir = mkdirIfNeeded(opts.log_dir if opts.log_dir else default_log_dir) script_dir = mkdirIfNeeded('batch/plot_emu') script_name = os.path.join( script_dir, group_name + '_' + selection + ("_{0}".format(systematic) if systematic else '') + '.sh') log_name = log_dir + '/' + group_name + '_' + selection + ( "_{0}".format(systematic) if systematic else '') + '.log' script_file = open(script_name, 'w') script_file.write( open(template).read().replace('%(opt)s', cmd_line_options).replace( '%(logfile)s', log_name).replace('%(jobname)s', group_name + '_' + selection).replace( '%(queue)s', opts.queue)) script_file.close() cmd = "sbatch %s" % script_name if verbose: print cmd out = getCommandOutput(cmd) if verbose: print out['stdout'] if out['stderr']: print out['stderr']
def main() : parser = optparse.OptionParser() parser.add_option('--from-address', default='*****@*****.**') parser.add_option('--to-address', default='*****@*****.**') parser.add_option('--message', default='all jobs done, bam!') parser.add_option('--polleverysec', default=60*5, type='int') parser.add_option('--subject', default='jobs done from `pwd`') parser.add_option('--user', default=os.environ['USER']) parser.add_option('--exec-cmd', help='on completion, execute this command') (opts, args) = parser.parse_args() receiver = opts.to_address username = opts.user message = opts.message subject = opts.subject everyNsec = opts.polleverysec nChecksDone = 0 countJobsCmd="qstat -u %(user)s | grep %(user)s | egrep '(Q|R)' | wc -l"% {'user':username} nJobsRunning = int(getCommandOutput(countJobsCmd)['stdout']) while nJobsRunning > 0 : sleep(everyNsec) nJobsRunning = int(getCommandOutput(countJobsCmd)['stdout']) nChecksDone += 1 preamble = "From: %s \n"%opts.from_address message = preamble+message message += ("\n(after %(nchk)d checks every %(nsec)d sec)" %{'nchk' : nChecksDone, 'nsec' : everyNsec}) subject = opts.subject mailCmd = ("echo -e \"%(msg)s\" | mail -s \"%(sbj)s\" %(dest)s" % {'dest' : receiver, 'msg' : message, 'sbj':subject}) print mailCmd getCommandOutput(mailCmd) if opts.exec_cmd: print "now executing {}".format(opts.exec_cmd) out = getCommandOutput(opts.exec_cmd) print out['stdout'] print out['stderr']
def runFill(opts) : batchMode = opts.batch inputFakeDir = opts.input_fake inputGenDir = opts.input_gen outputDir = opts.output_dir sysOption = opts.syst excludedSyst = opts.exclude verbose = opts.verbose if verbose : print "filling histos" mkdirIfNeeded(outputDir) systematics = ['NOM'] anySys = sysOption==None if sysOption=='fake' or anySys : systematics += systUtils.fakeSystVariations() if sysOption=='object' or anySys : systematics += systUtils.mcObjectVariations() if sysOption=='weight' or anySys : systematics += systUtils.mcWeightVariations() if sysOption and sysOption.count(',') : systematics = [s for s in systUtils.getAllVariations() if s in sysOption.split(',')] elif sysOption in systUtils.getAllVariations() : systematics = [sysOption] elif not anySys and len(systematics)==1 and sysOption!='NOM' : raise ValueError("Invalid syst %s"%str(sysOption)) if excludedSyst : systematics = [s for s in systematics if s not in filterWithRegexp(systematics, excludedSyst)] if verbose : print "about to loop over these systematics:\n %s"%str(systematics) for syst in systematics : if batchMode : newOptions = " --input-gen %s" % opts.input_gen newOptions += " --input-fake %s" % opts.input_fake newOptions += " --output-dir %s" % opts.output_dir newOptions += " --verbose %s" % opts.verbose newOptions += " --syst %s" % syst template = 'batch/templates/check_hft_fill.sh.template' script = "batch/hft_%s.sh"%syst scriptFile = open(script, 'w') scriptFile.write(open(template).read() .replace('%(opt)s', newOptions) .replace('%(logfile)s', 'log/hft/fill_'+syst+'.log') .replace('%(jobname)s', 'fill_'+syst)) scriptFile.close() cmd = "sbatch %s"%script if verbose : print cmd out = getCommandOutput(cmd) if verbose : print out['stdout'] if out['stderr'] : print out['stderr'] continue if verbose : print '---- filling ',syst samplesPerGroup = allSamplesAllGroups() [s.setSyst(syst) for g, samples in samplesPerGroup.iteritems() for s in samples] counters, histos = countAndFillHistos(samplesPerGroup=samplesPerGroup, syst=syst, verbose=verbose, outdir=outputDir) printCounters(counters) saveHistos(samplesPerGroup, histos, outputDir, verbose)
def __init__(self, args) : for key,value in args.iteritems() : setattr(self,key,value) if any(self.signalExampleToStack) : assert self.smOnly self.toPrint = [] self.ewkType = "function" if self.REwk else "var" self.label = "CMS Preliminary 2011 1.1 fb^{-1} #sqrt{s} = 7 TeV" self.obsLabel = "Data" if not hasattr(self, "toyNumber") else "Toy %d"%self.toyNumber self.plotsDir = "plots" utils.getCommandOutput("mkdir %s"%self.plotsDir) if not self.smOnly : self.signalDesc = "signal" self.signalDesc2 = "xs/xs^{nom} = %4.2e #pm %4.2e; #rho = %4.2f"%(self.wspace.var("f").getVal(), self.wspace.var("f").getError(), self.wspace.var("rhoSignal").getVal()) self.width1 = 2 self.width2 = 3 self.sm = r.kAzure+6 self.sig = r.kPink+7 self.ewk = r.kBlue+1 self.qcd = r.kGreen+3
def submit_batch_fill_job_per_group(group, opts): options_dict = vars(opts) group_name = group.name if hasattr(group, 'name') else group verbose = opts.verbose options_dict['group'] = group_name options_with_value = dict( (k, v) for k, v in options_dict.iteritems() if v and v is not True) # note to self: the line below assumes that the argument-less options have a default=False options_with_toggle = dict((k, v) for k, v in options_dict.iteritems() if v and v is True and k != "batch") def escape_regex(v): return v if v != '.*' else "'.*'" def back_to_dash(v): return v.replace('_', '-') cmd_line_options = ' '.join([ "--%s %s" % (back_to_dash(k), escape_regex(str(v))) for k, v in options_with_value.iteritems() ] + ["--%s" % back_to_dash(k) for k in options_with_toggle.keys()] + ['--just-fill']) template = 'batch/templates/plot_by_source.sh' default_log_dir = opts.output_dir.replace('out/', 'log/') if default_log_dir.count('/histos') == 1: default_log_dir = default_log_dir.replace('/histos', '') log_dir = mkdirIfNeeded(opts.log_dir if opts.log_dir else default_log_dir) script_dir = mkdirIfNeeded('batch/plot_by_source') script_name = os.path.join(script_dir, group_name + '.sh') log_name = log_dir + '/' + group_name + '.log' script_file = open(script_name, 'w') script_file.write( open(template).read().replace('%(opt)s', cmd_line_options).replace( '%(logfile)s', log_name).replace('%(jobname)s', group_name).replace('%(queue)s', opts.queue)) script_file.close() cmd = "sbatch %s" % script_name if verbose: print cmd out = getCommandOutput(cmd) if verbose: print out['stdout'] if out['stderr']: print out['stderr']
'outdir': outdir, 'sample': sample, 'tag': batchTag } outlog = outLogTemplate % { 'logdir': logdir, 'sample': sample, 'tag': batchTag } script = outScriptTemplate % {'batdir': batdir, 'sample': sample} fileExists = os.path.exists(script) if overwrite or not fileExists: fillInScriptTemplate(sample, input, output, otherOptions, script, template) elif fileExists: print "warning, not overwriting existing script '%s'" % script cmd = "qsub " \ "-j oe -V " \ "-N %(jobname)s " \ "-o %(outlog)s " \ " %(scripname)s" \ % \ {'jobname':"%s%s"%(sample, batchTag), 'outlog':outlog, 'scripname':script} print cmd if submit: out = getCommandOutput(cmd) if verbose: print out['stdout'] if not submit: print "This was a dry run; use '--submit' to actually submit the jobs"
def getGitSha(dir) : def extrackPkg(dir) : return os.path.basename(dir.rstrip(' /')) if not os.path.isdir("%s/.git"%dir) : return cmd = "git --git-dir=%s/.git --work-tree=%s describe"%(dir, dir) res = getCommandOutput(cmd) return "%s @ %s"%(extrackPkg(dir), res['stdout'].strip())
import sys import utils pat = sys.argv[1] nums = [int(f.replace(pat,'').replace('.root','')) for f in utils.getCommandOutput("ls %s*.root"%pat)["stdout"].split()] print ','.join([str(s) for s in sorted(set(range(max(nums))) - set(nums))])
continue group = dataset.group if dataset.isNotToBeMerged : continue if not re.search(group_regexp, group) : continue if not group : print "warning, invalid group '%s' for '%s'"%(group, rf) if verbose and group not in filenamesByGroup : print "adding group '%s'"%group filenamesByGroup[group].append(rf) if allBkg and dataset.isMcBackground : filenamesByGroup['allBkg'].append(rf) if allBkgButHf and dataset.isMcBackground and not dataset.isHeavyFlavor : filenamesByGroup['allBkgButHf'].append(rf) nGroupsToMerge = len(filenamesByGroup.keys()) groupCounter = 0 logFilename = outdir+"merge_%s.log"%datetime.date.today().strftime('%Y-%m-%d') logFile = open(logFilename, 'w') for group, files in filenamesByGroup.iteritems() : groupCounter += 1 if verbose : print "[%d/%d] %s (%d files)"%(groupCounter, nGroupsToMerge, group, len(files)) outfile = outdir+'/'+group+'_'+tag+'.root' if overwrite and os.path.isfile(outfile) : os.remove(outfile) if debug : print "hadd %s\n\t%s"%(outfile, '\n\t'.join(files)) cmd = "hadd %s %s" % (outfile, ' '.join(files)) out = 0 if dryrun else getCommandOutput(cmd) success = dryrun or out['returncode']==0 logFile.write(out['stdout']) if not success : print "'%s' failed..."%group if debug : print out['stderr']+'\n'+out['stdout'] logFile.close() if verbose : print "hadd commands logged to '%s'"%logFilename
if silentInstall: dl = 'n' elif lightInstall: dl = 'n' else: dl = raw_input("Enter Y/N: ") if dl == 'y' or dl == 'Y': archive = "glimmer-mg-0.3.1.tar.gz" os.system("curl -L ftp://ftp.cbcb.umd.edu/pub/data/metamos/%s -o %s" %(archive, archive)) os.system("tar -C ./Utilities/ -xvf %s" % archive) os.system("rm %s"%archive) os.system("python ./Utilities/glimmer-mg/install_glimmer.py") # check the number of files the DB currently is and see if we have the expected number dbResult = utils.getCommandOutput("perl ./Utilities/perl/update_blastdb.pl refseq_protein --numpartitions", False) if dbResult == "": print "Error: could not connect to NCBI, will not be installing refseq protein DB" else: (dbName, numPartitions) = dbResult.split("\t", 1) print "Checking whether %s is complete. Expecting %d partitions.\n"%(dbName, int(numPartitions)) numPartitions = int(numPartitions) - 1 if not os.path.exists("./Utilities/DB/refseq_protein.pal") or not os.path.exists("./Utilities/DB/refseq_protein.%02d.psq"%(int(numPartitions))) or not os.path.exists("./Utilities/DB/allprots.faa"): print "refseq protein DB not found or incomplete, needed for Annotate step, download now?" if silentInstall: dl = 'y' elif lightInstall: dl = 'n' else: dl = raw_input("Enter Y/N: ")
outFile.write(line) outFile.close() for sample in sampleNames : missList, regexUnmatch = not listExists(sample), not re.search(regexp, sample) if missList or regexUnmatch: msg = "# skipping %s (%s)" % (sample, 'no list' if missList else 'regex unmatch') print msg continue input = inputTemplate%{'sample':sample} output = outRootTemplate %{'outdir':outdir, 'sample':sample, 'tag':batchTag} outlog = outLogTemplate %{'logdir':logdir, 'sample':sample, 'tag':batchTag} script = outScriptTemplate%{'batdir':batdir, 'sample':sample} fileExists = os.path.exists(script) if overwrite or not fileExists : fillInScriptTemplate(sample, input, output, otherOptions, script, template) elif fileExists : print "warning, not overwriting existing script '%s'"%script cmd = "qsub " \ "-j oe -V " \ "-N %(jobname)s " \ "-o %(outlog)s " \ " %(scripname)s" \ % \ {'jobname':"%s%s"%(sample, batchTag), 'outlog':outlog, 'scripname':script} print cmd if submit : out = getCommandOutput(cmd) if verbose : print out['stdout'] if not submit : print "This was a dry run; use '--submit' to actually submit the jobs"