def main() :
parser = optparse.OptionParser()
parser.add_option('--email', default='*****@*****.**')
parser.add_option('--message', default='all jobs done, bam!')
parser.add_option('--polleverysec', default=60*5, type='int')
parser.add_option('--subject', default='All jobs done')
parser.add_option('--user', default=os.environ['USER'])
(opts, args) = parser.parse_args()
receiver = opts.email
username = opts.user
message = opts.message
subject = opts.subject
everyNsec = opts.polleverysec
nChecksDone = 0
countJobsCmd="qstat -u %(user)s | grep %(user)s | wc -l"% {'user':username}
nJobsRunning = int(getCommandOutput(countJobsCmd)['stdout'])
while nJobsRunning > 0 :
sleep(everyNsec)
nJobsRunning = int(getCommandOutput(countJobsCmd)['stdout'])
nChecksDone += 1
message = message+("\n(after %(nchk)d checks every %(nsec)d sec)"
%{'nchk' : nChecksDone, 'nsec' : everyNsec})
mailCmd = ("echo \"%(msg)s\" | mail -s \"%(sbj)s\" %(dest)s"
% {'dest' : receiver, 'msg' : message, 'sbj':subject})
getCommandOutput(mailCmd)
def main():
parser = optparse.OptionParser()
parser.add_option('--email', default='*****@*****.**')
parser.add_option('--message', default='all jobs done, bam!')
parser.add_option('--polleverysec', default=60 * 5, type='int')
parser.add_option('--subject', default='All jobs done')
parser.add_option('--user', default=os.environ['USER'])
(opts, args) = parser.parse_args()
receiver = opts.email
username = opts.user
message = opts.message
subject = opts.subject
everyNsec = opts.polleverysec
nChecksDone = 0
countJobsCmd = "qstat -u %(user)s | grep %(user)s | wc -l" % {
'user': username
}
nJobsRunning = int(getCommandOutput(countJobsCmd)['stdout'])
while nJobsRunning > 0:
sleep(everyNsec)
nJobsRunning = int(getCommandOutput(countJobsCmd)['stdout'])
nChecksDone += 1
message = message + ("\n(after %(nchk)d checks every %(nsec)d sec)" % {
'nchk': nChecksDone,
'nsec': everyNsec
})
mailCmd = ("echo \"%(msg)s\" | mail -s \"%(sbj)s\" %(dest)s" % {
'dest': receiver,
'msg': message,
'sbj': subject
})
getCommandOutput(mailCmd)
def git_info(path):
cmd = 'git describe --tags --dirty'
out = getCommandOutput(cmd, cwd=path)
tag_info = out['stdout'].strip()
cmd = 'git status'
modified_files = getCommandOutput(cmd, cwd=path)['stdout'].splitlines()
modified_files = [l for l in modified_files if 'modified:' in l]
return "{0} {1}".format(tag_info,
('\n'.join(['']+modified_files) if modified_files else ''))
def git_info(path):
cmd = 'git describe --tags --dirty'
out = getCommandOutput(cmd, cwd=path)
tag_info = out['stdout'].strip()
cmd = 'git status'
modified_files = getCommandOutput(cmd, cwd=path)['stdout'].splitlines()
modified_files = [l for l in modified_files if 'modified:' in l]
return "{0} {1}".format(
tag_info, ('\n'.join([''] + modified_files) if modified_files else ''))
def svn_info(path):
# get tag or last rev
cmd = 'svn info '+path
out = getCommandOutput(cmd)
url_origin = first([l for l in out['stdout'].splitlines() if 'URL: ' in l])
url_origin = url_origin.replace('/', ' ').split() if url_origin else []
tag = url_origin[url_origin.index('tags')+1] if 'tags' in url_origin else None
last_rev = first(first([l for l in out['stdout'].splitlines() if 'Last Changed Rev:' in l]).split()[::-1])
# get list of modified files
cmd = 'svn status '+path
out = getCommandOutput(cmd)
modified_files = [l for l in out['stdout'].splitlines() if l.startswith('M ')]
return "{0} {1}".format(tag if tag else last_rev,
('\n'.join(['']+modified_files) if modified_files else ''))
def main():
options = parse_options()
exe = options.executable
inputdf = options.input
include = options.include_regexp
exclude = options.exclude_regexp
tag = options.tag
verbose = options.verbose
submit = options.submit
datasets = dataset.build_all_datasets_from_dir_or_file(inputdf)
datasets = utils.filterWithRegexp (datasets, include, lambda _: _.name) if include else datasets
datasets = utils.excludeWithRegexp(datasets, exclude, lambda _: _.name) if exclude else datasets
for dset in datasets :
if not get_filelist(dset.name):
print "# skipping (missing filelist) {0}".format(dset.name)
continue
script = get_batch_script(dset, options)
if not script:
if verbose : print "skipping (do-not-overwrite) {0}".format(dset.name)
continue
cmd = "sbatch %s" % script
if verbose : print cmd
if submit :
out = utils.getCommandOutput(cmd)
if verbose : print out['stdout']
if not submit : print "This was a dry run; use '--submit' to actually submit the jobs"
def main():
options = parse_options()
exe = options.executable
inputdf = options.input
include = options.include_regexp
exclude = options.exclude_regexp
tag = options.tag
verbose = options.verbose
submit = options.submit
datasets = dataset.build_all_datasets_from_dir_or_file(inputdf)
datasets = utils.filterWithRegexp(
datasets, include, lambda _: _.name) if include else datasets
datasets = utils.excludeWithRegexp(
datasets, exclude, lambda _: _.name) if exclude else datasets
for dset in datasets:
if not get_filelist(dset.name):
print "# skipping (missing filelist) {0}".format(dset.name)
continue
script = get_batch_script(dset, options)
if not script:
if verbose:
print "skipping (do-not-overwrite) {0}".format(dset.name)
continue
cmd = "sbatch %s" % script
if verbose: print cmd
if submit:
out = utils.getCommandOutput(cmd)
if verbose: print out['stdout']
if not submit:
print "This was a dry run; use '--submit' to actually submit the jobs"
def submit_batch_fill_job_per_group(group, opts):
options_dict = vars(opts)
group_name = group.name if hasattr(group, 'name') else group
systematic = opts.syst if hasattr(opts, 'syst') and opts.syst else None
verbose = opts.verbose
options_dict['group'] = group_name
options_with_value = dict((k,v) for k,v in options_dict.iteritems() if v and v is not True)
# note to self: the line below assumes that the argument-less options have a default=False
options_with_toggle = dict((k,v) for k,v in options_dict.iteritems() if v and v is True and k!="batch")
def escape_regex(v) : return v if v!='.*' else "'.*'"
def back_to_dash(v) : return v.replace('_','-')
cmd_line_options = ' '.join(["--%s %s"%(back_to_dash(k), escape_regex(str(v)))
for k,v in options_with_value.iteritems()]
+["--%s"%back_to_dash(k) for k in options_with_toggle.keys()])
template = 'batch/templates/plot_emu.sh'
default_log_dir = opts.output_dir.replace('out/', 'log/')
if default_log_dir.count('/histos')==1:
default_log_dir = default_log_dir.replace('/histos','')
log_dir = mkdirIfNeeded(opts.log_dir if opts.log_dir else default_log_dir)
script_dir = mkdirIfNeeded('batch/plot_emu')
script_name = os.path.join(script_dir, group_name+("_{0}".format(systematic) if systematic else '')+'.sh')
log_name = log_dir+'/'+group_name+("_{0}".format(systematic) if systematic else '')+'.log'
script_file = open(script_name, 'w')
script_file.write(open(template).read()
.replace('%(opt)s', cmd_line_options)
.replace('%(logfile)s', log_name)
.replace('%(jobname)s', group_name)
.replace('%(queue)s', opts.queue))
script_file.close()
cmd = "sbatch %s"%script_name
if verbose : print cmd
out = getCommandOutput(cmd)
if verbose : print out['stdout']
if out['stderr'] : print out['stderr']
def GenerateVersionFile():
# TODO: fix it.
if HOST_OS == 'win': return
versionInfo = utils.getCommandOutput(
os.path.join('..', '..', 'dartium_tools', 'print_dart_version.sh'))
file = open(VERSION_FILE, 'w')
file.write(versionInfo)
file.close()
def GenerateVersionFile():
# TODO: fix it.
if HOST_OS == 'win': return
versionInfo = utils.getCommandOutput(os.path.join('..', '..',
'dartium_tools',
'print_dart_version.sh'))
file = open(VERSION_FILE, 'w')
file.write(versionInfo)
file.close()
def main():
parser = optparse.OptionParser()
parser.add_option('--from-address', default='*****@*****.**')
parser.add_option('--to-address', default='*****@*****.**')
parser.add_option('--message', default='all jobs done, bam!')
parser.add_option('--polleverysec', default=60 * 5, type='int')
parser.add_option('--subject', default='jobs done from `pwd`')
parser.add_option('--user', default=os.environ['USER'])
parser.add_option('--exec-cmd', help='on completion, execute this command')
(opts, args) = parser.parse_args()
receiver = opts.to_address
username = opts.user
message = opts.message
subject = opts.subject
everyNsec = opts.polleverysec
nChecksDone = 0
countJobsCmd = "qstat -u %(user)s | grep %(user)s | egrep '(Q|R)' | wc -l" % {
'user': username
}
nJobsRunning = int(getCommandOutput(countJobsCmd)['stdout'])
while nJobsRunning > 0:
sleep(everyNsec)
nJobsRunning = int(getCommandOutput(countJobsCmd)['stdout'])
nChecksDone += 1
preamble = "From: %s \n" % opts.from_address
message = preamble + message
message += ("\n(after %(nchk)d checks every %(nsec)d sec)" % {
'nchk': nChecksDone,
'nsec': everyNsec
})
subject = opts.subject
mailCmd = ("echo -e \"%(msg)s\" | mail -s \"%(sbj)s\" %(dest)s" % {
'dest': receiver,
'msg': message,
'sbj': subject
})
print mailCmd
getCommandOutput(mailCmd)
if opts.exec_cmd:
print "now executing {}".format(opts.exec_cmd)
out = getCommandOutput(opts.exec_cmd)
print out['stdout']
print out['stderr']
def merge(input_files=[], output_filename=''):
"hadd a list of input files; return name merged file"
input_files = [f for f in input_files if f is not output_filename]
cmd = "hadd %s %s" % (output_filename, ' '.join(input_files))
out = utils.getCommandOutput(cmd)
log.info(cmd)
if out['returncode']!=0:
log.info(out['stdout'])
log.error(out['stderr'])
return output_filename if out['returncode']==0 else None
def merge(input_files=[], output_filename=''):
"hadd a list of input files; return name merged file"
input_files = [f for f in input_files if f is not output_filename]
cmd = "hadd %s %s" % (output_filename, ' '.join(input_files))
out = utils.getCommandOutput(cmd)
log.info(cmd)
if out['returncode'] != 0:
log.info(out['stdout'])
log.error(out['stderr'])
return output_filename if out['returncode'] == 0 else None
def checkFileExists(file):
if isRemote(file):
if isSRAID(file):
file = utils.translateToSRAURL(settings, file)
result = utils.getCommandOutput("curl -L -I %s && echo $?"%(file), False)
if result == "":
return False
return True
else:
return os.path.exists(file)
def checkFileExists(file):
if isRemote(file):
if isSRAID(file):
file = utils.translateToSRAURL(settings, file)
result = utils.getCommandOutput("curl -L -I %s && echo $?" % (file),
False)
if result == "":
return False
return True
else:
return os.path.exists(file)
def svn_info(path):
# get tag or last rev
cmd = 'svn info ' + path
out = getCommandOutput(cmd)
url_origin = first([l for l in out['stdout'].splitlines() if 'URL: ' in l])
url_origin = url_origin.replace('/', ' ').split() if url_origin else []
tag = url_origin[url_origin.index('tags') +
1] if 'tags' in url_origin else None
last_rev = first(
first([
l for l in out['stdout'].splitlines() if 'Last Changed Rev:' in l
]).split()[::-1])
# get list of modified files
cmd = 'svn status ' + path
out = getCommandOutput(cmd)
modified_files = [
l for l in out['stdout'].splitlines() if l.startswith('M ')
]
return "{0} {1}".format(
tag if tag else last_rev,
('\n'.join([''] + modified_files) if modified_files else ''))
def getTagOrRev(dir) :
"tag if available, otherwise rev"
cmd = 'svn info '+dir
res = getCommandOutput(cmd)
lines = res['stdout'].split('\n')
urlLine = next((l for l in lines if l.startswith('URL:')), None)
revLine = next((l for l in lines if l.startswith('Revision:')), None)
if not urlLine or not revLine : return
pkg = extractPkg(urlLine)
tag = extractTag(urlLine)
rev = extractRev(revLine)
assert tag or rev, "cannot extract info for %s"%dir
return "%s @ %s"%(pkg, tag if tag else rev)
def submit_batch_fill_job_per_group_per_selection(group=None,
selection='',
opts=None):
"if we are processing cached selections, we can submit one job per selection"
options_dict = vars(opts)
group_name = group.name if hasattr(group, 'name') else group
systematic = opts.syst if hasattr(opts, 'syst') and opts.syst else None
verbose = opts.verbose
options_dict['group'] = group_name
options_dict['region'] = selection
options_dict['regions'] = None
options_with_value = dict(
(k, v) for k, v in options_dict.iteritems() if v and v is not True)
# note to self: the line below assumes that the argument-less options have a default=False
options_with_toggle = dict((k, v) for k, v in options_dict.iteritems()
if v and v is True and k != "batch")
def escape_regex(v):
return v if v != '.*' else "'.*'"
def back_to_dash(v):
return v.replace('_', '-')
cmd_line_options = ' '.join([
"--%s %s" % (back_to_dash(k), escape_regex(str(v)))
for k, v in options_with_value.iteritems()
] + ["--%s" % back_to_dash(k) for k in options_with_toggle.keys()])
template = 'batch/templates/plot_emu.sh'
default_log_dir = opts.output_dir.replace('out/', 'log/')
if default_log_dir.count('/histos') == 1:
default_log_dir = default_log_dir.replace('/histos', '')
log_dir = mkdirIfNeeded(opts.log_dir if opts.log_dir else default_log_dir)
script_dir = mkdirIfNeeded('batch/plot_emu')
script_name = os.path.join(
script_dir, group_name + '_' + selection +
("_{0}".format(systematic) if systematic else '') + '.sh')
log_name = log_dir + '/' + group_name + '_' + selection + (
"_{0}".format(systematic) if systematic else '') + '.log'
script_file = open(script_name, 'w')
script_file.write(
open(template).read().replace('%(opt)s', cmd_line_options).replace(
'%(logfile)s',
log_name).replace('%(jobname)s',
group_name + '_' + selection).replace(
'%(queue)s', opts.queue))
script_file.close()
cmd = "sbatch %s" % script_name
if verbose: print cmd
out = getCommandOutput(cmd)
if verbose: print out['stdout']
if out['stderr']: print out['stderr']
def main() :
parser = optparse.OptionParser()
parser.add_option('--from-address', default='*****@*****.**')
parser.add_option('--to-address', default='*****@*****.**')
parser.add_option('--message', default='all jobs done, bam!')
parser.add_option('--polleverysec', default=60*5, type='int')
parser.add_option('--subject', default='jobs done from `pwd`')
parser.add_option('--user', default=os.environ['USER'])
parser.add_option('--exec-cmd', help='on completion, execute this command')
(opts, args) = parser.parse_args()
receiver = opts.to_address
username = opts.user
message = opts.message
subject = opts.subject
everyNsec = opts.polleverysec
nChecksDone = 0
countJobsCmd="qstat -u %(user)s | grep %(user)s | egrep '(Q|R)' | wc -l"% {'user':username}
nJobsRunning = int(getCommandOutput(countJobsCmd)['stdout'])
while nJobsRunning > 0 :
sleep(everyNsec)
nJobsRunning = int(getCommandOutput(countJobsCmd)['stdout'])
nChecksDone += 1
preamble = "From: %s \n"%opts.from_address
message = preamble+message
message += ("\n(after %(nchk)d checks every %(nsec)d sec)"
%{'nchk' : nChecksDone, 'nsec' : everyNsec})
subject = opts.subject
mailCmd = ("echo -e \"%(msg)s\" | mail -s \"%(sbj)s\" %(dest)s"
% {'dest' : receiver, 'msg' : message, 'sbj':subject})
print mailCmd
getCommandOutput(mailCmd)
if opts.exec_cmd:
print "now executing {}".format(opts.exec_cmd)
out = getCommandOutput(opts.exec_cmd)
print out['stdout']
print out['stderr']
def runFill(opts) :
batchMode = opts.batch
inputFakeDir = opts.input_fake
inputGenDir = opts.input_gen
outputDir = opts.output_dir
sysOption = opts.syst
excludedSyst = opts.exclude
verbose = opts.verbose
if verbose : print "filling histos"
mkdirIfNeeded(outputDir)
systematics = ['NOM']
anySys = sysOption==None
if sysOption=='fake' or anySys : systematics += systUtils.fakeSystVariations()
if sysOption=='object' or anySys : systematics += systUtils.mcObjectVariations()
if sysOption=='weight' or anySys : systematics += systUtils.mcWeightVariations()
if sysOption and sysOption.count(',') : systematics = [s for s in systUtils.getAllVariations() if s in sysOption.split(',')]
elif sysOption in systUtils.getAllVariations() : systematics = [sysOption]
elif not anySys and len(systematics)==1 and sysOption!='NOM' : raise ValueError("Invalid syst %s"%str(sysOption))
if excludedSyst : systematics = [s for s in systematics if s not in filterWithRegexp(systematics, excludedSyst)]
if verbose : print "about to loop over these systematics:\n %s"%str(systematics)
for syst in systematics :
if batchMode :
newOptions = " --input-gen %s" % opts.input_gen
newOptions += " --input-fake %s" % opts.input_fake
newOptions += " --output-dir %s" % opts.output_dir
newOptions += " --verbose %s" % opts.verbose
newOptions += " --syst %s" % syst
template = 'batch/templates/check_hft_fill.sh.template'
script = "batch/hft_%s.sh"%syst
scriptFile = open(script, 'w')
scriptFile.write(open(template).read()
.replace('%(opt)s', newOptions)
.replace('%(logfile)s', 'log/hft/fill_'+syst+'.log')
.replace('%(jobname)s', 'fill_'+syst))
scriptFile.close()
cmd = "sbatch %s"%script
if verbose : print cmd
out = getCommandOutput(cmd)
if verbose : print out['stdout']
if out['stderr'] : print out['stderr']
continue
if verbose : print '---- filling ',syst
samplesPerGroup = allSamplesAllGroups()
[s.setSyst(syst) for g, samples in samplesPerGroup.iteritems() for s in samples]
counters, histos = countAndFillHistos(samplesPerGroup=samplesPerGroup, syst=syst, verbose=verbose, outdir=outputDir)
printCounters(counters)
saveHistos(samplesPerGroup, histos, outputDir, verbose)
def __init__(self, args) :
for key,value in args.iteritems() :
setattr(self,key,value)
if any(self.signalExampleToStack) : assert self.smOnly
self.toPrint = []
self.ewkType = "function" if self.REwk else "var"
self.label = "CMS Preliminary 2011 1.1 fb^{-1} #sqrt{s} = 7 TeV"
self.obsLabel = "Data" if not hasattr(self, "toyNumber") else "Toy %d"%self.toyNumber
self.plotsDir = "plots"
utils.getCommandOutput("mkdir %s"%self.plotsDir)
if not self.smOnly :
self.signalDesc = "signal"
self.signalDesc2 = "xs/xs^{nom} = %4.2e #pm %4.2e; #rho = %4.2f"%(self.wspace.var("f").getVal(), self.wspace.var("f").getError(), self.wspace.var("rhoSignal").getVal())
self.width1 = 2
self.width2 = 3
self.sm = r.kAzure+6
self.sig = r.kPink+7
self.ewk = r.kBlue+1
self.qcd = r.kGreen+3
def submit_batch_fill_job_per_group(group, opts):
options_dict = vars(opts)
group_name = group.name if hasattr(group, 'name') else group
verbose = opts.verbose
options_dict['group'] = group_name
options_with_value = dict(
(k, v) for k, v in options_dict.iteritems() if v and v is not True)
# note to self: the line below assumes that the argument-less options have a default=False
options_with_toggle = dict((k, v) for k, v in options_dict.iteritems()
if v and v is True and k != "batch")
def escape_regex(v):
return v if v != '.*' else "'.*'"
def back_to_dash(v):
return v.replace('_', '-')
cmd_line_options = ' '.join([
"--%s %s" % (back_to_dash(k), escape_regex(str(v)))
for k, v in options_with_value.iteritems()
] + ["--%s" % back_to_dash(k)
for k in options_with_toggle.keys()] + ['--just-fill'])
template = 'batch/templates/plot_by_source.sh'
default_log_dir = opts.output_dir.replace('out/', 'log/')
if default_log_dir.count('/histos') == 1:
default_log_dir = default_log_dir.replace('/histos', '')
log_dir = mkdirIfNeeded(opts.log_dir if opts.log_dir else default_log_dir)
script_dir = mkdirIfNeeded('batch/plot_by_source')
script_name = os.path.join(script_dir, group_name + '.sh')
log_name = log_dir + '/' + group_name + '.log'
script_file = open(script_name, 'w')
script_file.write(
open(template).read().replace('%(opt)s', cmd_line_options).replace(
'%(logfile)s',
log_name).replace('%(jobname)s',
group_name).replace('%(queue)s', opts.queue))
script_file.close()
cmd = "sbatch %s" % script_name
if verbose: print cmd
out = getCommandOutput(cmd)
if verbose: print out['stdout']
if out['stderr']: print out['stderr']
'outdir': outdir,
'sample': sample,
'tag': batchTag
}
outlog = outLogTemplate % {
'logdir': logdir,
'sample': sample,
'tag': batchTag
}
script = outScriptTemplate % {'batdir': batdir, 'sample': sample}
fileExists = os.path.exists(script)
if overwrite or not fileExists:
fillInScriptTemplate(sample, input, output, otherOptions, script,
template)
elif fileExists:
print "warning, not overwriting existing script '%s'" % script
cmd = "qsub " \
"-j oe -V " \
"-N %(jobname)s " \
"-o %(outlog)s " \
" %(scripname)s" \
% \
{'jobname':"%s%s"%(sample, batchTag), 'outlog':outlog, 'scripname':script}
print cmd
if submit:
out = getCommandOutput(cmd)
if verbose: print out['stdout']
if not submit:
print "This was a dry run; use '--submit' to actually submit the jobs"
def getGitSha(dir) :
def extrackPkg(dir) : return os.path.basename(dir.rstrip(' /'))
if not os.path.isdir("%s/.git"%dir) : return
cmd = "git --git-dir=%s/.git --work-tree=%s describe"%(dir, dir)
res = getCommandOutput(cmd)
return "%s @ %s"%(extrackPkg(dir), res['stdout'].strip())
import sys
import utils
pat = sys.argv[1]
nums = [int(f.replace(pat,'').replace('.root','')) for f in utils.getCommandOutput("ls %s*.root"%pat)["stdout"].split()]
print ','.join([str(s) for s in sorted(set(range(max(nums))) - set(nums))])
continue
group = dataset.group
if dataset.isNotToBeMerged : continue
if not re.search(group_regexp, group) : continue
if not group : print "warning, invalid group '%s' for '%s'"%(group, rf)
if verbose and group not in filenamesByGroup : print "adding group '%s'"%group
filenamesByGroup[group].append(rf)
if allBkg and dataset.isMcBackground : filenamesByGroup['allBkg'].append(rf)
if allBkgButHf and dataset.isMcBackground and not dataset.isHeavyFlavor :
filenamesByGroup['allBkgButHf'].append(rf)
nGroupsToMerge = len(filenamesByGroup.keys())
groupCounter = 0
logFilename = outdir+"merge_%s.log"%datetime.date.today().strftime('%Y-%m-%d')
logFile = open(logFilename, 'w')
for group, files in filenamesByGroup.iteritems() :
groupCounter += 1
if verbose :
print "[%d/%d] %s (%d files)"%(groupCounter, nGroupsToMerge, group, len(files))
outfile = outdir+'/'+group+'_'+tag+'.root'
if overwrite and os.path.isfile(outfile) : os.remove(outfile)
if debug : print "hadd %s\n\t%s"%(outfile, '\n\t'.join(files))
cmd = "hadd %s %s" % (outfile, ' '.join(files))
out = 0 if dryrun else getCommandOutput(cmd)
success = dryrun or out['returncode']==0
logFile.write(out['stdout'])
if not success : print "'%s' failed..."%group
if debug : print out['stderr']+'\n'+out['stdout']
logFile.close()
if verbose : print "hadd commands logged to '%s'"%logFilename
if silentInstall:
dl = 'n'
elif lightInstall:
dl = 'n'
else:
dl = raw_input("Enter Y/N: ")
if dl == 'y' or dl == 'Y':
archive = "glimmer-mg-0.3.1.tar.gz"
os.system("curl -L ftp://ftp.cbcb.umd.edu/pub/data/metamos/%s -o %s" %(archive, archive))
os.system("tar -C ./Utilities/ -xvf %s" % archive)
os.system("rm %s"%archive)
os.system("python ./Utilities/glimmer-mg/install_glimmer.py")
# check the number of files the DB currently is and see if we have the expected number
dbResult = utils.getCommandOutput("perl ./Utilities/perl/update_blastdb.pl refseq_protein --numpartitions", False)
if dbResult == "":
print "Error: could not connect to NCBI, will not be installing refseq protein DB"
else:
(dbName, numPartitions) = dbResult.split("\t", 1)
print "Checking whether %s is complete. Expecting %d partitions.\n"%(dbName, int(numPartitions))
numPartitions = int(numPartitions) - 1
if not os.path.exists("./Utilities/DB/refseq_protein.pal") or not os.path.exists("./Utilities/DB/refseq_protein.%02d.psq"%(int(numPartitions))) or not os.path.exists("./Utilities/DB/allprots.faa"):
print "refseq protein DB not found or incomplete, needed for Annotate step, download now?"
if silentInstall:
dl = 'y'
elif lightInstall:
dl = 'n'
else:
dl = raw_input("Enter Y/N: ")
outFile.write(line)
outFile.close()
for sample in sampleNames :
missList, regexUnmatch = not listExists(sample), not re.search(regexp, sample)
if missList or regexUnmatch:
msg = "# skipping %s (%s)" % (sample, 'no list' if missList else 'regex unmatch')
print msg
continue
input = inputTemplate%{'sample':sample}
output = outRootTemplate %{'outdir':outdir, 'sample':sample, 'tag':batchTag}
outlog = outLogTemplate %{'logdir':logdir, 'sample':sample, 'tag':batchTag}
script = outScriptTemplate%{'batdir':batdir, 'sample':sample}
fileExists = os.path.exists(script)
if overwrite or not fileExists :
fillInScriptTemplate(sample, input, output, otherOptions, script, template)
elif fileExists : print "warning, not overwriting existing script '%s'"%script
cmd = "qsub " \
"-j oe -V " \
"-N %(jobname)s " \
"-o %(outlog)s " \
" %(scripname)s" \
% \
{'jobname':"%s%s"%(sample, batchTag), 'outlog':outlog, 'scripname':script}
print cmd
if submit :
out = getCommandOutput(cmd)
if verbose : print out['stdout']
if not submit : print "This was a dry run; use '--submit' to actually submit the jobs"
