should_transfer_files = YES
when_to_transfer_output = ON_EXIT
transfer_input_files = %(input_files)s,cs_jobmap
+REQUIRED_OS = "rhel7"
+DesiredOS = REQUIRED_OS
Queue %(njobs)s
'''
batch_root = crab_dirs_root('combine_output_%i' % time.time())
if os.path.isdir(batch_root):
raise IOError('%s exists' % batch_root)
print batch_root
os.mkdir(batch_root)
os.mkdir(os.path.join(batch_root, 'inputs'))
save_git_status(os.path.join(batch_root, 'gitstatus'))
input_files = []
for x in ['signal_efficiency.py', 'datacard.py', 'limitsinput.root']:
nx = os.path.abspath(
os.path.join(batch_root, 'inputs', os.path.basename(x)))
shutil.copy2(x, nx)
input_files.append(nx)
input_files = ','.join(input_files)
f = ROOT.TFile('limitsinput.root')
years = ('2016', '2017', '2018') if include_2016 else ('2017', '2018')
samples = limitsinput.sample_iterator(
f,
require_years=years,
def submit(samples, ntracks, overlay_args, batch_name_ex=''):
testing = 'testing' in sys.argv
batch_name = 'ntk%i' % ntracks
batch_name += batch_name_ex
if testing:
batch_name += '_TEST'
work_area = '/uscms_data/d2/%s/crab_dirs/Overlay%s/%s' % (
os.environ['USER'], version, batch_name)
inputs_dir = os.path.join(work_area, 'inputs')
os.makedirs(inputs_dir)
save_git_status(os.path.join(work_area, 'gitstatus'))
cmssw_py = 'overlay.py'
cmssw_py_fn = os.path.join(inputs_dir, cmssw_py)
shutil.copy2(cmssw_py, cmssw_py_fn)
tool_path = os.path.join(
os.environ['CMSSW_BASE'], 'src/JMTucker/MFVNeutralino/test/MakeSamples'
) # JMTBAD put dummy.py and fixfjr.py somewhere better
config = Config()
config.General.transferLogs = True
config.General.transferOutputs = True
config.General.workArea = work_area
config.JobType.pluginName = 'PrivateMC'
config.JobType.psetName = os.path.join(tool_path, 'dummy.py')
config.JobType.sendPythonFolder = True
config.JobType.inputFiles = [
cmssw_py_fn, os.path.join(tool_path, 'fixfjr.py')
]
config.JobType.outputFiles = ['overlay.root']
config.Data.splitting = 'EventBased'
config.Data.unitsPerJob = 1
config.Data.publication = False
config.Site.storageSite = 'T3_US_FNALLPC'
config.Site.whitelist = 'T1_US_*,T2_US_*,T3_US_Baylor,T3_US_FIU,T3_US_FSU,T3_US_JHU,T3_US_Kansas,T3_US_NERSC,T3_US_NU,T3_US_NotreDame,T3_US_OSG,T3_US_OSU,T3_US_Omaha,T3_US_Princeton_ICSE,T3_US_PuertoRico,T3_US_Rice,T3_US_SDSC,T3_US_TAMU,T3_US_TTU,T3_US_UCD,T3_US_UCSB,T3_US_UMD'.split(
',')
for sample in samples:
njobs, per_last = max_njobs[(sample, ntracks)]
sh_vars = {
'per': per,
'njobs': njobs,
'per_m1': per - 1,
'per_last_m1': per_last - 1,
'ntracks': ntracks,
'sample': sample,
'overlay_args': overlay_args,
'cmssw_version': os.environ['CMSSW_VERSION'],
'cmssw_py': cmssw_py,
}
sh_fn = os.path.join(inputs_dir, 'run.%s.sh' % sample)
open(sh_fn, 'wt').write(sh_template % sh_vars)
os.chmod(sh_fn, 0755)
config.General.requestName = '%s_%s' % (batch_name, sample)
config.JobType.scriptExe = sh_fn
config.Data.totalUnits = njobs
print colors.boldwhite('%s, %s' % (batch_name, sample))
if not testing:
output = crab_command('submit', config=config)
print output['stdout']
else:
print config
print
def submit(samples, ntracks, overlay_args, batch_name_ex=''):
testing = 'testing' in sys.argv
batch_name = 'ntk%i' % ntracks
batch_name += batch_name_ex
if testing:
batch_name += '_TEST'
work_area = '/uscms_data/d2/%s/crab_dirs/Overlay%s/%s' % (os.environ['USER'], version, batch_name)
inputs_dir = os.path.join(work_area, 'inputs')
os.makedirs(inputs_dir)
save_git_status(os.path.join(work_area, 'gitstatus'))
cmssw_py = 'overlay.py'
cmssw_py_fn = os.path.join(inputs_dir, cmssw_py)
shutil.copy2(cmssw_py, cmssw_py_fn)
tool_path = os.path.join(os.environ['CMSSW_BASE'], 'src/JMTucker/MFVNeutralino/test/MakeSamples') # JMTBAD put dummy.py and fixfjr.py somewhere better
config = Config()
config.General.transferLogs = True
config.General.transferOutputs = True
config.General.workArea = work_area
config.JobType.pluginName = 'PrivateMC'
config.JobType.psetName = os.path.join(tool_path, 'dummy.py')
config.JobType.sendPythonFolder = True
config.JobType.inputFiles = [cmssw_py_fn, os.path.join(tool_path, 'fixfjr.py')]
config.JobType.outputFiles = ['overlay.root']
config.Data.splitting = 'EventBased'
config.Data.unitsPerJob = 1
config.Data.publication = False
config.Site.storageSite = 'T3_US_FNALLPC'
config.Site.whitelist = 'T1_US_*,T2_US_*,T3_US_Baylor,T3_US_FIU,T3_US_FSU,T3_US_JHU,T3_US_Kansas,T3_US_NERSC,T3_US_NU,T3_US_NotreDame,T3_US_OSG,T3_US_OSU,T3_US_Omaha,T3_US_Princeton_ICSE,T3_US_PuertoRico,T3_US_Rice,T3_US_SDSC,T3_US_TAMU,T3_US_TTU,T3_US_UCD,T3_US_UCSB,T3_US_UMD'.split(',')
for sample in samples:
njobs, per_last = max_njobs[(sample, ntracks)]
sh_vars = {
'per': per,
'njobs': njobs,
'per_m1': per - 1,
'per_last_m1': per_last - 1,
'ntracks': ntracks,
'sample': sample,
'overlay_args': overlay_args,
'cmssw_version': os.environ['CMSSW_VERSION'],
'cmssw_py': cmssw_py,
}
sh_fn = os.path.join(inputs_dir, 'run.%s.sh' % sample)
open(sh_fn, 'wt').write(sh_template % sh_vars)
os.chmod(sh_fn, 0755)
config.General.requestName = '%s_%s' % (batch_name, sample)
config.JobType.scriptExe = sh_fn
config.Data.totalUnits = njobs
print colors.boldwhite('%s, %s' % (batch_name, sample))
if not testing:
output = crab_command('submit', config=config)
print output['stdout']
else:
print config
print
def __init__(
self,
batch_name,
ex='',
testing='testing' in sys.argv or 'cs_testing' in sys.argv,
meat=cmsRun_meat,
pset_template_fn=sys.argv[0],
pset_modifier=None,
input_files=[],
output_files=[],
skip_output_files=[],
stageout_files=[], # can be "all" or "pool"
stageout_path='', # if / in it, does not try to generate
publish_name='',
dataset='main',
jdl_extras='',
_events=-1,
_njobs=None,
_fail=[],
):
self.nsubmits = -1
self.testing = testing
if type(meat) == file:
meat = meat.read()
elif type(meat) == str and os.path.isfile(meat):
meat = open(meat).read()
if 'meatexit=' not in meat:
raise ValueError('meatexit not set in meat?')
self.meat = meat
self.is_cmsRun = meat == self.cmsRun_meat
if '$' in pset_template_fn:
pset_template_fn = os.path.expandvars(pset_template_fn)
if '~' in pset_template_fn:
pset_template_fn = os.path.expanduser(pset_template_fn)
self.pset_template_fn = pset_template_fn
self.pset_modifier = pset_modifier
self.dataset = dataset
self._njobs = _njobs
for arg in sys.argv:
if arg.startswith('cs_name='):
batch_name = arg.replace('cs_name=', '')
break
if not set(self.batch_name_allowed).issuperset(
set(batch_name.replace('/', ''))):
raise ValueError(
'illegal batch name %s, allowed characters are letters, numbers, and _'
% batch_name)
nslash = batch_name.count('/')
if nslash > 1:
raise ValueError('only one slash allowed in batch name %s' %
batch_name)
elif nslash == 1:
batch_path = os.path.abspath(crab_dirs_root(batch_name))
batch_root = os.path.dirname(batch_path)
if os.path.exists(batch_root):
if not os.path.isdir(batch_root):
raise ValueError(
'slashy mode: batch_root %s exists but is not a dir?' %
batch_root)
for x in os.listdir(batch_root):
fx = os.path.join(batch_root, x)
if x == 'gitstatus' or x == 'inputs' or x == 'psets' or not os.path.isdir(
fx):
raise ValueError('bad slashy %s' % batch_path)
else:
os.mkdir(batch_root)
os.mkdir(batch_path)
if ex:
ex = str(ex) + '_'
self.ex_str = 'ex_%s%s' % (ex, str(int(time.time() * 1000)))
self.batch_name = batch_name
self.batch_dir = os.path.abspath(crab_dirs_root(batch_name))
self.inputs_dir = os.path.join(self.batch_dir, self.ex_str, 'inputs')
if os.path.exists(
self.inputs_dir
): # check inputs_dir instead of batch_dir since we might be from metasubmitter
raise ValueError(
'batch_dir %s already exists, refusing to clobber' %
self.batch_dir)
if not testing and self.get_proxy:
#print 'CondorSubmitter init: checking proxies, might ask for password twice (but you can skip it with ^C if you know what you are doing).'
crab_renew_proxy_if_needed()
self.get_proxy = False
#Have different username between UW-Madison and CERN;
#username = os.environ['USER']
username = '******'
self.timestamp = datetime.now()
#os.system('mkdir -p /tmp/%s' % username)
#print 'CondorSubmitter init: saving git status'
save_git_status(os.path.join(self.batch_dir, self.ex_str, 'gitstatus'))
os.mkdir(self.inputs_dir)
self.filelist_fn_pattern = os.path.join(self.inputs_dir, '%(name)s.sh')
self.jdl_fn_pattern = os.path.join(self.inputs_dir, '%(name)s.jdl')
self.pset_fn_pattern = os.path.join(self.inputs_dir, '%(name)s.py')
tarball_fn = os.path.join(self.inputs_dir, 'input.tgz')
#print 'CondorSubmitter init: making cmssw tarball'
make_tarball(tarball_fn, include_python=True, include_interface=True)
sh_fn = os.path.join(self.inputs_dir, 'run.sh')
if input_files:
input_bns = []
input_fns = []
for x in input_files:
bn = os.path.basename(x)
fn = os.path.abspath(os.path.join(self.inputs_dir, bn))
input_bns.append(bn)
input_fns.append(fn)
shutil.copy(x, fn)
input_bns = ',' + ','.join(input_bns)
input_fns = ',' + ','.join(input_fns)
else:
input_bns = ''
input_fns = ''
# JMTBAD if pset_modifier or cmsrun_args modifies the output filenames, we won't catch them
#print 'CondorSubmitter init: importing pset_template fn %s to get output filenames' % pset_template_fn
module_output_files = {}
if self.pset_template_fn:
module = imp.load_source('dummy', self.pset_template_fn)
module_output_files = find_output_files(module.process)
for l in module_output_files.itervalues():
for x in l:
if x not in skip_output_files:
output_files.append(x)
if type(stageout_files) == str:
stageout_which = stageout_files
if stageout_which == 'all':
stageout_files = output_files
elif stageout_which == 'pool':
stageout_files = [
x for x in module_output_files.get('PoolOutputModule', [])
if x not in skip_output_files
]
else:
raise ValueError("don't understand stageout_files = %s" %
stageout_which)
output_files = [x for x in output_files if x not in stageout_files]
for x in output_files + stageout_files:
if os.path.basename(x) != x:
raise ValueError('output file must have no slashes: %r' % x)
self.output_files = output_files = ' '.join(output_files)
self.stageout_files = stageout_files = ' '.join(stageout_files)
output_snippet = ''
if output_files:
#print 'CondorSubmitter init: output files are', output_files
output_snippet += self.output_template.replace(
'__OUTPUT_BNS__', output_files)
if stageout_files:
if '/' not in stageout_path:
stageout_user = username # JMTBAD use getUsernameFromSiteDB?
if stageout_path:
stageout_path = '/' + stageout_path
stageout_path = 'root://cmseos.fnal.gov//store/user/' + stageout_user + stageout_path
if not publish_name:
publish_name = batch_name.replace('/', '_')
stageout_path += '/$(<cs_primaryds)/' + publish_name + '/$(<cs_timestamp)/$(printf "%04i" $(($job/1000)) )'
print 'CondorSubmitter init: stageout files are', stageout_files
print 'CondorSubmitter init: stageout path is', stageout_path
output_snippet += self.stageout_template \
.replace('__STAGEOUT_BNS__', stageout_files) \
.replace('__STAGEOUT_PATH__', stageout_path)
self.sh_template = self.sh_template \
.replace('__INPUT_BNS__', input_bns) \
.replace('__MEAT__', self.meat) \
.replace('__OUTPUT_SNIPPET__', output_snippet)
self.jdl_template = self.jdl_template \
.replace('__SH_FN__', sh_fn) \
.replace('__TARBALL_FN__', tarball_fn) \
.replace('__INPUT_FNS__', input_fns) \
.replace('__EXTRAS__', jdl_extras)
self.pset_end_template = self.pset_end_template \
.replace('__MAX_EVENTS__', str(_events)) \
.replace('__FAIL_LIST__', repr(_fail))
open(sh_fn, 'wt').write(self.sh_template)
def submit(sample, ntracks, overlay_args, njobs=0, testing=False, batch_name_ex=''):
if njobs <= 0:
njobs, per_last = max_njobs[(sample, ntracks)]
else:
per_last = per
njobs_m1 = njobs - 1
per_m1 = per - 1
per_last_m1 = per_last - 1
batch_name = 'ntk%i' % ntracks
if 'deltasvgaus' in overlay_args:
batch_name += '_deltasvgaus'
if 'rest-of-event' in overlay_args:
batch_name += '_wevent'
batch_name += batch_name_ex
if testing:
batch_name += '_TEST'
print batch_name, sample
cmssw_py = 'overlay.py'
batch_dir = '/uscms_data/d2/tucker/overlay/%s/%s' % (batch_name, sample)
inputs_dir = os.path.join(batch_dir, 'inputs')
outputs_dir = os.path.join(batch_dir, 'outputs')
os.makedirs(inputs_dir)
os.makedirs(outputs_dir)
save_git_status(os.path.join(batch_dir, 'gitstatus'))
tarball_fn_base = 'input.tgz'
tarball_fn = os.path.join(inputs_dir, tarball_fn_base)
make_tarball(tarball_fn, include_python=True)
cmssw_py_fn = os.path.join(inputs_dir, cmssw_py)
shutil.copy2(cmssw_py, cmssw_py_fn)
sh_fn = os.path.join(inputs_dir, 'run.sh')
jdl_fn = os.path.join(inputs_dir, 'submit.jdl')
scram_arch = os.environ['SCRAM_ARCH']
cmssw_version = os.environ['CMSSW_VERSION']
sh = dedent('''
#!/bin/bash
workdir=$(pwd)
job=$1
echo date: $(date)
echo job: $job
echo pwd: $workdir
export SCRAM_ARCH=%(scram_arch)s
source /cvmfs/cms.cern.ch/cmsset_default.sh
scram project CMSSW %(cmssw_version)s 2>&1 > /dev/null
scramexit=$?
if [[ $scramexit -ne 0 ]]; then
echo scram exited with code $scramexit
exit $scramexit
fi
cd %(cmssw_version)s
tar xf ${workdir}/%(tarball_fn_base)s
cd src
eval `scram ru -sh`
scram b -j 2 2>&1 > /dev/null
if [[ $job -eq %(njobs_m1)i ]]; then
nev=%(per_last_m1)i
else
nev=%(per_m1)i
fi
set -x
for i in $(seq 0 $nev); do
cmsRun ${workdir}/%(cmssw_py)s +batch +which-event $((job*10+i)) +sample %(sample)s +ntracks %(ntracks)s %(overlay_args)s 2>&1
cmsexit=$?
if [[ $cmsexit -ne 0 ]]; then
echo cmsRun exited with code $cmsexit
exit $cmsexit
fi
done
set +x
hadd overlay.root overlay_*.root 2>&1
haddexit=$?
if [[ $haddexit -ne 0 ]]; then
echo hadd exited with code $haddexit
exit $haddexit
fi
mv overlay.root $workdir/overlay_${job}.root
''' % locals())
open(sh_fn, 'wt').write(sh)
os.chmod(sh_fn, 0755)
jdl = dedent('''
universe = vanilla
Executable = %(sh_fn)s
arguments = $(Process)
Output = overlay_$(Cluster)_$(Process).stdout
Error = overlay_$(Cluster)_$(Process).stderr
Log = overlay_$(Cluster)_$(Process).log
stream_output = false
stream_error = false
notification = never
should_transfer_files = YES
when_to_transfer_output = ON_EXIT
transfer_input_files = %(tarball_fn)s,%(cmssw_py_fn)s
Queue %(njobs)s
''' % locals())
open(jdl_fn, 'wt').write(jdl)
if not testing:
os.chdir(outputs_dir)
os.system('condor_submit < ' + jdl_fn)
os.chdir(pwd)
if os.path.exists(full_output_dir):
sys.exit('%s already exists' % full_output_dir)
####
from JMTucker.Tools.CRAB3ToolsBase import crab_dirs_root
from JMTucker.Tools.Year import year; assert year == 2017
from JMTucker.Tools.general import save_git_status
from JMTucker.Tools import colors
testing = 'testing' in sys.argv
work_area = crab_dirs_root('nstep_gs2reco_condor_%s' % which)
if os.path.isdir(work_area):
sys.exit('work_area %s exists' % work_area)
os.makedirs(work_area)
save_git_status(os.path.join(work_area, 'gitstatus'))
inputs_dir = os.path.join(work_area, 'inputs')
os.mkdir(inputs_dir)
open('year.txt', 'wt').write(str(year))
sh_fn = 'nstep_gs2reco_condor.sh'
input_files = 'todoify.sh rawhlt.py reco.py miniaod.py dynamicconf.py year.txt modify.py inputfns.txt minbias.py minbias_premix.txt.gz'.split()
for fn in [sh_fn] + input_files:
shutil.copy2(fn, inputs_dir)
sh_fn = os.path.join(inputs_dir, sh_fn)
input_files = ','.join([os.path.join(inputs_dir, x) for x in input_files])
todos = ' '.join('todo=' + x for x in todos)
from CRABClient.ClientExceptions import ConfigException as CRABConfigException
from JMTucker.Tools.CRAB3Tools import Config, crab_dirs_root, crab_command
from JMTucker.Tools.general import index_startswith, save_git_status
from JMTucker.Tools.CondorSubmitter import CondorSubmitter
from JMTucker.Tools.Sample import MCSample
from JMTucker.Tools import colors
testing = 'testing' in sys.argv
if ex and not ex.startswith('_'):
ex = '_' + ex
work_area = crab_dirs_root('nstep%s' % ex)
if not os.path.isdir(work_area):
os.makedirs(work_area)
gitstatus_dir = 'gitstatus_%s' % int(time() * 1000)
if not condor:
save_git_status(os.path.join(work_area, gitstatus_dir))
config = Config()
open('year.txt', 'wt').write(str(year))
to_rm = ['year.txt']
if output_level in ('minitree', 'ntuple'):
for x in 'ntuple.py', 'minitree.py':
to_rm.append(x)
os.system('cmsDumpPython.py ../%s > %s' % (x, x))
config.General.transferLogs = False
config.General.transferOutputs = True
config.General.workArea = work_area
def __init__(self,
batch_name,
ex = '',
testing = 'testing' in sys.argv or 'cs_testing' in sys.argv,
meat = cmsRun_meat,
pset_template_fn = sys.argv[0],
pset_modifier = None,
input_files = [],
output_files = [],
skip_output_files = [],
stageout_files = [], # can be "all" or "pool"
stageout_path = '', # if / in it, does not try to generate
publish_name = '',
dataset = 'main',
jdl_extras = '',
_events = -1,
_njobs = None,
_fail = [],
):
self.nsubmits = -1
self.testing = testing
if type(meat) == file:
meat = meat.read()
elif type(meat) == str and os.path.isfile(meat):
meat = open(meat).read()
if 'meatexit=' not in meat:
raise ValueError('meatexit not set in meat?')
self.meat = meat
self.is_cmsRun = meat == self.cmsRun_meat
if '$' in pset_template_fn:
pset_template_fn = os.path.expandvars(pset_template_fn)
if '~' in pset_template_fn:
pset_template_fn = os.path.expanduser(pset_template_fn)
self.pset_template_fn = pset_template_fn
self.pset_modifier = pset_modifier
self.dataset = dataset
self._njobs = _njobs
for arg in sys.argv:
if arg.startswith('cs_name='):
batch_name = arg.replace('cs_name=', '')
break
if not set(self.batch_name_allowed).issuperset(set(batch_name.replace('/', ''))):
raise ValueError('illegal batch name %s, allowed characters are letters, numbers, and _' % batch_name)
nslash = batch_name.count('/')
if nslash > 1:
raise ValueError('only one slash allowed in batch name %s' % batch_name)
elif nslash == 1:
batch_path = os.path.abspath(crab_dirs_root(batch_name))
batch_root = os.path.dirname(batch_path)
if os.path.exists(batch_root):
if not os.path.isdir(batch_root):
raise ValueError('slashy mode: batch_root %s exists but is not a dir?' % batch_root)
for x in os.listdir(batch_root):
fx = os.path.join(batch_root, x)
if x == 'gitstatus' or x == 'inputs' or x == 'psets' or not os.path.isdir(fx):
raise ValueError('bad slashy %s' % batch_path)
else:
os.mkdir(batch_root)
os.mkdir(batch_path)
if ex:
ex = str(ex) + '_'
self.ex_str = 'ex_%s%s' % (ex, str(int(time.time()*1000)))
self.batch_name = batch_name
self.batch_dir = os.path.abspath(crab_dirs_root(batch_name))
self.inputs_dir = os.path.join(self.batch_dir, self.ex_str, 'inputs')
if os.path.exists(self.inputs_dir): # check inputs_dir instead of batch_dir since we might be from metasubmitter
raise ValueError('batch_dir %s already exists, refusing to clobber' % self.batch_dir)
if not os.path.isdir(self.links_dir):
os.mkdir(self.links_dir)
for schedd in self.schedds:
schedd_d = os.path.join(self.links_dir, schedd)
if not os.path.isdir(schedd_d):
os.mkdir(schedd_d)
if not testing and self.get_proxy:
#print 'CondorSubmitter init: checking proxies, might ask for password twice (but you can skip it with ^C if you know what you are doing).'
crab_renew_proxy_if_needed()
self.get_proxy = False
username = os.environ['USER']
self.timestamp = datetime.now()
#os.system('mkdir -p /tmp/%s' % username)
#print 'CondorSubmitter init: saving git status'
save_git_status(os.path.join(self.batch_dir, self.ex_str, 'gitstatus'))
os.mkdir(self.inputs_dir)
self.filelist_fn_pattern = os.path.join(self.inputs_dir, '%(name)s.sh')
self.jdl_fn_pattern = os.path.join(self.inputs_dir, '%(name)s.jdl')
self.pset_fn_pattern = os.path.join(self.inputs_dir, '%(name)s.py')
tarball_fn = os.path.join(self.inputs_dir, 'input.tgz')
#print 'CondorSubmitter init: making cmssw tarball'
make_tarball(tarball_fn, include_python=True, include_interface=True)
sh_fn = os.path.join(self.inputs_dir, 'run.sh')
if input_files:
input_bns = []
input_fns = []
for x in input_files:
bn = os.path.basename(x)
fn = os.path.abspath(os.path.join(self.inputs_dir, bn))
input_bns.append(bn)
input_fns.append(fn)
shutil.copy(x, fn)
input_bns = ',' + ','.join(input_bns)
input_fns = ',' + ','.join(input_fns)
else:
input_bns = ''
input_fns = ''
# JMTBAD if pset_modifier or cmsrun_args modifies the output filenames, we won't catch them
#print 'CondorSubmitter init: importing pset_template fn %s to get output filenames' % pset_template_fn
module_output_files = {}
if self.pset_template_fn:
module = imp.load_source('dummy', self.pset_template_fn)
module_output_files = find_output_files(module.process)
for l in module_output_files.itervalues():
for x in l:
if x not in skip_output_files:
output_files.append(x)
if type(stageout_files) == str:
stageout_which = stageout_files
if stageout_which == 'all':
stageout_files = output_files
elif stageout_which == 'pool':
stageout_files = [x for x in module_output_files.get('PoolOutputModule', []) if x not in skip_output_files]
else:
raise ValueError("don't understand stageout_files = %s" % stageout_which)
output_files = [x for x in output_files if x not in stageout_files]
assert all(os.path.basename(x) == x for x in output_files + stageout_files)
self.output_files = output_files = ' '.join(output_files)
self.stageout_files = stageout_files = ' '.join(stageout_files)
output_snippet = ''
if output_files:
#print 'CondorSubmitter init: output files are', output_files
output_snippet += self.output_template.replace('__OUTPUT_BNS__', output_files)
if stageout_files:
if '/' not in stageout_path:
stageout_user = username # JMTBAD use getUsernameFromSiteDB?
if stageout_path:
stageout_path = '/' + stageout_path
stageout_path = 'root://cmseos.fnal.gov//store/user/' + stageout_user + stageout_path
if not publish_name:
publish_name = batch_name.replace('/', '_')
stageout_path += '/$(<cs_primaryds)/' + publish_name + '/$(<cs_timestamp)/$(printf "%04i" $(($job/1000)) )'
#print 'CondorSubmitter init: stageout files are', stageout_files
#print 'CondorSubmitter init: stageout path is', stageout_path
output_snippet += self.stageout_template \
.replace('__STAGEOUT_BNS__', stageout_files) \
.replace('__STAGEOUT_PATH__', stageout_path)
self.sh_template = self.sh_template \
.replace('__INPUT_BNS__', input_bns) \
.replace('__MEAT__', self.meat) \
.replace('__OUTPUT_SNIPPET__', output_snippet)
self.jdl_template = self.jdl_template \
.replace('__SH_FN__', sh_fn) \
.replace('__TARBALL_FN__', tarball_fn) \
.replace('__INPUT_FNS__', input_fns) \
.replace('__EXTRAS__', jdl_extras)
self.pset_end_template = self.pset_end_template \
.replace('__MAX_EVENTS__', str(_events)) \
.replace('__FAIL_LIST__', repr(_fail))
open(sh_fn, 'wt').write(self.sh_template)
def submit(njobs, template_type, min_ntracks, signal_sample, template_signal, samples):
global maked
global setuped
global i_submit
if not maked:
if os.system("make clean; make -j 16") != 0:
raise "no make"
raw_input("did the make go OK?")
maked = True
extra_name = ""
batch_root = "crab3/One2Two"
for x in sys.argv:
if x.startswith(batch_root):
extra_name = x.replace(batch_root + "_", "") + "_"
batch_root = x
break
dummy_pset_fn = os.path.join(batch_root, "dummy_pset.py")
files_needed = set()
files_needed.add(os.path.join(os.path.abspath("mfvo2t.exe")))
files_needed.add(os.path.join(os.path.abspath("filtertee.py")))
if not setuped:
os.system("mkdir -p %s" % batch_root)
save_git_status(os.path.join(batch_root, "gitstatus"))
open(dummy_pset_fn, "wt").write(dummy_pset)
setuped = True
batch_name = "%sTmp%s_Ntk%i_SigTmp%s_SigSam%s_Sam%s" % (
extra_name,
template_type,
min_ntracks,
template_signal,
"no" if signal_sample is None else "n%ix%i" % signal_sample,
samples,
)
files_needed.add("MultiJetPk2012.root.gz")
if template_signal > -100:
files_needed.add("backgrounds.tgz")
files_needed.add(sample_number_to_name[template_signal] + ".root.gz")
else:
files_needed.add("bigsigscan.root.gz")
extra_setup = ""
env = ["toythrower_min_ntracks=%i" % min_ntracks, "toythrower_template_signal=%i" % template_signal]
if no_save_plots:
env += ["templates_save_plots=0", "fitter_save_plots=0"]
if template_signal <= -101:
env.append("toythrower_use_only_data_sample=1")
env.append("sig_from_file_num=%i" % template_signal)
if template_signal >= -12 or (template_signal <= -101 and template_signal >= -126):
env.append("fitter_sig_limit_step=1")
if template_type == "PS":
env.append("templates_kind=phishift")
env.append("fitter_start_nuis0=2")
env.append("fitter_start_nuis1=0.003")
elif template_type == "SC":
env.append("templates_kind=simpleclear")
env.append("fitter_fix_nuis1=1")
env.append("fitter_start_nuis0=10")
env.append("fitter_start_nuis1=0")
if signal_sample is not None:
sig_samp, sig_scale = signal_sample
assert sig_samp < 0
if sig_samp > -100:
files_needed.add(sample_number_to_name[sig_samp] + ".root.gz")
else:
files_needed.add("bigsigscan.root.gz")
if sig_scale < 0:
env.append("ntoys=0")
env.append("process_data=1")
if sig_scale == -2:
env.append("seed=1")
env.append("fitter_i_limit_job=$JOB_NUM")
env.append("fitter_do_signif=0")
else:
assert njobs <= 20
else:
env.append("fitter_do_limit=0")
env.append("toythrower_injected_signal=%i" % sig_samp)
env.append("toythrower_injected_signal_scale=%f" % sig_scale)
else:
env.append("fitter_do_limit=0")
if type(samples) == int:
env.append("toythrower_sample_only=%i" % samples)
elif type(samples) == str and "500" in samples:
env.append("toythrower_use_qcd500=1")
elif "ttbarsyst" in samples:
which_syst = samples.replace("ttbarsyst", "")
files_needed.add("myttbar%s.root.gz" % which_syst)
extra_setup += (
"""
cd trees
ln -s myttbar%s.root bkgsyst.root
cd -
"""
% which_syst
)
env.append("toythrower_sample_only=99")
###
env = "\n".join("export mfvo2t_" + e for e in env)
unzip_files = []
for fn in files_needed:
if fn.endswith(".tgz"):
unzip_files.append("tar zxf %s" % fn)
elif fn.endswith(".gz"):
unzip_files.append("gunzip %s" % fn)
unzip_files = "\n".join(unzip_files)
tree_path = (
"/eos/uscms/store/user/tucker/mfvo2t_all_trees_444de711cdc630ddfe7cb6cd8f64ec8b46d09990_plussomettbarsyst"
)
files_needed = ", ".join('"%s"' % os.path.join(tree_path, f) for f in files_needed)
submit_dir = os.path.join(crab_submit_dir, "%05i" % i_submit)
os.mkdir(submit_dir)
os.symlink(os.readlink("crab3"), os.path.join(submit_dir, "crab3"))
open(os.path.join(submit_dir, "runme.sh"), "wt").write(script_template % locals())
open(os.path.join(submit_dir, "crabConfig.py"), "wt").write(crab_cfg % locals())
i_submit += 1
from time import time
from CRABClient.ClientExceptions import ConfigException as CRABConfigException
from JMTucker.Tools.CRAB3Tools import Config, crab_dirs_root, crab_command
from JMTucker.Tools.general import index_startswith, save_git_status
from JMTucker.Tools.CondorSubmitter import CondorSubmitter
from JMTucker.Tools.Sample import MCSample
from JMTucker.Tools import colors
testing = 'testing' in sys.argv
work_area = crab_dirs_root('nstep_%s%s' % (meta, ex))
if os.path.isdir(work_area):
sys.exit('work_area %s exists' % work_area)
os.makedirs(work_area)
gitstatus_dir = 'gitstatus_%s' % int(time()*1000)
if not condor:
save_git_status(os.path.join(work_area, gitstatus_dir))
config = Config()
open('year.txt', 'wt').write(str(year))
to_rm = ['year.txt']
if output_level in ('minitree', 'ntuple'):
for x in 'ntuple.py', 'minitree.py':
to_rm.append(x)
os.system('cmsDumpPython.py ../%s > %s' % (x,x))
config.General.transferLogs = False
config.General.transferOutputs = True
config.General.workArea = work_area
def submit(sample, testing=False):
per = 10
batch_name = sample
file_list_fn = os.path.abspath('filelist.%s.gz' % sample)
event_list_fn = os.path.abspath('vetolist.%s.gz' % sample)
nfiles = sum(1 for line in gzip.open(file_list_fn))
njobs = int_ceil(nfiles, per)
if testing:
batch_name += '_TEST'
njobs = 1
print batch_name
cmssw_py = 'pick_by_veto.py'
root_name = 'pick1vtx'
batch_root = os.path.join('/uscms_data/d2/tucker/condor_batch', root_name)
batch_dir = os.path.abspath(os.path.join(batch_root, batch_name))
inputs_dir = os.path.join(batch_dir, 'inputs')
outputs_dir = os.path.join(batch_dir, 'outputs')
os.makedirs(inputs_dir)
os.makedirs(outputs_dir)
save_git_status(os.path.join(batch_dir, 'gitstatus'))
tarball_fn_base = 'input.tgz'
tarball_fn = os.path.join(inputs_dir, tarball_fn_base)
make_tarball(tarball_fn, include_python=True)
cmssw_py_fn = os.path.join(inputs_dir, cmssw_py)
shutil.copy2(cmssw_py, cmssw_py_fn)
sh_fn = os.path.join(inputs_dir, 'run.sh')
jdl_fn = os.path.join(inputs_dir, 'submit.jdl')
scram_arch = os.environ['SCRAM_ARCH']
cmssw_version = os.environ['CMSSW_VERSION']
out_xrd = 'root://cmseos.fnal.gov/'
out_path = '/store/user/tucker/pick1vtx/%s' % sample
if os.system('eos %s mkdir -p %s' % (out_xrd, out_path)) != 0:
raise IOError('problem making %s' % out_path)
if any(os.system('eos %s ls %s/pick_%i.root 2>&1 > /dev/null' % (out_xrd, out_path, i)) == 0 for i in xrange(njobs)):
raise IOError('one or more output files already exist')
sh = dedent('''
#!/bin/bash
workdir=$(pwd)
job=$1
echo date: $(date)
echo job: $job
echo pwd: $workdir
export SCRAM_ARCH=%(scram_arch)s
source /cvmfs/cms.cern.ch/cmsset_default.sh
scram project CMSSW %(cmssw_version)s 2>&1
cd %(cmssw_version)s
tar xvf ${workdir}/%(tarball_fn_base)s
cd src
eval `scram ru -sh`
scram b -j 2
cd $workdir
cmsRun %(cmssw_py)s +job $job +per %(per)s +sample %(sample)s 2>&1
cmsexit=$?
if [[ $cmsexit -ne 0 ]]; then
echo cmsRun exited with code $cmsexit
exit $cmsexit
fi
new_fn=${out_xrd}${out_path}/pick_${job}.root
echo xrdcp to $new_fn
xrdcp pick.root $new_fn
xrdcpexit=$?
if [[ $xrdcpexit -ne 0 ]]; then
echo xrdcp exited with code $xrdcpexit
exit $xrdcpexit
fi
''' % locals())
open(sh_fn, 'wt').write(sh)
os.chmod(sh_fn, 0755)
jdl = dedent('''
universe = vanilla
Executable = %(sh_fn)s
arguments = $(Process)
Output = %(root_name)s_%(batch_name)s_$(Cluster)_$(Process).stdout
Error = %(root_name)s_%(batch_name)s_$(Cluster)_$(Process).stderr
Log = %(root_name)s_%(batch_name)s_$(Cluster)_$(Process).log
stream_output = false
stream_error = false
notification = never
should_transfer_files = YES
when_to_transfer_output = ON_EXIT
transfer_input_files = %(tarball_fn)s,%(cmssw_py_fn)s,%(file_list_fn)s,%(event_list_fn)s
Queue %(njobs)s
''' % locals())
open(jdl_fn, 'wt').write(jdl)
if not testing or testing == 'run':
os.chdir(outputs_dir)
os.system('condor_submit < ' + jdl_fn)
os.chdir(pwd)