def __init__(
self,
batch_name,
ex='',
testing='testing' in sys.argv or 'cs_testing' in sys.argv,
meat=cmsRun_meat,
pset_template_fn=sys.argv[0],
pset_modifier=None,
input_files=[],
output_files=[],
skip_output_files=[],
stageout_files=[], # can be "all" or "pool"
stageout_path='', # if / in it, does not try to generate
publish_name='',
dataset='main',
jdl_extras='',
_events=-1,
_njobs=None,
_fail=[],
):
self.nsubmits = -1
self.testing = testing
if type(meat) == file:
meat = meat.read()
elif type(meat) == str and os.path.isfile(meat):
meat = open(meat).read()
if 'meatexit=' not in meat:
raise ValueError('meatexit not set in meat?')
self.meat = meat
self.is_cmsRun = meat == self.cmsRun_meat
if '$' in pset_template_fn:
pset_template_fn = os.path.expandvars(pset_template_fn)
if '~' in pset_template_fn:
pset_template_fn = os.path.expanduser(pset_template_fn)
self.pset_template_fn = pset_template_fn
self.pset_modifier = pset_modifier
self.dataset = dataset
self._njobs = _njobs
for arg in sys.argv:
if arg.startswith('cs_name='):
batch_name = arg.replace('cs_name=', '')
break
if not set(self.batch_name_allowed).issuperset(
set(batch_name.replace('/', ''))):
raise ValueError(
'illegal batch name %s, allowed characters are letters, numbers, and _'
% batch_name)
nslash = batch_name.count('/')
if nslash > 1:
raise ValueError('only one slash allowed in batch name %s' %
batch_name)
elif nslash == 1:
batch_path = os.path.abspath(crab_dirs_root(batch_name))
batch_root = os.path.dirname(batch_path)
if os.path.exists(batch_root):
if not os.path.isdir(batch_root):
raise ValueError(
'slashy mode: batch_root %s exists but is not a dir?' %
batch_root)
for x in os.listdir(batch_root):
fx = os.path.join(batch_root, x)
if x == 'gitstatus' or x == 'inputs' or x == 'psets' or not os.path.isdir(
fx):
raise ValueError('bad slashy %s' % batch_path)
else:
os.mkdir(batch_root)
os.mkdir(batch_path)
if ex:
ex = str(ex) + '_'
self.ex_str = 'ex_%s%s' % (ex, str(int(time.time() * 1000)))
self.batch_name = batch_name
self.batch_dir = os.path.abspath(crab_dirs_root(batch_name))
self.inputs_dir = os.path.join(self.batch_dir, self.ex_str, 'inputs')
if os.path.exists(
self.inputs_dir
): # check inputs_dir instead of batch_dir since we might be from metasubmitter
raise ValueError(
'batch_dir %s already exists, refusing to clobber' %
self.batch_dir)
if not testing and self.get_proxy:
#print 'CondorSubmitter init: checking proxies, might ask for password twice (but you can skip it with ^C if you know what you are doing).'
crab_renew_proxy_if_needed()
self.get_proxy = False
#Have different username between UW-Madison and CERN;
#username = os.environ['USER']
username = '******'
self.timestamp = datetime.now()
#os.system('mkdir -p /tmp/%s' % username)
#print 'CondorSubmitter init: saving git status'
save_git_status(os.path.join(self.batch_dir, self.ex_str, 'gitstatus'))
os.mkdir(self.inputs_dir)
self.filelist_fn_pattern = os.path.join(self.inputs_dir, '%(name)s.sh')
self.jdl_fn_pattern = os.path.join(self.inputs_dir, '%(name)s.jdl')
self.pset_fn_pattern = os.path.join(self.inputs_dir, '%(name)s.py')
tarball_fn = os.path.join(self.inputs_dir, 'input.tgz')
#print 'CondorSubmitter init: making cmssw tarball'
make_tarball(tarball_fn, include_python=True, include_interface=True)
sh_fn = os.path.join(self.inputs_dir, 'run.sh')
if input_files:
input_bns = []
input_fns = []
for x in input_files:
bn = os.path.basename(x)
fn = os.path.abspath(os.path.join(self.inputs_dir, bn))
input_bns.append(bn)
input_fns.append(fn)
shutil.copy(x, fn)
input_bns = ',' + ','.join(input_bns)
input_fns = ',' + ','.join(input_fns)
else:
input_bns = ''
input_fns = ''
# JMTBAD if pset_modifier or cmsrun_args modifies the output filenames, we won't catch them
#print 'CondorSubmitter init: importing pset_template fn %s to get output filenames' % pset_template_fn
module_output_files = {}
if self.pset_template_fn:
module = imp.load_source('dummy', self.pset_template_fn)
module_output_files = find_output_files(module.process)
for l in module_output_files.itervalues():
for x in l:
if x not in skip_output_files:
output_files.append(x)
if type(stageout_files) == str:
stageout_which = stageout_files
if stageout_which == 'all':
stageout_files = output_files
elif stageout_which == 'pool':
stageout_files = [
x for x in module_output_files.get('PoolOutputModule', [])
if x not in skip_output_files
]
else:
raise ValueError("don't understand stageout_files = %s" %
stageout_which)
output_files = [x for x in output_files if x not in stageout_files]
for x in output_files + stageout_files:
if os.path.basename(x) != x:
raise ValueError('output file must have no slashes: %r' % x)
self.output_files = output_files = ' '.join(output_files)
self.stageout_files = stageout_files = ' '.join(stageout_files)
output_snippet = ''
if output_files:
#print 'CondorSubmitter init: output files are', output_files
output_snippet += self.output_template.replace(
'__OUTPUT_BNS__', output_files)
if stageout_files:
if '/' not in stageout_path:
stageout_user = username # JMTBAD use getUsernameFromSiteDB?
if stageout_path:
stageout_path = '/' + stageout_path
stageout_path = 'root://cmseos.fnal.gov//store/user/' + stageout_user + stageout_path
if not publish_name:
publish_name = batch_name.replace('/', '_')
stageout_path += '/$(<cs_primaryds)/' + publish_name + '/$(<cs_timestamp)/$(printf "%04i" $(($job/1000)) )'
print 'CondorSubmitter init: stageout files are', stageout_files
print 'CondorSubmitter init: stageout path is', stageout_path
output_snippet += self.stageout_template \
.replace('__STAGEOUT_BNS__', stageout_files) \
.replace('__STAGEOUT_PATH__', stageout_path)
self.sh_template = self.sh_template \
.replace('__INPUT_BNS__', input_bns) \
.replace('__MEAT__', self.meat) \
.replace('__OUTPUT_SNIPPET__', output_snippet)
self.jdl_template = self.jdl_template \
.replace('__SH_FN__', sh_fn) \
.replace('__TARBALL_FN__', tarball_fn) \
.replace('__INPUT_FNS__', input_fns) \
.replace('__EXTRAS__', jdl_extras)
self.pset_end_template = self.pset_end_template \
.replace('__MAX_EVENTS__', str(_events)) \
.replace('__FAIL_LIST__', repr(_fail))
open(sh_fn, 'wt').write(self.sh_template)
#!/usr/bin/env python
import sys, os, argparse
from JMTucker.Tools.general import typed_from_argv
from JMTucker.Tools.CMSSWTools import make_tarball
parser = argparse.ArgumentParser(description = 'cmsMakeTarball.py: make a tarball suitable for distribution to grid sites a la crab')
parser.add_argument('tarball_fn', help='The desired tarball fn.')
parser.add_argument('--verbose', action='store_true', help='Turn on verbosity.')
parser.add_argument('--include-bin', action='store_true', help='Whether to include the bin dirs.')
parser.add_argument('--no-include-python', action='store_false', dest='include_python', help='Whether to include the python dirs.')
parser.add_argument('--no-include-interface', action='store_false', dest='include_interface', help='Whether to include the interface dirs.')
options = parser.parse_args()
if os.path.exists(options.tarball_fn):
raise IOError('refusing to clobber %s' % options.tarball_fn)
make_tarball(options.tarball_fn,
verbose=options.verbose,
include_bin=options.include_bin,
include_python=options.include_python,
include_interface=options.include_interface)
#!/usr/bin/env python
import sys, os, argparse
from JMTucker.Tools.CMSSWTools import make_tarball
parser = argparse.ArgumentParser(description = 'cmsMakeTarball.py: make a tarball suitable for distribution to grid sites a la crab')
parser.add_argument('tarball_fn', help='The desired tarball fn.')
parser.add_argument('--verbose', action='store_true', help='Turn on verbosity.')
parser.add_argument('--include-bin', action='store_true', help='Whether to include the bin dirs.')
parser.add_argument('--no-include-python', action='store_false', dest='include_python', help='Whether to include the python dirs.')
parser.add_argument('--no-include-interface', action='store_false', dest='include_interface', help='Whether to include the interface dirs.')
parser.add_argument('--standalone', action='store_true', help='Instead of running make_tarball, dump this script to stdout in standalone version. (You still have to supply a dummy arg for the tarball name.)')
options = parser.parse_args()
if options.standalone:
import inspect
script = inspect.getsource(inspect.getmodule(inspect.currentframe()))
fcn = inspect.getsource(make_tarball)
print script.replace('from JMTucker.Tools.CMSSWTools import make_tarball\n', fcn)
else:
if os.path.exists(options.tarball_fn):
raise IOError('refusing to clobber %s' % options.tarball_fn)
make_tarball(options.tarball_fn,
verbose=options.verbose,
include_bin=options.include_bin,
include_python=options.include_python,
include_interface=options.include_interface)
def submit(sample, ntracks, overlay_args, njobs=0, testing=False, batch_name_ex=''):
if njobs <= 0:
njobs, per_last = max_njobs[(sample, ntracks)]
else:
per_last = per
njobs_m1 = njobs - 1
per_m1 = per - 1
per_last_m1 = per_last - 1
batch_name = 'ntk%i' % ntracks
if 'deltasvgaus' in overlay_args:
batch_name += '_deltasvgaus'
if 'rest-of-event' in overlay_args:
batch_name += '_wevent'
batch_name += batch_name_ex
if testing:
batch_name += '_TEST'
print batch_name, sample
cmssw_py = 'overlay.py'
batch_dir = '/uscms_data/d2/tucker/overlay/%s/%s' % (batch_name, sample)
inputs_dir = os.path.join(batch_dir, 'inputs')
outputs_dir = os.path.join(batch_dir, 'outputs')
os.makedirs(inputs_dir)
os.makedirs(outputs_dir)
save_git_status(os.path.join(batch_dir, 'gitstatus'))
tarball_fn_base = 'input.tgz'
tarball_fn = os.path.join(inputs_dir, tarball_fn_base)
make_tarball(tarball_fn, include_python=True)
cmssw_py_fn = os.path.join(inputs_dir, cmssw_py)
shutil.copy2(cmssw_py, cmssw_py_fn)
sh_fn = os.path.join(inputs_dir, 'run.sh')
jdl_fn = os.path.join(inputs_dir, 'submit.jdl')
scram_arch = os.environ['SCRAM_ARCH']
cmssw_version = os.environ['CMSSW_VERSION']
sh = dedent('''
#!/bin/bash
workdir=$(pwd)
job=$1
echo date: $(date)
echo job: $job
echo pwd: $workdir
export SCRAM_ARCH=%(scram_arch)s
source /cvmfs/cms.cern.ch/cmsset_default.sh
scram project CMSSW %(cmssw_version)s 2>&1 > /dev/null
scramexit=$?
if [[ $scramexit -ne 0 ]]; then
echo scram exited with code $scramexit
exit $scramexit
fi
cd %(cmssw_version)s
tar xf ${workdir}/%(tarball_fn_base)s
cd src
eval `scram ru -sh`
scram b -j 2 2>&1 > /dev/null
if [[ $job -eq %(njobs_m1)i ]]; then
nev=%(per_last_m1)i
else
nev=%(per_m1)i
fi
set -x
for i in $(seq 0 $nev); do
cmsRun ${workdir}/%(cmssw_py)s +batch +which-event $((job*10+i)) +sample %(sample)s +ntracks %(ntracks)s %(overlay_args)s 2>&1
cmsexit=$?
if [[ $cmsexit -ne 0 ]]; then
echo cmsRun exited with code $cmsexit
exit $cmsexit
fi
done
set +x
hadd overlay.root overlay_*.root 2>&1
haddexit=$?
if [[ $haddexit -ne 0 ]]; then
echo hadd exited with code $haddexit
exit $haddexit
fi
mv overlay.root $workdir/overlay_${job}.root
''' % locals())
open(sh_fn, 'wt').write(sh)
os.chmod(sh_fn, 0755)
jdl = dedent('''
universe = vanilla
Executable = %(sh_fn)s
arguments = $(Process)
Output = overlay_$(Cluster)_$(Process).stdout
Error = overlay_$(Cluster)_$(Process).stderr
Log = overlay_$(Cluster)_$(Process).log
stream_output = false
stream_error = false
notification = never
should_transfer_files = YES
when_to_transfer_output = ON_EXIT
transfer_input_files = %(tarball_fn)s,%(cmssw_py_fn)s
Queue %(njobs)s
''' % locals())
open(jdl_fn, 'wt').write(jdl)
if not testing:
os.chdir(outputs_dir)
os.system('condor_submit < ' + jdl_fn)
os.chdir(pwd)
def __init__(self,
batch_name,
ex = '',
testing = 'testing' in sys.argv or 'cs_testing' in sys.argv,
meat = cmsRun_meat,
pset_template_fn = sys.argv[0],
pset_modifier = None,
input_files = [],
output_files = [],
skip_output_files = [],
stageout_files = [], # can be "all" or "pool"
stageout_path = '', # if / in it, does not try to generate
publish_name = '',
dataset = 'main',
jdl_extras = '',
_events = -1,
_njobs = None,
_fail = [],
):
self.nsubmits = -1
self.testing = testing
if type(meat) == file:
meat = meat.read()
elif type(meat) == str and os.path.isfile(meat):
meat = open(meat).read()
if 'meatexit=' not in meat:
raise ValueError('meatexit not set in meat?')
self.meat = meat
self.is_cmsRun = meat == self.cmsRun_meat
if '$' in pset_template_fn:
pset_template_fn = os.path.expandvars(pset_template_fn)
if '~' in pset_template_fn:
pset_template_fn = os.path.expanduser(pset_template_fn)
self.pset_template_fn = pset_template_fn
self.pset_modifier = pset_modifier
self.dataset = dataset
self._njobs = _njobs
for arg in sys.argv:
if arg.startswith('cs_name='):
batch_name = arg.replace('cs_name=', '')
break
if not set(self.batch_name_allowed).issuperset(set(batch_name.replace('/', ''))):
raise ValueError('illegal batch name %s, allowed characters are letters, numbers, and _' % batch_name)
nslash = batch_name.count('/')
if nslash > 1:
raise ValueError('only one slash allowed in batch name %s' % batch_name)
elif nslash == 1:
batch_path = os.path.abspath(crab_dirs_root(batch_name))
batch_root = os.path.dirname(batch_path)
if os.path.exists(batch_root):
if not os.path.isdir(batch_root):
raise ValueError('slashy mode: batch_root %s exists but is not a dir?' % batch_root)
for x in os.listdir(batch_root):
fx = os.path.join(batch_root, x)
if x == 'gitstatus' or x == 'inputs' or x == 'psets' or not os.path.isdir(fx):
raise ValueError('bad slashy %s' % batch_path)
else:
os.mkdir(batch_root)
os.mkdir(batch_path)
if ex:
ex = str(ex) + '_'
self.ex_str = 'ex_%s%s' % (ex, str(int(time.time()*1000)))
self.batch_name = batch_name
self.batch_dir = os.path.abspath(crab_dirs_root(batch_name))
self.inputs_dir = os.path.join(self.batch_dir, self.ex_str, 'inputs')
if os.path.exists(self.inputs_dir): # check inputs_dir instead of batch_dir since we might be from metasubmitter
raise ValueError('batch_dir %s already exists, refusing to clobber' % self.batch_dir)
if not os.path.isdir(self.links_dir):
os.mkdir(self.links_dir)
for schedd in self.schedds:
schedd_d = os.path.join(self.links_dir, schedd)
if not os.path.isdir(schedd_d):
os.mkdir(schedd_d)
if not testing and self.get_proxy:
#print 'CondorSubmitter init: checking proxies, might ask for password twice (but you can skip it with ^C if you know what you are doing).'
crab_renew_proxy_if_needed()
self.get_proxy = False
username = os.environ['USER']
self.timestamp = datetime.now()
#os.system('mkdir -p /tmp/%s' % username)
#print 'CondorSubmitter init: saving git status'
save_git_status(os.path.join(self.batch_dir, self.ex_str, 'gitstatus'))
os.mkdir(self.inputs_dir)
self.filelist_fn_pattern = os.path.join(self.inputs_dir, '%(name)s.sh')
self.jdl_fn_pattern = os.path.join(self.inputs_dir, '%(name)s.jdl')
self.pset_fn_pattern = os.path.join(self.inputs_dir, '%(name)s.py')
tarball_fn = os.path.join(self.inputs_dir, 'input.tgz')
#print 'CondorSubmitter init: making cmssw tarball'
make_tarball(tarball_fn, include_python=True, include_interface=True)
sh_fn = os.path.join(self.inputs_dir, 'run.sh')
if input_files:
input_bns = []
input_fns = []
for x in input_files:
bn = os.path.basename(x)
fn = os.path.abspath(os.path.join(self.inputs_dir, bn))
input_bns.append(bn)
input_fns.append(fn)
shutil.copy(x, fn)
input_bns = ',' + ','.join(input_bns)
input_fns = ',' + ','.join(input_fns)
else:
input_bns = ''
input_fns = ''
# JMTBAD if pset_modifier or cmsrun_args modifies the output filenames, we won't catch them
#print 'CondorSubmitter init: importing pset_template fn %s to get output filenames' % pset_template_fn
module_output_files = {}
if self.pset_template_fn:
module = imp.load_source('dummy', self.pset_template_fn)
module_output_files = find_output_files(module.process)
for l in module_output_files.itervalues():
for x in l:
if x not in skip_output_files:
output_files.append(x)
if type(stageout_files) == str:
stageout_which = stageout_files
if stageout_which == 'all':
stageout_files = output_files
elif stageout_which == 'pool':
stageout_files = [x for x in module_output_files.get('PoolOutputModule', []) if x not in skip_output_files]
else:
raise ValueError("don't understand stageout_files = %s" % stageout_which)
output_files = [x for x in output_files if x not in stageout_files]
assert all(os.path.basename(x) == x for x in output_files + stageout_files)
self.output_files = output_files = ' '.join(output_files)
self.stageout_files = stageout_files = ' '.join(stageout_files)
output_snippet = ''
if output_files:
#print 'CondorSubmitter init: output files are', output_files
output_snippet += self.output_template.replace('__OUTPUT_BNS__', output_files)
if stageout_files:
if '/' not in stageout_path:
stageout_user = username # JMTBAD use getUsernameFromSiteDB?
if stageout_path:
stageout_path = '/' + stageout_path
stageout_path = 'root://cmseos.fnal.gov//store/user/' + stageout_user + stageout_path
if not publish_name:
publish_name = batch_name.replace('/', '_')
stageout_path += '/$(<cs_primaryds)/' + publish_name + '/$(<cs_timestamp)/$(printf "%04i" $(($job/1000)) )'
#print 'CondorSubmitter init: stageout files are', stageout_files
#print 'CondorSubmitter init: stageout path is', stageout_path
output_snippet += self.stageout_template \
.replace('__STAGEOUT_BNS__', stageout_files) \
.replace('__STAGEOUT_PATH__', stageout_path)
self.sh_template = self.sh_template \
.replace('__INPUT_BNS__', input_bns) \
.replace('__MEAT__', self.meat) \
.replace('__OUTPUT_SNIPPET__', output_snippet)
self.jdl_template = self.jdl_template \
.replace('__SH_FN__', sh_fn) \
.replace('__TARBALL_FN__', tarball_fn) \
.replace('__INPUT_FNS__', input_fns) \
.replace('__EXTRAS__', jdl_extras)
self.pset_end_template = self.pset_end_template \
.replace('__MAX_EVENTS__', str(_events)) \
.replace('__FAIL_LIST__', repr(_fail))
open(sh_fn, 'wt').write(self.sh_template)
def submit(sample, testing=False):
per = 10
batch_name = sample
file_list_fn = os.path.abspath('filelist.%s.gz' % sample)
event_list_fn = os.path.abspath('vetolist.%s.gz' % sample)
nfiles = sum(1 for line in gzip.open(file_list_fn))
njobs = int_ceil(nfiles, per)
if testing:
batch_name += '_TEST'
njobs = 1
print batch_name
cmssw_py = 'pick_by_veto.py'
root_name = 'pick1vtx'
batch_root = os.path.join('/uscms_data/d2/tucker/condor_batch', root_name)
batch_dir = os.path.abspath(os.path.join(batch_root, batch_name))
inputs_dir = os.path.join(batch_dir, 'inputs')
outputs_dir = os.path.join(batch_dir, 'outputs')
os.makedirs(inputs_dir)
os.makedirs(outputs_dir)
save_git_status(os.path.join(batch_dir, 'gitstatus'))
tarball_fn_base = 'input.tgz'
tarball_fn = os.path.join(inputs_dir, tarball_fn_base)
make_tarball(tarball_fn, include_python=True)
cmssw_py_fn = os.path.join(inputs_dir, cmssw_py)
shutil.copy2(cmssw_py, cmssw_py_fn)
sh_fn = os.path.join(inputs_dir, 'run.sh')
jdl_fn = os.path.join(inputs_dir, 'submit.jdl')
scram_arch = os.environ['SCRAM_ARCH']
cmssw_version = os.environ['CMSSW_VERSION']
out_xrd = 'root://cmseos.fnal.gov/'
out_path = '/store/user/tucker/pick1vtx/%s' % sample
if os.system('eos %s mkdir -p %s' % (out_xrd, out_path)) != 0:
raise IOError('problem making %s' % out_path)
if any(os.system('eos %s ls %s/pick_%i.root 2>&1 > /dev/null' % (out_xrd, out_path, i)) == 0 for i in xrange(njobs)):
raise IOError('one or more output files already exist')
sh = dedent('''
#!/bin/bash
workdir=$(pwd)
job=$1
echo date: $(date)
echo job: $job
echo pwd: $workdir
export SCRAM_ARCH=%(scram_arch)s
source /cvmfs/cms.cern.ch/cmsset_default.sh
scram project CMSSW %(cmssw_version)s 2>&1
cd %(cmssw_version)s
tar xvf ${workdir}/%(tarball_fn_base)s
cd src
eval `scram ru -sh`
scram b -j 2
cd $workdir
cmsRun %(cmssw_py)s +job $job +per %(per)s +sample %(sample)s 2>&1
cmsexit=$?
if [[ $cmsexit -ne 0 ]]; then
echo cmsRun exited with code $cmsexit
exit $cmsexit
fi
new_fn=${out_xrd}${out_path}/pick_${job}.root
echo xrdcp to $new_fn
xrdcp pick.root $new_fn
xrdcpexit=$?
if [[ $xrdcpexit -ne 0 ]]; then
echo xrdcp exited with code $xrdcpexit
exit $xrdcpexit
fi
''' % locals())
open(sh_fn, 'wt').write(sh)
os.chmod(sh_fn, 0755)
jdl = dedent('''
universe = vanilla
Executable = %(sh_fn)s
arguments = $(Process)
Output = %(root_name)s_%(batch_name)s_$(Cluster)_$(Process).stdout
Error = %(root_name)s_%(batch_name)s_$(Cluster)_$(Process).stderr
Log = %(root_name)s_%(batch_name)s_$(Cluster)_$(Process).log
stream_output = false
stream_error = false
notification = never
should_transfer_files = YES
when_to_transfer_output = ON_EXIT
transfer_input_files = %(tarball_fn)s,%(cmssw_py_fn)s,%(file_list_fn)s,%(event_list_fn)s
Queue %(njobs)s
''' % locals())
open(jdl_fn, 'wt').write(jdl)
if not testing or testing == 'run':
os.chdir(outputs_dir)
os.system('condor_submit < ' + jdl_fn)
os.chdir(pwd)