should_transfer_files = YES when_to_transfer_output = ON_EXIT transfer_input_files = %(input_files)s,cs_jobmap +REQUIRED_OS = "rhel7" +DesiredOS = REQUIRED_OS Queue %(njobs)s ''' batch_root = crab_dirs_root('combine_output_%i' % time.time()) if os.path.isdir(batch_root): raise IOError('%s exists' % batch_root) print batch_root os.mkdir(batch_root) os.mkdir(os.path.join(batch_root, 'inputs')) save_git_status(os.path.join(batch_root, 'gitstatus')) input_files = [] for x in ['signal_efficiency.py', 'datacard.py', 'limitsinput.root']: nx = os.path.abspath( os.path.join(batch_root, 'inputs', os.path.basename(x))) shutil.copy2(x, nx) input_files.append(nx) input_files = ','.join(input_files) f = ROOT.TFile('limitsinput.root') years = ('2016', '2017', '2018') if include_2016 else ('2017', '2018') samples = limitsinput.sample_iterator( f, require_years=years,
def submit(samples, ntracks, overlay_args, batch_name_ex=''): testing = 'testing' in sys.argv batch_name = 'ntk%i' % ntracks batch_name += batch_name_ex if testing: batch_name += '_TEST' work_area = '/uscms_data/d2/%s/crab_dirs/Overlay%s/%s' % ( os.environ['USER'], version, batch_name) inputs_dir = os.path.join(work_area, 'inputs') os.makedirs(inputs_dir) save_git_status(os.path.join(work_area, 'gitstatus')) cmssw_py = 'overlay.py' cmssw_py_fn = os.path.join(inputs_dir, cmssw_py) shutil.copy2(cmssw_py, cmssw_py_fn) tool_path = os.path.join( os.environ['CMSSW_BASE'], 'src/JMTucker/MFVNeutralino/test/MakeSamples' ) # JMTBAD put dummy.py and fixfjr.py somewhere better config = Config() config.General.transferLogs = True config.General.transferOutputs = True config.General.workArea = work_area config.JobType.pluginName = 'PrivateMC' config.JobType.psetName = os.path.join(tool_path, 'dummy.py') config.JobType.sendPythonFolder = True config.JobType.inputFiles = [ cmssw_py_fn, os.path.join(tool_path, 'fixfjr.py') ] config.JobType.outputFiles = ['overlay.root'] config.Data.splitting = 'EventBased' config.Data.unitsPerJob = 1 config.Data.publication = False config.Site.storageSite = 'T3_US_FNALLPC' config.Site.whitelist = 'T1_US_*,T2_US_*,T3_US_Baylor,T3_US_FIU,T3_US_FSU,T3_US_JHU,T3_US_Kansas,T3_US_NERSC,T3_US_NU,T3_US_NotreDame,T3_US_OSG,T3_US_OSU,T3_US_Omaha,T3_US_Princeton_ICSE,T3_US_PuertoRico,T3_US_Rice,T3_US_SDSC,T3_US_TAMU,T3_US_TTU,T3_US_UCD,T3_US_UCSB,T3_US_UMD'.split( ',') for sample in samples: njobs, per_last = max_njobs[(sample, ntracks)] sh_vars = { 'per': per, 'njobs': njobs, 'per_m1': per - 1, 'per_last_m1': per_last - 1, 'ntracks': ntracks, 'sample': sample, 'overlay_args': overlay_args, 'cmssw_version': os.environ['CMSSW_VERSION'], 'cmssw_py': cmssw_py, } sh_fn = os.path.join(inputs_dir, 'run.%s.sh' % sample) open(sh_fn, 'wt').write(sh_template % sh_vars) os.chmod(sh_fn, 0755) config.General.requestName = '%s_%s' % (batch_name, sample) config.JobType.scriptExe = sh_fn config.Data.totalUnits = njobs print colors.boldwhite('%s, %s' % (batch_name, sample)) if not testing: output = crab_command('submit', config=config) print output['stdout'] else: print config print
def submit(samples, ntracks, overlay_args, batch_name_ex=''): testing = 'testing' in sys.argv batch_name = 'ntk%i' % ntracks batch_name += batch_name_ex if testing: batch_name += '_TEST' work_area = '/uscms_data/d2/%s/crab_dirs/Overlay%s/%s' % (os.environ['USER'], version, batch_name) inputs_dir = os.path.join(work_area, 'inputs') os.makedirs(inputs_dir) save_git_status(os.path.join(work_area, 'gitstatus')) cmssw_py = 'overlay.py' cmssw_py_fn = os.path.join(inputs_dir, cmssw_py) shutil.copy2(cmssw_py, cmssw_py_fn) tool_path = os.path.join(os.environ['CMSSW_BASE'], 'src/JMTucker/MFVNeutralino/test/MakeSamples') # JMTBAD put dummy.py and fixfjr.py somewhere better config = Config() config.General.transferLogs = True config.General.transferOutputs = True config.General.workArea = work_area config.JobType.pluginName = 'PrivateMC' config.JobType.psetName = os.path.join(tool_path, 'dummy.py') config.JobType.sendPythonFolder = True config.JobType.inputFiles = [cmssw_py_fn, os.path.join(tool_path, 'fixfjr.py')] config.JobType.outputFiles = ['overlay.root'] config.Data.splitting = 'EventBased' config.Data.unitsPerJob = 1 config.Data.publication = False config.Site.storageSite = 'T3_US_FNALLPC' config.Site.whitelist = 'T1_US_*,T2_US_*,T3_US_Baylor,T3_US_FIU,T3_US_FSU,T3_US_JHU,T3_US_Kansas,T3_US_NERSC,T3_US_NU,T3_US_NotreDame,T3_US_OSG,T3_US_OSU,T3_US_Omaha,T3_US_Princeton_ICSE,T3_US_PuertoRico,T3_US_Rice,T3_US_SDSC,T3_US_TAMU,T3_US_TTU,T3_US_UCD,T3_US_UCSB,T3_US_UMD'.split(',') for sample in samples: njobs, per_last = max_njobs[(sample, ntracks)] sh_vars = { 'per': per, 'njobs': njobs, 'per_m1': per - 1, 'per_last_m1': per_last - 1, 'ntracks': ntracks, 'sample': sample, 'overlay_args': overlay_args, 'cmssw_version': os.environ['CMSSW_VERSION'], 'cmssw_py': cmssw_py, } sh_fn = os.path.join(inputs_dir, 'run.%s.sh' % sample) open(sh_fn, 'wt').write(sh_template % sh_vars) os.chmod(sh_fn, 0755) config.General.requestName = '%s_%s' % (batch_name, sample) config.JobType.scriptExe = sh_fn config.Data.totalUnits = njobs print colors.boldwhite('%s, %s' % (batch_name, sample)) if not testing: output = crab_command('submit', config=config) print output['stdout'] else: print config print
def __init__( self, batch_name, ex='', testing='testing' in sys.argv or 'cs_testing' in sys.argv, meat=cmsRun_meat, pset_template_fn=sys.argv[0], pset_modifier=None, input_files=[], output_files=[], skip_output_files=[], stageout_files=[], # can be "all" or "pool" stageout_path='', # if / in it, does not try to generate publish_name='', dataset='main', jdl_extras='', _events=-1, _njobs=None, _fail=[], ): self.nsubmits = -1 self.testing = testing if type(meat) == file: meat = meat.read() elif type(meat) == str and os.path.isfile(meat): meat = open(meat).read() if 'meatexit=' not in meat: raise ValueError('meatexit not set in meat?') self.meat = meat self.is_cmsRun = meat == self.cmsRun_meat if '$' in pset_template_fn: pset_template_fn = os.path.expandvars(pset_template_fn) if '~' in pset_template_fn: pset_template_fn = os.path.expanduser(pset_template_fn) self.pset_template_fn = pset_template_fn self.pset_modifier = pset_modifier self.dataset = dataset self._njobs = _njobs for arg in sys.argv: if arg.startswith('cs_name='): batch_name = arg.replace('cs_name=', '') break if not set(self.batch_name_allowed).issuperset( set(batch_name.replace('/', ''))): raise ValueError( 'illegal batch name %s, allowed characters are letters, numbers, and _' % batch_name) nslash = batch_name.count('/') if nslash > 1: raise ValueError('only one slash allowed in batch name %s' % batch_name) elif nslash == 1: batch_path = os.path.abspath(crab_dirs_root(batch_name)) batch_root = os.path.dirname(batch_path) if os.path.exists(batch_root): if not os.path.isdir(batch_root): raise ValueError( 'slashy mode: batch_root %s exists but is not a dir?' % batch_root) for x in os.listdir(batch_root): fx = os.path.join(batch_root, x) if x == 'gitstatus' or x == 'inputs' or x == 'psets' or not os.path.isdir( fx): raise ValueError('bad slashy %s' % batch_path) else: os.mkdir(batch_root) os.mkdir(batch_path) if ex: ex = str(ex) + '_' self.ex_str = 'ex_%s%s' % (ex, str(int(time.time() * 1000))) self.batch_name = batch_name self.batch_dir = os.path.abspath(crab_dirs_root(batch_name)) self.inputs_dir = os.path.join(self.batch_dir, self.ex_str, 'inputs') if os.path.exists( self.inputs_dir ): # check inputs_dir instead of batch_dir since we might be from metasubmitter raise ValueError( 'batch_dir %s already exists, refusing to clobber' % self.batch_dir) if not testing and self.get_proxy: #print 'CondorSubmitter init: checking proxies, might ask for password twice (but you can skip it with ^C if you know what you are doing).' crab_renew_proxy_if_needed() self.get_proxy = False #Have different username between UW-Madison and CERN; #username = os.environ['USER'] username = '******' self.timestamp = datetime.now() #os.system('mkdir -p /tmp/%s' % username) #print 'CondorSubmitter init: saving git status' save_git_status(os.path.join(self.batch_dir, self.ex_str, 'gitstatus')) os.mkdir(self.inputs_dir) self.filelist_fn_pattern = os.path.join(self.inputs_dir, '%(name)s.sh') self.jdl_fn_pattern = os.path.join(self.inputs_dir, '%(name)s.jdl') self.pset_fn_pattern = os.path.join(self.inputs_dir, '%(name)s.py') tarball_fn = os.path.join(self.inputs_dir, 'input.tgz') #print 'CondorSubmitter init: making cmssw tarball' make_tarball(tarball_fn, include_python=True, include_interface=True) sh_fn = os.path.join(self.inputs_dir, 'run.sh') if input_files: input_bns = [] input_fns = [] for x in input_files: bn = os.path.basename(x) fn = os.path.abspath(os.path.join(self.inputs_dir, bn)) input_bns.append(bn) input_fns.append(fn) shutil.copy(x, fn) input_bns = ',' + ','.join(input_bns) input_fns = ',' + ','.join(input_fns) else: input_bns = '' input_fns = '' # JMTBAD if pset_modifier or cmsrun_args modifies the output filenames, we won't catch them #print 'CondorSubmitter init: importing pset_template fn %s to get output filenames' % pset_template_fn module_output_files = {} if self.pset_template_fn: module = imp.load_source('dummy', self.pset_template_fn) module_output_files = find_output_files(module.process) for l in module_output_files.itervalues(): for x in l: if x not in skip_output_files: output_files.append(x) if type(stageout_files) == str: stageout_which = stageout_files if stageout_which == 'all': stageout_files = output_files elif stageout_which == 'pool': stageout_files = [ x for x in module_output_files.get('PoolOutputModule', []) if x not in skip_output_files ] else: raise ValueError("don't understand stageout_files = %s" % stageout_which) output_files = [x for x in output_files if x not in stageout_files] for x in output_files + stageout_files: if os.path.basename(x) != x: raise ValueError('output file must have no slashes: %r' % x) self.output_files = output_files = ' '.join(output_files) self.stageout_files = stageout_files = ' '.join(stageout_files) output_snippet = '' if output_files: #print 'CondorSubmitter init: output files are', output_files output_snippet += self.output_template.replace( '__OUTPUT_BNS__', output_files) if stageout_files: if '/' not in stageout_path: stageout_user = username # JMTBAD use getUsernameFromSiteDB? if stageout_path: stageout_path = '/' + stageout_path stageout_path = 'root://cmseos.fnal.gov//store/user/' + stageout_user + stageout_path if not publish_name: publish_name = batch_name.replace('/', '_') stageout_path += '/$(<cs_primaryds)/' + publish_name + '/$(<cs_timestamp)/$(printf "%04i" $(($job/1000)) )' print 'CondorSubmitter init: stageout files are', stageout_files print 'CondorSubmitter init: stageout path is', stageout_path output_snippet += self.stageout_template \ .replace('__STAGEOUT_BNS__', stageout_files) \ .replace('__STAGEOUT_PATH__', stageout_path) self.sh_template = self.sh_template \ .replace('__INPUT_BNS__', input_bns) \ .replace('__MEAT__', self.meat) \ .replace('__OUTPUT_SNIPPET__', output_snippet) self.jdl_template = self.jdl_template \ .replace('__SH_FN__', sh_fn) \ .replace('__TARBALL_FN__', tarball_fn) \ .replace('__INPUT_FNS__', input_fns) \ .replace('__EXTRAS__', jdl_extras) self.pset_end_template = self.pset_end_template \ .replace('__MAX_EVENTS__', str(_events)) \ .replace('__FAIL_LIST__', repr(_fail)) open(sh_fn, 'wt').write(self.sh_template)
def submit(sample, ntracks, overlay_args, njobs=0, testing=False, batch_name_ex=''): if njobs <= 0: njobs, per_last = max_njobs[(sample, ntracks)] else: per_last = per njobs_m1 = njobs - 1 per_m1 = per - 1 per_last_m1 = per_last - 1 batch_name = 'ntk%i' % ntracks if 'deltasvgaus' in overlay_args: batch_name += '_deltasvgaus' if 'rest-of-event' in overlay_args: batch_name += '_wevent' batch_name += batch_name_ex if testing: batch_name += '_TEST' print batch_name, sample cmssw_py = 'overlay.py' batch_dir = '/uscms_data/d2/tucker/overlay/%s/%s' % (batch_name, sample) inputs_dir = os.path.join(batch_dir, 'inputs') outputs_dir = os.path.join(batch_dir, 'outputs') os.makedirs(inputs_dir) os.makedirs(outputs_dir) save_git_status(os.path.join(batch_dir, 'gitstatus')) tarball_fn_base = 'input.tgz' tarball_fn = os.path.join(inputs_dir, tarball_fn_base) make_tarball(tarball_fn, include_python=True) cmssw_py_fn = os.path.join(inputs_dir, cmssw_py) shutil.copy2(cmssw_py, cmssw_py_fn) sh_fn = os.path.join(inputs_dir, 'run.sh') jdl_fn = os.path.join(inputs_dir, 'submit.jdl') scram_arch = os.environ['SCRAM_ARCH'] cmssw_version = os.environ['CMSSW_VERSION'] sh = dedent(''' #!/bin/bash workdir=$(pwd) job=$1 echo date: $(date) echo job: $job echo pwd: $workdir export SCRAM_ARCH=%(scram_arch)s source /cvmfs/cms.cern.ch/cmsset_default.sh scram project CMSSW %(cmssw_version)s 2>&1 > /dev/null scramexit=$? if [[ $scramexit -ne 0 ]]; then echo scram exited with code $scramexit exit $scramexit fi cd %(cmssw_version)s tar xf ${workdir}/%(tarball_fn_base)s cd src eval `scram ru -sh` scram b -j 2 2>&1 > /dev/null if [[ $job -eq %(njobs_m1)i ]]; then nev=%(per_last_m1)i else nev=%(per_m1)i fi set -x for i in $(seq 0 $nev); do cmsRun ${workdir}/%(cmssw_py)s +batch +which-event $((job*10+i)) +sample %(sample)s +ntracks %(ntracks)s %(overlay_args)s 2>&1 cmsexit=$? if [[ $cmsexit -ne 0 ]]; then echo cmsRun exited with code $cmsexit exit $cmsexit fi done set +x hadd overlay.root overlay_*.root 2>&1 haddexit=$? if [[ $haddexit -ne 0 ]]; then echo hadd exited with code $haddexit exit $haddexit fi mv overlay.root $workdir/overlay_${job}.root ''' % locals()) open(sh_fn, 'wt').write(sh) os.chmod(sh_fn, 0755) jdl = dedent(''' universe = vanilla Executable = %(sh_fn)s arguments = $(Process) Output = overlay_$(Cluster)_$(Process).stdout Error = overlay_$(Cluster)_$(Process).stderr Log = overlay_$(Cluster)_$(Process).log stream_output = false stream_error = false notification = never should_transfer_files = YES when_to_transfer_output = ON_EXIT transfer_input_files = %(tarball_fn)s,%(cmssw_py_fn)s Queue %(njobs)s ''' % locals()) open(jdl_fn, 'wt').write(jdl) if not testing: os.chdir(outputs_dir) os.system('condor_submit < ' + jdl_fn) os.chdir(pwd)
if os.path.exists(full_output_dir): sys.exit('%s already exists' % full_output_dir) #### from JMTucker.Tools.CRAB3ToolsBase import crab_dirs_root from JMTucker.Tools.Year import year; assert year == 2017 from JMTucker.Tools.general import save_git_status from JMTucker.Tools import colors testing = 'testing' in sys.argv work_area = crab_dirs_root('nstep_gs2reco_condor_%s' % which) if os.path.isdir(work_area): sys.exit('work_area %s exists' % work_area) os.makedirs(work_area) save_git_status(os.path.join(work_area, 'gitstatus')) inputs_dir = os.path.join(work_area, 'inputs') os.mkdir(inputs_dir) open('year.txt', 'wt').write(str(year)) sh_fn = 'nstep_gs2reco_condor.sh' input_files = 'todoify.sh rawhlt.py reco.py miniaod.py dynamicconf.py year.txt modify.py inputfns.txt minbias.py minbias_premix.txt.gz'.split() for fn in [sh_fn] + input_files: shutil.copy2(fn, inputs_dir) sh_fn = os.path.join(inputs_dir, sh_fn) input_files = ','.join([os.path.join(inputs_dir, x) for x in input_files]) todos = ' '.join('todo=' + x for x in todos)
from CRABClient.ClientExceptions import ConfigException as CRABConfigException from JMTucker.Tools.CRAB3Tools import Config, crab_dirs_root, crab_command from JMTucker.Tools.general import index_startswith, save_git_status from JMTucker.Tools.CondorSubmitter import CondorSubmitter from JMTucker.Tools.Sample import MCSample from JMTucker.Tools import colors testing = 'testing' in sys.argv if ex and not ex.startswith('_'): ex = '_' + ex work_area = crab_dirs_root('nstep%s' % ex) if not os.path.isdir(work_area): os.makedirs(work_area) gitstatus_dir = 'gitstatus_%s' % int(time() * 1000) if not condor: save_git_status(os.path.join(work_area, gitstatus_dir)) config = Config() open('year.txt', 'wt').write(str(year)) to_rm = ['year.txt'] if output_level in ('minitree', 'ntuple'): for x in 'ntuple.py', 'minitree.py': to_rm.append(x) os.system('cmsDumpPython.py ../%s > %s' % (x, x)) config.General.transferLogs = False config.General.transferOutputs = True config.General.workArea = work_area
def __init__(self, batch_name, ex = '', testing = 'testing' in sys.argv or 'cs_testing' in sys.argv, meat = cmsRun_meat, pset_template_fn = sys.argv[0], pset_modifier = None, input_files = [], output_files = [], skip_output_files = [], stageout_files = [], # can be "all" or "pool" stageout_path = '', # if / in it, does not try to generate publish_name = '', dataset = 'main', jdl_extras = '', _events = -1, _njobs = None, _fail = [], ): self.nsubmits = -1 self.testing = testing if type(meat) == file: meat = meat.read() elif type(meat) == str and os.path.isfile(meat): meat = open(meat).read() if 'meatexit=' not in meat: raise ValueError('meatexit not set in meat?') self.meat = meat self.is_cmsRun = meat == self.cmsRun_meat if '$' in pset_template_fn: pset_template_fn = os.path.expandvars(pset_template_fn) if '~' in pset_template_fn: pset_template_fn = os.path.expanduser(pset_template_fn) self.pset_template_fn = pset_template_fn self.pset_modifier = pset_modifier self.dataset = dataset self._njobs = _njobs for arg in sys.argv: if arg.startswith('cs_name='): batch_name = arg.replace('cs_name=', '') break if not set(self.batch_name_allowed).issuperset(set(batch_name.replace('/', ''))): raise ValueError('illegal batch name %s, allowed characters are letters, numbers, and _' % batch_name) nslash = batch_name.count('/') if nslash > 1: raise ValueError('only one slash allowed in batch name %s' % batch_name) elif nslash == 1: batch_path = os.path.abspath(crab_dirs_root(batch_name)) batch_root = os.path.dirname(batch_path) if os.path.exists(batch_root): if not os.path.isdir(batch_root): raise ValueError('slashy mode: batch_root %s exists but is not a dir?' % batch_root) for x in os.listdir(batch_root): fx = os.path.join(batch_root, x) if x == 'gitstatus' or x == 'inputs' or x == 'psets' or not os.path.isdir(fx): raise ValueError('bad slashy %s' % batch_path) else: os.mkdir(batch_root) os.mkdir(batch_path) if ex: ex = str(ex) + '_' self.ex_str = 'ex_%s%s' % (ex, str(int(time.time()*1000))) self.batch_name = batch_name self.batch_dir = os.path.abspath(crab_dirs_root(batch_name)) self.inputs_dir = os.path.join(self.batch_dir, self.ex_str, 'inputs') if os.path.exists(self.inputs_dir): # check inputs_dir instead of batch_dir since we might be from metasubmitter raise ValueError('batch_dir %s already exists, refusing to clobber' % self.batch_dir) if not os.path.isdir(self.links_dir): os.mkdir(self.links_dir) for schedd in self.schedds: schedd_d = os.path.join(self.links_dir, schedd) if not os.path.isdir(schedd_d): os.mkdir(schedd_d) if not testing and self.get_proxy: #print 'CondorSubmitter init: checking proxies, might ask for password twice (but you can skip it with ^C if you know what you are doing).' crab_renew_proxy_if_needed() self.get_proxy = False username = os.environ['USER'] self.timestamp = datetime.now() #os.system('mkdir -p /tmp/%s' % username) #print 'CondorSubmitter init: saving git status' save_git_status(os.path.join(self.batch_dir, self.ex_str, 'gitstatus')) os.mkdir(self.inputs_dir) self.filelist_fn_pattern = os.path.join(self.inputs_dir, '%(name)s.sh') self.jdl_fn_pattern = os.path.join(self.inputs_dir, '%(name)s.jdl') self.pset_fn_pattern = os.path.join(self.inputs_dir, '%(name)s.py') tarball_fn = os.path.join(self.inputs_dir, 'input.tgz') #print 'CondorSubmitter init: making cmssw tarball' make_tarball(tarball_fn, include_python=True, include_interface=True) sh_fn = os.path.join(self.inputs_dir, 'run.sh') if input_files: input_bns = [] input_fns = [] for x in input_files: bn = os.path.basename(x) fn = os.path.abspath(os.path.join(self.inputs_dir, bn)) input_bns.append(bn) input_fns.append(fn) shutil.copy(x, fn) input_bns = ',' + ','.join(input_bns) input_fns = ',' + ','.join(input_fns) else: input_bns = '' input_fns = '' # JMTBAD if pset_modifier or cmsrun_args modifies the output filenames, we won't catch them #print 'CondorSubmitter init: importing pset_template fn %s to get output filenames' % pset_template_fn module_output_files = {} if self.pset_template_fn: module = imp.load_source('dummy', self.pset_template_fn) module_output_files = find_output_files(module.process) for l in module_output_files.itervalues(): for x in l: if x not in skip_output_files: output_files.append(x) if type(stageout_files) == str: stageout_which = stageout_files if stageout_which == 'all': stageout_files = output_files elif stageout_which == 'pool': stageout_files = [x for x in module_output_files.get('PoolOutputModule', []) if x not in skip_output_files] else: raise ValueError("don't understand stageout_files = %s" % stageout_which) output_files = [x for x in output_files if x not in stageout_files] assert all(os.path.basename(x) == x for x in output_files + stageout_files) self.output_files = output_files = ' '.join(output_files) self.stageout_files = stageout_files = ' '.join(stageout_files) output_snippet = '' if output_files: #print 'CondorSubmitter init: output files are', output_files output_snippet += self.output_template.replace('__OUTPUT_BNS__', output_files) if stageout_files: if '/' not in stageout_path: stageout_user = username # JMTBAD use getUsernameFromSiteDB? if stageout_path: stageout_path = '/' + stageout_path stageout_path = 'root://cmseos.fnal.gov//store/user/' + stageout_user + stageout_path if not publish_name: publish_name = batch_name.replace('/', '_') stageout_path += '/$(<cs_primaryds)/' + publish_name + '/$(<cs_timestamp)/$(printf "%04i" $(($job/1000)) )' #print 'CondorSubmitter init: stageout files are', stageout_files #print 'CondorSubmitter init: stageout path is', stageout_path output_snippet += self.stageout_template \ .replace('__STAGEOUT_BNS__', stageout_files) \ .replace('__STAGEOUT_PATH__', stageout_path) self.sh_template = self.sh_template \ .replace('__INPUT_BNS__', input_bns) \ .replace('__MEAT__', self.meat) \ .replace('__OUTPUT_SNIPPET__', output_snippet) self.jdl_template = self.jdl_template \ .replace('__SH_FN__', sh_fn) \ .replace('__TARBALL_FN__', tarball_fn) \ .replace('__INPUT_FNS__', input_fns) \ .replace('__EXTRAS__', jdl_extras) self.pset_end_template = self.pset_end_template \ .replace('__MAX_EVENTS__', str(_events)) \ .replace('__FAIL_LIST__', repr(_fail)) open(sh_fn, 'wt').write(self.sh_template)
def submit(njobs, template_type, min_ntracks, signal_sample, template_signal, samples): global maked global setuped global i_submit if not maked: if os.system("make clean; make -j 16") != 0: raise "no make" raw_input("did the make go OK?") maked = True extra_name = "" batch_root = "crab3/One2Two" for x in sys.argv: if x.startswith(batch_root): extra_name = x.replace(batch_root + "_", "") + "_" batch_root = x break dummy_pset_fn = os.path.join(batch_root, "dummy_pset.py") files_needed = set() files_needed.add(os.path.join(os.path.abspath("mfvo2t.exe"))) files_needed.add(os.path.join(os.path.abspath("filtertee.py"))) if not setuped: os.system("mkdir -p %s" % batch_root) save_git_status(os.path.join(batch_root, "gitstatus")) open(dummy_pset_fn, "wt").write(dummy_pset) setuped = True batch_name = "%sTmp%s_Ntk%i_SigTmp%s_SigSam%s_Sam%s" % ( extra_name, template_type, min_ntracks, template_signal, "no" if signal_sample is None else "n%ix%i" % signal_sample, samples, ) files_needed.add("MultiJetPk2012.root.gz") if template_signal > -100: files_needed.add("backgrounds.tgz") files_needed.add(sample_number_to_name[template_signal] + ".root.gz") else: files_needed.add("bigsigscan.root.gz") extra_setup = "" env = ["toythrower_min_ntracks=%i" % min_ntracks, "toythrower_template_signal=%i" % template_signal] if no_save_plots: env += ["templates_save_plots=0", "fitter_save_plots=0"] if template_signal <= -101: env.append("toythrower_use_only_data_sample=1") env.append("sig_from_file_num=%i" % template_signal) if template_signal >= -12 or (template_signal <= -101 and template_signal >= -126): env.append("fitter_sig_limit_step=1") if template_type == "PS": env.append("templates_kind=phishift") env.append("fitter_start_nuis0=2") env.append("fitter_start_nuis1=0.003") elif template_type == "SC": env.append("templates_kind=simpleclear") env.append("fitter_fix_nuis1=1") env.append("fitter_start_nuis0=10") env.append("fitter_start_nuis1=0") if signal_sample is not None: sig_samp, sig_scale = signal_sample assert sig_samp < 0 if sig_samp > -100: files_needed.add(sample_number_to_name[sig_samp] + ".root.gz") else: files_needed.add("bigsigscan.root.gz") if sig_scale < 0: env.append("ntoys=0") env.append("process_data=1") if sig_scale == -2: env.append("seed=1") env.append("fitter_i_limit_job=$JOB_NUM") env.append("fitter_do_signif=0") else: assert njobs <= 20 else: env.append("fitter_do_limit=0") env.append("toythrower_injected_signal=%i" % sig_samp) env.append("toythrower_injected_signal_scale=%f" % sig_scale) else: env.append("fitter_do_limit=0") if type(samples) == int: env.append("toythrower_sample_only=%i" % samples) elif type(samples) == str and "500" in samples: env.append("toythrower_use_qcd500=1") elif "ttbarsyst" in samples: which_syst = samples.replace("ttbarsyst", "") files_needed.add("myttbar%s.root.gz" % which_syst) extra_setup += ( """ cd trees ln -s myttbar%s.root bkgsyst.root cd - """ % which_syst ) env.append("toythrower_sample_only=99") ### env = "\n".join("export mfvo2t_" + e for e in env) unzip_files = [] for fn in files_needed: if fn.endswith(".tgz"): unzip_files.append("tar zxf %s" % fn) elif fn.endswith(".gz"): unzip_files.append("gunzip %s" % fn) unzip_files = "\n".join(unzip_files) tree_path = ( "/eos/uscms/store/user/tucker/mfvo2t_all_trees_444de711cdc630ddfe7cb6cd8f64ec8b46d09990_plussomettbarsyst" ) files_needed = ", ".join('"%s"' % os.path.join(tree_path, f) for f in files_needed) submit_dir = os.path.join(crab_submit_dir, "%05i" % i_submit) os.mkdir(submit_dir) os.symlink(os.readlink("crab3"), os.path.join(submit_dir, "crab3")) open(os.path.join(submit_dir, "runme.sh"), "wt").write(script_template % locals()) open(os.path.join(submit_dir, "crabConfig.py"), "wt").write(crab_cfg % locals()) i_submit += 1
from time import time from CRABClient.ClientExceptions import ConfigException as CRABConfigException from JMTucker.Tools.CRAB3Tools import Config, crab_dirs_root, crab_command from JMTucker.Tools.general import index_startswith, save_git_status from JMTucker.Tools.CondorSubmitter import CondorSubmitter from JMTucker.Tools.Sample import MCSample from JMTucker.Tools import colors testing = 'testing' in sys.argv work_area = crab_dirs_root('nstep_%s%s' % (meta, ex)) if os.path.isdir(work_area): sys.exit('work_area %s exists' % work_area) os.makedirs(work_area) gitstatus_dir = 'gitstatus_%s' % int(time()*1000) if not condor: save_git_status(os.path.join(work_area, gitstatus_dir)) config = Config() open('year.txt', 'wt').write(str(year)) to_rm = ['year.txt'] if output_level in ('minitree', 'ntuple'): for x in 'ntuple.py', 'minitree.py': to_rm.append(x) os.system('cmsDumpPython.py ../%s > %s' % (x,x)) config.General.transferLogs = False config.General.transferOutputs = True config.General.workArea = work_area
def submit(sample, testing=False): per = 10 batch_name = sample file_list_fn = os.path.abspath('filelist.%s.gz' % sample) event_list_fn = os.path.abspath('vetolist.%s.gz' % sample) nfiles = sum(1 for line in gzip.open(file_list_fn)) njobs = int_ceil(nfiles, per) if testing: batch_name += '_TEST' njobs = 1 print batch_name cmssw_py = 'pick_by_veto.py' root_name = 'pick1vtx' batch_root = os.path.join('/uscms_data/d2/tucker/condor_batch', root_name) batch_dir = os.path.abspath(os.path.join(batch_root, batch_name)) inputs_dir = os.path.join(batch_dir, 'inputs') outputs_dir = os.path.join(batch_dir, 'outputs') os.makedirs(inputs_dir) os.makedirs(outputs_dir) save_git_status(os.path.join(batch_dir, 'gitstatus')) tarball_fn_base = 'input.tgz' tarball_fn = os.path.join(inputs_dir, tarball_fn_base) make_tarball(tarball_fn, include_python=True) cmssw_py_fn = os.path.join(inputs_dir, cmssw_py) shutil.copy2(cmssw_py, cmssw_py_fn) sh_fn = os.path.join(inputs_dir, 'run.sh') jdl_fn = os.path.join(inputs_dir, 'submit.jdl') scram_arch = os.environ['SCRAM_ARCH'] cmssw_version = os.environ['CMSSW_VERSION'] out_xrd = 'root://cmseos.fnal.gov/' out_path = '/store/user/tucker/pick1vtx/%s' % sample if os.system('eos %s mkdir -p %s' % (out_xrd, out_path)) != 0: raise IOError('problem making %s' % out_path) if any(os.system('eos %s ls %s/pick_%i.root 2>&1 > /dev/null' % (out_xrd, out_path, i)) == 0 for i in xrange(njobs)): raise IOError('one or more output files already exist') sh = dedent(''' #!/bin/bash workdir=$(pwd) job=$1 echo date: $(date) echo job: $job echo pwd: $workdir export SCRAM_ARCH=%(scram_arch)s source /cvmfs/cms.cern.ch/cmsset_default.sh scram project CMSSW %(cmssw_version)s 2>&1 cd %(cmssw_version)s tar xvf ${workdir}/%(tarball_fn_base)s cd src eval `scram ru -sh` scram b -j 2 cd $workdir cmsRun %(cmssw_py)s +job $job +per %(per)s +sample %(sample)s 2>&1 cmsexit=$? if [[ $cmsexit -ne 0 ]]; then echo cmsRun exited with code $cmsexit exit $cmsexit fi new_fn=${out_xrd}${out_path}/pick_${job}.root echo xrdcp to $new_fn xrdcp pick.root $new_fn xrdcpexit=$? if [[ $xrdcpexit -ne 0 ]]; then echo xrdcp exited with code $xrdcpexit exit $xrdcpexit fi ''' % locals()) open(sh_fn, 'wt').write(sh) os.chmod(sh_fn, 0755) jdl = dedent(''' universe = vanilla Executable = %(sh_fn)s arguments = $(Process) Output = %(root_name)s_%(batch_name)s_$(Cluster)_$(Process).stdout Error = %(root_name)s_%(batch_name)s_$(Cluster)_$(Process).stderr Log = %(root_name)s_%(batch_name)s_$(Cluster)_$(Process).log stream_output = false stream_error = false notification = never should_transfer_files = YES when_to_transfer_output = ON_EXIT transfer_input_files = %(tarball_fn)s,%(cmssw_py_fn)s,%(file_list_fn)s,%(event_list_fn)s Queue %(njobs)s ''' % locals()) open(jdl_fn, 'wt').write(jdl) if not testing or testing == 'run': os.chdir(outputs_dir) os.system('condor_submit < ' + jdl_fn) os.chdir(pwd)