Ejemplo n.º 1
0
should_transfer_files = YES
when_to_transfer_output = ON_EXIT
transfer_input_files = %(input_files)s,cs_jobmap
+REQUIRED_OS = "rhel7"
+DesiredOS = REQUIRED_OS
Queue %(njobs)s
'''

batch_root = crab_dirs_root('combine_output_%i' % time.time())
if os.path.isdir(batch_root):
    raise IOError('%s exists' % batch_root)
print batch_root
os.mkdir(batch_root)
os.mkdir(os.path.join(batch_root, 'inputs'))

save_git_status(os.path.join(batch_root, 'gitstatus'))

input_files = []
for x in ['signal_efficiency.py', 'datacard.py', 'limitsinput.root']:
    nx = os.path.abspath(
        os.path.join(batch_root, 'inputs', os.path.basename(x)))
    shutil.copy2(x, nx)
    input_files.append(nx)
input_files = ','.join(input_files)

f = ROOT.TFile('limitsinput.root')

years = ('2016', '2017', '2018') if include_2016 else ('2017', '2018')
samples = limitsinput.sample_iterator(
    f,
    require_years=years,
Ejemplo n.º 2
0
def submit(samples, ntracks, overlay_args, batch_name_ex=''):
    testing = 'testing' in sys.argv

    batch_name = 'ntk%i' % ntracks
    batch_name += batch_name_ex
    if testing:
        batch_name += '_TEST'

    work_area = '/uscms_data/d2/%s/crab_dirs/Overlay%s/%s' % (
        os.environ['USER'], version, batch_name)

    inputs_dir = os.path.join(work_area, 'inputs')
    os.makedirs(inputs_dir)
    save_git_status(os.path.join(work_area, 'gitstatus'))

    cmssw_py = 'overlay.py'
    cmssw_py_fn = os.path.join(inputs_dir, cmssw_py)
    shutil.copy2(cmssw_py, cmssw_py_fn)

    tool_path = os.path.join(
        os.environ['CMSSW_BASE'], 'src/JMTucker/MFVNeutralino/test/MakeSamples'
    )  # JMTBAD put dummy.py and fixfjr.py somewhere better

    config = Config()
    config.General.transferLogs = True
    config.General.transferOutputs = True
    config.General.workArea = work_area
    config.JobType.pluginName = 'PrivateMC'
    config.JobType.psetName = os.path.join(tool_path, 'dummy.py')
    config.JobType.sendPythonFolder = True
    config.JobType.inputFiles = [
        cmssw_py_fn, os.path.join(tool_path, 'fixfjr.py')
    ]
    config.JobType.outputFiles = ['overlay.root']
    config.Data.splitting = 'EventBased'
    config.Data.unitsPerJob = 1
    config.Data.publication = False
    config.Site.storageSite = 'T3_US_FNALLPC'
    config.Site.whitelist = 'T1_US_*,T2_US_*,T3_US_Baylor,T3_US_FIU,T3_US_FSU,T3_US_JHU,T3_US_Kansas,T3_US_NERSC,T3_US_NU,T3_US_NotreDame,T3_US_OSG,T3_US_OSU,T3_US_Omaha,T3_US_Princeton_ICSE,T3_US_PuertoRico,T3_US_Rice,T3_US_SDSC,T3_US_TAMU,T3_US_TTU,T3_US_UCD,T3_US_UCSB,T3_US_UMD'.split(
        ',')

    for sample in samples:
        njobs, per_last = max_njobs[(sample, ntracks)]
        sh_vars = {
            'per': per,
            'njobs': njobs,
            'per_m1': per - 1,
            'per_last_m1': per_last - 1,
            'ntracks': ntracks,
            'sample': sample,
            'overlay_args': overlay_args,
            'cmssw_version': os.environ['CMSSW_VERSION'],
            'cmssw_py': cmssw_py,
        }
        sh_fn = os.path.join(inputs_dir, 'run.%s.sh' % sample)
        open(sh_fn, 'wt').write(sh_template % sh_vars)
        os.chmod(sh_fn, 0755)

        config.General.requestName = '%s_%s' % (batch_name, sample)
        config.JobType.scriptExe = sh_fn
        config.Data.totalUnits = njobs

        print colors.boldwhite('%s, %s' % (batch_name, sample))
        if not testing:
            output = crab_command('submit', config=config)
            print output['stdout']
        else:
            print config
            print
Ejemplo n.º 3
0
def submit(samples, ntracks, overlay_args, batch_name_ex=''):
    testing = 'testing' in sys.argv

    batch_name = 'ntk%i' % ntracks
    batch_name += batch_name_ex
    if testing:
        batch_name += '_TEST'

    work_area = '/uscms_data/d2/%s/crab_dirs/Overlay%s/%s' % (os.environ['USER'], version, batch_name)
    
    inputs_dir = os.path.join(work_area, 'inputs')
    os.makedirs(inputs_dir)
    save_git_status(os.path.join(work_area, 'gitstatus'))

    cmssw_py = 'overlay.py'
    cmssw_py_fn = os.path.join(inputs_dir, cmssw_py)
    shutil.copy2(cmssw_py, cmssw_py_fn)
    
    tool_path = os.path.join(os.environ['CMSSW_BASE'], 'src/JMTucker/MFVNeutralino/test/MakeSamples') # JMTBAD put dummy.py and fixfjr.py somewhere better

    config = Config()
    config.General.transferLogs = True
    config.General.transferOutputs = True
    config.General.workArea = work_area
    config.JobType.pluginName = 'PrivateMC'
    config.JobType.psetName = os.path.join(tool_path, 'dummy.py')
    config.JobType.sendPythonFolder = True
    config.JobType.inputFiles = [cmssw_py_fn, os.path.join(tool_path, 'fixfjr.py')]
    config.JobType.outputFiles = ['overlay.root']
    config.Data.splitting = 'EventBased'
    config.Data.unitsPerJob = 1
    config.Data.publication = False
    config.Site.storageSite = 'T3_US_FNALLPC'
    config.Site.whitelist = 'T1_US_*,T2_US_*,T3_US_Baylor,T3_US_FIU,T3_US_FSU,T3_US_JHU,T3_US_Kansas,T3_US_NERSC,T3_US_NU,T3_US_NotreDame,T3_US_OSG,T3_US_OSU,T3_US_Omaha,T3_US_Princeton_ICSE,T3_US_PuertoRico,T3_US_Rice,T3_US_SDSC,T3_US_TAMU,T3_US_TTU,T3_US_UCD,T3_US_UCSB,T3_US_UMD'.split(',')
    
    for sample in samples:
        njobs, per_last = max_njobs[(sample, ntracks)]
        sh_vars = {
            'per': per,
            'njobs': njobs,
            'per_m1': per - 1,
            'per_last_m1': per_last - 1,
            'ntracks': ntracks,
            'sample': sample,
            'overlay_args': overlay_args,
            'cmssw_version': os.environ['CMSSW_VERSION'],
            'cmssw_py': cmssw_py,
            }
        sh_fn = os.path.join(inputs_dir, 'run.%s.sh' % sample)
        open(sh_fn, 'wt').write(sh_template % sh_vars)
        os.chmod(sh_fn, 0755)

        config.General.requestName = '%s_%s' % (batch_name, sample)
        config.JobType.scriptExe = sh_fn
        config.Data.totalUnits = njobs

        print colors.boldwhite('%s, %s' % (batch_name, sample))
        if not testing:
            output = crab_command('submit', config=config)
            print output['stdout']
        else:
            print config
            print
Ejemplo n.º 4
0
    def __init__(
        self,
        batch_name,
        ex='',
        testing='testing' in sys.argv or 'cs_testing' in sys.argv,
        meat=cmsRun_meat,
        pset_template_fn=sys.argv[0],
        pset_modifier=None,
        input_files=[],
        output_files=[],
        skip_output_files=[],
        stageout_files=[],  # can be "all" or "pool"
        stageout_path='',  # if / in it, does not try to generate
        publish_name='',
        dataset='main',
        jdl_extras='',
        _events=-1,
        _njobs=None,
        _fail=[],
    ):

        self.nsubmits = -1

        self.testing = testing

        if type(meat) == file:
            meat = meat.read()
        elif type(meat) == str and os.path.isfile(meat):
            meat = open(meat).read()
        if 'meatexit=' not in meat:
            raise ValueError('meatexit not set in meat?')
        self.meat = meat
        self.is_cmsRun = meat == self.cmsRun_meat

        if '$' in pset_template_fn:
            pset_template_fn = os.path.expandvars(pset_template_fn)
        if '~' in pset_template_fn:
            pset_template_fn = os.path.expanduser(pset_template_fn)
        self.pset_template_fn = pset_template_fn
        self.pset_modifier = pset_modifier
        self.dataset = dataset
        self._njobs = _njobs

        for arg in sys.argv:
            if arg.startswith('cs_name='):
                batch_name = arg.replace('cs_name=', '')
                break

        if not set(self.batch_name_allowed).issuperset(
                set(batch_name.replace('/', ''))):
            raise ValueError(
                'illegal batch name %s, allowed characters are letters, numbers, and _'
                % batch_name)

        nslash = batch_name.count('/')
        if nslash > 1:
            raise ValueError('only one slash allowed in batch name %s' %
                             batch_name)
        elif nslash == 1:
            batch_path = os.path.abspath(crab_dirs_root(batch_name))
            batch_root = os.path.dirname(batch_path)
            if os.path.exists(batch_root):
                if not os.path.isdir(batch_root):
                    raise ValueError(
                        'slashy mode: batch_root %s exists but is not a dir?' %
                        batch_root)
                for x in os.listdir(batch_root):
                    fx = os.path.join(batch_root, x)
                    if x == 'gitstatus' or x == 'inputs' or x == 'psets' or not os.path.isdir(
                            fx):
                        raise ValueError('bad slashy %s' % batch_path)
            else:
                os.mkdir(batch_root)
                os.mkdir(batch_path)

        if ex:
            ex = str(ex) + '_'
        self.ex_str = 'ex_%s%s' % (ex, str(int(time.time() * 1000)))

        self.batch_name = batch_name
        self.batch_dir = os.path.abspath(crab_dirs_root(batch_name))
        self.inputs_dir = os.path.join(self.batch_dir, self.ex_str, 'inputs')
        if os.path.exists(
                self.inputs_dir
        ):  # check inputs_dir instead of batch_dir since we might be from metasubmitter
            raise ValueError(
                'batch_dir %s already exists, refusing to clobber' %
                self.batch_dir)

        if not testing and self.get_proxy:
            #print 'CondorSubmitter init: checking proxies, might ask for password twice (but you can skip it with ^C if you know what you are doing).'
            crab_renew_proxy_if_needed()
            self.get_proxy = False

        #Have different username between UW-Madison and CERN;
        #username = os.environ['USER']
        username = '******'
        self.timestamp = datetime.now()
        #os.system('mkdir -p /tmp/%s' % username)

        #print 'CondorSubmitter init: saving git status'
        save_git_status(os.path.join(self.batch_dir, self.ex_str, 'gitstatus'))

        os.mkdir(self.inputs_dir)

        self.filelist_fn_pattern = os.path.join(self.inputs_dir, '%(name)s.sh')
        self.jdl_fn_pattern = os.path.join(self.inputs_dir, '%(name)s.jdl')
        self.pset_fn_pattern = os.path.join(self.inputs_dir, '%(name)s.py')

        tarball_fn = os.path.join(self.inputs_dir, 'input.tgz')
        #print 'CondorSubmitter init: making cmssw tarball'
        make_tarball(tarball_fn, include_python=True, include_interface=True)

        sh_fn = os.path.join(self.inputs_dir, 'run.sh')

        if input_files:
            input_bns = []
            input_fns = []
            for x in input_files:
                bn = os.path.basename(x)
                fn = os.path.abspath(os.path.join(self.inputs_dir, bn))
                input_bns.append(bn)
                input_fns.append(fn)
                shutil.copy(x, fn)
            input_bns = ',' + ','.join(input_bns)
            input_fns = ',' + ','.join(input_fns)
        else:
            input_bns = ''
            input_fns = ''

        # JMTBAD if pset_modifier or cmsrun_args modifies the output filenames, we won't catch them
        #print 'CondorSubmitter init: importing pset_template fn %s to get output filenames' % pset_template_fn
        module_output_files = {}
        if self.pset_template_fn:
            module = imp.load_source('dummy', self.pset_template_fn)
            module_output_files = find_output_files(module.process)
            for l in module_output_files.itervalues():
                for x in l:
                    if x not in skip_output_files:
                        output_files.append(x)

        if type(stageout_files) == str:
            stageout_which = stageout_files
            if stageout_which == 'all':
                stageout_files = output_files
            elif stageout_which == 'pool':
                stageout_files = [
                    x for x in module_output_files.get('PoolOutputModule', [])
                    if x not in skip_output_files
                ]
            else:
                raise ValueError("don't understand stageout_files = %s" %
                                 stageout_which)
        output_files = [x for x in output_files if x not in stageout_files]

        for x in output_files + stageout_files:
            if os.path.basename(x) != x:
                raise ValueError('output file must have no slashes: %r' % x)
        self.output_files = output_files = ' '.join(output_files)
        self.stageout_files = stageout_files = ' '.join(stageout_files)

        output_snippet = ''

        if output_files:
            #print 'CondorSubmitter init: output files are', output_files
            output_snippet += self.output_template.replace(
                '__OUTPUT_BNS__', output_files)

        if stageout_files:
            if '/' not in stageout_path:
                stageout_user = username  # JMTBAD use getUsernameFromSiteDB?
                if stageout_path:
                    stageout_path = '/' + stageout_path
                stageout_path = 'root://cmseos.fnal.gov//store/user/' + stageout_user + stageout_path
                if not publish_name:
                    publish_name = batch_name.replace('/', '_')
                stageout_path += '/$(<cs_primaryds)/' + publish_name + '/$(<cs_timestamp)/$(printf "%04i" $(($job/1000)) )'

            print 'CondorSubmitter init: stageout files are', stageout_files
            print 'CondorSubmitter init: stageout path is', stageout_path

            output_snippet += self.stageout_template \
                .replace('__STAGEOUT_BNS__',  stageout_files) \
                .replace('__STAGEOUT_PATH__', stageout_path)

        self.sh_template = self.sh_template \
            .replace('__INPUT_BNS__',  input_bns) \
            .replace('__MEAT__', self.meat) \
            .replace('__OUTPUT_SNIPPET__', output_snippet)

        self.jdl_template = self.jdl_template \
            .replace('__SH_FN__',      sh_fn) \
            .replace('__TARBALL_FN__', tarball_fn) \
            .replace('__INPUT_FNS__',  input_fns) \
            .replace('__EXTRAS__',     jdl_extras)

        self.pset_end_template = self.pset_end_template \
            .replace('__MAX_EVENTS__', str(_events)) \
            .replace('__FAIL_LIST__', repr(_fail))

        open(sh_fn, 'wt').write(self.sh_template)
Ejemplo n.º 5
0
def submit(sample, ntracks, overlay_args, njobs=0, testing=False, batch_name_ex=''):
    if njobs <= 0:
        njobs, per_last = max_njobs[(sample, ntracks)]
    else:
        per_last = per
    njobs_m1 = njobs - 1
    per_m1 = per - 1
    per_last_m1 = per_last - 1

    batch_name = 'ntk%i' % ntracks
    if 'deltasvgaus' in overlay_args:
        batch_name += '_deltasvgaus'
    if 'rest-of-event' in overlay_args:
        batch_name += '_wevent'
    batch_name += batch_name_ex
    if testing:
        batch_name += '_TEST'

    print batch_name, sample

    cmssw_py = 'overlay.py'
    
    batch_dir = '/uscms_data/d2/tucker/overlay/%s/%s' % (batch_name, sample)
    
    inputs_dir = os.path.join(batch_dir, 'inputs')
    outputs_dir = os.path.join(batch_dir, 'outputs')
    os.makedirs(inputs_dir)
    os.makedirs(outputs_dir)
    
    save_git_status(os.path.join(batch_dir, 'gitstatus'))
    
    tarball_fn_base = 'input.tgz'
    tarball_fn = os.path.join(inputs_dir, tarball_fn_base)
    make_tarball(tarball_fn, include_python=True)
    
    cmssw_py_fn = os.path.join(inputs_dir, cmssw_py)
    shutil.copy2(cmssw_py, cmssw_py_fn)
    
    sh_fn = os.path.join(inputs_dir, 'run.sh')
    jdl_fn = os.path.join(inputs_dir, 'submit.jdl')
    
    scram_arch = os.environ['SCRAM_ARCH']
    cmssw_version = os.environ['CMSSW_VERSION']
    
    sh = dedent('''
    #!/bin/bash
    
    workdir=$(pwd)
    job=$1
    
    echo date: $(date)
    echo job: $job
    echo pwd: $workdir
    
    export SCRAM_ARCH=%(scram_arch)s
    source /cvmfs/cms.cern.ch/cmsset_default.sh
    
    scram project CMSSW %(cmssw_version)s 2>&1 > /dev/null
    scramexit=$?
    if [[ $scramexit -ne 0 ]]; then
        echo scram exited with code $scramexit
        exit $scramexit
    fi

    cd %(cmssw_version)s
    tar xf ${workdir}/%(tarball_fn_base)s
    cd src
    eval `scram ru -sh`
    scram b -j 2 2>&1 > /dev/null

    if [[ $job -eq %(njobs_m1)i ]]; then
        nev=%(per_last_m1)i
    else
        nev=%(per_m1)i
    fi

    set -x
    for i in $(seq 0 $nev); do
        cmsRun ${workdir}/%(cmssw_py)s +batch +which-event $((job*10+i)) +sample %(sample)s +ntracks %(ntracks)s %(overlay_args)s 2>&1
        cmsexit=$?
        if [[ $cmsexit -ne 0 ]]; then
            echo cmsRun exited with code $cmsexit
            exit $cmsexit
        fi
    done
    set +x

    hadd overlay.root overlay_*.root 2>&1
    haddexit=$?
    if [[ $haddexit -ne 0 ]]; then
        echo hadd exited with code $haddexit
        exit $haddexit
    fi
    mv overlay.root $workdir/overlay_${job}.root
    ''' % locals())

    open(sh_fn, 'wt').write(sh)
    os.chmod(sh_fn, 0755)
    
    jdl = dedent('''
    universe = vanilla
    Executable = %(sh_fn)s
    arguments = $(Process)
    Output = overlay_$(Cluster)_$(Process).stdout
    Error = overlay_$(Cluster)_$(Process).stderr
    Log = overlay_$(Cluster)_$(Process).log
    stream_output = false
    stream_error  = false
    notification  = never
    should_transfer_files   = YES
    when_to_transfer_output = ON_EXIT
    transfer_input_files = %(tarball_fn)s,%(cmssw_py_fn)s
    Queue %(njobs)s
    ''' % locals())
    
    open(jdl_fn, 'wt').write(jdl)

    if not testing:
        os.chdir(outputs_dir)
        os.system('condor_submit < ' + jdl_fn)
        os.chdir(pwd)
if os.path.exists(full_output_dir):
    sys.exit('%s already exists' % full_output_dir)

####

from JMTucker.Tools.CRAB3ToolsBase import crab_dirs_root
from JMTucker.Tools.Year import year; assert year == 2017
from JMTucker.Tools.general import save_git_status
from JMTucker.Tools import colors

testing = 'testing' in sys.argv
work_area = crab_dirs_root('nstep_gs2reco_condor_%s' % which)
if os.path.isdir(work_area):
    sys.exit('work_area %s exists' % work_area)
os.makedirs(work_area)
save_git_status(os.path.join(work_area, 'gitstatus'))

inputs_dir = os.path.join(work_area, 'inputs')
os.mkdir(inputs_dir)

open('year.txt', 'wt').write(str(year))

sh_fn = 'nstep_gs2reco_condor.sh'
input_files = 'todoify.sh rawhlt.py reco.py miniaod.py dynamicconf.py year.txt modify.py inputfns.txt minbias.py minbias_premix.txt.gz'.split()
for fn in [sh_fn] + input_files:
    shutil.copy2(fn, inputs_dir)

sh_fn = os.path.join(inputs_dir, sh_fn)
input_files = ','.join([os.path.join(inputs_dir, x) for x in input_files])

todos = ' '.join('todo=' + x for x in todos)
Ejemplo n.º 7
0
from CRABClient.ClientExceptions import ConfigException as CRABConfigException
from JMTucker.Tools.CRAB3Tools import Config, crab_dirs_root, crab_command
from JMTucker.Tools.general import index_startswith, save_git_status
from JMTucker.Tools.CondorSubmitter import CondorSubmitter
from JMTucker.Tools.Sample import MCSample
from JMTucker.Tools import colors

testing = 'testing' in sys.argv
if ex and not ex.startswith('_'):
    ex = '_' + ex
work_area = crab_dirs_root('nstep%s' % ex)
if not os.path.isdir(work_area):
    os.makedirs(work_area)
gitstatus_dir = 'gitstatus_%s' % int(time() * 1000)
if not condor:
    save_git_status(os.path.join(work_area, gitstatus_dir))

config = Config()

open('year.txt', 'wt').write(str(year))

to_rm = ['year.txt']

if output_level in ('minitree', 'ntuple'):
    for x in 'ntuple.py', 'minitree.py':
        to_rm.append(x)
        os.system('cmsDumpPython.py ../%s > %s' % (x, x))

config.General.transferLogs = False
config.General.transferOutputs = True
config.General.workArea = work_area
Ejemplo n.º 8
0
    def __init__(self,
                 batch_name,
                 ex = '',
                 testing = 'testing' in sys.argv or 'cs_testing' in sys.argv,
                 meat = cmsRun_meat,
                 pset_template_fn = sys.argv[0],
                 pset_modifier = None,
                 input_files = [],
                 output_files = [],
                 skip_output_files = [],
                 stageout_files = [], # can be "all" or "pool"
                 stageout_path = '', # if / in it, does not try to generate
                 publish_name = '',
                 dataset = 'main',
                 jdl_extras = '',
                 _events = -1,
                 _njobs = None,
                 _fail = [],
                 ):

        self.nsubmits = -1

        self.testing = testing

        if type(meat) == file:
            meat = meat.read()
        elif type(meat) == str and os.path.isfile(meat):
            meat = open(meat).read()
        if 'meatexit=' not in meat:
            raise ValueError('meatexit not set in meat?')
        self.meat = meat
        self.is_cmsRun = meat == self.cmsRun_meat

        if '$' in pset_template_fn:
            pset_template_fn =  os.path.expandvars(pset_template_fn)
        if '~' in pset_template_fn:
            pset_template_fn = os.path.expanduser(pset_template_fn)
        self.pset_template_fn = pset_template_fn
        self.pset_modifier = pset_modifier
        self.dataset = dataset
        self._njobs = _njobs

        for arg in sys.argv:
            if arg.startswith('cs_name='):
                batch_name = arg.replace('cs_name=', '')
                break

        if not set(self.batch_name_allowed).issuperset(set(batch_name.replace('/', ''))):
            raise ValueError('illegal batch name %s, allowed characters are letters, numbers, and _' % batch_name)

        nslash = batch_name.count('/')
        if nslash > 1:
            raise ValueError('only one slash allowed in batch name %s' % batch_name)
        elif nslash == 1:
            batch_path = os.path.abspath(crab_dirs_root(batch_name))
            batch_root = os.path.dirname(batch_path)
            if os.path.exists(batch_root):
                if not os.path.isdir(batch_root):
                    raise ValueError('slashy mode: batch_root %s exists but is not a dir?' % batch_root)
                for x in os.listdir(batch_root):
                    fx = os.path.join(batch_root, x)
                    if x == 'gitstatus' or x == 'inputs' or x == 'psets' or not os.path.isdir(fx):
                        raise ValueError('bad slashy %s' % batch_path)
            else:
                os.mkdir(batch_root)
                os.mkdir(batch_path)

        if ex:
            ex = str(ex) + '_'
        self.ex_str = 'ex_%s%s' % (ex, str(int(time.time()*1000)))

        self.batch_name = batch_name
        self.batch_dir = os.path.abspath(crab_dirs_root(batch_name))
        self.inputs_dir = os.path.join(self.batch_dir, self.ex_str, 'inputs')
        if os.path.exists(self.inputs_dir): # check inputs_dir instead of batch_dir since we might be from metasubmitter
            raise ValueError('batch_dir %s already exists, refusing to clobber' % self.batch_dir)

        if not os.path.isdir(self.links_dir):
            os.mkdir(self.links_dir)
        for schedd in self.schedds:
            schedd_d = os.path.join(self.links_dir, schedd)
            if not os.path.isdir(schedd_d):
                os.mkdir(schedd_d)

        if not testing and self.get_proxy:
            #print 'CondorSubmitter init: checking proxies, might ask for password twice (but you can skip it with ^C if you know what you are doing).'
            crab_renew_proxy_if_needed()
            self.get_proxy = False

        username = os.environ['USER']
        self.timestamp = datetime.now()
        #os.system('mkdir -p /tmp/%s' % username)

        #print 'CondorSubmitter init: saving git status'
        save_git_status(os.path.join(self.batch_dir, self.ex_str, 'gitstatus'))

        os.mkdir(self.inputs_dir)

        self.filelist_fn_pattern = os.path.join(self.inputs_dir, '%(name)s.sh')
        self.jdl_fn_pattern = os.path.join(self.inputs_dir, '%(name)s.jdl')
        self.pset_fn_pattern = os.path.join(self.inputs_dir, '%(name)s.py')

        tarball_fn = os.path.join(self.inputs_dir, 'input.tgz')
        #print 'CondorSubmitter init: making cmssw tarball'
        make_tarball(tarball_fn, include_python=True, include_interface=True)

        sh_fn = os.path.join(self.inputs_dir, 'run.sh')

        if input_files:
            input_bns = []
            input_fns = []
            for x in input_files:
                bn = os.path.basename(x)
                fn = os.path.abspath(os.path.join(self.inputs_dir, bn))
                input_bns.append(bn)
                input_fns.append(fn)
                shutil.copy(x, fn)
            input_bns = ',' + ','.join(input_bns)
            input_fns = ',' + ','.join(input_fns)
        else:
            input_bns = ''
            input_fns = ''

        # JMTBAD if pset_modifier or cmsrun_args modifies the output filenames, we won't catch them
        #print 'CondorSubmitter init: importing pset_template fn %s to get output filenames' % pset_template_fn
        module_output_files = {}
        if self.pset_template_fn:
            module = imp.load_source('dummy', self.pset_template_fn)
            module_output_files = find_output_files(module.process)
            for l in module_output_files.itervalues():
                for x in l:
                    if x not in skip_output_files:
                        output_files.append(x)

        if type(stageout_files) == str:
            stageout_which = stageout_files
            if stageout_which == 'all':
                stageout_files = output_files
            elif stageout_which == 'pool':
                stageout_files = [x for x in module_output_files.get('PoolOutputModule', []) if x not in skip_output_files]
            else:
                raise ValueError("don't understand stageout_files = %s" % stageout_which)
        output_files = [x for x in output_files if x not in stageout_files]

        assert all(os.path.basename(x) == x for x in output_files + stageout_files)
        self.output_files   = output_files   = ' '.join(output_files)
        self.stageout_files = stageout_files = ' '.join(stageout_files)

        output_snippet = ''

        if output_files:
            #print 'CondorSubmitter init: output files are', output_files
            output_snippet += self.output_template.replace('__OUTPUT_BNS__', output_files)

        if stageout_files:
            if '/' not in stageout_path:
                stageout_user = username # JMTBAD use getUsernameFromSiteDB?
                if stageout_path:
                    stageout_path = '/' + stageout_path
                stageout_path = 'root://cmseos.fnal.gov//store/user/' + stageout_user + stageout_path
                if not publish_name:
                    publish_name = batch_name.replace('/', '_')
                stageout_path += '/$(<cs_primaryds)/' + publish_name + '/$(<cs_timestamp)/$(printf "%04i" $(($job/1000)) )'

            #print 'CondorSubmitter init: stageout files are', stageout_files
            #print 'CondorSubmitter init: stageout path is', stageout_path

            output_snippet += self.stageout_template \
                .replace('__STAGEOUT_BNS__',  stageout_files) \
                .replace('__STAGEOUT_PATH__', stageout_path)

        self.sh_template = self.sh_template \
            .replace('__INPUT_BNS__',  input_bns) \
            .replace('__MEAT__', self.meat) \
            .replace('__OUTPUT_SNIPPET__', output_snippet)

        self.jdl_template = self.jdl_template \
            .replace('__SH_FN__',      sh_fn) \
            .replace('__TARBALL_FN__', tarball_fn) \
            .replace('__INPUT_FNS__',  input_fns) \
            .replace('__EXTRAS__',     jdl_extras)

        self.pset_end_template = self.pset_end_template \
            .replace('__MAX_EVENTS__', str(_events)) \
            .replace('__FAIL_LIST__', repr(_fail))

        open(sh_fn, 'wt').write(self.sh_template)
Ejemplo n.º 9
0
def submit(njobs, template_type, min_ntracks, signal_sample, template_signal, samples):
    global maked
    global setuped
    global i_submit

    if not maked:
        if os.system("make clean; make -j 16") != 0:
            raise "no make"
        raw_input("did the make go OK?")
        maked = True

    extra_name = ""
    batch_root = "crab3/One2Two"
    for x in sys.argv:
        if x.startswith(batch_root):
            extra_name = x.replace(batch_root + "_", "") + "_"
            batch_root = x
            break

    dummy_pset_fn = os.path.join(batch_root, "dummy_pset.py")

    files_needed = set()
    files_needed.add(os.path.join(os.path.abspath("mfvo2t.exe")))
    files_needed.add(os.path.join(os.path.abspath("filtertee.py")))

    if not setuped:
        os.system("mkdir -p %s" % batch_root)
        save_git_status(os.path.join(batch_root, "gitstatus"))
        open(dummy_pset_fn, "wt").write(dummy_pset)
        setuped = True

    batch_name = "%sTmp%s_Ntk%i_SigTmp%s_SigSam%s_Sam%s" % (
        extra_name,
        template_type,
        min_ntracks,
        template_signal,
        "no" if signal_sample is None else "n%ix%i" % signal_sample,
        samples,
    )

    files_needed.add("MultiJetPk2012.root.gz")
    if template_signal > -100:
        files_needed.add("backgrounds.tgz")
        files_needed.add(sample_number_to_name[template_signal] + ".root.gz")
    else:
        files_needed.add("bigsigscan.root.gz")

    extra_setup = ""

    env = ["toythrower_min_ntracks=%i" % min_ntracks, "toythrower_template_signal=%i" % template_signal]

    if no_save_plots:
        env += ["templates_save_plots=0", "fitter_save_plots=0"]

    if template_signal <= -101:
        env.append("toythrower_use_only_data_sample=1")
        env.append("sig_from_file_num=%i" % template_signal)

    if template_signal >= -12 or (template_signal <= -101 and template_signal >= -126):
        env.append("fitter_sig_limit_step=1")

    if template_type == "PS":
        env.append("templates_kind=phishift")
        env.append("fitter_start_nuis0=2")
        env.append("fitter_start_nuis1=0.003")
    elif template_type == "SC":
        env.append("templates_kind=simpleclear")
        env.append("fitter_fix_nuis1=1")
        env.append("fitter_start_nuis0=10")
        env.append("fitter_start_nuis1=0")

    if signal_sample is not None:
        sig_samp, sig_scale = signal_sample
        assert sig_samp < 0
        if sig_samp > -100:
            files_needed.add(sample_number_to_name[sig_samp] + ".root.gz")
        else:
            files_needed.add("bigsigscan.root.gz")

        if sig_scale < 0:
            env.append("ntoys=0")
            env.append("process_data=1")
            if sig_scale == -2:
                env.append("seed=1")
                env.append("fitter_i_limit_job=$JOB_NUM")
                env.append("fitter_do_signif=0")
            else:
                assert njobs <= 20
        else:
            env.append("fitter_do_limit=0")
            env.append("toythrower_injected_signal=%i" % sig_samp)
            env.append("toythrower_injected_signal_scale=%f" % sig_scale)
    else:
        env.append("fitter_do_limit=0")

    if type(samples) == int:
        env.append("toythrower_sample_only=%i" % samples)
    elif type(samples) == str and "500" in samples:
        env.append("toythrower_use_qcd500=1")
    elif "ttbarsyst" in samples:
        which_syst = samples.replace("ttbarsyst", "")
        files_needed.add("myttbar%s.root.gz" % which_syst)
        extra_setup += (
            """
cd trees
ln -s myttbar%s.root bkgsyst.root
cd -
"""
            % which_syst
        )
        env.append("toythrower_sample_only=99")

    ###

    env = "\n".join("export mfvo2t_" + e for e in env)

    unzip_files = []
    for fn in files_needed:
        if fn.endswith(".tgz"):
            unzip_files.append("tar zxf %s" % fn)
        elif fn.endswith(".gz"):
            unzip_files.append("gunzip %s" % fn)
    unzip_files = "\n".join(unzip_files)
    tree_path = (
        "/eos/uscms/store/user/tucker/mfvo2t_all_trees_444de711cdc630ddfe7cb6cd8f64ec8b46d09990_plussomettbarsyst"
    )
    files_needed = ", ".join('"%s"' % os.path.join(tree_path, f) for f in files_needed)

    submit_dir = os.path.join(crab_submit_dir, "%05i" % i_submit)
    os.mkdir(submit_dir)
    os.symlink(os.readlink("crab3"), os.path.join(submit_dir, "crab3"))
    open(os.path.join(submit_dir, "runme.sh"), "wt").write(script_template % locals())
    open(os.path.join(submit_dir, "crabConfig.py"), "wt").write(crab_cfg % locals())

    i_submit += 1
Ejemplo n.º 10
0
from time import time
from CRABClient.ClientExceptions import ConfigException as CRABConfigException
from JMTucker.Tools.CRAB3Tools import Config, crab_dirs_root, crab_command
from JMTucker.Tools.general import index_startswith, save_git_status
from JMTucker.Tools.CondorSubmitter import CondorSubmitter
from JMTucker.Tools.Sample import MCSample
from JMTucker.Tools import colors

testing = 'testing' in sys.argv
work_area = crab_dirs_root('nstep_%s%s' % (meta, ex))
if os.path.isdir(work_area):
    sys.exit('work_area %s exists' % work_area)
os.makedirs(work_area)
gitstatus_dir = 'gitstatus_%s' % int(time()*1000)
if not condor:
    save_git_status(os.path.join(work_area, gitstatus_dir))

config = Config()

open('year.txt', 'wt').write(str(year))

to_rm = ['year.txt']

if output_level in ('minitree', 'ntuple'):
    for x in 'ntuple.py', 'minitree.py':
        to_rm.append(x)
        os.system('cmsDumpPython.py ../%s > %s' % (x,x))

config.General.transferLogs = False
config.General.transferOutputs = True
config.General.workArea = work_area
Ejemplo n.º 11
0
def submit(sample, testing=False):
    per = 10
    batch_name = sample
    file_list_fn = os.path.abspath('filelist.%s.gz' % sample)
    event_list_fn = os.path.abspath('vetolist.%s.gz' % sample)
    nfiles = sum(1 for line in gzip.open(file_list_fn))
    njobs = int_ceil(nfiles, per)

    if testing:
        batch_name += '_TEST'
        njobs = 1

    print batch_name

    cmssw_py = 'pick_by_veto.py'

    root_name = 'pick1vtx'
    batch_root = os.path.join('/uscms_data/d2/tucker/condor_batch', root_name)
    batch_dir = os.path.abspath(os.path.join(batch_root, batch_name))
    
    inputs_dir = os.path.join(batch_dir, 'inputs')
    outputs_dir = os.path.join(batch_dir, 'outputs')
    os.makedirs(inputs_dir)
    os.makedirs(outputs_dir)
    
    save_git_status(os.path.join(batch_dir, 'gitstatus'))
    
    tarball_fn_base = 'input.tgz'
    tarball_fn = os.path.join(inputs_dir, tarball_fn_base)
    make_tarball(tarball_fn, include_python=True)
    
    cmssw_py_fn = os.path.join(inputs_dir, cmssw_py)
    shutil.copy2(cmssw_py, cmssw_py_fn)
    
    sh_fn = os.path.join(inputs_dir, 'run.sh')
    jdl_fn = os.path.join(inputs_dir, 'submit.jdl')
    
    scram_arch = os.environ['SCRAM_ARCH']
    cmssw_version = os.environ['CMSSW_VERSION']

    out_xrd = 'root://cmseos.fnal.gov/'
    out_path = '/store/user/tucker/pick1vtx/%s' % sample
    if os.system('eos %s mkdir -p %s' % (out_xrd, out_path)) != 0:
        raise IOError('problem making %s' % out_path)

    if any(os.system('eos %s ls %s/pick_%i.root 2>&1 > /dev/null' % (out_xrd, out_path, i)) == 0 for i in xrange(njobs)):
        raise IOError('one or more output files already exist')

    sh = dedent('''
    #!/bin/bash
    
    workdir=$(pwd)
    job=$1

    echo date: $(date)
    echo job: $job
    echo pwd: $workdir
    
    export SCRAM_ARCH=%(scram_arch)s
    source /cvmfs/cms.cern.ch/cmsset_default.sh
    
    scram project CMSSW %(cmssw_version)s 2>&1
    cd %(cmssw_version)s
    tar xvf ${workdir}/%(tarball_fn_base)s
    cd src
    eval `scram ru -sh`
    scram b -j 2
    cd $workdir
    
    cmsRun %(cmssw_py)s +job $job +per %(per)s +sample %(sample)s 2>&1
    cmsexit=$?
    if [[ $cmsexit -ne 0 ]]; then
        echo cmsRun exited with code $cmsexit
        exit $cmsexit
    fi

    new_fn=${out_xrd}${out_path}/pick_${job}.root
    echo xrdcp to $new_fn
    xrdcp pick.root $new_fn
    xrdcpexit=$?
    if [[ $xrdcpexit -ne 0 ]]; then
        echo xrdcp exited with code $xrdcpexit
        exit $xrdcpexit
    fi
    ''' % locals())
    
    open(sh_fn, 'wt').write(sh)
    os.chmod(sh_fn, 0755)
    
    jdl = dedent('''
    universe = vanilla
    Executable = %(sh_fn)s
    arguments = $(Process)
    Output = %(root_name)s_%(batch_name)s_$(Cluster)_$(Process).stdout
    Error = %(root_name)s_%(batch_name)s_$(Cluster)_$(Process).stderr
    Log = %(root_name)s_%(batch_name)s_$(Cluster)_$(Process).log
    stream_output = false
    stream_error  = false
    notification  = never
    should_transfer_files   = YES
    when_to_transfer_output = ON_EXIT
    transfer_input_files = %(tarball_fn)s,%(cmssw_py_fn)s,%(file_list_fn)s,%(event_list_fn)s
    Queue %(njobs)s
    ''' % locals())
    
    open(jdl_fn, 'wt').write(jdl)

    if not testing or testing == 'run':
        os.chdir(outputs_dir)
        os.system('condor_submit < ' + jdl_fn)
        os.chdir(pwd)