def build_snapshot(N, modify=False):
if not modify:
logger.info('Cleaning up staging areas...')
rmtree(workdir)
makedirs(workdir)
rmtree(logdir)
makedirs(logdir)
else:
logger.warning(
"Partially rebuilding work area on request...hope you know what you're doing"
)
logger.info('Acquiring configuration...')
do('wget -nv -O %s/list.cfg %s' % (workdir, panda_cfg))
fin = open(outcfg.replace('local.cfg', 'list.cfg'))
samples = jm.convert_catalog(list(fin), as_dict=True)
keys = sorted(samples)
if len(keys) == 0:
logger.error('Configuration is empty - exiting!')
sys.exit(1)
to_write = [samples[k].get_config(N, suffix='_%i') for k in keys]
with open(outcfg, 'w') as fout:
for i, c in enumerate(chain.from_iterable(to_write)):
fout.write(c % (i, i))
logger.info('Submission will have %i jobs' % i)
do('cp -v {0}/list.cfg {0}/list_all.cfg'.format(workdir))
do('cp -v {0}/local.cfg {0}/local_all.cfg'.format(workdir))
logger.info('Tarring up CMSSW...')
cwd = os.getcwd()
os.chdir(cmssw_base)
do('tar --exclude-vcs -chzf cmssw.tgz src python biglib bin lib objs test external'
)
do('mv -v cmssw.tgz %s' % workdir)
logger.info('Creating executable...')
os.chdir('%s/src/PandaAnalysis/T3/inputs/' % (cmssw_base))
do('cp -v %s %s/skim.py' % (getenv('SUBMIT_TMPL'), workdir))
do('chmod 775 %s/skim.py' % (workdir))
logger.info('Finalizing work area...')
jm.issue_proxy()
do('cp -v /tmp/x509up_u%i %s/x509up' % (os.getuid(), workdir))
do('cp -v %s/src/PandaAnalysis/T3/inputs/exec.sh %s' %
(cmssw_base, workdir))
logger.info('Taking a snapshot of work area...')
do('cp -rvT %s %s/workdir' % (workdir, outdir))
import argparse
import subprocess
from re import sub
from os import getenv
from PandaCore.Tools.Misc import PInfo
from PandaCore.Tools.job_management import DataSample,convert_catalog
workdir = getenv('SUBMIT_WORKDIR')
parser = argparse.ArgumentParser(description='convert configuration')
parser.add_argument('--infile',type=str,default=None)
parser.add_argument('--outfile',type=str,default=None)
parser.add_argument('--nfiles',type=int,default=None)
args = parser.parse_args()
fin = open(args.infile)
samples = convert_catalog(list(fin),as_dict=True)
fout = open(args.outfile,'w')
keys = sorted(samples)
counter=0
for k in keys:
sample = samples[k]
configs = sample.get_config(args.nfiles,suffix='_%i')
for c in configs:
fout.write(c%(counter,counter))
counter += 1
PInfo('buildConfig.py','Submission will have %i jobs'%(counter))
fout.close()