def iter_arrays_qcd(N):
xss = [
0.23 * 7025.0, 620.6, 59.06, 18.21, 7.5, 0.6479, 0.08715, 0.005242,
0.0001349, 3.276
]
ntuple_path = 'root://cmseos.fnal.gov//store/user/lpcsusyhad/SVJ2017/boosted/ecfntuples/'
samples = [
'Feb15_qcd_BTV-RunIIFall18GS-00024_1_cfg',
'Feb15_qcd_BTV-RunIIFall18GS-00025_1_cfg',
'Feb15_qcd_BTV-RunIIFall18GS-00026_1_cfg',
'Feb15_qcd_BTV-RunIIFall18GS-00027_1_cfg',
'Feb15_qcd_BTV-RunIIFall18GS-00029_1_cfg',
'Feb15_qcd_BTV-RunIIFall18GS-00030_1_cfg',
'Feb15_qcd_BTV-RunIIFall18GS-00031_1_cfg',
'Feb15_qcd_BTV-RunIIFall18GS-00032_1_cfg',
'Feb15_qcd_BTV-RunIIFall18GS-00033_1_cfg',
'Feb15_qcd_BTV-RunIIFall18GS-00051_1_cfg',
]
rootfiles = [
seutils.ls_wildcard(ntuple_path + s + '/*.root') for s in samples
]
yield from uptools.iter_arrays_weighted(N,
xss,
rootfiles,
treepath='gensubntupler/tree')
def init_sig_2016_nohtcut(mz, **kwargs):
name = 'mz' + str(mz)
rootfiles = seutils.ls_wildcard(
'root://cmseos.fnal.gov//store/user/lpcsusyhad/SVJ2017/boosted/treemaker/nohtcut_year2016/*mz{}*/*.root'
.format(int(mz)))
set_default_branches(kwargs)
signal = svjflatanalysis.dataset.SignalDataset(name, rootfiles, **kwargs)
signal.xs = SIGNAL_CROSSSECTIONS[mz]
return signal
def init_sig_ht1000(mz, **kwargs):
name = 'mz' + str(mz)
rootfiles = seutils.ls_wildcard(
'root://cmseos.fnal.gov//store/user/klijnsma/semivis/treemakerht1000/mz{0}_*.root'
.format(int(mz)))
if not 'max_entries' in kwargs: kwargs['max_entries'] = None
set_default_branches(kwargs)
signal = svjflatanalysis.dataset.SignalDataset(name, rootfiles, **kwargs)
signal.xs = SIGNAL_CROSSSECTIONS[mz] # * HT1000_EFFICIENCIES[mz]
return signal
def init_sig_jetpt250(mz, year, **kwargs):
name = 'mz' + str(mz)
rootfiles = seutils.ls_wildcard(
'root://cmseos.fnal.gov//store/user/lpcsusyhad/SVJ2017/boosted/treemaker/jetpt250*mz{}*year{}*/*.root'
.format(int(mz), year))
kwargs['branches'] = kwargs.get(
'branches', []) + svjflatanalysis.arrayutils.nonnested_branches(
b'JetsAK15', add_subjets=True)
if not 'max_entries' in kwargs: kwargs['max_entries'] = None
signal = svjflatanalysis.dataset.SignalDataset(name, rootfiles, **kwargs)
signal.xs = SIGNAL_CROSSSECTIONS_PYTHIA_SARA[mz]
return signal
def du():
parser = ParserMultipleRemotePaths()
parser.add_argument('-s',
'--sort',
action='store_true',
help='Sorts by size instead (default is by name)')
args = parser.parse_args(expand_wildcards=False)
for path in args.paths:
inodes = seutils.ls_wildcard(path,
stat=True,
implementation=args.implementation)
if args.sort: inodes.sort(key=lambda inode: -inode.size)
for inode in inodes:
print('{0:<8} {1}'.format(inode.size_human, inode.path))
def make_npzs_signal(directory):
N = seutils.root.sum_dataset(directory, treepath='gensubntupler/tree')
signal = seutils.ls_wildcard(directory + '/*.root')
signal_name = osp.basename(osp.dirname(signal[0]))
train_outdir = 'data/train/raw/' + signal_name
test_outdir = 'data/test/raw/' + signal_name
n_total = 0
i_event_good = 0
for i_event, event in tqdm.tqdm(enumerate(
uptools.iter_events(signal, treepath='gensubntupler/tree',
nmax=N)),
total=N):
outdir = test_outdir if i_event_good % 5 == 0 else train_outdir
succeeded = ntup_to_npz_signal(event, outdir + f'/{i_event_good}.npz')
if succeeded: i_event_good += 1
n_total += 1
print(
'Total events turned into npzs: {}/{} ({} failures due to 0 Z-energy)'
.format(i_event_good, n_total, n_total - i_event_good))
def make_npzs_signal():
directory = 'root://cmseos.fnal.gov//store/user/lpcsusyhad/SVJ2017/boosted/ecfntuples/Mar16_mz150_rinv0.1_mdark10'
N = seutils.root.sum_dataset(directory, treepath='gensubntupler/tree')
signal = seutils.ls_wildcard(directory + '/*.root')
signal_name = osp.basename(osp.dirname(signal[0]))
train_outdir = 'data/train/raw/' + signal_name
test_outdir = 'data/test/raw/' + signal_name
n_total = 0
i_event_good = 0
for i_event, event in tqdm.tqdm(enumerate(
uptools.iter_events(signal, treepath='gensubntupler/tree',
nmax=N)),
total=N):
outdir = test_outdir if i_event_good % 5 == 0 else train_outdir
succeeded = ntup_to_npz_signal(event, outdir + f'/{i_event_good}.npz')
if succeeded: i_event_good += 1
n_total += 1
print(
'Total events turned into npzs: {}/{} ({} failures due to 0 Z-energy)'
.format(i_event_good, n_total, n_total - i_event_good))
def expand_path(path, log=log):
log('Path to expand: %s' % path)
# For debugging only:
if COMPLETION_TEST_MODE:
try:
log('dirname={}'.format(seutils.path.dirname(path)))
log('lfn={}'.format(seutils.path.get_lfn(path)))
except Exception:
pass
# Do expansion to the registerd default mgms
if not (seutils.path.is_valid_path(path)):
log('%s is not valid' % path)
matching_mgms = [
mgm for mgm in DEFAULT_MGMS if fnmatch.fnmatch(mgm, path + '*')
]
return format_matches(path, matching_mgms, log=log)
log('Applying normpath')
path = seutils.path.normpath(path) #+ ('' if path.endswith('/') else '/')
log('path=%s' % path)
if seutils.path.get_depth(path) <= 1:
log('path=%s has depth <= 1' % path)
matching_mgms = [
mgm for mgm in DEFAULT_MGMS if fnmatch.fnmatch(mgm, path + '*')
]
return format_matches(path, matching_mgms, log=log)
# Do expansion
log('%s is valid, fetching contents' % path)
if path.endswith('/'):
log('Doing seutils.listdir("%s")' % path)
contents = seutils.listdir(path)
else:
log('Doing seutils.ls_wildcard("%s*")' % path)
contents = seutils.ls_wildcard(path +
('' if path.endswith('*') else '*'))
return format_matches(path, contents, add_trailing_slash=True, log=log)
def ls():
parser = ParserMultipleRemotePaths()
parser.add_argument('-l',
'--long',
action='store_true',
help='Include mtime and size in the output')
parser.add_argument(
'-s',
'--sort',
type=str,
choices=['name', 'date', 'size'],
default='name',
help=
'Only works if -l is passed as well. Possibility to sort the output by date or size as well'
)
args = parser.parse_args(expand_wildcards=False)
if not args.long and args.sort != 'name':
seutils.logger.warning('Option --sort ignored (use --long as well)')
if not args.paths: args.paths = ['']
for path in args.paths:
if '*' in path:
output = seutils.ls_wildcard(path,
stat=args.long,
implementation=args.implementation)
else:
output = seutils.ls(path,
stat=args.long,
implementation=args.implementation)
if args.long:
if args.sort == 'date':
output.sort(key=lambda inode: inode.modtime, reverse=True)
elif args.sort == 'size':
output.sort(key=lambda inode: -inode.size)
for inode in output:
print('{} {:<8} {}'.format(
inode.modtime.strftime('%Y-%m-%d %H:%M'), inode.size_human,
inode.path))
else:
print('\n'.join(output))
def parse_args(self, *args, **kwargs):
expand_wildcards = kwargs.pop('expand_wildcards', True)
allow_zero_paths = kwargs.pop('allow_zero_paths', False)
disallow_wildcards = kwargs.pop('disallow_wildcards', False)
parsed_args = super(ParserMultipleRemotePaths,
self).parse_args(*args, **kwargs)
paths = []
for path in parsed_args.paths:
if not seutils.path.has_protocol(path):
raise TypeError('Path {0} is not remote'.format(path))
if disallow_wildcards and '*' in path:
raise TypeError('Wildcards not allowed')
if expand_wildcards:
paths.extend(
seutils.ls_wildcard(
path, implementation=parsed_args.implementation))
else:
paths.append(path)
parsed_args.paths = paths
if not allow_zero_paths and len(paths) == 0:
raise TypeError('Pass at least one path')
return parsed_args
def init_bkgs_triggered(category=None, **kwargs):
bkgs = []
path = osp.join(triggered_path, 'Autumn18.*')
bkg_rootfiles = seutils.ls_wildcard(path) if os.uname()[1].endswith(
'.fnal.gov') else glob.glob(path)
kwargs['treename'] = 'PreSelection'
for bkg_rootfile in bkg_rootfiles:
if category:
if not category.lower() in bkg_rootfile.lower():
logger.info('Skipping %s', bkg_rootfile)
continue
logger.info('Loading %s', bkg_rootfile)
name = osp.basename(bkg_rootfile).replace('.root', '')
bkg = svjflatanalysis.dataset.BackgroundDataset(
name, [bkg_rootfile], **kwargs)
bkg.get_xs()
# Take into account the trigger efficiencies
if not name in NOCUTS_TRIGGER_PLUS_JETPT550_EFF_BKG:
logger.error('FIXME: efficiency for %s not calculated yet!', name)
continue
bkg.xs *= NOCUTS_TRIGGER_PLUS_JETPT550_EFF_BKG[name]
bkgs.append(bkg)
return bkgs
def test_ls_wildcard(impl):
seutils.ls_wildcard('root://foo.bar.gov//foo/*/*', implementation=impl) == ['root://foo.bar.gov//foo/bar/test.file']
seutils.ls_wildcard('root://foo.bar.gov//foo', implementation=impl) == ['root://foo.bar.gov//foo']
seutils.ls_wildcard('root://foo.bar.gov//foo/', implementation=impl) == ['root://foo.bar.gov//foo']
'cmsconnect_blacklist',
['*_RU_*', '*FNAL*', '*IFCA*', '*KIPT*', 'T3_IT_Trieste', 'T2_TR_METU', 'T2_US_Vanderbilt', 'T2_PK_NCP']
)
for mz in [150, 250]:
for rinv in [0.001, 0.1, 0.3]:
submit(mz=mz, rinv=rinv)
"""# endsubmit
import qondor, seutils, os.path as osp, os, argparse
parser = argparse.ArgumentParser()
parser.add_argument('-t', '--test', action='store_true')
args = parser.parse_args()
genfiles = seutils.ls_wildcard(
'root://cmseos.fnal.gov//store/user/lpcsusyhad/SVJ2017/boosted/gen/'
'genjetpt250_Jan25_mz{mz:.0f}_mdark10_rinv{rinv}/*.root'.format(
mz=qondor.scope.mz, rinv=qondor.scope.rinv))
cmssw = qondor.svj.init_cmssw(
'root://cmseos.fnal.gov//store/user/lpcsusyhad/SVJ2017/boosted/svjproduction-tarballs/'
'CMSSW_10_2_21_efb554a_el7_gen_2018_Feb02_ecfntupler.tar.gz')
rundir = osp.join(cmssw.cmssw_src, 'SVJ/Production/test')
processed_genfiles = []
for i, genfile in enumerate(genfiles):
if args.test and i == 2: break
local_rootfile = osp.join(rundir, 'step1.root')
if osp.isfile(local_rootfile): os.remove(local_rootfile)
seutils.cp(genfile, local_rootfile)
cmssw.run_commands([
'cd {0}'.format(rundir), 'cmsRun runSVJ.py'
