def generate_signal_json(version=-1): """generate private signal file list json""" paramsubdirs = eosls(EOSPATH_SIG) json_4mu, json_2mu2e = {}, {} for subdir in paramsubdirs: if 'MDp-0p8' in subdir or 'MDp-2p5' in subdir: continue # skipping unrequested darkphoton mass points if '4Mu' in subdir: key = subdir.replace('SIDM_BsTo2DpTo4Mu_', '').split('_ctau')[0].replace('MBs', 'mXX').replace( 'MDp', 'mA') key += '_lxy-300' # mXX-1000_mA-0p25_lxy-300 timestampdirs = eosls(join(EOSPATH_SIG, subdir)) timestampdirs = sorted( timestampdirs, key=lambda x: datetime.strptime(x, "%y%m%d_%H%M%S")) latest = join(EOSPATH_SIG, subdir, timestampdirs[version]) json_4mu[key] = [ f for f in eosfindfile(latest) if '/failed/' not in f ] if '2Mu2e' in subdir: key = subdir.replace('SIDM_BsTo2DpTo2Mu2e_', '').split('_ctau')[0].replace('MBs', 'mXX').replace( 'MDp', 'mA') key += '_lxy-300' timestampdirs = eosls(join(EOSPATH_SIG, subdir)) timestampdirs = sorted( timestampdirs, key=lambda x: datetime.strptime(x, "%y%m%d_%H%M%S")) latest = join(EOSPATH_SIG, subdir, timestampdirs[version]) json_2mu2e[key] = [ f for f in eosfindfile(latest) if '/failed/' not in f ] ## samples with new naming for subdir in eosls(EOSPATH_SIG2['4mu']): key = subdir.split('_ctau')[0] # mXX-100_mA-5_lxy-0p3 timestampdirs = eosls(join(EOSPATH_SIG2['4mu'], subdir)) timestampdirs = sorted( timestampdirs, key=lambda x: datetime.strptime(x, "%y%m%d_%H%M%S")) latest = join(EOSPATH_SIG2['4mu'], subdir, timestampdirs[version]) json_4mu[key] = [f for f in eosfindfile(latest) if '/failed/' not in f] for subdir in eosls(EOSPATH_SIG2['2mu2e']): key = subdir.split('_ctau')[0] # mXX-100_mA-5_lxy-0p3 timestampdirs = eosls(join(EOSPATH_SIG2['2mu2e'], subdir)) timestampdirs = sorted( timestampdirs, key=lambda x: datetime.strptime(x, "%y%m%d_%H%M%S")) latest = join(EOSPATH_SIG2['2mu2e'], subdir, timestampdirs[version]) json_2mu2e[key] = [ f for f in eosfindfile(latest) if '/failed/' not in f ] with open(f'signal_4mu_v2{version}.json', 'w') as outf: outf.write(json.dumps(json_4mu, indent=4)) with open(f'signal_2mu2e_v2{version}.json', 'w') as outf: outf.write(json.dumps(json_2mu2e, indent=4))
def list_files(dir, pattern=None): """ wrapper of `eos find` subcommand to list files recursively, excluding failed crab job output :param str dir: eos directory :param str pattern: restrict pattern of file names :return: a list of file names :rtype: list """ return [ f for f in eosfindfile(dir, pattern=pattern) if f and "/failed" not in f ]
def generate_background_json(): generated = dict() for group in EOSPATHS_BKG: generated[group] = {} for tag in EOSPATHS_BKG[group]: generated[group][tag] = [] for path in EOSPATHS_BKG[group][tag]: timestampdirs = eosls(path) timestampdirs = sorted( timestampdirs, key=lambda x: datetime.strptime(x, "%y%m%d_%H%M%S") ) latest = join(path, timestampdirs[-1]) for filepath in eosfindfile(latest): if "/failed/" in filepath: continue # filter out those in *failed* folder generated[group][tag].append(filepath) with open("backgrounds.json", "w") as outf: outf.write(json.dumps(generated, indent=4))
#!/usr/bin/env python """generate data sample list, until proper sample management tool show up. """ import json import concurrent.futures import uproot from FireHydrant.Tools.commonhelpers import eosls, eosfindfile # This is control region events. EOSPATHS = dict( A= "/store/user/pimeloni/ffNtuple4trigger/2018/SingleMuon/Run2018A-17Sep2018-v2/190903_170031", B= "/store/user/pimeloni/ffNtuple4trigger/2018/SingleMuon/Run2018B-17Sep2018-v1/190903_170117", C= "/store/user/pimeloni/ffNtuple4trigger/2018/SingleMuon/Run2018C-17Sep2018-v1/190903_170204", D= "/store/user/pimeloni/ffNtuple4trigger/2018/SingleMuon/Run2018D-22Jan2019-v2/190903_170249", ) REDIRECTOR = "root://cmseos.fnal.gov/" if __name__ == "__main__": datasets = {k: eosfindfile(v) for k, v in EOSPATHS.items() } #json.load(open("trigger_data.json")) with open("trigger_data.json", "w") as outf: outf.write(json.dumps(datasets, indent=4))
#!/usr/bin/env python """generate data sample list, until proper sample management tool show up. """ import json from FireHydrant.Tools.commonhelpers import eosls, eosfindfile # This is control region events. EOSPATHS = dict( A= "/store/group/lpcmetx/SIDM/ffNtuple/2018/DoubleMuon/Run2018A-17Sep2018-v2/190701_165735", B= "/store/group/lpcmetx/MCSIDM/ffNtuple/2018/DoubleMuon/Run2018B-17Sep2018-v1/190625_233608", C= "/store/group/lpcmetx/MCSIDM/ffNtuple/2018/DoubleMuon/Run2018C-17Sep2018-v1/190625_233628", D= "/store/group/lpcmetx/MCSIDM/ffNtuple/2018/DoubleMuon/Run2018D-PromptReco-v2/190625_233354", ) REDIRECTOR = "root://cmseos.fnal.gov/" if __name__ == "__main__": datasets = {k: eosfindfile(v) for k, v in EOSPATHS.items()} with open("control_data2018.json", "w") as outf: outf.write(json.dumps(datasets, indent=4))
def list_files(dir): "remove crab failed files" return [f for f in eosfindfile(dir) if "/failed" not in f]