def generate_signal_files():
"""generate private signal file list json"""
print("[generate_signal_files]")
paramsubdirs = eosls(EOSPATH_SIG)
json_4mu, json_2mu2e = {}, {}
for subdir in paramsubdirs:
if 'MDp-0p8' in subdir or 'MDp-2p5' in subdir:
continue # skipping unrequested darkphoton mass points
if '4Mu' in subdir:
key = subdir.replace('SIDM_BsTo2DpTo4Mu_', '').split('_ctau')[0]\
.replace('MBs', 'mXX').replace('MDp', 'mA')
key += '_lxy-300' # mXX-1000_mA-0p25_lxy-300
json_4mu[key] = latest_files(join(EOSPATH_SIG, subdir))
if '2Mu2e' in subdir:
key = subdir.replace('SIDM_BsTo2DpTo2Mu2e_', '').split('_ctau')[0]\
.replace('MBs', 'mXX').replace('MDp', 'mA')
key += '_lxy-300'
json_2mu2e[key] = latest_files(join(EOSPATH_SIG, subdir))
## samples with new naming
for subdir in eosls(EOSPATH_SIG2['4mu']):
key = subdir.split('_ctau')[0] # mXX-100_mA-5_lxy-0p3
json_4mu[key] = latest_files(join(EOSPATH_SIG2['4mu'], subdir))
for subdir in eosls(EOSPATH_SIG2['2mu2e']):
key = subdir.split('_ctau')[0] # mXX-100_mA-5_lxy-0p3
json_2mu2e[key] = latest_files(join(EOSPATH_SIG2['2mu2e'], subdir))
return json_4mu, json_2mu2e
def generate_background_files(skim=False, forscale=False):
print(f"[generate_background_files(skim={skim}, forscale={forscale})]")
_dirmap = EOSPATHS_BKGSKIM if skim else EOSPATHS_BKG
if forscale:
_dirmap = EOSPATHS_BKGAOD
## get max(latest) timestamp
## Note: for scale, it will scan AOD skim submissions, we do not put limit on
## submission timestamps as it can span over a longer time.
if forscale is False:
timestamps = []
for group in _dirmap:
for tag in _dirmap[group]:
for path in _dirmap[group][tag]:
timestamps.append(
sorted(eosls(path),
key=lambda x: datetime.strptime(
x, "%y%m%d_%H%M%S"))[-1])
maxts = sorted(timestamps,
key=lambda x: datetime.strptime(x, "%y%m%d_%H%M%S"))[-1]
maxts = datetime.strptime(maxts, "%y%m%d_%H%M%S")
generated = dict()
for group in _dirmap:
generated[group] = {}
for tag in _dirmap[group]:
generated[group][tag] = []
for path in _dirmap[group][tag]:
if forscale is False:
if (maxts - last_submit_timestamp(path)).seconds > 60 * 60:
continue
generated[group][tag].extend(
latest_files(path, pattern='*ffNtuple*.root'))
return generated
def generate_signal_json(version=-1):
"""generate private signal file list json"""
paramsubdirs = eosls(EOSPATH_SIG)
json_4mu, json_2mu2e = {}, {}
for subdir in paramsubdirs:
if 'MDp-0p8' in subdir or 'MDp-2p5' in subdir:
continue # skipping unrequested darkphoton mass points
if '4Mu' in subdir:
key = subdir.replace('SIDM_BsTo2DpTo4Mu_',
'').split('_ctau')[0].replace('MBs',
'mXX').replace(
'MDp', 'mA')
key += '_lxy-300' # mXX-1000_mA-0p25_lxy-300
timestampdirs = eosls(join(EOSPATH_SIG, subdir))
timestampdirs = sorted(
timestampdirs,
key=lambda x: datetime.strptime(x, "%y%m%d_%H%M%S"))
latest = join(EOSPATH_SIG, subdir, timestampdirs[version])
json_4mu[key] = [
f for f in eosfindfile(latest) if '/failed/' not in f
]
if '2Mu2e' in subdir:
key = subdir.replace('SIDM_BsTo2DpTo2Mu2e_',
'').split('_ctau')[0].replace('MBs',
'mXX').replace(
'MDp', 'mA')
key += '_lxy-300'
timestampdirs = eosls(join(EOSPATH_SIG, subdir))
timestampdirs = sorted(
timestampdirs,
key=lambda x: datetime.strptime(x, "%y%m%d_%H%M%S"))
latest = join(EOSPATH_SIG, subdir, timestampdirs[version])
json_2mu2e[key] = [
f for f in eosfindfile(latest) if '/failed/' not in f
]
## samples with new naming
for subdir in eosls(EOSPATH_SIG2['4mu']):
key = subdir.split('_ctau')[0] # mXX-100_mA-5_lxy-0p3
timestampdirs = eosls(join(EOSPATH_SIG2['4mu'], subdir))
timestampdirs = sorted(
timestampdirs, key=lambda x: datetime.strptime(x, "%y%m%d_%H%M%S"))
latest = join(EOSPATH_SIG2['4mu'], subdir, timestampdirs[version])
json_4mu[key] = [f for f in eosfindfile(latest) if '/failed/' not in f]
for subdir in eosls(EOSPATH_SIG2['2mu2e']):
key = subdir.split('_ctau')[0] # mXX-100_mA-5_lxy-0p3
timestampdirs = eosls(join(EOSPATH_SIG2['2mu2e'], subdir))
timestampdirs = sorted(
timestampdirs, key=lambda x: datetime.strptime(x, "%y%m%d_%H%M%S"))
latest = join(EOSPATH_SIG2['2mu2e'], subdir, timestampdirs[version])
json_2mu2e[key] = [
f for f in eosfindfile(latest) if '/failed/' not in f
]
with open(f'signal_4mu_v2{version}.json', 'w') as outf:
outf.write(json.dumps(json_4mu, indent=4))
with open(f'signal_2mu2e_v2{version}.json', 'w') as outf:
outf.write(json.dumps(json_2mu2e, indent=4))
def generate_background_json():
generated = dict()
for group in EOSPATHS_BKG:
generated[group] = {}
for tag in EOSPATHS_BKG[group]:
generated[group][tag] = []
for path in EOSPATHS_BKG[group][tag]:
timestampdirs = eosls(path)
timestampdirs = sorted(
timestampdirs, key=lambda x: datetime.strptime(x, "%y%m%d_%H%M%S")
)
latest = join(path, timestampdirs[-1])
for filepath in eosfindfile(latest):
if "/failed/" in filepath:
continue # filter out those in *failed* folder
generated[group][tag].append(filepath)
with open("backgrounds.json", "w") as outf:
outf.write(json.dumps(generated, indent=4))
def latest_files(parentPathOfTimestamps, pattern=None):
"""
list file names of latest batch job submission
:param str parentPathOfTimestamps: eos directory up to timestamp level
:param str pattern: restrict pattern of file names
:return: a list of file names
:rtype: list
"""
try:
timestampdirs = eosls(parentPathOfTimestamps)
timestampdirs = sorted(
timestampdirs, key=lambda x: datetime.strptime(x, "%y%m%d_%H%M%S"))
latest = join(parentPathOfTimestamps, timestampdirs[-1])
return list_files(latest, pattern=pattern)
except Exception as e:
print(e)
print(
f"Error when stat eos path: {parentPathOfTimestamps}! Empty list returned"
)
return []
def latest_files(parentPathOfTimestamps):
timestampdirs = eosls(parentPathOfTimestamps)
timestampdirs = sorted(timestampdirs, key=lambda x: datetime.strptime(x, "%y%m%d_%H%M%S"))
latest = join(parentPathOfTimestamps, timestampdirs[-1])
return list_files(latest)
def last_submit_timestamp(parentPathOfTimestamps):
timestamps = eosls(parentPathOfTimestamps)
last_ts = sorted(timestamps,
key=lambda x: datetime.strptime(x, "%y%m%d_%H%M%S"))[-1]
return datetime.strptime(last_ts, "%y%m%d_%H%M%S")
