def findMatchingDirs(box_data_path):
matching_dir_pairs = []
if box_data_path == '':
for dpm_dir in dirs:
print(dpm_dir)
match = re.search(
'^(.*/)dpm_.*?/(ip_offset_XYZDXDYDZ_.*)/.*/\d*/\d*-\d*_(.*cut)/.*/(binning_\d*)/merge_data$',
dpm_dir)
pattern = '^' + match.group(1) + 'box_.*?' + match.group(
2) + '.*' + match.group(3) + '/.*' + match.group(
4) + '/merge_data$'
# print pattern
for box_dir in box_dirs:
# print box_dir
box_match = re.search(pattern, box_dir)
if box_match:
matching_dir_pairs.append([dpm_dir, box_dir])
break
else:
for dpm_dir in dirs:
# attempt to find directory with same binning
print('checking for matching directory for ' + dpm_dir)
match = re.search('^.*(binning_\d*)/.*$', dpm_dir)
if match:
dir_searcher = general.DirectorySearcher([match.group(1)])
dir_searcher.searchListOfDirectories(box_data_path,
box_acc_glob_pattern)
correct_dirs = dir_searcher.getListOfDirectories()
if correct_dirs:
matching_dir_pairs.append([dpm_dir, correct_dirs[0]])
return matching_dir_pairs
parser.add_argument(
'dirname',
metavar='dirname_to_scan',
type=str,
nargs=1,
help=
'Name of directory to scan recursively for lmd data files and call merge!')
parser.add_argument('--dir_pattern',
metavar='path name pattern',
type=str,
default='.*',
help='')
args = parser.parse_args()
patterns = []
patterns.append(args.dir_pattern)
dir_searcher = general.DirectorySearcher(patterns)
dir_searcher.searchListOfDirectories(args.dirname[0], glob_pattern)
dirs = dir_searcher.getListOfDirectories()
bashcommand = default = os.getenv(
'LMDFIT_BUILD_PATH') + '/bin/plotIPDistribution'
for dir in dirs:
bashcommand += ' ' + dir
print bashcommand
returnvalue = subprocess.call(bashcommand.split())
parser.add_argument('--general_dimension_bins_high',
metavar='general_dimension_bins_high',
type=int,
default=300,
help='binning of data max')
parser.add_argument('--general_dimension_bins_step',
metavar='general_dimension_bins_step',
type=int,
default=100,
help='binning of data stepsize')
args = parser.parse_args()
failed_submit_commands = []
dir_searcher = general.DirectorySearcher([args.dir_pattern])
dir_searcher.searchListOfDirectories(args.dirname[0], ['filelist_', '.txt'])
dirs = dir_searcher.getListOfDirectories()
config_modifier = general.ConfigModifier()
config = config_modifier.loadConfig(args.config_url[0])
config_paths = []
for bins in range(args.general_dimension_bins_low,
args.general_dimension_bins_high + 1,
args.general_dimension_bins_step):
if 'general_data' in config:
subconfig = config['general_data']
if 'primary_dimension' in subconfig:
def simulateDataOnHimster(scenario):
tasks_to_remove = []
lab_momentum = scenario.momentum
for simulation_task in scenario.simulation_info_lists:
dir_path = simulation_task[0]
sim_type = simulation_task[1]
state = simulation_task[2]
last_state = simulation_task[3]
print('running simulation of type ' + str(sim_type) + ' and path (' +
dir_path + ') at state=' + str(state) + '/' + str(last_state))
data_keywords = []
data_pattern = ''
cut_keyword = ''
if scenario.use_xy_cut:
cut_keyword += 'xy_'
if scenario.use_m_cut:
cut_keyword += 'm_'
if cut_keyword == '':
cut_keyword += 'un'
cut_keyword += 'cut_real'
merge_keywords = ['merge_data', 'binning_300']
if 'v' in sim_type:
data_keywords = ['uncut', 'bunches', 'binning_300']
data_pattern = 'lmd_vertex_data_'
elif 'a' in sim_type:
data_keywords = [cut_keyword, 'bunches', 'binning_300']
data_pattern = 'lmd_data_'
else:
data_keywords = [cut_keyword, 'bunches', 'binning_300']
data_pattern = 'lmd_res_data_'
# 1. simulate data
if state == 1:
os.chdir(lmd_fit_script_path)
status_code = 1
if 'er' in sim_type:
found_dirs = []
if dir_path != '':
temp_dir_searcher = general.DirectorySearcher(
['box', data_keywords[0]])
temp_dir_searcher.searchListOfDirectories(
dir_path, 'Lumi_TrksQA_')
found_dirs = temp_dir_searcher.getListOfDirectories()
if found_dirs:
status_code = wasSimulationSuccessful(
found_dirs[0], 'Lumi_TrksQA_*.root')
elif last_state < 1:
# then lets simulate!
# this command runs the full sim software with box gen data
# to generate the acceptance and resolution information
# for this sample
# note: beam tilt and divergence are not necessary here,
# because that is handled completely by the model
ip_info_dict = scenario.rec_ip_info
max_xy_shift = math.sqrt(ip_info_dict['ip_offset_x'] ** 2 +
ip_info_dict['ip_offset_y'] ** 2)
max_xy_shift = float('{0:.2f}'.format(
round(float(max_xy_shift), 2)))
gen_par = general.createGeneralRunParameters(
box_num_events_per_sample,
box_num_samples, lab_momentum)
sim_par = simulation.createSimulationParameters('box')
sim_par['theta_min_in_mrad'] -= max_xy_shift
sim_par['theta_max_in_mrad'] += max_xy_shift
sim_par.update(gen_par)
rec_par = reconstruction.createReconstructionParameters()
rec_par['use_xy_cut'] = scenario.use_xy_cut
rec_par['use_m_cut'] = scenario.use_m_cut
rec_par['reco_ip_offset'] = [ip_info_dict['ip_offset_x'],
ip_info_dict['ip_offset_y'],
ip_info_dict['ip_offset_z']]
rec_par.update(gen_par)
# alignment part
# if alignement matrices were specified, we used them as a mis-alignment
# and alignment for the box simulations
align_par = alignment.createAlignmentParameters()
if 'alignment_matrices_path' in scen.alignment_parameters:
align_par['misalignment_matrices_path'] = scen.alignment_parameters['alignment_matrices_path']
align_par['alignment_matrices_path'] = scen.alignment_parameters['alignment_matrices_path']
# update the sim and reco par dicts
sim_par.update(align_par)
rec_par.update(align_par)
(dir_path, is_finished) = simulation.startSimulationAndReconstruction(
sim_par, align_par, rec_par, use_devel_queue=args.use_devel_queue)
simulation_task[0] = dir_path
scenario.acc_and_res_dir_path = dir_path
if is_finished:
state += 1
last_state += 1
elif 'a' in sim_type:
found_dirs = []
status_code = 1
if dir_path != '':
temp_dir_searcher = general.DirectorySearcher(
['dpm_elastic', data_keywords[0]])
temp_dir_searcher.searchListOfDirectories(
dir_path, 'Lumi_TrksQA_')
found_dirs = temp_dir_searcher.getListOfDirectories()
if found_dirs:
status_code = wasSimulationSuccessful(
found_dirs[0], 'Lumi_TrksQA_*.root')
elif last_state < state:
# then lets do reco
# this command runs the track reco software on the
# elastic scattering data with the estimated ip position
# note: beam tilt and divergence are not used here because
# only the last reco steps are rerun of the track reco
ip_info_dict = scenario.rec_ip_info
# TODO: save digi files instead of mc files!!
# we are either in the base dir or an "aligned" subdirectory,
# apply dirty hack here:
simParamFile = scenario.dir_path+'/../sim_params.config'
if not os.path.exists(simParamFile):
simParamFile = scenario.dir_path+'/../../sim_params.config'
with open(simParamFile, 'r') as json_file:
sim_par = json.load(json_file)
with open(scenario.dir_path + '/reco_params.config', 'r') as json_file:
rec_par = json.load(json_file)
rec_par['use_xy_cut'] = scenario.use_xy_cut
rec_par['use_m_cut'] = scenario.use_m_cut
rec_par['reco_ip_offset'] = [ip_info_dict['ip_offset_x'],
ip_info_dict['ip_offset_y'],
ip_info_dict['ip_offset_z']]
if (num_samples > 0 and
rec_par['num_samples'] > num_samples):
rec_par['num_samples'] = num_samples
sim_par['num_samples'] = num_samples
# dirname = os.path.dirname(scenario.dir_path)
(dir_path, is_finished) = simulation.startSimulationAndReconstruction(
sim_par, alignment.getAlignmentParameters(rec_par),
rec_par, use_devel_queue=args.use_devel_queue)
# (dir_path, is_finished) = reconstruction.startReconstruction(
# rec_par, alignment.getAlignmentParameters(rec_par),
# dirname, use_devel_queue=args.use_devel_queue)
simulation_task[0] = dir_path
scen.filtered_dir_path = dir_path
if is_finished:
state += 1
last_state += 1
else:
# just skip simulation for vertex data... we always have that..
print('skipping simulation step...')
status_code = 0
if status_code == 0:
print('found simulation files, skipping')
state = 2
last_state = 1
elif status_code > 0:
print('still waiting for himster simulation jobs for ' +
sim_type + ' data to complete...')
else:
# ok something went wrong there, exit this scenario and
# push on bad scenario stack
last_state = -1
# 2. create data (that means bunch data, create data objects)
if state == 2:
# check if data objects already exists and skip!
temp_dir_searcher = general.DirectorySearcher(data_keywords)
temp_dir_searcher.searchListOfDirectories(dir_path, data_pattern)
found_dirs = temp_dir_searcher.getListOfDirectories()
status_code = 1
if found_dirs:
status_code = wasSimulationSuccessful(
found_dirs[0], data_pattern + "*", True)
elif last_state < state:
os.chdir(lmd_fit_script_path)
# bunch data
bashcommand = 'python makeMultipleFileListBunches.py '\
'--files_per_bunch 10 --maximum_number_of_files ' + \
str(num_samples) + ' ' + dir_path
returnvalue = subprocess.call(bashcommand.split())
# create data
if 'a' in sim_type:
el_cs = scenario.elastic_pbarp_integrated_cross_secion_in_mb
bashcommand = 'python createMultipleLmdData.py '\
+ ' --dir_pattern ' + data_keywords[0] + ' ' + str(
lab_momentum) + ' ' + sim_type + ' ' + dir_path\
+ ' ../dataconfig_xy.json'
if el_cs:
bashcommand += ' --elastic_cross_section ' + str(el_cs)
else:
bashcommand = 'python createMultipleLmdData.py '\
+ '--dir_pattern ' + data_keywords[0] + ' '\
+ str(lab_momentum) + ' '\
+ sim_type + ' ' + dir_path + ' ../dataconfig_xy.json'
print(bashcommand)
returnvalue = subprocess.call(bashcommand.split())
last_state = last_state + 1
if status_code == 0:
print('skipping bunching and data object creation...')
state = 3
last_state = 2
elif status_code > 0:
print('still waiting for himster simulation jobs for ' +
sim_type + ' data to complete...')
else:
# ok something went wrong there, exit this scenario and
# push on bad scenario stack
print("ERROR: Something went wrong with the cluster jobs! "
"This scenario will be pushed onto the dead stack, "
"and is no longer processed.")
last_state = -1
# 3. merge data
if state == 3:
# check first if merged data already exists and skip it!
temp_dir_searcher = general.DirectorySearcher(merge_keywords)
temp_dir_searcher.searchListOfDirectories(dir_path, data_pattern)
found_dirs = temp_dir_searcher.getListOfDirectories()
if not found_dirs:
os.chdir(lmd_fit_script_path)
# merge data
if 'a' in sim_type:
bashcommand = 'python mergeMultipleLmdData.py'\
+ ' --dir_pattern ' + data_keywords[0]\
+ ' --num_samples ' + str(bootstrapped_num_samples)\
+ ' ' + sim_type + ' ' + dir_path
else:
bashcommand = 'python mergeMultipleLmdData.py'\
+ ' --dir_pattern '+data_keywords[0]\
+ ' ' + sim_type + ' ' + dir_path
returnvalue = subprocess.call(bashcommand.split())
state = 4
simulation_task[2] = state
simulation_task[3] = last_state
if simulation_task[3] == -1:
scenario.is_broken = True
break
if simulation_task[2] == 4:
tasks_to_remove.append(simulation_task)
print("Task is finished and will be removed from list!")
for x in tasks_to_remove:
del scenario.simulation_info_lists[
scenario.simulation_info_lists.index(x)]
return scenario
def lumiDetermination(scen):
dir_path = scen.dir_path
state = scen.state
last_state = scen.last_state
# open file
if os.path.exists(dir_path + "/../../elastic_cross_section.txt"):
print("Found an elastic cross section file!")
with open(dir_path + "/../../elastic_cross_section.txt") as f:
content = f.readlines()
scen.elastic_pbarp_integrated_cross_secion_in_mb = float(
content[0])
f.close()
else:
print(f'ERROR! Can not find elastic cross section file! The determined Luminosity will be wrong!\n')
sys.exit()
print('processing scenario ' + dir_path + ' at step ' + str(state))
# TODO: Not sure if later on the lab momentum has to be extracted from the data
m = re.search('(\d*?\.\d*?)GeV', dir_path)
momentum = float(m.group(1))
scen.momentum = momentum
with open(scen.dir_path + '/reco_params.config', 'r') as json_file:
rec_par = json.load(json_file)
scen.alignment_parameters = alignment.getAlignmentParameters(rec_par)
finished = False
# 1. create vertex data (that means bunch data, create data objects and merge)
if state == 1:
if len(scen.simulation_info_lists) == 0:
scen.simulation_info_lists.append([dir_path, 'v', 1, 0])
scen = simulateDataOnHimster(scen)
if scen.is_broken:
dead_scenario_stack.append(scen)
return
if len(scen.simulation_info_lists) == 0:
state += 1
last_state += 1
if state == 2:
# check if ip was already determined
if scen.use_ip_determination:
temp_dir_searcher = general.DirectorySearcher(
['merge_data', 'binning_300'])
temp_dir_searcher.searchListOfDirectories(dir_path, 'reco_ip.json')
found_dirs = temp_dir_searcher.getListOfDirectories()
if not found_dirs:
# 2. determine offset on the vertex data sample
os.chdir(lmd_fit_bin_path)
temp_dir_searcher = general.DirectorySearcher(
['merge_data', 'binning_300'])
temp_dir_searcher.searchListOfDirectories(
dir_path, ['lmd_vertex_data_', 'of1.root'])
found_dirs = temp_dir_searcher.getListOfDirectories()
bashcommand = './determineBeamOffset -p ' + \
found_dirs[0] + ' -c ' + '../../vertex_fitconfig.json'
returnvalue = subprocess.call(bashcommand.split())
ip_rec_file = found_dirs[0] + '/reco_ip.json'
else:
ip_rec_file = found_dirs[0] + '/reco_ip.json'
file_content = open(ip_rec_file)
ip_rec_data = json.load(file_content)
scen.rec_ip_info['ip_offset_x'] = float('{0:.3f}'.format(
round(float(ip_rec_data["ip_x"]), 3))) # in cm
scen.rec_ip_info['ip_offset_y'] = float(
'{0:.3f}'.format(round(float(ip_rec_data["ip_y"]), 3)))
scen.rec_ip_info['ip_offset_z'] = float(
'{0:.3f}'.format(round(float(ip_rec_data["ip_z"]), 3)))
print("Finished IP determination for this scenario!")
else:
scen.rec_ip_info['ip_offset_x'] = 0.0
scen.rec_ip_info['ip_offset_y'] = 0.0
scen.rec_ip_info['ip_offset_z'] = 0.0
print("Skipped IP determination for this scenario!")
state += 1
last_state += 1
if state == 3:
# 3a. track filter the dpm data using the ip values and create ang
# dist objects
# (that again means bunch -> create -> merge)
# 3b. generate acceptance and resolution with these reconstructed ip
# values
# (that means simulation + bunching + creating data objects + merging)
if len(scen.simulation_info_lists) == 0:
scen.simulation_info_lists.append(['', 'a', 1, 0])
scen.simulation_info_lists.append(['', 'er', 1, 0])
scen = simulateDataOnHimster(scen)
if scen.is_broken:
dead_scenario_stack.append(scen)
return
if len(scen.simulation_info_lists) == 0:
state += 1
last_state += 1
if state == 4:
# 4. runLmdFit!
temp_dir_searcher = general.DirectorySearcher(
['merge_data', 'binning_300'])
temp_dir_searcher.searchListOfDirectories(
scen.filtered_dir_path, 'lmd_fitted_data')
found_dirs = temp_dir_searcher.getListOfDirectories()
if not found_dirs:
os.chdir(lmd_fit_script_path)
print('running lmdfit!')
cut_keyword = ''
if scen.use_xy_cut:
cut_keyword += 'xy_'
if scen.use_m_cut:
cut_keyword += 'm_'
if cut_keyword == '':
cut_keyword += 'un'
cut_keyword += 'cut_real '
bashcommand = 'python doMultipleLuminosityFits.py '\
'--forced_box_gen_data ' + scen.acc_and_res_dir_path + \
' ' + scen.filtered_dir_path + ' ' + cut_keyword + \
lmd_fit_path + '/' + args.fit_config
returnvalue = subprocess.call(bashcommand.split())
print('this scenario is fully processed!!!')
finished = True
# if we are in an intermediate step then push on the waiting stack and
# increase step state
if not finished:
scen.state = state
scen.last_state = last_state
waiting_scenario_stack.append(scen)
parser.add_argument('--use_devel_queue', action='store_true',
help='If flag is set, the devel queue is used')
args = parser.parse_args()
lmd_fit_script_path = os.path.dirname(os.path.realpath(__file__))
lmd_fit_path = os.path.dirname(lmd_fit_script_path)
lmd_fit_bin_path = os.getenv('LMDFIT_BUILD_PATH') + '/bin'
num_samples = args.num_samples
bootstrapped_num_samples = args.bootstrapped_num_samples
box_num_samples = args.box_num_samples
box_num_events_per_sample = args.box_num_events_per_sample
# first lets try to find all directories and their status/step
dir_searcher = general.DirectorySearcher(['dpm_elastic', 'uncut'])
dir_searcher.searchListOfDirectories(
args.base_output_data_dir, 'Lumi_TrksQA_')
dirs = dir_searcher.getListOfDirectories()
print(dirs)
# at first assign each scenario the first step and push on the active stack
for dir in dirs:
scen = Scenario(dir)
print("creating scenario:", dir)
if args.disable_xy_cut or ("/no_alignment_correction" in dir and "no_geo_misalignment/" not in dir):
print("Disabling xy cut!")
scen.use_xy_cut = False # for testing purposes
if args.disable_m_cut or ("/no_alignment_correction" in dir and "no_geo_misalignment/" not in dir):
if args.type[0].find('r') >= 0:
data_type_list.append(
DataTypeInfo('r', ' -f lmd_res_data_\\d*.root',
['lmd_res_data_', '.root']))
if args.type[0].find('h') >= 0:
data_type_list.append(
DataTypeInfo('h', ' -f lmd_res_data_\\d*.root',
['lmd_res_data_', '.root']))
if args.type[0].find('v') >= 0:
data_type_list.append(
DataTypeInfo('v', ' -f lmd_vertex_data_\\d*.root',
['lmd_vertex_data_', '.root']))
patterns = []
patterns.append(args.dir_pattern)
# to avoid merge_data directories to be used recursively
# we forbid the occurence merge_data in path name
dir_searcher = general.DirectorySearcher(patterns, 'merge_data')
for data_type_info in data_type_list:
dir_searcher.searchListOfDirectories(args.dirname[0],
data_type_info.glob_pattern)
dirs = dir_searcher.getListOfDirectories()
for dir in dirs:
print('starting merge for ' + dir)
bashcommand = default = os.getenv('LMDFIT_BUILD_PATH') + '/bin/mergeLmdData -p ' + dir + ' -t ' + data_type_info.data_type + \
' -n ' + str(args.num_samples) + ' -s ' + \
str(args.sample_size) + data_type_info.pattern
print(bashcommand)
returnvalue = subprocess.call(bashcommand.split())