def make_particle_mask(db, ids, particle, multiprocess=True):
# find the particle code to make mask for
particle_code = get_particle_code(particle)
# Split the candidates into chunks for multiprocessing
if multiprocess:
ids_chunked = np.array_split(ids, AVAILABLE_CORES)
packed = make_multiprocess_pack(ids_chunked, db, particle_code)
with Pool(AVAILABLE_CORES) as p:
accepted_lists = p.map(find_particles, packed)
# Combine again
mask = sorted(flatten_list_of_lists(accepted_lists))
# x = np.arange(len(mask))
# d = {'x': [x], 'y': [mask]}
# d['savefig'] = get_project_root()+'/MASK'+str(len(mask))+'_TESTLOL.png'
# _ = make_plot(d)
else:
raise ValueError(
'make_particle_mask: Only multiprocessing solution implemented')
mask_name = particle
return mask, mask_name
def assign_energy_weights_multiprocess(
ids,
db,
interpolator_quadratic,
true_key=['true_primary_energy'],
debug=False
):
# Create packs - loop over all events
n_chunks = len(ids)//AVAILABLE_CORES
chunks = np.array_split(ids, n_chunks)
packed = make_multiprocess_pack(
chunks, interpolator_quadratic, true_key, db
)
# Multiprocess and recombine
with Pool(AVAILABLE_CORES) as p:
weights = p.map(calc_weights_multiprocess, packed)
weights_combined = flatten_list_of_lists(weights)
# # Make a dictionary with weights - put nan where mask didn't apply
# event_ids = db.ids
# all_weights_dict = {str(event_id): np.nan for event_id in event_ids}
# weights_dict = {
# str(event_id): weight for event_id, weight in zip(ids, weights_combined)
# }
# all_weights_dict.update(weights_dict)
return weights_combined
def make_dom_interval_mask(db,
ids,
min_doms,
max_doms,
multiprocess=True,
dom_mask='SplitInIcePulses'):
# Split the candidates into chunks for multiprocessing
if multiprocess:
ids_chunked = np.array_split(ids, AVAILABLE_CORES)
packed = make_multiprocess_pack(ids_chunked, db, min_doms, max_doms,
dom_mask)
with Pool(AVAILABLE_CORES) as p:
accepted_lists = p.map(find_dom_interval_passed_cands, packed)
# Combine again
mask = sorted(flatten_list_of_lists(accepted_lists))
else:
raise ValueError(
'make_dom_interval_mask: Only multiprocessing solution implemented'
)
# save it
mask_name = 'dom_interval_%s_min%d_max%d' % (dom_mask, min_doms, max_doms)
return mask, mask_name
def make_energy_interval_mask(db,
ids,
min_energy,
max_energy,
multiprocess=True):
# Split the candidates into chunks for multiprocessing
if multiprocess:
ids_chunked = np.array_split(ids, AVAILABLE_CORES)
packed = make_multiprocess_pack(ids_chunked, db, min_energy,
max_energy)
with Pool(AVAILABLE_CORES) as p:
accepted_lists = p.map(find_energy_interval_passed_cands, packed)
# Combine again
mask = sorted(flatten_list_of_lists(accepted_lists))
else:
raise ValueError(
'make_energy_interval_mask: Only multiprocessing solution implemented'
)
print(len(mask) / len(ids))
# save it
mask_name = 'energy_interval_min%.1f_max%.1f' % (min_energy, max_energy)
return mask, mask_name
def assign_energy_weights_multiprocess(masks,
dataset_path,
interpolator_quadratic,
true_key='true_primary_energy',
debug=False):
available_cores = cpu_count()
n_tot = get_n_tot_pickles(dataset_path)
# * Create packs - loop over all events
# * Get mask
mask_all = np.array(load_pickle_mask(dataset_path, masks))
if debug:
mask_all = mask_all[:1000]
chunks = np.array_split(mask_all, available_cores)
interpolator_list = [interpolator_quadratic] * len(chunks)
keys_list = [true_key] * len(chunks)
paths = [dataset_path] * len(chunks)
n_per_dir_list = [get_n_events_per_dir(dataset_path)] * len(chunks)
subprocess_id = list(range(available_cores))
packed = zip(chunks, interpolator_list, keys_list, paths, n_per_dir_list,
subprocess_id)
# * Multiprocess and recombine
with Pool(available_cores) as p:
weights = p.map(calc_weights_multiprocess, packed)
weights_combined = flatten_list_of_lists(weights)
# * Make a list of weights - put nan where mask didn't apply
weights_list = np.array([np.nan] * n_tot)
weights_list[mask_all] = np.array(weights_combined)
return weights_list
def make_particle_mask(data_path, dirs, particle, multiprocess=True):
particle_code = get_particle_code(particle)
# * Split the candidates into chunks for multiprocessing
if multiprocess:
available_cores = cpu_count()
dirs_chunked = np.array_split(dirs, available_cores)
particle_codes = [particle_code]*len(dirs_chunked)
packed = zip(dirs_chunked, particle_codes)
with Pool(available_cores) as p:
accepted_lists = p.map(find_particles, packed)
# * Combine again
mask = sorted(flatten_list_of_lists(accepted_lists))
else:
raise ValueError('make_particle_mask: Only multiprocessing solution implemented')
# * save it
mask_path = data_path+'/masks/%s.pickle'%(particle)
pickle.dump(mask, open(mask_path, 'wb'))
return mask_path
def make_dom_interval_mask(data_path, dirs, min_doms, max_doms, multiprocess=True, dom_mask='SplitInIcePulses'):
# * Split the candidates into chunks for multiprocessing
if multiprocess:
available_cores = cpu_count()
dirs_chunked = np.array_split(dirs, available_cores)
min_doms_list = [min_doms]*len(dirs_chunked)
max_doms_list = [max_doms]*len(dirs_chunked)
dom_mask_list = [dom_mask]*len(dirs_chunked)
subprocess_ids = np.arange(len(dirs_chunked))
packed = zip(dirs_chunked, min_doms_list, max_doms_list, dom_mask_list, subprocess_ids)
with Pool(available_cores) as p:
accepted_lists = p.map(find_dom_interval_passed_cands, packed)
# * Combine again
mask = sorted(flatten_list_of_lists(accepted_lists))
else:
raise ValueError('make_dom_interval_mask: Only multiprocessing solution implemented')
# * save it
mask_path = data_path+'/masks/dom_interval_%s_min%d_max%d.pickle'%(dom_mask, min_doms, max_doms)
pickle.dump(mask, open(mask_path, 'wb'))
return mask_path
def assign_energy_balanced_weights_multiprocess(
ids,
db,
interpolator_quadratic,
true_key=['true_primary_energy'],
debug=False
):
# Create packs - loop over all events
# n_chunks = len(ids)//AVAILABLE_CORES
chunks = np.array_split(ids, AVAILABLE_CORES)
packed = make_multiprocess_pack(
chunks, interpolator_quadratic, true_key, db
)
# Multiprocess and recombine
with Pool(AVAILABLE_CORES) as p:
weights = p.map(calc_weights_multiprocess, packed)
weights_combined = flatten_list_of_lists(weights)
return weights_combined
def make_energy_interval_mask(data_path, dirs, min_energy, max_energy, multiprocess=True):
# * Split the candidates into chunks for multiprocessing
if multiprocess:
available_cores = cpu_count()
dirs_chunked = np.array_split(dirs, available_cores)
min_energy_list = [min_energy]*len(dirs_chunked)
max_energy_list = [max_energy]*len(dirs_chunked)
packed = zip(dirs_chunked, min_energy_list, max_energy_list)
with Pool(available_cores) as p:
accepted_lists = p.map(find_energy_interval_passed_cands, packed)
# * Combine again
mask = sorted(flatten_list_of_lists(accepted_lists))
else:
raise ValueError('make_energy_interval_mask: Only multiprocessing solution implemented')
# * save it
mask_path = data_path+'/masks/energy_interval_min%.1f_max%.1f.pickle'%(min_energy, max_energy)
pickle.dump(mask, open(mask_path, 'wb'))
return mask_path
