def __init__(
self,
energy_bins_file=join(RETRO_DIR, "data", "muon_secondaries_light_output/energy_bins.npy"),
histograms_file=join(RETRO_DIR, "data", "muon_secondaries_light_output/histograms.npy"),
):
self.interpolators = []
"""linear interpolation function for each curve in histarray"""
self.average_track_lengths = []
"""the "native" muon length for each curve in histarray"""
# Load files for interpolating light output vs. track length for energy
# ranges.
self.energy_bins = np.load(expand(energy_bins_file))
self.histarray = np.load(expand(histograms_file))
for curve in self.histarray:
curvex = []
curveE = []
for tup in curve:
curvex.append(tup[0])
curveE.append(tup[1])
curvex = np.array(curvex)
curveE = np.array(curveE)
interpolator = interpolate.interp1d(
curvex,
curveE,
kind="linear",
bounds_error=False,
fill_value=(curveE[0], curveE[-1]),
)
self.interpolators.append(interpolator)
self.average_track_lengths.append(
float(curve[-1][0] + (curve[-1][0] - curve[-2][0]) / 2)
)
def remove_dimension(input_file, output_file, dim_name):
"""
Parameters
----------
input_file : str
Path to input file (table)
output_file : str
Path to output file (table)
dim_name : str
Dimension to remove from the intput table
"""
input_file = expand(input_file)
output_file = expand(output_file)
input_dir = dirname(input_file)
output_dir = dirname(output_file)
if abspath(output_dir) == abspath(input_dir):
raise ValueError("Will not allow output dir to be same as input dir")
if not isdir(output_dir):
mkdir(output_dir)
input_table = np.load(input_file, mmap_mode="r")
input_binning = np.load(join(input_dir, "binning.npy"))
dim_num = [i for i, n in enumerate(input_binning.dtype.names) if n == dim_name][0]
output_binning = input_binning[
[n for n in input_binning.dtype.names if n != dim_name]
]
# Save the binning to the output directory
np.save(join(output_dir, "binning.npy"), output_binning)
# Legacy way of storing bin edges: store each dim individually
for d_name in output_binning.dtype.names:
bin_edges_fpath = join(output_dir, "{}_bin_edges.npy".format(d_name))
np.save(bin_edges_fpath, output_binning[d_name])
# If we find the removed dimension's bin edges in output dir, remove that file
bin_edges_fpath = join(output_dir, "{}_bin_edges.npy".format(dim_name))
if isfile(bin_edges_fpath):
remove(bin_edges_fpath)
output_shape = tuple(n for i, n in enumerate(input_table.shape) if i != dim_num)
output_table = np.empty(shape=output_shape, dtype=input_table.dtype)
#output_table = np.memmap(
# output_file, dtype=input_table.dtype, mode="w+", shape=output_shape
#)
# Perform the summation over the dimension to be removed
# Note that setting dtype to float64 causes accumulator to be double
# precision, even if output table is not
input_table.sum(axis=dim_num, dtype=np.float64, out=output_table)
np.save(output_file, output_table)
def combine_table_clusters(
cluster_file,
source_path_proto,
basedir,
cluster_idx,
t_is_residual_time=None,
overwrite=False,
):
"""Combine clustered tables together.
The file should be Numpy .npy format and contain a structured Numpy array
with field 'label'; subsequent fields are used to format the source table
filename prototype (e.g. fields 'string' and 'dom' can be used if prototype
is "table_{string}_{dom}.fits").
Output table files are stored to a `cl{cluster_idx}` subdirectory within
the specified directory.
Parameters
----------
cluster_file : str
source_path_proto : str
basedir : str
cluster_idx : int
t_is_residual_time : bool, optional
overwrite : bool, optional
"""
cluster_file = expand(cluster_file)
outdir = join(expand(basedir), 'cl{}'.format(cluster_idx))
clusters = np.load(cluster_file)
labels = clusters['label']
members = clusters[labels == cluster_idx]
assert len(members) > 0
omkeys = np.empty(len(members), dtype=OMKEY_T)
if 'dom' in members.dtype.names:
omkeys[['string', 'om']] = members[['string', 'dom']]
omkeys[:]['pmt'] = 0
else:
omkeys[['string', 'om', 'pmt']] = members[['string', 'om', 'pmt']]
table_fpaths = []
names = members.dtype.names
for member in members:
pathspec = expand(source_path_proto.format(**dict(zip(names, member))))
fpaths = glob(pathspec)
if len(fpaths) == 0:
raise ValueError('Cannot file(s) "{}"'.format(pathspec))
table_fpaths.extend(fpaths)
combine_clsim_tables(
table_fpaths=table_fpaths,
t_is_residual_time=t_is_residual_time,
outdir=outdir,
overwrite=overwrite,
)
np.save(join(outdir, 'omkeys.npy'), omkeys)
def find_unique_gcds():
"""Find unique GCD files in data"""
with open(expand("~/all_data_gcd_files.txt"), "r") as f:
fpaths = [expand(l.strip()) for l in f.readlines()]
original_num_files = len(fpaths)
root_infos = {}
original_size = 0
final_size = 0
for fpath in fpaths:
base = basename(fpath)
size = getsize(fpath)
fname_info = GENERIC_I3_FNAME_RE.match(base).groupdict()
root = fname_info["base"]
compext = fname_info.get("compext", None)
original_size += size
if root not in root_infos:
root_infos[root] = []
final_size += size
root_infos[root].append(
dict(
fpath=fpath,
base=base,
root=root,
compext=compext,
size=size,
))
root_infos = OrderedDict([
(rn, root_infos[rn])
for rn in sorted(root_infos.keys(), key=nsort_key_func)
])
final_num_files = len(root_infos)
#unequal_sizes = False
#for rn, finfos in root_infos.items():
# file_paths = []
# file_sizes = []
# for fpath, fsize in finfos:
# file_paths.append(fsize)
# file_sizes.append(fsize)
# file_sizes = np.array(file_sizes)
# if not np.all(file_sizes == file_sizes[0]):
# unequal_sizes = True
# for file_path, file_size in finfos:
# print("{:14d} b : {}".format(file_size, file_path))
print("original number of files = {}, final = {}".format(
original_num_files, final_num_files))
print("original size = {:.0f} GiB, final size = {:.0f} GiB".format(
original_size / (1024**3), final_size / (1024**3)))
return root_infos
def load_stacked_tables(
self,
stacked_tables_meta_fpath,
stacked_tables_fpath,
stacked_t_indep_tables_fpath,
mmap_tables=False,
mmap_t_indep=False,
):
if self.is_stacked is not None:
assert self.is_stacked
stacked_tables_meta_fpath = expand(stacked_tables_meta_fpath)
stacked_tables_fpath = expand(stacked_tables_fpath)
stacked_t_indep_tables_fpath = expand(stacked_t_indep_tables_fpath)
tables_mmap_mode = 'r' if mmap_tables else None
t_indep_mmap_mode = 'r' if mmap_t_indep else None
self.table_meta = load_pickle(stacked_tables_meta_fpath)
self.tables = np.load(stacked_tables_fpath, mmap_mode=tables_mmap_mode)
self.tables.setflags(write=False, align=True, uic=False)
num_tables = self.tables.shape[0]
self.t_is_residual_time = bool(
self.table_meta.get('t_is_residual_time', False))
self.t_indep_tables = np.load(stacked_t_indep_tables_fpath,
mmap_mode=t_indep_mmap_mode)
self.t_indep_tables.setflags(write=False, align=True, uic=False)
assert self.t_indep_tables.shape[0] == num_tables
self.sd_idx_table_indexer = deepcopy(
self.table_meta['sd_idx_table_indexer'])
self.sd_idx_table_indexer.setflags(write=False, align=True, uic=False)
self.loaded_sd_indices = np.where(self.sd_idx_table_indexer >= 0)[0]
self.n_photons_per_table = self.table_meta['n_photons_per_table']
# Note that in creating the stacked tables, each indiividual table
# is scaled such that the effective number of photons used to generate
# the table is one (to avoid different norms across the tables if
# different number of photons was used originally to create each).
self.table_norm, self.t_indep_table_norm = get_table_norm(
avg_angsens=self.avg_angsens,
quantum_efficiency=1,
norm_version=self.norm_version,
**{k: self.table_meta[k]
for k in TABLE_NORM_KEYS})
self.table_norms = [self.table_norm] * num_tables
self.t_indep_table_norms = [self.t_indep_table_norm] * num_tables
self.is_stacked = True
def __init__(self,
tables_dir,
hash_val,
geom,
use_directionality,
ic_exponent=1,
dc_exponent=1,
naming_version=None):
# Translation and validation of args
tables_dir = expand(tables_dir)
assert isdir(tables_dir)
assert len(geom.shape) == 3
assert isinstance(use_directionality, bool)
assert ic_exponent >= 0
assert dc_exponent >= 0
if naming_version is None:
naming_version = len(RETRO_DOM_TABLE_FNAME_PROTO) - 1
self.naming_version = naming_version
self.dom_table_fname_proto = RETRO_DOM_TABLE_FNAME_PROTO[
naming_version]
self.tables_dir = tables_dir
self.hash_val = hash_val
self.geom = geom
self.use_directionality = use_directionality
self.ic_exponent = ic_exponent
self.dc_exponent = dc_exponent
self.tables = {'ic': {}, 'dc': {}}
self.bin_edges = {'ic': {}, 'dc': {}}
def load_angsens_model(model):
"""Load an angular sensitivity model.
Note that this tries to look in current directory, in Retro data
directory, and in the $I3_SRC directory--if it is defined--for the model
named `model` _and_ `model` prefixed by "as." if it is not already.
Returns
-------
angsens_poly : numpy.polynomial.Polynomial
avg_angsens : float
"""
models = [model]
if not basename(model).startswith('as.'):
models += [join(dirname(model), 'as.' + basename(model))]
possible_dirs = [
'.',
join(RETRO_DIR, 'data'),
]
if 'I3_SRC' in os.environ:
possible_dirs.append('$I3_SRC/ice-models/resources/models/angsens')
possible_paths = []
for model_name in models:
possible_paths += [join(d, model_name) for d in possible_dirs]
coeffs_loaded = False
for path in possible_paths:
path = expand(path)
if not isfile(path):
continue
# The first number in the file is approximately equal (but greater
# than) the peak in the distribution, used for scaling before
# rejection sampling, so is useless for us (and makes simulation
# less efficient).
poly_coeffs = np.loadtxt(path)[1:]
coeffs_loaded = True
break
if not coeffs_loaded:
raise ValueError('Could not load hole ice model at any of\n{}'.format(
possible_paths))
# We want coszen = -1 to correspond to upgoing particles, but angular
# sensitivity is given w.r.t. the DOM axis (which points "down" towards
# earth, and therefore is rotated 180-deg). So rotate the coszen
# polynomial about cz=0 by negating the odd coefficients (coeffs are in
# ascending powers of "x").
flipped_coeffs = np.empty_like(poly_coeffs)
flipped_coeffs[0::2] = poly_coeffs[0::2]
flipped_coeffs[1::2] = -poly_coeffs[1::2]
angsens_poly = np.polynomial.Polynomial(flipped_coeffs, domain=(-1, 1))
integral_poly = angsens_poly.integ(m=1)
avg_angsens = (integral_poly(1) - integral_poly(-1)) / 2
return angsens_poly, avg_angsens
def find_problematic_pulses(indir, pulse_series):
"""Find missing, bad, or old extracted pulse series and print the paths of
the corresponding events directories.
Parameters
----------
indir : str
pulse_series : str or iterable thereof
"""
if isinstance(pulse_series, str):
pulse_series = [pulse_series]
indir = expand(indir)
for dirpath, dirs_, files in walk(indir, followlinks=True):
if "events.npy" in files:
dirs_.clear()
else:
dirs_.sort(key=nsort_key_func)
files.sort(key=nsort_key_func)
for fname in files:
match = OSCNEXT_FNAME_RE.match(fname)
if not match:
continue
i3f_dname = join(dirpath, match.groupdict()["basename"])
if isdir(i3f_dname):
if not isfile(join(i3f_dname, "events.npy")):
print(i3f_dname)
else:
print(i3f_dname)
continue
sys.stderr.write(".")
sys.stderr.flush()
# If any one of the named pulse series are missing or bad, record
# the path and move on without checking the other pulse series
for ps_name in pulse_series:
pulses_fpath = join(dirpath, "pulses", ps_name + ".pkl")
if not isfile(pulses_fpath):
print(dirpath)
break
try:
pulses = load_pickle(pulses_fpath)
if len(pulses) > 0 and "flags" not in pulses[0][0][1].dtype.names:
print(dirpath)
break
except Exception:
print(dirpath)
break
def get_estimate(fpath, verbose=True):
"""Get estimate from llhp.npy file"""
fpath = expand(fpath)
llhp = np.load(fpath)
estimate = estimate_from_llhp(llhp)
if verbose:
for dim, est in estimate.items():
mean, low, high = est['mean'], est['low'], est['high']
print(
'{:s} : mean = {:9.3f} ; 95% interval = [{:9.3f}, {:9.3f}] {}'.
format(dim.rjust(20), mean, low, high, UNITS[dim]))
return llhp, estimate
def centralize_gcds(root_infos, gcd_dir=GCD_DIR):
"""Move GCD files to a single directory, if they don't already exist there.
Compression extensions should be ignored, so only one version of each GCD
exists.
Parameters
----------
root_infos : mapping
gcd_dir : str, optional
"""
gcd_dir = expand(gcd_dir)
mkdir(gcd_dir)
existing_fnames = os.listdir(gcd_dir)
existing_roots = set()
for fname in existing_fnames:
match = GENERIC_I3_FNAME_RE.match(fname)
if not match:
continue
groupdict = match.groupdict()
existing_roots.add(groupdict["base"])
for root, infos in root_infos.items():
for info in infos:
is_link = islink(info["fpath"])
is_file = isfile(info["fpath"])
if is_link:
if is_file: # link to an existing file
if root not in existing_roots:
shutil.copy2(info["fpath"],
gcd_dir,
follow_symlinks=True)
existing_roots.add(root)
else: # bad link (to nothing, or to a directory)
if not isdir(info["fpath"]):
print(f'os.remove({info["fpath"]})')
os.remove(info["fpath"])
else:
if root in existing_roots:
if is_file:
print(f'os.remove({info["fpath"]})')
os.remove(info["fpath"])
else:
print(f'shutil.move({info["fpath"]}, {gcd_dir})')
shutil.move(info["fpath"], gcd_dir)
existing_roots.add(root)
def main():
"""Main function for calling summarize_clsim_table as a script"""
t0 = time()
args = parse_args()
kwargs = vars(args)
table_fpaths = []
for fpath in kwargs.pop('table-fpaths'):
table_fpaths.extend(glob(expand(fpath)))
for fpath in table_fpaths:
kwargs['table_fpath'] = fpath
summarize_clsim_table(**kwargs)
total_time = time() - t0
if len(table_fpaths) > 1:
avg = np.round(total_time / len(table_fpaths), 3)
wstderr('Average time to summarize tables: {} s/table\n'.format(avg))
def concatenate_recos_and_save(outfile, **kwargs):
"""Concatenate recos and save to a file.
Parameters
----------
outfile : str
**kwargs
Arguments passed to `concatenate_recos`
"""
outfile = expand(outfile)
out_array = concatenate_recos(**kwargs)
outdir = dirname(outfile)
if not isdir(outdir):
mkdir(outdir)
np.save(outfile, out_array)
sys.stdout.write('Saved concatenated array to "{}"\n'.format(outfile))
def main():
"""Load file specified on command line and print results of parsing it."""
with open(expand(sys.argv[1]), 'r') as f:
contents = f.readlines()
try:
points, errors = parse(contents)
except ValueError:
print('Failed to parse file "{}"'.format(sys.argv[1]))
raise
for dim in 't x y z track_zenith track_azimuth track_energy cascade_energy'.split():
pt = points[dim]
sd = errors[dim]
if dim in ['track_zenith', 'track_azimuth']:
pt = np.rad2deg(pt)
sd = np.rad2deg(sd)
print('{:14s} = {:8.3f} +/- {:5.1f} {}'.format(dim, pt, sd, UNITS[dim]))
def setup_tdi_tables(tdi=None, mmap=False):
"""Load and instantiate (Cherenkov) TDI tables.
Parameters
----------
tdi : sequence of strings, optional
Path to TDI tables' `ckv_tdi_table.npy` files, or paths to
directories containing those files; one entry per TDI table
mmap : bool
Returns
-------
tdi_tables : tuple of 0 or more numpy arrays
tdi_metas : tuple of 0 or more OrderedDicts
"""
if tdi is None:
return (), ()
mmap_mode = 'r' if mmap else None
tdi_tables = []
tdi_metas = []
for tdi_ in tdi:
if tdi_ is None:
continue
tdi_ = expand(tdi_)
if isdir(tdi_):
tdi_ = join(tdi_, 'ckv_tdi_table.npy')
print('Loading and instantiating TDI table at "{}"'.format(tdi_))
be = load_pickle(join(dirname(tdi_), 'tdi_bin_edges.pkl'))
meta = load_pickle(join(dirname(tdi_), 'tdi_metadata.pkl'))
meta['bin_edges'] = be
tdi_table = np.load(tdi_, mmap_mode=mmap_mode)
tdi_metas.append(meta)
tdi_tables.append(tdi_table)
return tuple(tdi_tables), tuple(tdi_metas)
def find_gcds_in_dirs(dirs, gcd_fname_re, recurse=True):
"""Find data run GCD files in directories.
Parameters
----------
dirs : str or iterable thereof
recurse : bool
Returns
-------
data_run_gcds : dict
Keys are <tuple>(<str>2-digit season, <str>run number) and values are
<str> path to corresponding GCD file
"""
if isinstance(dirs, str):
dirs = [dirs]
dirs = [expand(rootdir) for rootdir in dirs]
data_run_gcds = {}
for rootdir in dirs:
for dirpath, subdirs, files in walk(rootdir):
if recurse:
subdirs.sort(key=nsort_key_func)
else:
del subdirs[:]
files.sort(key=nsort_key_func)
for fname in files:
gcd_match = gcd_fname_re.match(fname)
if gcd_match:
gcd_groupdict = gcd_match.groupdict()
# get 2 digit year
year = "{:02d}".format(int(gcd_groupdict["year"]) % 2000)
key = (year, gcd_groupdict["run"])
# prefer "levelXpassY_* gcd files
if key in data_run_gcds and gcd_groupdict["pass"] is None:
continue
data_run_gcds[key] = join(dirpath, fname)
return data_run_gcds
def generate_stacked_tables(outdir, dom_tables_kw):
"""Stack a set of tables into a single numpy array for use of all tables in
Numba.
Currently, only ckv_templ_compr tables are supported.
Parameters
----------
outdir : string
Path ot directory into which the three resulting files will be stored.
dom_tables_kw : mapping
As returned by retro.init_obj.parse_args
"""
if dom_tables_kw['dom_tables_kind'] != 'ckv_templ_compr':
raise NotImplementedError(
'"{}" tables not supported; only "ckv_templ_compr"'
.format(dom_tables_kw['dom_tables_kind'])
)
# Use the convenience function to load the single-DOM tables into a
# retro_5d_tables.Retro5DTables object, and then we can use the loaded
# tables from there.
dom_tables = init_obj.setup_dom_tables(**dom_tables_kw)
assert np.all(dom_tables.sd_idx_table_indexer >= 0)
table_meta = OrderedDict()
table_meta['table_kind'] = dom_tables.table_kind
table_meta['sd_idx_table_indexer'] = dom_tables.sd_idx_table_indexer
table_meta.update(dom_tables.table_meta)
table_meta['n_photons'] = 1.0
table_meta['n_photons_per_table'] = np.array(dom_tables.n_photons_per_table)
outdir = expand(outdir)
mkdir(outdir)
fpath = join(outdir, 'stacked_{}_meta.pkl'.format(dom_tables.table_name))
sys.stdout.write('Writing metadata to "{}" ...'.format(fpath))
sys.stdout.flush()
pickle.dump(
table_meta,
file(fpath, 'wb'),
protocol=pickle.HIGHEST_PROTOCOL
)
sys.stdout.write(' done.\n')
sys.stdout.flush()
if dom_tables.compute_t_indep_exp:
# Renormalize to 1 photon
stacked_t_indep_tables = np.stack(
[tbl/n for tbl, n in zip(dom_tables.t_indep_tables, dom_tables.n_photons_per_table)]
)
fpath = join(
outdir,
'stacked_{}.npy'.format(dom_tables.t_indep_table_name)
)
sys.stdout.write('Writing stacked t_indep tables to "{}" ...'
.format(fpath))
sys.stdout.flush()
np.save(fpath, stacked_t_indep_tables)
sys.stdout.write(' done.\n')
sys.stdout.flush()
# Renormalize to 1 photon
for template_map, n_photons in zip(dom_tables.tables, dom_tables.n_photons_per_table):
template_map['weight'] /= n_photons
stacked_tables = np.stack(dom_tables.tables)
fpath = join(outdir, 'stacked_{}.npy'.format(dom_tables.table_name))
sys.stdout.write('Writing stacked tables to "{}" ...'.format(fpath))
sys.stdout.flush()
np.save(fpath, stacked_tables)
sys.stdout.write(' done.\n')
sys.stdout.flush()
def load_clsim_table_minimal(fpath, mmap=False, include_overflow=False):
"""Load a CLSim table from disk (optionally compressed with zstd).
Similar to the `load_clsim_table` function but the full table, including
under/overflow bins, is kept and no normalization or further processing is
performed on the table data besides populating the ouptput OrderedDict.
Parameters
----------
fpath : string
Path to file to be loaded. If the file has extension 'zst', 'zstd', or
'zstandard', the file will be decompressed using the `python-zstandard`
Python library before passing to `fits` for interpreting.
mmap : bool, optional
Whether to memory map the table
include_overflow : bool, optional
By default, overflow bins (if present) are removed
Returns
-------
table : OrderedDict
"""
t0 = time()
table = OrderedDict()
fpath = expand(fpath)
if DEBUG:
wstderr('Loading table from {} ...\n'.format(fpath))
if isdir(fpath):
indir = fpath
if mmap:
mmap_mode = 'r'
else:
mmap_mode = None
for rel_fpath in listdir(indir):
key, ext = splitext(rel_fpath)
abs_fpath = join(indir, rel_fpath)
if not (isfile(abs_fpath) and ext == '.npy'):
continue
if DEBUG:
wstderr(' loading {} from "{}" ...'.format(key, abs_fpath))
t1 = time()
val = np.load(abs_fpath, mmap_mode=mmap_mode)
# Pull "small" things (less than 10 MiB) into memory so we don't
# have too many file handles open due to memory mapping
if mmap and val.nbytes < 10 * 1024**2:
val = np.copy(val)
table[key] = val
if DEBUG:
wstderr(' ({} ms)\n'.format(np.round((time() - t1) * 1e3, 3)))
elif isfile(fpath):
from astropy.io import fits
fobj = get_decompressd_fobj(fpath)
pf_table = None
try:
pf_table = fits.open(fobj, mode='readonly', memmap=mmap)
header = pf_table[0].header # pylint: disable=no-member
table['table_shape'] = np.array(pf_table[0].data.shape, dtype=int) # pylint: disable=no-member
table['group_refractive_index'] = set_explicit_dtype(
force_little_endian(header['_i3_n_group']))
table['phase_refractive_index'] = set_explicit_dtype(
force_little_endian(header['_i3_n_phase']))
n_dims = len(table['table_shape'])
new_style = False
axnames = [None] * n_dims
binning = [None] * n_dims
for key in header.keys():
if not key.startswith('_i3_ax_'):
continue
new_style = True
axnum = header[key]
axname = key[len('_i3_ax_'):]
be0 = header['_i3_{}_min'.format(axname)]
be1 = header['_i3_{}_max'.format(axname)]
n_bins = header['_i3_{}_n_bins'.format(axname)]
power = header.get('_i3_{}_power'.format(axname), 1)
bin_edges = force_little_endian(pf_table[axnum + 1].data) # pylint: disable=no-member
assert np.isclose(bin_edges[0],
be0), '%f .. %f' % (be0, bin_edges[0])
assert np.isclose(bin_edges[-1],
be1), '%f .. %f' % (be1, bin_edges[-1])
assert len(bin_edges) == n_bins + 1, '%d vs. %d' % (
len(bin_edges), n_bins + 1)
assert np.allclose(
bin_edges,
powerspace(start=be0,
stop=be1,
num=n_bins + 1,
power=power),
)
axnames[axnum] = axname
binning[axnum] = bin_edges
if not new_style:
if n_dims == 5:
axnames = [
'r', 'costheta', 't', 'costhetadir', 'deltaphidir'
]
elif n_dims == 6:
axnames = [
'r', 'costheta', 'phi', 't', 'costhetadir',
'deltaphidir'
]
else:
raise NotImplementedError(
'{}-dimensional table not handled for old-style CLSim'
' tables'.format(n_dims))
binning = [
force_little_endian(pf_table[i + 1].data).flat
for i in range(len(axnames))
] # pylint: disable=no-member
for axnum, (axname, bin_edges) in enumerate(zip(axnames, binning)):
assert axname is not None, 'missing axis %d name' % axnum
assert bin_edges is not None, 'missing axis %d binning' % axnum
dtype = np.dtype([(axname, np.float64, dim.size)
for axname, dim in zip(axnames, binning)])
table['binning'] = np.array(tuple(binning), dtype=dtype)
for keyroot in GENERIC_KEYS:
keyname = '_i3_' + keyroot
if keyname in header:
val = force_little_endian(header[keyname])
if keyroot in (
't_is_residual_time',
'disable_tilt',
'disable_anisotropy',
):
val = np.bool8(val)
else:
val = set_explicit_dtype(val)
table[keyroot] = val
# Get string values from keys that have a prefix preceded by the
# value all in the key (I3 software had issues saving strings as
# values in the header "dict" so the workaround was to store the
# string value in this way)
for infix in INFIX_KEYS:
keyroot = '_i3_' + infix + '_'
for keyname in header.keys():
if not keyname.startswith(keyroot):
continue
val = keyname[len(keyroot):]
table[infix] = np.string0(val)
if include_overflow:
slicer = (slice(None), ) * n_dims
else:
slicer = (slice(1, -1), ) * n_dims
table['table'] = force_little_endian(pf_table[0].data[slicer]) # pylint: disable=no-member
wstderr(' (load took {} s)\n'.format(np.round(time() - t0, 3)))
except:
wstderr('ERROR: Failed to load "{}"\n'.format(fpath))
raise
finally:
del pf_table
if hasattr(fobj, 'close'):
fobj.close()
del fobj
else: # fpath is neither dir nor file
raise ValueError('Table does not exist at path "{}"'.format(fpath))
if 'step_length' not in table:
table['step_length'] = 1
if 't_is_residual_time' not in table:
table['t_is_residual_time'] = True
if DEBUG:
wstderr(' Total time to load: {} s\n'.format(np.round(time() - t0,
3)))
return table
def generate_time_indep_tables(table,
outdir=None,
kinds=('clsim', 'ckv'),
overwrite=False):
"""Generate and save to disk time independent table(s) from the original
CLSim table and/or a Cherenkov table.
Parameters
----------
table : string
outdir : string, optional
kinds : string, optional
overwrite : bool, optional
Returns
-------
t_indep_table : numpy.ndarray of size (n_r, n_costheta, n_costhetadir, n_deltaphidir)
"""
if isinstance(kinds, basestring):
kinds = [kinds]
kinds = [k.strip().lower() for k in kinds]
clsim_table_path = None
ckv_table_path = None
table = expand(table)
if outdir is None:
if isdir(table):
outdir = table
elif table.endswith('.npy'):
outdir = dirname(table)
elif table.endswith('.fits'):
outdir = table.rstrip('.fits')
if isfile(table):
table_basename = basename(table)
if table_basename == 'table.npy' or table_basename.endswith('.fits'):
clsim_table_path = table
elif table_basename == 'ckv_table.npy':
ckv_table_path = table
elif isdir(table):
if 'clsim' in kinds and isfile(join(table, 'table.npy')):
clsim_table_path = table
if 'ckv' in kinds and isfile(join(table, 'ckv_table.npy')):
ckv_table_path = table
t_indep_table_exists = False
if 'clsim' in kinds and isfile(join(outdir, 't_indep_table.npy')):
t_indep_table_exists = True
t_indep_ckv_table_exists = False
if 'ckv' in kinds and isfile(join(outdir, 't_indep_ckv_table.npy')):
t_indep_ckv_table_exists = True
if 'clsim' in kinds and (overwrite or not t_indep_table_exists):
if clsim_table_path is None:
raise ValueError(
'Told to generate t-indep table from CLSim table but CLSim'
' table does not exist.')
print('generating t_indep_table')
mkdir(outdir)
t0 = time.time()
clsim_table = load_clsim_table_minimal(clsim_table_path, mmap=True)
t1 = time.time()
if retro.DEBUG:
print('loaded clsim table in {:.3f} s'.format(t1 - t0))
t_indep_table = clsim_table['table'][1:-1, 1:-1, 1:-1, 1:-1,
1:-1].sum(axis=2)
t2 = time.time()
if retro.DEBUG:
print('summed over t-axis in {:.3f} s'.format(t2 - t1))
np.save(join(outdir, 't_indep_table.npy'), t_indep_table)
t3 = time.time()
if retro.DEBUG:
print('saved t_indep_table.npy to disk in {:.3f} s'.format(t3 -
t2))
del clsim_table, t_indep_table
if 'ckv' in kinds and (overwrite or not t_indep_ckv_table_exists):
if ckv_table_path is None:
raise ValueError(
'Told to generate t-indep table from ckv table but ckv'
' table does not exist.')
print('generating t_indep_ckv_table')
mkdir(outdir)
t0 = time.time()
ckv_table = load_ckv_table(ckv_table_path, mmap=True)
t1 = time.time()
if retro.DEBUG:
print('loaded ckv table in {:.3f} s'.format(t1 - t0))
t_indep_ckv_table = ckv_table['ckv_table'].sum(axis=2)
t2 = time.time()
if retro.DEBUG:
print('summed over t-axis in {:.3f} s'.format(t2 - t1))
np.save(join(outdir, 't_indep_ckv_table.npy'), t_indep_ckv_table)
t3 = time.time()
if retro.DEBUG:
print('saved t_indep_table.npy to disk in {:.3f} s'.format(t3 -
t2))
del ckv_table, t_indep_ckv_table
def load_t_r_theta_table(fpath,
depth_idx,
scale=1,
exponent=1,
photon_info=None):
"""Extract info from a file containing a (t, r, theta)-binned Retro table.
Parameters
----------
fpath : string
Path to FITS file corresponding to the passed ``depth_idx``.
depth_idx : int
Depth index (e.g. from 0 to 59)
scale : float
Scaling factor to apply to the photon survival probability from the
table, e.g. for quantum efficiency. This is applied _before_
`exponent`. See `Notes` for more info.
exponent : float >= 0, optional
Modify probabilties in the table by ``prob = 1 - (1 - prob)**exponent``
to allow for up- and down-scaling the efficiency of the DOMs. This is
applied to each DOM's table _after_ `scale`. See `Notes` for more
info.
photon_info : None or RetroPhotonInfo namedtuple of dicts
If None, creates a new RetroPhotonInfo namedtuple with empty dicts to
fill. If one is provided, the existing component dictionaries are
updated.
Returns
-------
photon_info : RetroPhotonInfo namedtuple of dicts
Tuple fields are 'survival_prob', 'theta', 'phi', and 'length'. Each
dict is keyed by `depth_idx` and values are the arrays loaded
from the FITS file.
bin_edges : TimeSphCoord namedtuple
Each element of the tuple is an array of bin edges.
Notes
-----
The parameters `scale` and `exponent` modify a table's probability `P` by::
P = 1 - (1 - P*scale)**exponent
This allows for `scale` (which must be from 0 to 1) to be used for e.g.
quantum efficiency--which always reduces the detection probability--and
`exponent` (which must be 0 or greater) to be used as a systematic that
modifies the post-`scale` probabilities up and down while keeping them
valid (i.e., between 0 and 1). Larger values of `scale` (i.e., closer to 1)
indicate a more efficient DOM. Likewise, values of `exponent` greater than
one scale up the DOM efficiency, while values of `exponent` between 0 and 1
scale the efficiency down.
"""
# pylint: disable=no-member
from astropy.io import fits
assert 0 <= scale <= 1
assert exponent >= 0
if photon_info is None:
empty_dicts = []
for _ in RetroPhotonInfo._fields:
empty_dicts.append({})
photon_info = RetroPhotonInfo(*empty_dicts)
with fits.open(expand(fpath)) as table:
data = force_little_endian(table[0].data)
if scale == exponent == 1:
photon_info.survival_prob[depth_idx] = data
else:
photon_info.survival_prob[depth_idx] = (
1 - (1 - data * scale)**exponent)
photon_info.theta[depth_idx] = force_little_endian(table[1].data)
photon_info.deltaphi[depth_idx] = force_little_endian(table[2].data)
photon_info.length[depth_idx] = force_little_endian(table[3].data)
# Note that we invert (reverse and multiply by -1) time edges; also,
# no phi edges are defined in these tables.
data = force_little_endian(table[4].data)
t = -data[::-1]
r = force_little_endian(table[5].data)
# Previously used the following to get "agreement" w/ raw photon sim
#r_volumes = np.square(0.5 * (r[1:] + r[:-1]))
#r_volumes = (0.5 * (r[1:] + r[:-1]))**2 * (r[1:] - r[:-1])
r_volumes = 0.25 * (r[1:]**3 - r[:-1]**3)
photon_info.survival_prob[depth_idx] /= r_volumes[np.newaxis, :,
np.newaxis]
photon_info.time_indep_survival_prob[depth_idx] = np.sum(
photon_info.survival_prob[depth_idx], axis=0)
theta = force_little_endian(table[6].data)
bin_edges = TimeSphCoord(t=t,
r=r,
theta=theta,
phi=np.array([], dtype=t.dtype))
return photon_info, bin_edges
def plotit(toplot, outdir=None):
"""
Parameters
----------
toplot : sequence of (events, str) tuples
Each events should be
"""
plot_recos = [
#"LineFit_DC",
#"L5_SPEFit11",
"retro_crs_prefit",
"retro_mn8d",
"Pegleg_Fit_MN",
][::-1]
plot_params = [
"x",
"y",
"z",
"time",
"azimuth",
"zenith",
"coszen",
"energy",
"track_energy",
"track_zenith",
"track_coszen",
"track_azimuth",
]
use_weights = False
n_bins = 71
n_ebins = 10
ebin_edges = np.logspace(0, 3, n_ebins + 1)
lower_disp_pctile, upper_disp_pctile = 2.5, 97.5
lower_pct, upper_pct = 25, 75
iq_pct = upper_pct - lower_pct
longest_recolen = 0
for reco in plot_recos:
longest_recolen = max(longest_recolen, len(reco))
recos = toplot[reco]["recos"]
truth = toplot[reco]["truth"]
if recos.dtype.names and "run_time" in recos.dtype.names:
print("Mean run time, {}: {:.1f} s; mean energy = {:.1f}".format(
reco, recos["run_time"].mean(), truth["energy"].mean()))
#nx = int(np.ceil(np.round(np.sqrt(len(plot_params)*2)/2)*2))
nx = 4
ny = int(np.ceil(len(plot_params) * 4 / nx))
f = 1.5
fig, axes = plt.subplots(ny,
nx,
figsize=(4 * f * nx, 3 * f * ny),
dpi=120,
squeeze=False)
axit = iter(axes.flat)
for param in plot_params:
err_lower, err_upper = np.inf, -np.inf
lower, upper = np.inf, -np.inf
stuff = OrderedDict()
plabel = LABELS[param] if param in LABELS else param
ulabel = " ({})".format(UNITS[param]) if param in UNITS else ""
bare_ulabel = " {}".format(UNITS[param]) if param in UNITS else ""
for reco in plot_recos:
rinfo = toplot[reco]
try:
if param == "energy":
track_energy = get_pval(rinfo["recos"], "track_energy")
cascade_energy = get_pval(rinfo["recos"], "cascade_energy")
recos = track_energy + 2. * cascade_energy
else:
recos = get_pval(rinfo["recos"], param)
truth = get_pval(rinfo["truth"], param)
except:
print('exception', param, reco)
continue
recos = get_point_estimate(recos,
estimator="median",
expect_scalar=False)
if not np.all(np.isfinite(recos)):
n_nonfinite = np.count_nonzero(
np.logical_not(np.isfinite(recos)))
print('not all finite: {}, {}: {} / {} not finite'.format(
param, reco, n_nonfinite, recos.size))
continue
weight = rinfo['truth']['weight']
if "azimuth" in param:
error = (recos - truth + np.pi) % (2 * np.pi) - np.pi
elif "energy" in param:
error = recos / truth - 1
else:
error = recos - truth
stuff[reco] = (recos, truth, error, weight)
if use_weights:
lower_, upper_ = weighted_percentile(
error[np.isfinite(error)],
[lower_disp_pctile, upper_disp_pctile],
weight,
)
else:
lower_, upper_ = np.percentile(
error[np.isfinite(error)],
[lower_disp_pctile, upper_disp_pctile],
)
err_lower = np.nanmin([lower_, err_lower])
err_upper = np.nanmax([upper_, err_upper])
for array in (truth, recos):
mask = np.isfinite(array)
if "energy" in param:
mask &= array > 0
if use_weights:
lower_, upper_ = weighted_percentile(
array[mask],
[lower_disp_pctile, upper_disp_pctile],
weight,
)
else:
lower_, upper_ = np.percentile(
array[mask],
[lower_disp_pctile, upper_disp_pctile],
)
lower = np.nanmin([lower_, lower])
upper = np.nanmax([upper_, upper])
# -- Plot raw distributions -- #
ax = next(axit)
if "energy" in param:
bins = np.logspace(np.log10(lower), np.log10(upper), n_bins)
xscale = "log"
else:
bins = np.linspace(lower, upper, n_bins)
xscale = "linear"
mask = np.isfinite(truth)
nf = np.count_nonzero(mask)
if nf != mask.size:
print(param, "truth", nf, mask.size)
for reco, (recos, truth, error, weight) in stuff.items():
mask = np.isfinite(recos)
nf = np.count_nonzero(mask)
if nf != mask.size:
print(reco, recos, nf, mask.size)
if use_weights:
pc_lower, median, pc_upper = weighted_percentile(
recos[mask],
[lower_pct, 50, upper_pct],
weight,
)
else:
pc_lower, median, pc_upper = np.percentile(
recos[mask], [lower_pct, 50, upper_pct])
iq = pc_upper - pc_lower
try:
_, _, out = ax.hist(
recos[mask],
weights=weight[mask] if use_weights else None,
bins=bins,
#label="{}".format(reco.rjust(longest_recolen)),
histtype='step',
lw=1,
)
except:
print(
reco,
param,
np.all(np.isfinite(recos)),
np.nanmin(recos),
np.nanmax(recos),
lower,
upper,
)
raise
recos, truth, error, weight = stuff.values()[0]
pc_lower, median, pc_upper = np.percentile(truth[mask],
[lower_pct, 50, upper_pct])
iq = pc_upper - pc_lower
_, _, out = ax.hist(
truth[mask],
weights=weight[mask] if use_weights else None,
bins=bins,
label="MC truth", #"{}".format("truth".rjust(longest_recolen)),
histtype='step',
lw=1.5,
edgecolor='k',
zorder=-10,
)
ax.set_xlim(lower, upper)
ax.set_xscale(xscale)
leg = ax.legend(loc="lower center", frameon=False)
leg._legend_box.align = "left"
plt.setp(leg.get_title(), family='monospace')
ax.set_yticks([])
ax.set_title("{} distribution, all E".format(param))
# -- Plot errors across all events -- #
ax = next(axit)
bins = np.linspace(err_lower, err_upper, n_bins)
for reco, (recos, truth, error, weight) in stuff.items():
mask = np.isfinite(error)
nf = np.count_nonzero(mask)
if nf != mask.size:
print(param, reco, "error", nf, mask.size)
if use_weights:
pc_lower, median, pc_upper = weighted_percentile(
error[mask],
[lower_pct, 50, upper_pct],
weight,
)
else:
pc_lower, median, pc_upper = np.percentile(
error[mask], [lower_pct, 50, upper_pct])
iq = pc_upper - pc_lower
try:
_, _, out = ax.hist(
error[mask],
weights=weight[mask] if use_weights else None,
bins=bins,
label="{} {:6.2f} {:6.2f}".format(
reco.rjust(longest_recolen), median, iq),
histtype='step',
lw=1,
)
except:
print(
reco,
param,
np.all(np.isfinite(error)),
np.nanmin(error),
np.nanmax(error),
lower,
upper,
)
raise
ax.set_xlim(err_lower, err_upper)
ax.set_ylim(0, ax.get_ylim()[1] * 1.3)
leg = ax.legend(
loc="upper right",
title=("{} {} {}".format(" " * longest_recolen, "median",
"IQ {:2d}%".format(iq_pct))),
markerfirst=False,
frameon=False,
framealpha=0.2,
prop=dict(family='monospace'),
)
leg._legend_box.align = "left"
plt.setp(leg.get_title(), family='monospace')
ax.set_yticks([])
if "energy" in param:
title = "fract {} error, all E".format(param)
else:
title = "{} error, all E".format(param)
ax.set_title(title)
# -- Plot errors vs. true energy -- #
ax = next(axit)
colors = ['C{}'.format(i) for i in range(8)]
colors_iter = iter(colors)
for reco, (recos, truth, error, weight) in stuff.items():
mask = np.isfinite(error)
true_en = get_pval(toplot[reco]['truth'], "energy")
idxs = np.digitize(true_en, ebin_edges) - 1
pc_l = []
pc_u = []
for idx in range(n_ebins):
bin_error = error[(idxs == idx) & mask]
if use_weights:
pc_l_, med_, pc_u_ = weighted_percentile(
bin_error, [lower_pct, 50, upper_pct], weight)
else:
pc_l_, med_, pc_u_ = np.percentile(
bin_error, [lower_pct, 50, upper_pct])
pc_l.append(pc_l_)
pc_u.append(pc_u_)
color = next(colors_iter)
#ax.fill_between(
# x=ebin_edges,
# y1=[pc_l[0]] + pc_l,
# y2=[pc_u[0]] + pc_u,
# interpolate=False,
# step="post",
# facecolor='none',
# edgecolor=color,
#)
ax.step(
x=ebin_edges,
y=[pc_l[0]] + pc_l,
#facecolor='none',
color=color,
)
ax.step(
x=ebin_edges,
y=[pc_u[0]] + pc_u,
#facecolor='none',
color=color,
)
ax.set_xscale('log')
ax.set_xlim(ebin_edges[0], ebin_edges[-1])
#if "energy" in param:
# title = "fractional {} error [{}, {}]% vs. true E".format(
# param, lower_pct, upper_pct
# )
#else:
# title = "{} error [{}, {}]% vs true E".format(
# param, lower_pct, upper_pct
# )
if "energy" in param:
title = "Fractional error [{}, {}]% vs. true E".format(
lower_pct, upper_pct)
ax.set_ylabel("Fractional {} error".format(plabel))
else:
title = "Error [{}, {}]% vs true E".format(
lower_pct, upper_pct)
ax.set_ylabel("{} error{}".format(plabel, ulabel))
ax.set_title(title)
# -- Plot error WIDTHS vs. true energy -- #
ax = next(axit)
colors_iter = iter(colors)
for reco, (recos, truth, error, weight) in stuff.items():
mask = np.isfinite(error)
if use_weights:
pc_l_, pc_u_ = weighted_percentile(error[mask],
[lower_pct, upper_pct],
weights=weight[mask])
else:
pc_l_, pc_u_ = np.percentile(error[mask],
[lower_pct, upper_pct])
overall_iq_width = pc_u_ - pc_l_
true_en = get_pval(toplot[reco]['truth'], "energy")
idxs = np.digitize(true_en, ebin_edges) - 1
widths = []
for idx in range(n_ebins):
bin_error = error[(idxs == idx) & mask]
if use_weights:
pc_l_, med_, pc_u_ = weighted_percentile(
bin_error, [lower_pct, 50, upper_pct], weight)
else:
pc_l_, med_, pc_u_ = np.percentile(
bin_error, [lower_pct, 50, upper_pct])
widths.append(pc_u_ - pc_l_)
color = next(colors_iter)
ax.step(
x=ebin_edges,
y=[widths[0]] + widths,
#facecolor='none',
color=color,
)
ax.plot(
ebin_edges[[0, -1]],
[overall_iq_width] * 2,
linestyle='--',
color=color,
label=r"IQ{}% width $\forall$ E = {:.2f}{}".format(
iq_pct, overall_iq_width, bare_ulabel),
)
ax.set_xscale('log')
ax.set_xlim(ebin_edges[0], ebin_edges[-1])
if "energy" in param:
title = "Fractional error IQ{}% width vs true E".format(iq_pct)
ax.set_ylabel("Fractional {} error width".format(plabel))
else:
title = "Error IQ{}% width vs true E".format(iq_pct)
ax.set_ylabel("{} error width{}".format(plabel, ulabel))
ax.set_title(title)
ax.legend(loc="best", frameon=False)
for ax in axit:
ax.remove()
fig.tight_layout(h_pad=1, w_pad=0.01)
#fig.subplots_adjust(wspace=0.1, hspace=0.25)
plt.draw()
plt.show()
if outdir is not None:
fbasename = join(expand(outdir), "distributions")
fig.savefig(fbasename + ".pdf")
fig.savefig(fbasename + ".png", dpi=120)
def combine_tdi_tiles(
source_dir,
dest_dir,
table_hash,
gcd,
bin_edges_file,
tile_spec_file,
):
"""Combine individual time-independent tiles (one produced per DOM) into a single
TDI table.
Parameters
----------
source_dir : str
dest_dir : str
bin_edges_file : str
tile_spec_file : str
"""
source_dir = expand(source_dir)
dest_dir = expand(dest_dir)
gcd = expand(gcd)
bin_edges_file = expand(bin_edges_file)
tile_spec_file = expand(tile_spec_file)
mkdir(dest_dir)
assert isdir(source_dir)
assert isfile(bin_edges_file)
assert isfile(tile_spec_file)
gcd = extract_gcd(gcd)
bin_edges = load_pickle(bin_edges_file)
x_edges = bin_edges['x']
y_edges = bin_edges['y']
z_edges = bin_edges['z']
ctdir_edges = bin_edges['costhetadir']
phidir_edges = bin_edges['phidir']
n_x = len(x_edges) - 1
n_y = len(y_edges) - 1
n_z = len(z_edges) - 1
n_ctdir = len(ctdir_edges) - 1
n_phidir = len(phidir_edges) - 1
n_dir_bins = n_ctdir * n_phidir
x_bw = (x_edges.max() - x_edges.min()) / n_x
y_bw = (y_edges.max() - y_edges.min()) / n_y
z_bw = (z_edges.max() - z_edges.min()) / n_z
bin_vol = x_bw * y_bw * z_bw
ctdir_min = ctdir_edges.min()
ctdir_max = ctdir_edges.max()
phidir_min = phidir_edges.min()
phidir_max = phidir_edges.max()
with open(tile_spec_file, 'r') as f:
tile_specs = [l.strip() for l in f.readlines()]
table = np.zeros(shape=(n_x, n_y, n_z, n_ctdir, n_phidir),
dtype=np.float32)
# Slice all table dimensions to exclude {under,over}flow bins
central_slice = (slice(1, -1), ) * 5
angsens_model = None
ice_model = None
disable_tilt = None
disable_anisotropy = None
n_phase = None
n_group = None
tiles_info = []
for tile_spec in tile_specs:
info = None
try:
fields = tile_spec.split()
info = OrderedDict()
info['tbl_idx'] = int(fields[0])
info['string'] = int(fields[1])
info['dom'] = int(fields[2])
info['seed'] = int(fields[3])
info['n_events'] = int(fields[4])
info['x_min'] = float(fields[5])
info['x_max'] = float(fields[6])
info['n_x'] = int(fields[7])
info['y_min'] = float(fields[8])
info['y_max'] = float(fields[9])
info['n_y'] = int(fields[10])
info['z_min'] = float(fields[11])
info['z_max'] = float(fields[12])
info['n_z'] = int(fields[13])
info['n_ctdir'] = int(fields[14])
info['n_phidir'] = int(fields[15])
tiles_info.append(info)
tile_fpath = glob(
join(
source_dir, 'clsim_table_set'
'_{table_hash}'
'_tile_{tbl_idx}'
'_string_{string}'
'_dom_{dom}'
'_seed_{seed}'
'_n_{n_events}'
'.fits'.format(table_hash=table_hash, **info)))[0]
try:
fits_table = fits.open(tile_fpath,
mode='readonly',
memmap=True)
except:
wstderr('Failed on tile_fpath "{}"'.format(tile_fpath))
raise
primary = fits_table[0]
header = primary.header # pylint: disable=no-member
keys = header.keys()
this_gcd_i3_md5 = extract_meta_from_keys(keys, '_i3_gcd_i3_md5_')
assert this_gcd_i3_md5 == gcd['source_gcd_i3_md5'], \
'this: {}, ref: {}'.format(this_gcd_i3_md5, gcd['source_gcd_i3_md5'])
this_angsens_model = extract_meta_from_keys(keys, '_i3_angsens_')
if angsens_model is None:
angsens_model = this_angsens_model
_, avg_angsens = load_angsens_model(angsens_model)
else:
assert this_angsens_model == angsens_model
this_table_hash = extract_meta_from_keys(keys, '_i3_hash_')
assert this_table_hash == table_hash
this_ice_model = extract_meta_from_keys(keys, '_i3_ice_')
if ice_model is None:
ice_model = this_ice_model
else:
assert this_ice_model == ice_model
this_disable_anisotropy = header['_i3_disable_anisotropy']
if disable_anisotropy is None:
disable_anisotropy = this_disable_anisotropy
else:
assert this_disable_anisotropy == disable_anisotropy
this_disable_tilt = header['_i3_disable_tilt']
if disable_tilt is None:
disable_tilt = this_disable_tilt
else:
assert this_disable_tilt == disable_tilt
this_n_phase = header['_i3_n_phase']
if n_phase is None:
n_phase = this_n_phase
else:
assert this_n_phase == n_phase
this_n_group = header['_i3_n_group']
if n_group is None:
n_group = this_n_group
else:
assert this_n_group == n_group
assert info['n_ctdir'] == n_ctdir
assert info['n_phidir'] == n_phidir
assert np.isclose(header['_i3_costhetadir_min'], ctdir_min)
assert np.isclose(header['_i3_costhetadir_max'], ctdir_max)
assert np.isclose(header['_i3_phidir_min'], phidir_min)
assert np.isclose(header['_i3_phidir_max'], phidir_max)
n_photons = header['_i3_n_photons']
n_dir_bins = info['n_ctdir'] * info['n_phidir']
this_x_bw = (info['x_max'] - info['x_min']) / info['n_x']
this_y_bw = (info['y_max'] - info['y_min']) / info['n_y']
this_z_bw = (info['z_max'] - info['z_min']) / info['n_z']
assert this_x_bw == x_bw
assert this_y_bw == y_bw
assert this_z_bw == z_bw
assert np.any(np.isclose(info['x_min'], x_edges))
assert np.any(np.isclose(info['x_max'], x_edges))
assert np.any(np.isclose(info['y_min'], y_edges))
assert np.any(np.isclose(info['y_max'], y_edges))
assert np.any(np.isclose(info['z_min'], z_edges))
assert np.any(np.isclose(info['z_max'], z_edges))
quantum_efficiency = 0.25 * gcd['rde'][info['string'] - 1,
info['dom'] - 1]
norm = n_dir_bins * quantum_efficiency * avg_angsens / (n_photons *
bin_vol)
if np.isnan(norm):
print('\nTile {} norm is nan!'.format(info['tbl_idx']))
print(' quantum_efficiency = {}, n_photons = {}'.format(
quantum_efficiency, n_photons))
elif norm == 0:
print('\nTile {} norm is 0'.format(info['tbl_idx']))
x_start = np.digitize(info['x_min'] + x_bw / 2, x_edges) - 1
x_stop = np.digitize(info['x_max'] - x_bw / 2, x_edges)
y_start = np.digitize(info['y_min'] + y_bw / 2, y_edges) - 1
y_stop = np.digitize(info['y_max'] - y_bw / 2, y_edges)
z_start = np.digitize(info['z_min'] + z_bw / 2, z_edges) - 1
z_stop = np.digitize(info['z_max'] - z_bw / 2, z_edges)
# NOTE: comparison excludes norm = 0 _and_ norm = NaN
if norm > 0:
assert not np.isnan(norm)
table[x_start:x_stop, y_start:y_stop,
z_start:z_stop, :, :] += (
norm * primary.data[central_slice] # pylint: disable=no-member
)
except:
wstderr('Failed on tile_spec {}'.format(tile_spec))
if info is not None:
wstderr('Info:\n{}'.format(info))
raise
wstderr('.')
wstderr('\n')
metadata = OrderedDict()
metadata['table_hash'] = table_hash
metadata['disable_tilt'] = disable_tilt
metadata['disable_anisotropy'] = disable_anisotropy
metadata['gcd'] = gcd
metadata['angsens_model'] = angsens_model
metadata['ice_model'] = ice_model
metadata['n_phase'] = n_phase
metadata['n_group'] = n_group
metadata['tiles_info'] = tiles_info
outdir = join(
dest_dir, 'tdi_table_{}_tilt_{}_anisotropy_{}'.format(
table_hash,
'off' if disable_tilt else 'on',
'off' if disable_anisotropy else 'on',
))
mkdir(outdir)
name = 'tdi_table.npy'
outfpath = join(outdir, name)
wstdout('saving table to "{}"\n'.format(outfpath))
np.save(outfpath, table)
#outfpath = join(outdir, 'tdi_bin_edges.json')
#wstdout('saving bin edges to "{}"\n'.format(outfpath))
#json.dump(
# bin_edges,
# file(outfpath, 'w'),
# sort_keys=False,
# indent=2,
#)
outfpath = join(outdir, 'tdi_bin_edges.pkl')
wstdout('saving bin edges to "{}"\n'.format(outfpath))
pickle.dump(
bin_edges,
open(outfpath, 'wb'),
protocol=pickle.HIGHEST_PROTOCOL,
)
#outfpath = join(outdir, 'tdi_metadata.json')
#wstdout('saving metadata to "{}"\n'.format(outfpath))
#json.dump(
# metadata,
# file(outfpath, 'w'),
# sort_keys=False,
# indent=2,
#)
outfpath = join(outdir, 'tdi_metadata.pkl')
wstdout('saving metadata to "{}"\n'.format(outfpath))
pickle.dump(
metadata,
open(outfpath, 'wb'),
protocol=pickle.HIGHEST_PROTOCOL,
)
def extract_gcd_frames(g_frame,
c_frame,
d_frame,
retro_gcd_dir,
metadata=None):
"""Extract GCD info to Python/Numpy-readable objects stored to a central
GCD directory, subdirs of which are named by the hex md5sum of each
extracted GCD file.
Parameters
----------
g_frame : icecube.icetray.I3Frame with stop I3Frame.Geometry
c_frame : icecube.icetray.I3Frame with stop I3Frame.Calibration
d_frame : icecube.icetray.I3Frame with stop I3Frame.DetectorStatus
retro_gcd_dir : string
metadata : None or mapping, optional
If non-empty mapping (e.g., OrderedDict) is provided, the contents are
written to the gcd file's subdirectory inside retro_gcd_dir as
"metadata.json"
Returns
-------
gcd_md5_hex : len-32 string of chars 0-9 and/or a-f
MD5 sum of _only_ the G, C, and D frames (in that order) dumped to an
uncompressed i3 file. Note that this can result in a hash value
different from hashing the original GCD file if other frames were
present besides the GCD frames (such as an I frame, or Q/P/etc. if the
GCD is embedded in a data i3 file)
"""
from icecube.dataio import I3File # pylint: disable=import-outside-toplevel
retro_gcd_dir = expand(retro_gcd_dir)
# Create root dir for gcd subdirs if necessary
if not isdir(retro_gcd_dir):
mkdir(retro_gcd_dir)
# Add a vaguely useful README to gcd root dir
readme_fpath = join(retro_gcd_dir, "README")
if not isfile(readme_fpath):
with io.open(readme_fpath, "w", encoding="utf-8") as fhandle:
fhandle.write(GCD_README.strip() + "\n")
# Find md5sum of an uncompressed GCD file created by these G, C, & D frames
tempdir_path = mkdtemp(suffix="gcd")
try:
gcd_i3file_path = join(tempdir_path, "gcd.i3")
gcd_i3file = I3File(gcd_i3file_path, "w")
gcd_i3file.push(g_frame)
gcd_i3file.push(c_frame)
gcd_i3file.push(d_frame)
gcd_i3file.close()
gcd_md5_hex = get_file_md5(gcd_i3file_path)
finally:
try:
rmtree(tempdir_path)
except Exception:
pass
this_gcd_dir_path = join(retro_gcd_dir, gcd_md5_hex)
if isdir(this_gcd_dir_path):
# already extracted this GCD
sys.stderr.write(
"Already extracted GCD with md5sum {}\n".format(gcd_md5_hex))
return gcd_md5_hex
tempdir_path = mkdtemp(suffix="." + gcd_md5_hex)
try:
# Extract GCD info into Python/Numpy-readable things
gcd_info = OrderedDict()
gcd_info["I3Geometry"] = extract_i3_geometry(g_frame)
gcd_info["I3Calibration"] = extract_i3_calibration(c_frame)
gcd_info["I3DetectorStatus"] = extract_i3_detector_status(d_frame)
gcd_info.update(extract_bad_doms_lists(d_frame))
# Write info to files. Preferable to write a single array to a .npy file;
# second most preferable is to write multiple arrays to (compressed) .npz
# file (faster to load than pkl files); finally, I3DetectorStatus _has_ to
# be stored as pickle to preserve varying-length items.
for key, val in gcd_info.items():
if isinstance(val, Mapping):
if key == "I3DetectorStatus":
key_fpath = join(tempdir_path, key + ".pkl")
with io.open(key_fpath, "wb") as fhandle:
pickle.dump(val,
fhandle,
protocol=pickle.HIGHEST_PROTOCOL)
else:
np.savez_compressed(join(tempdir_path, key + ".npz"),
**val)
else:
assert isinstance(val, np.ndarray)
np.save(join(tempdir_path, key + ".npy"), val)
if metadata:
metadata_fpath = join(tempdir_path, "metadata.json")
with open(metadata_fpath, "w") as fhandle:
json.dump(metadata, fhandle, sort_keys=False, indent=4)
try:
copytree(tempdir_path, this_gcd_dir_path)
except OSError as err:
if err.errno != errno.EEXIST:
raise
finally:
try:
rmtree(tempdir_path)
except Exception:
pass
return gcd_md5_hex
def extract_gcd_files(gcd_files, retro_gcd_dir, verbosity=0):
"""
Parameters
----------
gcd_files : string or iterable thereof
retro_gcd_dir : string
Path to communal Retro-extracted GCD dir
verbosity : int in [0, 1]
Returns
-------
gcd_md5_hexs : len(gcd_files)-list of strings
"""
# Import here so module can be read without access to IceCube software
from icecube.dataio import I3File # pylint: disable=no-name-in-module, import-outside-toplevel
from icecube.icetray import I3Frame # pylint: disable=no-name-in-module, import-outside-toplevel
if isinstance(gcd_files, string_types):
gcd_files = [gcd_files]
gcd_md5_hexs = []
for gcd_fpath in gcd_files:
gcd_fpath = expand(gcd_fpath)
i3f = I3File(gcd_fpath)
gcd_frames = OrderedDict()
while i3f.more():
frame = i3f.pop_frame()
if frame.Stop == I3Frame.Geometry:
if "g_frame" in gcd_frames:
raise ValueError(
'GCD file "{}" contains multiple G frames'.format(
gcd_fpath))
gcd_frames["g_frame"] = frame
elif frame.Stop == I3Frame.Calibration:
if "c_frame" in gcd_frames:
raise ValueError(
'GCD file "{}" contains multiple C frames'.format(
gcd_fpath))
gcd_frames["c_frame"] = frame
elif frame.Stop == I3Frame.DetectorStatus:
if "d_frame" in gcd_frames:
raise ValueError(
'GCD file "{}" contains multiple D frames'.format(
gcd_fpath))
gcd_frames["d_frame"] = frame
for frame_type in ("g", "c", "d"):
if "{}_frame".format(frame_type) not in gcd_frames:
raise ValueError('No {} frame found in GCD file "{}"'.format(
frame_type, gcd_fpath))
metadata = OrderedDict()
metadata["extracted_on_fqdn"] = socket.getfqdn()
metadata["path_to_sourcefile"] = abspath(gcd_fpath)
metadata["sourcefile_md5sum"] = get_file_md5(gcd_fpath)
try:
gcd_md5_hex = extract_gcd_frames(retro_gcd_dir=retro_gcd_dir,
metadata=metadata,
**gcd_frames)
except Exception:
sys.stderr.write(
'failed to extract GCD file "{}"\n'.format(gcd_fpath))
raise
gcd_md5_hexs.append(gcd_md5_hex)
if verbosity:
sys.stdout.write("{} {}\n".format(gcd_md5_hex, gcd_fpath))
return gcd_md5_hexs
def find_files_to_extract(roots,
overwrite,
find_gcd_in_dir=False,
data_run_gcds=None):
"""Find missing, bad, or old extracted pulse series and print the paths of
the corresponding events directories.
Parameters
----------
roots : str
overwrite : bool
find_gcd_in_dir : bool or str
If True, search for a data run's GCD file in the same directory as each
i3 data file that is found. If False, do not search for a data run's
GCD file.
If `find_gcd_in_dir` is a string, interpret as a directory path; search
for a data run's GCD file in that directory.
Note that Monte Carlo i3 files will not return a `gcd_fpath`, as it is
difficult to logically ascertain which GCD was used for a MC run and
where it exists.
data_run_gcds : dict or None, optional
Keys must be <tuple>(<str>2-digit IC86 season, <str>Run number). Each
value is a string full path to the corresponding GCD file.
Yields
------
fpath : str
Full path to data/MC i3 file
gcd_fpath : str
Full path to GCD file corresponding to the data i3 file at fpath
fname_groupdict : dict
As returned by OSCNEXT_I3_FNAME_RE.match(...).groupdict()
"""
if isinstance(roots, str):
roots = [roots]
roots = [expand(root) for root in roots]
# If `find_gcd_in_dir` is a string, interpret as a directory and search for
# GCD's in that directory (recursively)
found_data_run_gcds = None
if isinstance(find_gcd_in_dir, str):
find_gcd_in_dir = expand(find_gcd_in_dir)
assert isdir(find_gcd_in_dir), str(find_gcd_in_dir)
found_data_run_gcds = find_gcds_in_dirs(find_gcd_in_dir,
gcd_fname_re=DATA_GCD_FNAME_RE,
recurse=True)
def get_i3_events_file_info(dirpath, fname):
"""Closure to find only i3 events file names and, in that case, grab a
relevant GCD file (if file contains data events and such a GCD can be
found in `dirpath`), and return the info extracted from the file name.
Parameters
----------
dirpath : str
Fully qualified path to file's directory
fname : str
(basename) of the file (i.e., excluding any directories)
Returns
-------
retval : None or 3-tuple
Returns `None` if the file is determined to not be an i3 events
file (based on filename alone). Otherwise, returns .. ::
fpath : str
fully qualified (including directories) path to the i3
events file
gcd_fpath : str or None
fully qualified (including directories) path to a relevant
GCD file found in the same dir, or None if none is found
fname_groupdict : mapping
Filename info as returned by regex
"""
fname_match = OSCNEXT_I3_FNAME_RE.match(fname)
if not fname_match:
return None
fname_groupdict = fname_match.groupdict()
i3_retro_dir = join(dirpath, fname_groupdict["basename"])
if (not overwrite and isdir(i3_retro_dir)
and isfile(join(i3_retro_dir, "events.npy"))):
return None
fpath = join(dirpath, fname)
gcd_fpath = None
if fname_groupdict["kind"] == "data":
key = (fname_groupdict["season"], fname_groupdict["run"])
if data_run_gcds:
gcd_fpath = data_run_gcds.get(key, None)
if gcd_fpath is None and found_data_run_gcds:
gcd_fpath = found_data_run_gcds.get(key, None)
if gcd_fpath is None and thisdir_data_run_gcds:
gcd_fpath = thisdir_data_run_gcds.get(key, None)
return fpath, gcd_fpath, fname_groupdict
for root in roots:
if isfile(root):
retval = get_i3_events_file_info(dirpath=dirname(root),
fname=basename(root))
if retval is not None:
yield retval
continue
for dirpath, dirs, files in walk(root, followlinks=True):
if "events.npy" in files:
# No need to recurse into an existing retro events directory,
# so clear out remaining directories
del dirs[:]
continue
dirs.sort(key=nsort_key_func)
files.sort(key=nsort_key_func)
# If `find_gcd_in_dir` is True (i.e., not a string and not False),
# look in current directory for all data-run GCD files
thisdir_data_run_gcds = None
if find_gcd_in_dir is True:
thisdir_data_run_gcds = find_gcds_in_dirs(
dirpath, gcd_fname_re=DATA_GCD_FNAME_RE, recurse=False)
for fname in files:
retval = get_i3_events_file_info(dirpath=dirpath, fname=fname)
if retval is not None:
yield retval
def main(description=__doc__):
"""Script interface to `extract_events` function: Parse command line args
and call function."""
hostname = gethostname()
dflt = {}
if hostname in ["schwyz", "luzern", "uri", "unterwalden"]:
sim_gcd_dir = "/data/icecube/gcd"
dflt["retro_gcd_dir"] = "/data/icecube/retro_gcd"
dflt["data_gcd_dir"] = "/data/icecube/gcd"
dflt["procs"] = cpu_count()
elif hostname.endswith(".aci.ics.psu.edu"):
sim_gcd_dir = "/gpfs/group/dfc13/default/gcd/mc"
dflt["retro_gcd_dir"] = "/gpfs/group/dfc13/default/retro_gcd"
dflt["data_gcd_dir"] = None
dflt["procs"] = 1
else: # wisconsin?
sim_gcd_dir = "/data/sim/DeepCore/2018/pass2/gcd"
dflt["retro_gcd_dir"] = "~/retro_gcd"
dflt["data_gcd_dir"] = None
dflt["procs"] = 1
raise ValueError("Unknown host: {}".format(hostname))
dflt["sim_gcd"] = join(
expand(sim_gcd_dir),
"GeoCalibDetectorStatus_AVG_55697-57531_PASS2_SPE_withScaledNoise.i3.gz",
)
parser = ArgumentParser(
description=description,
formatter_class=ArgumentDefaultsHelpFormatter,
)
parser.add_argument(
"--roots",
nargs="+",
required=True,
help=
"""i3 file(s) and/or directories to search for i3 files to extract""",
)
parser.add_argument(
"--overwrite",
action="store_true",
help="""If event was already extracted, overwrite existing object(s)
(existing files will not be deleted, so excess objects not specified
here will not be removed)""",
)
parser.add_argument(
"--retro-gcd-dir",
required=False,
default=dflt["retro_gcd_dir"], #dflt.get("retro_gcd_dir", None),
help="""Directory into which to store any extracted GCD info""",
)
parser.add_argument(
"--sim-gcd",
required=False,
default=dflt.get("sim_gcd", None),
help="""Specify an external GCD file or md5sum (as returned by
`retro.i3processing.extract_gcd_frames`, i.e., the md5sum of an
uncompressed i3 file containing _only_ the G, C, and D frames). It is
not required to specify --gcd if the G, C, and D frames are embedded in
all files specified by --i3-files. Any GCD frames within said files
will also take precedent if --gcd _is_ specified.""",
)
parser.add_argument(
"--data-gcd-dir",
required=False,
default=dflt.get("data_gcd_dir", None),
help="""If data GCDs all live in one directory, specify it here.""",
)
parser.add_argument(
"--outdir",
required=False,
help="""Directory into which to store the extracted directories and
files. If not specified, the directory where each i3 file is stored is
used (a leaf directory is created with the same name as each i3 file
but with .i3 and any compression extensions removed).""")
#parser.add_argument(
# "--photons",
# nargs="+",
# default=[],
# help="""Photon series names to extract from each event""",
#)
parser.add_argument(
"--pulses",
required=False,
nargs="+",
default=["SRTTWOfflinePulsesDC", "SplitInIcePulses"],
help="""Pulse series names to extract from each event""",
)
parser.add_argument(
"--recos",
required=False,
nargs="+",
help="""Reco names to extract from each event. If not specified,
"L5_SPEFit11", "LineFit_DC", and "retro_crs_prefit" (if the file name
matches L6 processing or above) are extracted.""",
)
parser.add_argument(
"--triggers",
nargs="+",
default=["I3TriggerHierarchy"],
help="""Trigger hierarchy names to extract from each event""",
)
parser.add_argument(
"--no-truth",
action="store_true",
help="""Do not extract truth information from Monte Carlo events""",
)
parser.add_argument(
"--additional-keys",
default=None,
nargs="+",
help="""Additional keys to extract from event I3 frame""",
)
parser.add_argument(
"--procs",
default=dflt["procs"],
type=int,
help="""Number of (sub)processes to use for converting files""",
)
args = parser.parse_args()
kwargs = vars(args)
find_func_kwargs = {
k: kwargs.pop(k)
for k in getargspec(find_files_to_extract).args if k in kwargs
}
no_truth = kwargs.pop("no_truth")
data_gcd_dir = kwargs.pop("data_gcd_dir", None)
sim_gcd = kwargs.pop("sim_gcd", None)
procs = kwargs.pop("procs", None)
if data_gcd_dir:
data_run_gcds = find_gcds_in_dirs(data_gcd_dir,
gcd_fname_re=DATA_GCD_FNAME_RE)
else:
data_run_gcds = None
kwargs["additional_keys"] = kwargs.pop("additional_keys", None)
if not kwargs["additional_keys"]:
from processing.samples.oscNext.verification.general_mc_data_harvest_and_plot import (
L5_VARS, L6_VARS, L7_VARS)
pool = Pool(procs)
requests = []
for fpath, gcd_fpath, fname_groupdict in find_files_to_extract(
find_gcd_in_dir=True, data_run_gcds=data_run_gcds,
**find_func_kwargs):
print(fpath)
extract_events_kwargs = deepcopy(kwargs)
extract_events_kwargs["i3_files"] = [fpath]
is_data = fname_groupdict["kind"].lower() == "data"
if is_data:
assert gcd_fpath is not None
extract_events_kwargs["gcd"] = gcd_fpath
else:
extract_events_kwargs["truth"] = not is_data and not no_truth
extract_events_kwargs["gcd"] = sim_gcd
level = int(fname_groupdict["level"])
if "recos" not in extract_events_kwargs or not extract_events_kwargs[
"recos"]:
recos = []
if level >= 5:
recos.extend(["LineFit_DC", "L5_SPEFit11"])
if level >= 6:
recos.append("retro_crs_prefit")
extract_events_kwargs["recos"] = recos
if not extract_events_kwargs["additional_keys"]:
additional_keys = []
if level >= 5:
additional_keys.extend(L5_VARS.keys())
if level >= 6:
additional_keys.extend(L6_VARS.keys())
if level >= 7:
additional_keys.extend(L7_VARS.keys())
extract_events_kwargs["additional_keys"] = sorted(additional_keys)
requests.append((
extract_events_kwargs,
pool.apply_async(wrapped_extract_events, tuple(),
extract_events_kwargs),
))
failed_i3_files = []
for extract_events_kwargs, async_result in requests:
retval = async_result.get()
if not retval:
failed_i3_files.append(extract_events_kwargs["i3_files"])
pool.close()
pool.join()
if failed_i3_files:
for failure in chain(*failed_i3_files):
print('"{}"'.format(failure))
print("\n{} failures out of {} i3 files found that needed to be extracted".
format(len(failed_i3_files), len(requests)))
def generate_clsim_table(
outdir,
gcd,
ice_model,
angular_sensitivity,
disable_tilt,
disable_anisotropy,
string,
dom,
n_events,
seed,
coordinate_system,
binning,
tableset_hash=None,
tile=None,
overwrite=False,
compress=False,
):
"""Generate a CLSim table.
See wiki.icecube.wisc.edu/index.php/Ice for information about ice models.
Parameters
----------
outdir : string
gcd : string
ice_model : str
E.g. "spice_mie", "spice_lea", ...
angular_sensitivity : str
E.g. "h2-50cm", "9" (which is equivalent to "new25" because, like, duh)
disable_tilt : bool
Whether to force no layer tilt in simulation (if tilt is present in
bulk ice model; otherwise, this has no effect)
disable_anisotropy : bool
Whether to force no bulk ice anisotropy (if anisotropy is present in
bulk ice model; otherwise, this has no effect)
string : int in [1, 86]
dom : int in [1, 60]
n_events : int > 0
Note that the number of photons is much larger than the number of
events (related to the "brightness" of the defined source).
seed : int in [0, 2**32)
Seed for CLSim's random number generator
coordinate_system : string in {"spherical", "cartesian"}
If spherical, base coordinate system is .. ::
(r, theta, phi, t, costhetadir, (optionally abs)deltaphidir)
If Cartesian, base coordinate system is .. ::
(x, y, z, costhetadir, phidir)
but if any of the coordinate axes are specified to have 0 bins, they
will be omitted (but the overall order is maintained).
binning : mapping
If `coordinate_system` is "spherical", keys should be:
"n_r_bins"
"n_t_bins"
"n_costheta_bins"
"n_phi_bins"
"n_costhetadir_bins"
"n_deltaphidir_bins"
"r_max"
"r_power"
"t_max"
"t_power"
"deltaphidir_power"
If `coordinate_system` is "cartesian", keys should be:
"n_x_bins"
"n_y_bins"
"n_z_bins"
"n_costhetadir_bins"
"n_phidir_bins"
"x_min"
"x_max"
"y_min"
"y_max"
"z_min"
"z_max"
tableset_hash : str, optional
Specify if the table is a tile used to generate a larger table
tile : int >= 0, optional
Specify if the table is a tile used to generate a larger table
overwrite : bool, optional
Whether to overwrite an existing table (default: False)
compress : bool, optional
Whether to pass the resulting table through zstandard compression
(default: True)
Raises
------
ValueError
If `compress` is True but `zstd` command-line utility cannot be found
AssertionError, ValueError
If illegal argument values are passed
ValueError
If `overwrite` is False and a table already exists at the target path
Notes
-----
Binnings are as follows:
* Radial binning is regular in the space of r**(1/r_power), with
`n_r_bins` spanning from 0 to `r_max` meters.
* Time binning is regular in the space of t**(1/t_power), with
`n_t_bins` spanning from 0 to `t_max` nanoseconds.
* Position zenith angle is binned regularly in the cosine of the zenith
angle with `n_costhetadir_bins` spanning from -1 to +1.
* Position azimuth angle is binned regularly, with `n_phi_bins`
spanning from -pi to pi radians.
* Photon directionality zenith angle (relative to IcedCube coordinate
system) is binned regularly in cosine-zenith space, with
`n_costhetadir_bins` spanning from `costhetadir_min` to
`costhetadir_max`
* Photon directionality azimuth angle; sometimes assumed to be
symmetric about line from DOM to the center of the bin, so is binned
as an absolute value, i.e., from 0 to pi radians. Otherwise, binned
from -np.pi to +np.pi
The following are forced upon the above binning specifications (and
remaining parameters are specified as arguments to the function)
* t_min = 0 (ns)
* r_min = 0 (m)
* costheta_min = -1
* costheta_max = 1
* phi_min = -pi (rad)
* phi_max = pi (rad)
* costhetadir_min = -1
* costhetadir_max = 1
* deltaphidir_min = 0 (rad)
* deltaphidir_min = pi (rad)
"""
assert isinstance(n_events, Integral) and n_events > 0
assert isinstance(seed, Integral) and 0 <= seed < 2**32
assert ((tableset_hash is not None and tile is not None)
or (tableset_hash is None and tile is None))
n_bins_per_dim = []
for key, val in binning.items():
if not key.startswith('n_'):
continue
assert isinstance(val, Integral), '{} not an integer'.format(key)
assert val >= 0, '{} must be >= 0'.format(key)
n_bins_per_dim.append(val)
# Note: + 2 accounts for under & overflow bins in each dimension
n_bins = np.product([n + 2 for n in n_bins_per_dim if n > 0])
assert n_bins > 0
#if n_bins > 2**32:
# raise ValueError(
# 'The flattened bin index in CLSim is represented by uint32 which'
# ' has a max of 4 294 967 296, but the binning specified comes to'
# ' {} bins ({} times too many).'
# .format(n_bins, n_bins / 2**32)
# )
ice_model = ice_model.strip()
angular_sensitivity = angular_sensitivity.strip()
# For now, hole ice model is hard-coded in our CLSim branch; see
# clsim/private/clsim/I3CLSimLightSourceToStepConverterFlasher.cxx
# in the branch you're using to check that this is correct
assert angular_sensitivity == 'flasher_p1_0.30_p2_-1'
gcd_info = extract_gcd(gcd)
if compress and not any(
access(join(path, 'zstd'), X_OK)
for path in environ['PATH'].split(pathsep)):
raise ValueError('`zstd` command not found in path')
outdir = expand(outdir)
mkdir(outdir)
axes = OrderedDict()
binning_kw = OrderedDict()
# Note that the actual binning in CLSim is performed using float32, so we
# first "truncate" all values to that precision. However, the `LinearAxis`
# function requires Python floats (which are 64 bits), so we have to
# convert all values to to `float` when passing as kwargs to `LinearAxis`
# (and presumably the values will be re-truncated to float32 within the
# CLsim code somewhere). Hopefully following this procedure, the values
# actually used within CLSim are what we want...? CLSim is stupid.
ftype = np.float32
if coordinate_system == 'spherical':
binning['t_min'] = ftype(0) # ns
binning['r_min'] = ftype(0) # meters
costheta_min = ftype(-1.0)
costheta_max = ftype(1.0)
# See
# clsim/resources/kernels/spherical_coordinates.c.cl
# in the branch you're using to check that the following are correct
phi_min = ftype(3.0543261766433716e-01)
phi_max = ftype(6.5886182785034180e+00)
binning['costhetadir_min'] = ftype(-1.0)
binning['costhetadir_max'] = ftype(1.0)
binning['deltaphidir_min'] = ftype(-3.1808626651763916e+00)
binning['deltaphidir_max'] = ftype(3.1023228168487549e+00)
if binning['n_r_bins'] > 0:
assert isinstance(binning['r_power'],
Integral) and binning['r_power'] > 0
r_binning_kw = OrderedDict([
('min', float(binning['r_min'])),
('max', float(binning['r_max'])),
('n_bins', int(binning['n_r_bins'])),
])
if binning['r_power'] == 1:
axes['r'] = LinearAxis(**r_binning_kw)
else:
r_binning_kw['power'] = int(binning['r_power'])
axes['r'] = PowerAxis(**r_binning_kw)
binning_kw['r'] = r_binning_kw
if binning['n_costheta_bins'] > 0:
costheta_binning_kw = OrderedDict([
('min', float(costheta_min)),
('max', float(costheta_max)),
('n_bins', int(binning['n_costheta_bins'])),
])
axes['costheta'] = LinearAxis(**costheta_binning_kw)
binning_kw['costheta'] = costheta_binning_kw
if binning['n_phi_bins'] > 0:
phi_binning_kw = OrderedDict([
('min', float(phi_min)),
('max', float(phi_max)),
('n_bins', int(binning['n_phi_bins'])),
])
axes['phi'] = LinearAxis(**phi_binning_kw)
binning_kw['phi'] = phi_binning_kw
if binning['n_t_bins'] > 0:
assert isinstance(binning['t_power'],
Integral) and binning['t_power'] > 0
t_binning_kw = OrderedDict([
('min', float(binning['t_min'])),
('max', float(binning['t_max'])),
('n_bins', int(binning['n_t_bins'])),
])
if binning['t_power'] == 1:
axes['t'] = LinearAxis(**t_binning_kw)
else:
t_binning_kw['power'] = int(binning['t_power'])
axes['t'] = PowerAxis(**t_binning_kw)
binning_kw['t'] = t_binning_kw
if binning['n_costhetadir_bins'] > 0:
costhetadir_binning_kw = OrderedDict([
('min', float(binning['costhetadir_min'])),
('max', float(binning['costhetadir_max'])),
('n_bins', int(binning['n_costhetadir_bins'])),
])
axes['costhetadir'] = LinearAxis(**costhetadir_binning_kw)
binning_kw['costhetadir'] = costhetadir_binning_kw
if binning['n_deltaphidir_bins'] > 0:
assert (isinstance(binning['deltaphidir_power'], Integral)
and binning['deltaphidir_power'] > 0)
deltaphidir_binning_kw = OrderedDict([
('min', float(binning['deltaphidir_min'])),
('max', float(binning['deltaphidir_max'])),
('n_bins', int(binning['n_deltaphidir_bins'])),
])
if binning['deltaphidir_power'] == 1:
axes['deltaphidir'] = LinearAxis(**deltaphidir_binning_kw)
else:
deltaphidir_binning_kw['power'] = int(
binning['deltaphidir_power'])
axes['deltaphidir'] = PowerAxis(**deltaphidir_binning_kw)
binning_kw['deltaphidir'] = deltaphidir_binning_kw
elif coordinate_system == 'cartesian':
binning['t_min'] = ftype(0) # ns
binning['costhetadir_min'], binning['costhetadir_max'] = ftype(
-1.0), ftype(1.0)
binning['phidir_min'], binning['phidir_max'] = ftype(-np.pi), ftype(
np.pi) # rad
if binning['n_x_bins'] > 0:
x_binning_kw = OrderedDict([
('min', float(binning['x_min'])),
('max', float(binning['x_max'])),
('n_bins', int(binning['n_x_bins'])),
])
axes['x'] = LinearAxis(**x_binning_kw)
binning_kw['x'] = x_binning_kw
if binning['n_y_bins'] > 0:
y_binning_kw = OrderedDict([
('min', float(binning['y_min'])),
('max', float(binning['y_max'])),
('n_bins', int(binning['n_y_bins'])),
])
axes['y'] = LinearAxis(**y_binning_kw)
binning_kw['y'] = y_binning_kw
if binning['n_z_bins'] > 0:
z_binning_kw = OrderedDict([
('min', float(binning['z_min'])),
('max', float(binning['z_max'])),
('n_bins', int(binning['n_z_bins'])),
])
axes['z'] = LinearAxis(**z_binning_kw)
binning_kw['z'] = z_binning_kw
if binning['n_t_bins'] > 0:
assert isinstance(binning['t_power'],
Integral) and binning['t_power'] > 0
t_binning_kw = OrderedDict([
('min', float(binning['t_min'])),
('max', float(binning['t_max'])),
('n_bins', int(binning['n_t_bins'])),
])
if binning['t_power'] == 1:
axes['t'] = LinearAxis(**t_binning_kw)
else:
t_binning_kw['power'] = int(binning['t_power'])
axes['t'] = PowerAxis(**t_binning_kw)
binning_kw['t'] = t_binning_kw
if binning['n_costhetadir_bins'] > 0:
costhetadir_binning_kw = OrderedDict([
('min', float(binning['costhetadir_min'])),
('max', float(binning['costhetadir_max'])),
('n_bins', int(binning['n_costhetadir_bins'])),
])
axes['costhetadir'] = LinearAxis(**costhetadir_binning_kw)
binning_kw['costhetadir'] = costhetadir_binning_kw
if binning['n_phidir_bins'] > 0:
phidir_binning_kw = OrderedDict([
('min', float(binning['phidir_min'])),
('max', float(binning['phidir_max'])),
('n_bins', int(binning['n_phidir_bins'])),
])
axes['phidir'] = LinearAxis(**phidir_binning_kw)
binning_kw['phidir'] = phidir_binning_kw
binning_order = BINNING_ORDER[coordinate_system]
missing_dims = set(axes.keys()).difference(binning_order)
if missing_dims:
raise ValueError(
'`binning_order` specified is {} but is missing dimension(s) {}'.
format(binning_order, missing_dims))
axes_ = OrderedDict()
binning_kw_ = OrderedDict()
for dim in binning_order:
if dim in axes:
axes_[dim] = axes[dim]
binning_kw_[dim] = binning_kw[dim]
axes = axes_
binning_kw = binning_kw_
# NOTE: use SphericalAxes even if we're actually binning Cartesian since we
# don't care how it handles e.g. volumes, and Cartesian isn't implemented
# in CLSim yet
axes = SphericalAxes(axes.values())
# Construct metadata initially with items that will be hashed
metadata = OrderedDict([
('source_gcd_i3_md5', gcd_info['source_gcd_i3_md5']),
('coordinate_system', coordinate_system), ('binning_kw', binning_kw),
('ice_model', ice_model), ('angular_sensitivity', angular_sensitivity),
('disable_tilt', disable_tilt),
('disable_anisotropy', disable_anisotropy)
])
# TODO: this is hard-coded in our branch of CLSim; make parameter & fix here!
if 't' in binning:
metadata['t_is_residual_time'] = True
if tableset_hash is None:
hash_val = hash_obj(metadata, fmt='hex')[:8]
print('derived hash:', hash_val)
else:
hash_val = tableset_hash
print('tableset_hash:', hash_val)
metadata['hash_val'] = hash_val
if tile is not None:
metadata['tile'] = tile
dom_spec = OrderedDict([('string', string), ('dom', dom)])
if 'depth_idx' in dom_spec and ('subdet' in dom_spec
or 'string' in dom_spec):
if 'subdet' in dom_spec:
dom_spec['string'] = dom_spec.pop('subdet')
string = dom_spec['string']
depth_idx = dom_spec['depth_idx']
if isinstance(string, str):
subdet = dom_spec['subdet'].lower()
dom_x, dom_y = 0, 0
ic_avg_z, dc_avg_z = get_average_dom_z_coords(gcd_info['geo'])
if string == 'ic':
dom_z = ic_avg_z[depth_idx]
elif string == 'dc':
dom_z = dc_avg_z[depth_idx]
else:
raise ValueError('Unrecognized subdetector {}'.format(subdet))
else:
dom_x, dom_y, dom_z = gcd_info['geo'][string - 1, depth_idx]
metadata['string'] = string
metadata['depth_idx'] = depth_idx
if tile is not None:
raise ValueError(
'Cannot produce tiled tables using "depth_idx"-style table groupings;'
' use "string"/"dom"-style tables instead.')
clsim_table_fname_proto = CLSIM_TABLE_FNAME_PROTO[1]
clsim_table_metaname_proto = CLSIM_TABLE_METANAME_PROTO[0]
print('Subdetector {}, depth index {} (z_avg = {} m)'.format(
subdet, depth_idx, dom_z))
elif 'string' in dom_spec and 'dom' in dom_spec:
string = dom_spec['string']
dom = dom_spec['dom']
dom_x, dom_y, dom_z = gcd_info['geo'][string - 1, dom - 1]
metadata['string'] = string
metadata['dom'] = dom
if tile is None:
clsim_table_fname_proto = CLSIM_TABLE_FNAME_PROTO[2]
clsim_table_metaname_proto = CLSIM_TABLE_METANAME_PROTO[1]
else:
clsim_table_fname_proto = CLSIM_TABLE_TILE_FNAME_PROTO[-1]
clsim_table_metaname_proto = CLSIM_TABLE_TILE_METANAME_PROTO[-1]
print(
'GCD = "{}"\nString {}, dom {}: (x, y, z) = ({}, {}, {}) m'.format(
gcd, string, dom, dom_x, dom_y, dom_z))
else:
raise ValueError('Cannot understand `dom_spec` {}'.format(dom_spec))
# Until someone figures out DOM tilt and ice column / bubble column / cable
# orientations for sure, we'll just set DOM orientation to zenith=pi,
# azimuth=0.
dom_zenith = np.pi
dom_azimuth = 0.0
# Now add other metadata items that are useful but not used for hashing
metadata['dom_x'] = dom_x
metadata['dom_y'] = dom_y
metadata['dom_z'] = dom_z
metadata['dom_zenith'] = dom_zenith
metadata['dom_azimuth'] = dom_azimuth
metadata['seed'] = seed
metadata['n_events'] = n_events
metapath = join(outdir, clsim_table_metaname_proto.format(**metadata))
tablepath = join(outdir, clsim_table_fname_proto.format(**metadata))
# Save metadata as a JSON file (so it's human-readable by any tool, not
# just Python--in contrast to e.g. pickle files)
json.dump(metadata, file(metapath, 'w'), sort_keys=False, indent=4)
print('=' * 80)
print('Metadata for the table set was written to\n "{}"'.format(metapath))
print('Table will be written to\n "{}"'.format(tablepath))
print('=' * 80)
exists_at = []
for fpath in [tablepath, tablepath + '.zst']:
if isfile(fpath):
exists_at.append(fpath)
if exists_at:
names = ', '.join('"{}"'.format(fp) for fp in exists_at)
if overwrite:
print('WARNING! Deleting existing table(s) at ' + names)
for fpath in exists_at:
remove(fpath)
else:
raise ValueError('Table(s) already exist at {}; not'
' overwriting.'.format(names))
print('')
tray = I3Tray()
tray.AddSegment(
TabulateRetroSources,
'TabulateRetroSources',
source_gcd_i3_md5=gcd_info['source_gcd_i3_md5'],
binning_kw=binning_kw,
axes=axes,
ice_model=ice_model,
angular_sensitivity=angular_sensitivity,
disable_tilt=disable_tilt,
disable_anisotropy=disable_anisotropy,
hash_val=hash_val,
dom_spec=dom_spec,
dom_x=dom_x,
dom_y=dom_y,
dom_z=dom_z,
dom_zenith=dom_zenith,
dom_azimuth=dom_azimuth,
seed=seed,
n_events=n_events,
tablepath=tablepath,
tile=tile,
record_errors=False,
)
logging.set_level_for_unit('I3CLSimStepToTableConverter', 'TRACE')
logging.set_level_for_unit('I3CLSimTabulatorModule', 'DEBUG')
logging.set_level_for_unit('I3CLSimLightSourceToStepConverterGeant4',
'TRACE')
logging.set_level_for_unit('I3CLSimLightSourceToStepConverterFlasher',
'TRACE')
tray.Execute()
tray.Finish()
if compress:
print('Compressing table with zstandard via command line')
print(' zstd -1 --rm "{}"'.format(tablepath))
subprocess.check_call(['zstd', '-1', '--rm', tablepath])
print('done.')
def load_ckv_table_compr(fpath, mmap):
"""Load a Cherenkov table from disk.
Parameters
----------
fpath : string
Path to directory containing the table's .npy files.
mmap : bool
Whether to memory map the table (if it's stored in a directory
containing .npy files).
Returns
-------
table : OrderedDict
Items are
- 'n_photons' :
- 'group_refractive_index' :
- 'phase_refractive_index' :
- 'r_bin_edges' :
- 'costheta_bin_edges' :
- 't_bin_edges' :
- 'costhetadir_bin_edges' :
- 'deltaphidir_bin_edges' :
- 'ckv_template_map' : np.ndarray
- 't_indep_ckv_table' : np.ndarray (if available)
"""
fpath = expand(fpath)
table = OrderedDict()
if mmap:
mmap_mode = 'r'
else:
mmap_mode = None
if DEBUG:
wstderr('Loading ckv table from {} ...\n'.format(fpath))
if isfile(fpath):
assert basename(fpath) == 'ckv_table.npy'
fpath = dirname(fpath)
t0 = time()
indir = fpath
for key in CKV_TABLE_KEYS + ['t_indep_ckv_table']:
fpath = join(indir, key + '.npy')
if DEBUG:
wstderr(' loading {} from "{}" ...'.format(key, fpath))
t1 = time()
if isfile(fpath):
table[key] = np.load(fpath)
elif key != 't_indep_ckv_table':
raise ValueError(
'Could not find file "{}" for loading table key "{}"'
.format(fpath, key)
)
if DEBUG:
wstderr(' ({} ms)\n'.format(np.round((time() - t1)*1e3, 3)))
if DEBUG:
wstderr(' Total time to load: {} s\n'.format(np.round(time() - t0, 3)))
return table
def setup_dom_tables(
dom_tables_kind,
dom_tables_fname_proto,
gcd,
angsens_model,
norm_version,
use_sd_indices=const.ALL_STRS_DOMS,
step_length=1.0,
num_phi_samples=None,
ckv_sigma_deg=None,
template_library=None,
compute_t_indep_exp=True,
use_directionality=True,
no_noise=False,
force_no_mmap=False,
):
"""Instantiate and load single-DOM tables
"""
print('Instantiating and loading single-DOM tables')
t0 = time.time()
# TODO: set mmap based on memory?
if force_no_mmap:
mmap = False
else:
mmap = 'uncompr' in dom_tables_kind
if dom_tables_kind in ['raw_templ_compr', 'ckv_templ_compr']:
template_library = np.load(expand(template_library))
else:
template_library = None
gcd = extract_gcd(gcd)
if no_noise:
gcd['noise'] = np.zeros_like(gcd['noise'])
# Instantiate single-DOM tables class
dom_tables = Retro5DTables(
table_kind=dom_tables_kind,
geom=gcd['geo'],
rde=gcd['rde'],
noise_rate_hz=gcd['noise'],
angsens_model=angsens_model,
compute_t_indep_exp=compute_t_indep_exp,
use_directionality=use_directionality,
norm_version=norm_version,
num_phi_samples=num_phi_samples,
ckv_sigma_deg=ckv_sigma_deg,
template_library=template_library,
use_sd_indices=use_sd_indices
)
if '{subdet' in dom_tables_fname_proto:
doms = const.ALL_DOMS
for subdet in ['ic', 'dc']:
if subdet == 'ic':
strings = const.IC_STRS
else:
strings = const.DC_STRS
for dom in doms:
fpath = dom_tables_fname_proto.format(
subdet=subdet, dom=dom, depth_idx=dom-1
)
shared_table_sd_indices = []
for string in strings:
sd_idx = const.get_sd_idx(string=string, dom=dom)
if sd_idx not in use_sd_indices:
continue
shared_table_sd_indices.append(sd_idx)
if not shared_table_sd_indices:
continue
dom_tables.load_table(
fpath=fpath,
sd_indices=shared_table_sd_indices,
step_length=step_length,
mmap=mmap
)
elif '{string}' in dom_tables_fname_proto:
raise NotImplementedError('dom_tables_fname_proto with {string} not'
' implemented')
elif '{string_idx}' in dom_tables_fname_proto:
raise NotImplementedError('dom_tables_fname_proto with {string_idx}'
' not implemented')
else:
stacked_tables_fpath = expand(join(
dom_tables_fname_proto,
'stacked_{}.npy'.format(dom_tables.table_name)
))
stacked_tables_meta_fpath = expand(join(
dom_tables_fname_proto,
'stacked_{}_meta.pkl'.format(dom_tables.table_name)
))
stacked_t_indep_tables_fpath = expand(join(
dom_tables_fname_proto,
'stacked_{}.npy'.format(dom_tables.t_indep_table_name)
))
dom_tables.load_stacked_tables(
stacked_tables_meta_fpath=stacked_tables_meta_fpath,
stacked_tables_fpath=stacked_tables_fpath,
stacked_t_indep_tables_fpath=stacked_t_indep_tables_fpath,
mmap_t_indep=mmap
)
print(' -> {:.3f} s\n'.format(time.time() - t0))
return dom_tables
def summarize_clsim_table(table_fpath,
table=None,
save_summary=True,
outdir=None):
"""
Parameters
----------
table_fpath : string
Path to table (or just the table's filename if `outdir` is specified)
table : mapping, optional
If the table has already been loaded, it can be passed here to avoid
re-loading the table.
save_summary : bool
Whether to save the table summary to disk.
outdir : string, optional
If `save_summary` is True, write the summary to this directory. If
`outdir` is not specified and `save_summary` is True, the summary will
be written to the same directory that contains `table_fpath`.
Returns
-------
table
See `load_clsim_table` for details of the data structure
summary : OrderedDict
"""
t_start = time()
if save_summary:
from pisa.utils.jsons import from_json, to_json
table_fpath = expand(table_fpath)
srcdir, clsim_fname = dirname(table_fpath), basename(table_fpath)
invalid_fname = False
try:
fname_info = interpret_clsim_table_fname(clsim_fname)
except ValueError:
invalid_fname = True
fname_info = {}
if outdir is None:
outdir = srcdir
outdir = expand(outdir)
mkdir(outdir)
if invalid_fname:
metapath = None
else:
metaname = (CLSIM_TABLE_METANAME_PROTO[-1].format(
hash_val=fname_info['hash_val']))
metapath = join(outdir, metaname)
if metapath and isfile(metapath):
meta = from_json(metapath)
else:
meta = dict()
if table is None:
table = load_clsim_table(table_fpath)
summary = OrderedDict()
for key in table.keys():
if key == 'table':
continue
summary[key] = table[key]
if fname_info:
for key in ('hash_val', 'string', 'depth_idx', 'seed'):
summary[key] = fname_info[key]
# TODO: Add hole ice info when added to tray_kw_to_hash
if meta:
summary['n_events'] = meta['tray_kw_to_hash']['NEvents']
summary['ice_model'] = meta['tray_kw_to_hash']['IceModel']
summary['tilt'] = not meta['tray_kw_to_hash']['DisableTilt']
for key, val in meta.items():
if key.endswith('_binning_kw'):
summary[key] = val
elif 'fname_version' in fname_info and fname_info['fname_version'] == 1:
summary['n_events'] = fname_info['n_events']
summary['ice_model'] = 'spice_mie'
summary['tilt'] = False
summary['r_binning_kw'] = dict(min=0.0, max=400.0, n_bins=200, power=2)
summary['costheta_binning_kw'] = dict(min=-1, max=1, n_bins=40)
summary['t_binning_kw'] = dict(min=0.0, max=3000.0, n_bins=300)
summary['costhetadir_binning_kw'] = dict(min=-1, max=1, n_bins=20)
summary['deltaphidir_binning_kw'] = dict(min=0.0, max=np.pi, n_bins=20)
# Save marginal distributions and info to file
norm = (
1 / table['n_photons'] /
(SPEED_OF_LIGHT_M_PER_NS / table['phase_refractive_index'] *
np.mean(np.diff(table['t_bin_edges'])))
#* table['angular_acceptance_fract']
* (len(table['costheta_bin_edges']) - 1))
summary['norm'] = norm
dim_names = ('r', 'costheta', 't', 'costhetadir', 'deltaphidir')
n_dims = len(table['table_shape'])
assert n_dims == len(dim_names)
# Apply norm to underflow and overflow so magnitudes can be compared
# relative to plotted marginal distributions
for flow, idx in product(('underflow', 'overflow'), iter(range(n_dims))):
summary[flow][idx] = summary[flow][idx] * norm
wstderr('Finding marginal distributions...\n')
wstderr(' masking off zeros in table...')
t0 = time()
nonzero_table = np.ma.masked_equal(table['table'], 0)
wstderr(' ({} ms)\n'.format(np.round((time() - t0) * 1e3, 3)))
t0_marg = time()
summary['dimensions'] = OrderedDict()
for keep_axis, ax_name in zip(tuple(range(n_dims)), dim_names):
remove_axes = list(range(n_dims))
remove_axes.pop(keep_axis)
remove_axes = tuple(remove_axes)
axis = OrderedDict()
wstderr(' mean across non-{} axes...'.format(ax_name))
t0 = time()
axis['mean'] = norm * np.asarray(
np.mean(table['table'], axis=remove_axes))
wstderr(' ({} s)\n'.format(np.round(time() - t0, 3)))
wstderr(' median across non-{} axes...'.format(ax_name))
t0 = time()
axis['median'] = norm * np.asarray(
np.ma.median(nonzero_table, axis=remove_axes))
wstderr(' ({} s)\n'.format(np.round(time() - t0, 3)))
wstderr(' max across non-{} axes...'.format(ax_name))
t0 = time()
axis['max'] = norm * np.asarray(
np.max(table['table'], axis=remove_axes))
wstderr(' ({} s)\n'.format(np.round(time() - t0, 3)))
summary['dimensions'][ax_name] = axis
wstderr(' Total time to find marginal distributions: {} s\n'.format(
np.round(time() - t0_marg, 3)))
if save_summary:
ext = None
base_fname = clsim_fname
while ext not in ('', '.fits'):
base_fname, ext = splitext(base_fname)
ext = ext.lower()
outfpath = join(outdir, base_fname + '_summary.json.bz2')
to_json(summary, outfpath)
print('saved summary to "{}"'.format(outfpath))
wstderr('Time to summarize table: {} s\n'.format(
np.round(time() - t_start, 3)))
return table, summary
def scan_llh(dom_tables_kw, hypo_kw, events_kw, scan_kw):
"""Script "main" function"""
t00 = time.time()
scan_values = []
for dim in HYPO_PARAMS_T._fields:
val_str = ''.join(scan_kw.pop(dim))
val_str = val_str.lower().replace('pi', format(np.pi, '.17e'))
scan_values.append(hrlist2list(val_str))
dom_tables = init_obj.setup_dom_tables(**dom_tables_kw)
hypo_handler = init_obj.setup_discrete_hypo(**hypo_kw)
events_generator = init_obj.get_events(**events_kw)
# Pop 'outdir' from `scan_kw` since we don't want to store this info in
# the metadata dict.
outdir = expand(scan_kw.pop('outdir'))
mkdir(outdir)
print('Scanning paramters')
t0 = time.time()
fast_llh = True
if fast_llh:
get_llh = dom_tables._get_llh
dom_info = dom_tables.dom_info
tables = dom_tables.tables
table_norm = dom_tables.table_norm
t_indep_tables = dom_tables.t_indep_tables
t_indep_table_norm = dom_tables.t_indep_table_norm
sd_idx_table_indexer = dom_tables.sd_idx_table_indexer
metric_kw = {}
def metric_wrapper(hypo, hits, hits_indexer, unhit_sd_indices,
time_window):
sources = hypo_handler.get_sources(hypo)
return get_llh(sources=sources,
hits=hits,
hits_indexer=hits_indexer,
unhit_sd_indices=unhit_sd_indices,
sd_idx_table_indexer=sd_idx_table_indexer,
time_window=time_window,
dom_info=dom_info,
tables=tables,
table_norm=table_norm,
t_indep_tables=t_indep_tables,
t_indep_table_norm=t_indep_table_norm)
else:
metric_kw = dict(dom_tables=dom_tables, tdi_table=None)
get_llh = likelihood.get_llh
def metric_wrapper(hypo, **metric_kw):
sources = hypo_handler.get_sources(hypo)
return get_llh(sources=sources, **metric_kw)
n_points_total = 0
metric_vals = []
for _, event in events_generator:
hits = event['hits']
hits_indexer = event['hits_indexer']
hits_summary = event['hits_summary']
metric_kw['hits'] = hits
metric_kw['hits_indexer'] = hits_indexer
hit_sd_indices = hits_indexer['sd_idx']
unhit_sd_indices = np.array(sorted(
ALL_STRS_DOMS_SET.difference(hit_sd_indices)),
dtype=np.uint32)
metric_kw['unhit_sd_indices'] = unhit_sd_indices
metric_kw['time_window'] = np.float32(
hits_summary['time_window_stop'] -
hits_summary['time_window_start'])
t1 = time.time()
metric_vals.append(scan(scan_values, metric_wrapper, metric_kw))
dt = time.time() - t1
n_points = metric_vals[-1].size
n_points_total += n_points
print(' ---> {:.3f} s, {:d} points ({:.3f} ms per LLH)'.format(
dt, n_points, dt / n_points * 1e3))
dt = time.time() - t0
info = OrderedDict([
('hypo_params', HYPO_PARAMS_T._fields),
('scan_values', scan_values),
('metric_name', 'llh'),
('metric_vals', metric_vals),
('scan_kw', sort_dict(scan_kw)),
('dom_tables_kw', sort_dict(dom_tables_kw)),
('hypo_kw', sort_dict(hypo_kw)),
('events_kw', sort_dict(events_kw)),
])
outfpath = join(outdir, 'scan.pkl')
print('Saving results in pickle file, path "{}"'.format(outfpath))
pickle.dump(info, open(outfpath, 'wb'), protocol=pickle.HIGHEST_PROTOCOL)
print('Total time to scan: {:.3f} s; {:.3f} ms avg per LLH'.format(
time.time() - t00, dt / n_points_total * 1e3))
return metric_vals, info
