def apply_ratio_scale(orig_maps, key1, key2, ratio_scale, is_flux_scale, int_type = None):
'''
Scales the ratio of the entries of two maps, conserving the total.
'''
if is_flux_scale: log_str = 'flux'
else: log_str = 'event rate (%s)'%int_type
if not is_flux_scale:
# we have maps of event counts of a certain interaction type
orig_sum = orig_maps[key1][int_type]['map'] + orig_maps[key2][int_type]['map']
orig_total1 = orig_maps[key1][int_type]['map'].sum()
orig_total2 = orig_maps[key2][int_type]['map'].sum()
orig_ratio = orig_maps[key1][int_type]['map'] / orig_maps[key2][int_type]['map']
else:
# we have flux_maps
orig_sum = orig_maps[key1]['map'] + orig_maps[key2]['map']
orig_total1 = orig_maps[key1]['map'].sum()
orig_total2 = orig_maps[key2]['map'].sum()
orig_ratio = orig_maps[key1]['map'] / orig_maps[key2]['map']
# conserved total:
scaled_map2 = orig_sum / (1 + ratio_scale*orig_ratio)
scaled_map1 = ratio_scale*orig_ratio*scaled_map2
logging.trace(' %s / %s %s ratio before scaling: %.3f'%(key1, key2, log_str,
orig_total1/orig_total2))
logging.trace(' %s / %s %s ratio after scaling with %.2f: %.3f'%(key1, key2, log_str,
ratio_scale, scaled_map1.sum()/scaled_map2.sum()))
return scaled_map1, scaled_map2
def apply_function(self):
for container in self.data:
# update uncertainty first, before the weights are changed. This step is skipped in event mode
if self.error_method == "sumw2":
# If computing uncertainties in events mode, warn that
# hs error propagation will be skipped
if self.data.representation == 'events':
logging.trace(
'WARNING: running stage in events mode. Hypersurface error propagation will be IGNORED.'
)
elif self.propagate_uncertainty:
container["errors"] = container["weights"] * container[
"hs_scales_uncertainty"]
else:
container["errors"] *= container["hs_scales"]
container.mark_changed('errors')
# Update weights according to hypersurfaces
container["weights"] = np.clip(container["weights"] *
container["hs_scales"],
a_min=0,
a_max=np.inf)
def apply_ratio_scale(orig_maps, key1, key2, ratio_scale, is_flux_scale, int_type=None):
"""
Scales the ratio of the entries of two maps, conserving the total.
"""
if is_flux_scale: log_str = 'flux'
else: log_str = 'event rate (%s)'%int_type
if not is_flux_scale:
# we have maps of event counts of a certain interaction type
orig_sum = orig_maps[key1][int_type]['map'] + orig_maps[key2][int_type]['map']
orig_total1 = orig_maps[key1][int_type]['map'].sum()
orig_total2 = orig_maps[key2][int_type]['map'].sum()
orig_ratio = orig_maps[key1][int_type]['map'] / orig_maps[key2][int_type]['map']
else:
# we have flux_maps
orig_sum = orig_maps[key1]['map'] + orig_maps[key2]['map']
orig_total1 = orig_maps[key1]['map'].sum()
orig_total2 = orig_maps[key2]['map'].sum()
orig_ratio = orig_maps[key1]['map'] / orig_maps[key2]['map']
# conserved total:
scaled_map2 = orig_sum / (1 + ratio_scale*orig_ratio)
scaled_map1 = ratio_scale*orig_ratio*scaled_map2
logging.trace(' %s / %s %s ratio before scaling: %.3f'%(key1, key2, log_str,
orig_total1/orig_total2))
logging.trace(' %s / %s %s ratio after scaling with %.2f: %.3f'%(key1, key2, log_str,
ratio_scale, scaled_map1.sum()/scaled_map2.sum()))
return scaled_map1, scaled_map2
def flatten_map(template, channel='all'):
"""
Takes a final level true (expected) template of trck/cscd, and returns a
single flattened map of trck appended to cscd, with all zero bins
removed.
"""
logging.trace("Getting flattened map of channel: %s"%channel)
if channel == 'all':
cscd = template['cscd']['map'].flatten()
trck = template['trck']['map'].flatten()
fmap = np.append(cscd, trck)
elif channel == 'trck':
trck = template[channel]['map'].flatten()
fmap = np.array(trck)
#fmap = np.array(fmap)[np.nonzero(fmap)]
elif channel == 'cscd':
cscd = template[channel]['map'].flatten()
fmap = np.array(cscd)
#fmap = np.array(fmap)[np.nonzero(fmap)]
elif channel == 'no_pid':
cscd = template['cscd']['map'].flatten()
trck = template['trck']['map'].flatten()
fmap = cscd + trck
#fmap = np.array(fmap)[np.nonzero(fmap)]
else:
raise ValueError(
"channel: '%s' not implemented! Allowed: ['all', 'trck', 'cscd', 'no_pid']"
%channel)
fmap = np.array(fmap)[np.nonzero(fmap)]
return fmap
def next(self):
"""Iterate through lines in the file(s).
Returns
-------
line : string
The next line from the current file.
fpname : string
The `fpname` of the file from which the line was gotten.
lineno : int
The line number in the file.
"""
if not self._iter_stack:
self._cleanup()
raise StopIteration
try:
record = self._iter_stack[-1]
record['line'] = next(record['fp'])
record['lineno'] += 1
return record
except StopIteration:
record = self._iter_stack.pop()
logging.trace(('Finished processing "{fpname:s}" with {lineno:d}'
' line(s)').format(**record))
return next(self)
except:
self._cleanup()
raise
def get_inv_eff(self, signal_data=None, gen_data=None):
this_hash = hash_obj(
[self.true_binning.hash, self.output_str, 'inv_eff'],
full_hash=self.full_hash)
assert len(set([signal_data is None, gen_data is None])) == 1
if signal_data is None and gen_data is None:
if self.inv_eff_hash == this_hash:
logging.trace('Loading inv eff from mem cache')
return self._inv_eff
if this_hash in self.disk_cache:
logging.debug('Loading inv eff histogram from disk cache.')
inv_eff = self.disk_cache[this_hash]
else:
raise ValueError(
'inverse efficiency histogram with correct hash not found '
'in disk_cache')
else:
this_hash = hash_obj([this_hash, self.fit_hash],
full_hash=self.full_hash)
if self.inv_eff_hash == this_hash:
logging.trace('Loading inv eff from mem cache')
return self._inv_eff
inv_eff = self._get_inv_eff(signal_data, gen_data,
self.true_binning, self.output_str)
if self.disk_cache is not None:
if this_hash not in self.disk_cache:
logging.debug('Caching inv eff histogram to disk.')
self.disk_cache[this_hash] = inv_eff
self.inv_eff_hash = this_hash
self._inv_eff = inv_eff
return inv_eff
def flatten_map(template,chan='all'):
'''
Takes a final level true (expected) template of trck/cscd, and returns a
single flattened map of trck appended to cscd, with all zero bins
removed.
'''
logging.trace("Getting flattened map of chan: %s"%chan)
if chan == 'all':
cscd = template['cscd']['map'].flatten()
trck = template['trck']['map'].flatten()
fmap = np.append(cscd,trck)
elif chan == 'trck':
trck = template[chan]['map'].flatten()
fmap = np.array(trck)
#fmap = np.array(fmap)[np.nonzero(fmap)]
elif chan == 'cscd':
cscd = template[chan]['map'].flatten()
fmap = np.array(cscd)
#fmap = np.array(fmap)[np.nonzero(fmap)]
elif chan == 'no_pid':
cscd = template['cscd']['map'].flatten()
trck = template['trck']['map'].flatten()
fmap = cscd + trck
#fmap = np.array(fmap)[np.nonzero(fmap)]
else:
raise ValueError("chan: '%s' not implemented! Allowed: ['all', 'trck', 'cscd','no_pid']")
fmap = np.array(fmap)[np.nonzero(fmap)]
return fmap
def _compute_outputs(self, inputs=None):
"""Apply basic cuts and compute histograms for output channels."""
logging.debug('Entering sample._compute_outputs')
self.config = from_file(self.params['data_sample_config'].value)
name = self.config.get('general', 'name')
logging.trace('{0} sample sample_hash = '
'{1}'.format(name, self.sample_hash))
self.load_sample_events()
if self.params['keep_criteria'].value is not None:
# TODO(shivesh)
raise NotImplementedError(
'needs check to make sure this works in a DistributionMaker'
)
self._data.applyCut(self.params['keep_criteria'].value)
self._data.update_hash()
if self.output_events:
return self._data
outputs = []
if self.neutrinos:
trans_nu_data = self._data.transform_groups(
self._output_nu_groups
)
for fig in trans_nu_data.keys():
outputs.append(trans_nu_data.histogram(
kinds = fig,
binning = self.output_binning,
weights_col = 'pisa_weight',
errors = True,
name = str(NuFlavIntGroup(fig)),
))
if self.muons:
outputs.append(self._data.histogram(
kinds = 'muons',
binning = self.output_binning,
weights_col = 'pisa_weight',
errors = True,
name = 'muons',
tex = r'\rm{muons}'
))
if self.noise:
outputs.append(self._data.histogram(
kinds = 'noise',
binning = self.output_binning,
weights_col = 'pisa_weight',
errors = True,
name = 'noise',
tex = r'\rm{noise}'
))
name = self.config.get('general', 'name')
return MapSet(maps=outputs, name=name)
def apply_function(self):
for container in self.data:
# update uncertainty first, before the weights are changed. This step is skipped in event mode
if self.error_method == "sumw2":
# If computing uncertainties in events mode, warn that
# hs error propagation will be skipped
if self.data.data_specs == 'events':
logging.trace(
'WARNING: running stage in events mode. Hypersurface error propagation will be IGNORED.'
)
elif self.propagate_uncertainty:
calc_uncertainty(
container["weights"].get(WHERE),
container["hs_scales_uncertainty"].get(WHERE),
container["errors"].get(WHERE),
)
container['errors'].mark_changed()
else:
vectorizer.imul(container["hs_scales"],
out=container["errors"])
container['errors'].mark_changed()
# Update weights according to hypersurfaces
propagate_hs_scales(container["weights"].get(WHERE),
container["hs_scales"].get(WHERE),
container["weights"].get(WHERE))
container['weights'].mark_changed()
def link_containers(self, key, names):
"""Link containers together. When containers are linked, they are
treated as a single (virtual) container for binned data
Parameters
----------
key : str
name of linked object
names : list
name of containers to be linked under the given key
"""
# intersection of names for linking and available names
link_names = set(names) & set(self.names)
if len(link_names) < len(names):
logging.warning(
"Skipping containers %s in linking, as those are not present" %
(set(names) - set(self.names)))
containers = [self.__getitem__(name) for name in link_names]
logging.trace('Linking containers %s into %s' % (link_names, key))
new_container = VirtualContainer(key, containers)
self.linked_containers.append(new_container)
def create_response(self,
reco_norm_data=None,
true_norm_data=None,
data=None):
"""Create the response object from the signal data."""
unfold_bg = self.params['unfold_bg'].value
unfold_eff = self.params['unfold_eff'].value
unfold_unweighted = self.params['unfold_unweighted'].value
this_hash = hash_obj([
self.reco_binning.hash, self.true_binning.hash, unfold_bg,
unfold_eff, unfold_unweighted, self.output_str, 'response'
],
full_hash=self.full_hash)
assert len(
set([reco_norm_data is None, true_norm_data is None,
data is None])) == 1
if reco_norm_data is None and true_norm_data is None and data is None:
if self.response_hash == this_hash:
logging.trace('Loading response from mem cache')
return self._response
else:
try:
del self._response
except:
pass
if this_hash in self.disk_cache:
logging.debug('Loading response from disk cache.')
response = self.disk_cache[this_hash]
else:
raise ValueError(
'response object with correct hash not found in disk_cache'
)
else:
this_hash = hash_obj([this_hash, self.fit_hash] +
list(self.params.values),
full_hash=self.full_hash)
if self.response_hash == this_hash:
logging.debug('Loading response from mem cache')
return self._response
else:
try:
del self._response
del self.t_th1d
except:
pass
# Truth histogram also gets returned if response matrix is created
response, self.t_th1d = self._create_response(
reco_norm_data, true_norm_data, data, self.reco_binning,
self.true_binning)
if self.disk_cache is not None:
if this_hash not in self.disk_cache:
logging.debug('Caching response object to disk.')
self.disk_cache[this_hash] = response
self.response_hash = this_hash
self._response = response
return response
def test_container():
"""Unit tests for Container class."""
# NOTE: Right now the numbers are tuned so that the weights are identical
# per bin. If you change binning that's likely not the case anymore and you
# inevitably end up with averaged values over bins, which are then not
# equal to the individual weights anymore when those are not identical per
# bin
n_evts = 10000
x = np.linspace(0, 100, n_evts, dtype=FTYPE)
y = np.linspace(0, 100, n_evts, dtype=FTYPE)
w = np.tile(np.arange(100, dtype=FTYPE) + 0.5, (100, 1)).T.ravel()
container = Container('test', 'events')
container['x'] = x
container['y'] = y
container['w'] = w
binning_x = OneDimBinning(name='x',
num_bins=100,
is_lin=True,
domain=[0, 100])
binning_y = OneDimBinning(name='y',
num_bins=100,
is_lin=True,
domain=[0, 100])
binning = MultiDimBinning([binning_x, binning_y])
logging.trace('Testing container and translation methods')
container.representation = binning
bx = container['x']
m = np.meshgrid(binning.midpoints[0].m, binning.midpoints[1].m)[1].ravel()
assert np.allclose(bx, m, **ALLCLOSE_KW), f'test:\n{bx}\n!= ref:\n{m}'
# array repr
container.representation = 'events'
array_weights = container['w']
assert np.allclose(array_weights, w,
**ALLCLOSE_KW), f'test:\n{array_weights}\n!= ref:\n{w}'
# binned repr
container.representation = binning
diag = np.diag(np.arange(100) + 0.5)
bd = container['w']
h = container.get_hist('w')
assert np.allclose(bd, diag.ravel(),
**ALLCLOSE_KW), f'test:\n{bd}\n!= ref:\n{diag.ravel()}'
assert np.allclose(h[0], diag,
**ALLCLOSE_KW), f'test:\n{h[0]}\n!= ref:\n{diag}'
assert h[1] == binning, f'test:\n{h[1]}\n!= ref:\n{binning}'
# augment to array repr again
container.representation = 'events'
a = container['w']
assert np.allclose(a, w, **ALLCLOSE_KW), f'test:\n{a}\n!= ref:\n{w}'
def _compute_outputs(self, inputs=None):
"""Compute histograms for output channels."""
logging.debug('Entering fit._compute_outputs')
if not isinstance(inputs, Data):
raise AssertionError('inputs is not a Data object, instead is '
'type {0}'.format(type(inputs)))
self.weight_hash = deepcopy(inputs.metadata['weight_hash'])
logging.trace('{0} fit weight_hash = '
'{1}'.format(inputs.metadata['name'], self.weight_hash))
logging.trace('{0} fit fit_hash = '
'{1}'.format(inputs.metadata['name'], self.fit_hash))
self._data = inputs
self.reweight()
if self.output_events:
return self._data
outputs = []
if self.neutrinos:
trans_nu_data = self._data.transform_groups(
self._output_nu_groups
)
for fig in trans_nu_data.iterkeys():
outputs.append(
trans_nu_data.histogram(
kinds=fig,
binning=self.output_binning,
weights_col='pisa_weight',
errors=True,
name=str(NuFlavIntGroup(fig)),
)
)
if self.muons:
outputs.append(
self._data.histogram(
kinds='muons',
binning=self.output_binning,
weights_col='pisa_weight',
errors=True,
name='muons',
tex=text2tex('muons')
)
)
if self.noise:
outputs.append(
self._data.histogram(
kinds='noise',
binning=self.output_binning,
weights_col='pisa_weight',
errors=True,
name='noise',
tex=text2tex('noise')
)
)
return MapSet(maps=outputs, name=self._data.metadata['name'])
def get_flux_maps(flux_service, ebins, czbins, nue_numu_ratio, nu_nubar_ratio,
energy_scale, atm_delta_index, **kwargs):
"""
Get a set of flux maps for the different primaries.
\params:
* flux_service -
* ebins/czbins - energy/coszenith bins to calculate flux
* nue_numu_ratio - systematic to be a proxy for the realistic
Flux_nue/Flux_numu and Flux_nuebar/Flux_numubar ratios,
keeping both the total flux from neutrinos and antineutrinos
constant. The adjusted ratios are given by
"nue_numu_ratio * original ratio".
* nu_nubar_ratio - systematic to be a proxy for the
neutrino/anti-neutrino production/cross section ratio.
* energy_scale - factor to scale energy bin centers by
* atm_delta_index - change in spectral index from fiducial
"""
# Be verbose on input
params = get_params()
report_params(params, units=[''])
# Initialize return dict
maps = {'params': params}
for prim in primaries:
# Get the flux for this primary
maps[prim] = {
'ebins': ebins,
'czbins': czbins,
'map': flux_service.get_flux(ebins * energy_scale, czbins, prim)
}
# be a bit verbose
logging.trace("Total flux of %s is %u [s^-1 m^-2]" %
(prim, maps[prim]['map'].sum()))
# now scale the nue(bar) / numu(bar) flux ratios, keeping the total
# Flux (nue + numu, nue_bar + numu_bar) constant, or return unscaled maps:
scaled_maps = apply_nue_numu_ratio(
maps, nue_numu_ratio) if nue_numu_ratio != 1.0 else maps
# now scale the nu(e/mu) / nu(e/mu)bar event count ratios, keeping the total
# (nue + nuebar etc.) constant
if nu_nubar_ratio != 1.:
scaled_maps = apply_nu_nubar_ratio(scaled_maps, nu_nubar_ratio)
median_energy = get_median_energy(maps['numu'])
if atm_delta_index != 0.0:
scaled_maps = apply_delta_index(scaled_maps, atm_delta_index,
median_energy)
return scaled_maps
def __init__(self, ebins, czbins):
"""
Parameters needed to instantiate any oscillation service:
* ebins: Energy bin edges
* czbins: cos(zenith) bin edges
If further member variables are needed, extend this method.
"""
logging.trace('Instantiating %s' % self.__class__.__name__)
self.ebins = np.array(ebins)
self.czbins = np.array(czbins)
for ax in [self.ebins, self.czbins]:
if (len(np.shape(ax)) != 1):
raise IndexError('Axes must be 1d! ' + str(np.shape(ax)))
def __init__(self, ebins, czbins):
"""
Parameters needed to instantiate any oscillation service:
* ebins: Energy bin edges
* czbins: cos(zenith) bin edges
If further member variables are needed, extend this method.
"""
logging.trace('Instantiating %s'%self.__class__.__name__)
self.ebins = np.array(ebins)
self.czbins = np.array(czbins)
for ax in [self.ebins, self.czbins]:
if (len(np.shape(ax)) != 1):
raise IndexError('Axes must be 1d! '+str(np.shape(ax)))
def get_flux_maps(flux_service, ebins, czbins, nue_numu_ratio, nu_nubar_ratio,
energy_scale, atm_delta_index,**kwargs):
"""
Get a set of flux maps for the different primaries.
\params:
* flux_service -
* ebins/czbins - energy/coszenith bins to calculate flux
* nue_numu_ratio - systematic to be a proxy for the realistic
Flux_nue/Flux_numu and Flux_nuebar/Flux_numubar ratios,
keeping both the total flux from neutrinos and antineutrinos
constant. The adjusted ratios are given by
"nue_numu_ratio * original ratio".
* nu_nubar_ratio - systematic to be a proxy for the
neutrino/anti-neutrino production/cross section ratio.
* energy_scale - factor to scale energy bin centers by
* atm_delta_index - change in spectral index from fiducial
"""
# Be verbose on input
params = get_params()
report_params(params, units = [''])
# Initialize return dict
maps = {'params': params}
for prim in primaries:
# Get the flux for this primary
maps[prim] = {'ebins': ebins,
'czbins': czbins,
'map': flux_service.get_flux(ebins*energy_scale,czbins,prim)}
# be a bit verbose
logging.trace("Total flux of %s is %u [s^-1 m^-2]"%
(prim,maps[prim]['map'].sum()))
# now scale the nue(bar) / numu(bar) flux ratios, keeping the total
# Flux (nue + numu, nue_bar + numu_bar) constant, or return unscaled maps:
scaled_maps = apply_nue_numu_ratio(maps, nue_numu_ratio) if nue_numu_ratio != 1.0 else maps
# now scale the nu(e/mu) / nu(e/mu)bar event count ratios, keeping the total
# (nue + nuebar etc.) constant
if nu_nubar_ratio != 1.:
scaled_maps = apply_nu_nubar_ratio(scaled_maps, nu_nubar_ratio)
median_energy = get_median_energy(maps['numu'])
if atm_delta_index != 0.0:
scaled_maps = apply_delta_index(scaled_maps, atm_delta_index, median_energy)
return scaled_maps
def check_reco_dist_consistency(self, dist_list):
"""Enforces correct normalisation of resolution functions."""
logging.trace(
" Verifying correct normalisation of resolution function.")
# Obtain list of all distributions. The sum of their relative weights
# should yield 1.
frac_sum = np.zeros_like(dist_list[0]['fraction'])
for dist_dict in dist_list:
frac_sum += dist_dict['fraction']
if not recursiveEquality(frac_sum, np.ones_like(frac_sum)):
err_msg = ("Total normalisation of resolution function is off"
" (fractions do not add up to 1).")
raise ValueError(err_msg)
return True
def array_to_binned(self, key, src_representation, dest_representation):
"""Histogram data array into binned data
Parameters
----------
key : str
src_representation : str
dest_representation : MultiDimBinning
#averaged : bool
# if True, the histogram entries are averages of the numbers that
# end up in a given bin. This for example must be used when oscillation
# probabilities are translated.....otherwise we end up with probability*count
# per bin
Notes
-----
right now, CPU-only
"""
# TODO: make work for n-dim
logging.trace('Transforming %s array to binned data' % (key))
assert src_representation in self.array_representations
assert isinstance(dest_representation, MultiDimBinning)
if not dest_representation.is_irregular:
sample = []
dimensions = []
for d in dest_representation:
if d.is_log:
self.representation = "log_events"
sample.append(self[d.name])
dimensions.append(
OneDimBinning(d.name,
domain=np.log(d.domain.m),
num_bins=d.num_bins))
else:
self.representation = "events"
sample.append(self[d.name])
dimensions.append(d)
hist_binning = MultiDimBinning(dimensions)
else:
self.representation = src_representation
sample = [self[name] for name in dest_representation.names]
hist_binning = dest_representation
self.representation = src_representation
weights = self[key]
hist = histogram(sample, weights, hist_binning, averaged=True)
return hist
def saveDict(data,fh):
"""
Saves a data dictionary to filehandle (or group), fh, as long as the
group to be written to the hdf5 file is one of:
[np.ndarray, list, float, int, bool, str, None]
"""
for k,v in data.items():
if type(v) == dict:
group = fh.create_group(k)
logging.trace(" creating group %s"%k)
saveDict(v,group)
else:
saveNonDict(k, v, fh)
return
def compute(self):
# simplest caching algorithm: don't compute if params didn't change
new_param_hash = self.params.values_hash
if new_param_hash == self.param_hash:
logging.trace("cached output")
return
if self.calc_mode is not None:
self.data.representation = self.calc_mode
if self.profile:
start_t = time()
self.compute_function()
end_t = time()
self.calc_times.append(end_t - start_t)
else:
self.compute_function()
self.param_hash = new_param_hash
def compute(self):
if len(self.params) == 0 and len(self.output_calc_keys) == 0:
return
# simplest caching algorithm: don't compute if params didn't change
new_param_hash = self.params.values_hash
if new_param_hash == self.param_hash:
logging.trace("cached output")
return
self.data.data_specs = self.input_specs
# convert any inputs if necessary:
if self.mode[:2] == "EB":
for container in self.data:
for key in self.input_calc_keys:
container.array_to_binned(key, self.calc_specs)
elif self.mode == "EBE":
for container in self.data:
for key in self.input_calc_keys:
container.binned_to_array(key)
#elif self.mode == "BBE":
# for container in self.data:
# for key in self.input_calc_keys:
# container.binned_to_array(key)
self.data.data_specs = self.calc_specs
self.compute_function()
self.param_hash = new_param_hash
# convert any outputs if necessary:
if self.mode[1:] == "EB":
for container in self.data:
for key in self.output_calc_keys:
container.array_to_binned(key, self.output_specs)
elif self.mode[1:] == "BE":
for container in self.data:
for key in self.output_calc_keys:
container.binned_to_array(key)
def extend_binning_for_coszen(self, ext_low=-3., ext_high=+3.):
"""
Check whether `coszen_flipback` can be applied to the stage's
coszen output binning and return an extended binning spanning [-3, +3]
if that is the case.
"""
logging.trace("Preparing binning for flipback of reco kernel at"
" coszen boundaries of physical range.")
cz_edges_out = self.output_binning['reco_coszen'].bin_edges.magnitude
coszen_range = self.output_binning['reco_coszen'].range.magnitude
n_cz_out = self.output_binning['reco_coszen'].size
coszen_step = coszen_range / n_cz_out
# we need to check for possible contributions from (-3, -1) and
# (1, 3) in coszen
assert ext_high > ext_low
ext_range = ext_high - ext_low
extended = np.linspace(ext_low, ext_high,
int(ext_range / coszen_step) + 1)
# We cannot flipback if we don't have -1 & +1 as (part of extended)
# bin edges. This could happen if 1 is a multiple of the output bin
# size, but the original edges themselves are not a multiple of that
# size.
for bound in (-1., +1.):
comp = [recursiveEquality(bound, e) for e in extended]
assert np.any(comp)
# Perform one final check: original edges subset of extended ones?
for coszen in cz_edges_out:
comp = [recursiveEquality(coszen, e) for e in extended]
assert np.any(comp)
# Binning seems fine - we can proceed
ext_cent = (extended[1:] + extended[:-1]) / 2.
flipback_mask = ((ext_cent < -1.) | (ext_cent > +1.))
keep = np.where((ext_cent > cz_edges_out[0])
& (ext_cent < cz_edges_out[-1]))[0]
cz_edges_out = extended
logging.trace(" -> temporary coszen bin edges:\n%s" % cz_edges_out)
return cz_edges_out, flipback_mask, keep
def apply_delta_index(flux_maps, delta_index, egy_med):
"""
Applies the spectral index systematic to the flux maps by scaling
each bin with (egy_cen/egy_med)^(-delta_index), preserving the total
integral flux Note that only the numu/numu_bar are scaled, because
the nue_numu_ratio will handle the systematic on the nue flux.
"""
for flav in ['numu','numu_bar']:
ecen = get_bin_centers(flux_maps[flav]['ebins'])
scale = np.power((ecen/egy_med),delta_index)
flux_map = flux_maps[flav]['map']
total_flux = flux_map.sum()
logging.trace("flav: %s, total counts before scale: %f"%(flav,total_flux))
scaled_flux = (flux_map.T*scale).T
scaled_flux *= (total_flux/scaled_flux.sum())
flux_maps[flav]['map'] = scaled_flux
logging.trace("flav: %s, total counts after scale: %f"%
(flav,flux_maps[flav]['map'].sum()))
return flux_maps
def select_params(self, selections, error_on_missing=False):
"""Apply the `selections` to contained ParamSet.
Parameters
----------
selections : string or iterable
error_on_missing : bool
"""
try:
self._param_selector.select_params(selections, error_on_missing=True)
except KeyError:
msg = "Not all of the selections %s found in this stage." % (selections,)
if error_on_missing:
# logging.error(msg)
raise
logging.trace(msg)
else:
logging.trace(
"`selections` = %s yielded `params` = %s" % (selections, self.params)
)
def binned_to_array(self, key, src_representation, dest_representation):
"""Augmented binned data to array data"""
logging.trace('Transforming %s binned to array data' % (key))
self.representation = src_representation
weights = self[key]
if not src_representation.is_irregular:
logging.trace(
f"Container `{self.name}`: regularized lookup for {key}")
sample = []
dimensions = []
for d in src_representation:
if d.is_log:
self.representation = "log_events"
sample.append(self[d.name])
dimensions.append(
OneDimBinning(d.name,
domain=np.log(d.domain.m),
num_bins=d.num_bins))
else:
self.representation = "events"
sample.append(self[d.name])
dimensions.append(d)
hist_binning = MultiDimBinning(dimensions)
else:
logging.trace(
f"Container `{self.name}`: irregular lookup for {key}")
self.representation = dest_representation
sample = [self[name] for name in src_representation.names]
hist_binning = src_representation
return lookup(sample, weights, hist_binning)
def apply_delta_index(flux_maps, delta_index, egy_med):
"""
Applies the spectral index systematic to the flux maps by scaling
each bin with (egy_cen/egy_med)^(-delta_index), preserving the total
integral flux Note that only the numu/numu_bar are scaled, because
the nue_numu_ratio will handle the systematic on the nue flux.
"""
for flav in ['numu', 'numu_bar']:
ecen = get_bin_centers(flux_maps[flav]['ebins'])
scale = np.power((ecen / egy_med), delta_index)
flux_map = flux_maps[flav]['map']
total_flux = flux_map.sum()
logging.trace("flav: %s, total counts before scale: %f" %
(flav, total_flux))
scaled_flux = (flux_map.T * scale).T
scaled_flux *= (total_flux / scaled_flux.sum())
flux_maps[flav]['map'] = scaled_flux
logging.trace("flav: %s, total counts after scale: %f" %
(flav, flux_maps[flav]['map'].sum()))
return flux_maps
def _histogram(events, binning, weights=None, errors=False, **kwargs):
"""Histogram the events given the input binning."""
logging.trace('Histogramming')
bin_names = binning.names
bin_edges = [edges.m for edges in binning.bin_edges]
for name in bin_names:
if name not in events:
raise AssertionError('Input events object does not have '
'key {0}'.format(name))
sample = [events[colname] for colname in bin_names]
hist, edges = np.histogramdd(sample=sample,
weights=weights,
bins=bin_edges)
if errors:
hist2, edges = np.histogramdd(sample=sample,
weights=np.square(weights),
bins=bin_edges)
hist = unp.uarray(hist, np.sqrt(hist2))
return Map(hist=hist, binning=binning, **kwargs)
def _derive_transforms_hash(self, nominal_transforms_hash=None):
"""Compute a hash that uniquely identifies the transforms that will be
produced from the current configuration. Note that this hash needs only
to be valid for this run (i.e., it is a volatile hash).
This implementation returns a hash from the current parameters' values.
"""
id_objects = []
h = self.params.values_hash
logging.trace("self.params.values_hash = %s" % h)
id_objects.append(h)
# Grab any provided nominal transforms hash, or derive it again
if nominal_transforms_hash is None:
nominal_transforms_hash = self._derive_nominal_transforms_hash()
# If a valid hash has been gotten, include it
if nominal_transforms_hash is not None:
id_objects.append(nominal_transforms_hash)
for attr in sorted(self._attrs_to_hash):
val = getattr(self, attr)
if hasattr(val, "hash"):
attr_hash = val.hash
elif self.full_hash:
norm_val = normQuant(val)
attr_hash = hash_obj(norm_val, full_hash=self.full_hash)
else:
attr_hash = hash_obj(val, full_hash=self.full_hash)
id_objects.append(attr_hash)
# If any hashes are missing (i.e, None), invalidate the entire hash
if any([(h is None) for h in id_objects]):
transforms_hash = None
else:
transforms_hash = hash_obj(id_objects, full_hash=self.full_hash)
return transforms_hash, nominal_transforms_hash
def get_gradients(param, hypo_maker, test_vals):
"""Use the template maker to create all the templates needed
to obtain the gradients in a given parameter.
Parameters
----------
param : str
Name of parameter w.r.t. which we are calculating binwise
template changes
hypo_maker : DistributionMaker
Needs to hold the parameter `param` in its `ParamSet`
test_vals : sequence with units
Values of the parameter `param` to probe, i.e., generate templates for
Returns
-------
pmaps : dict
Dictionary with `test_vals` as keys and resulting templates' 'total'
nominal values
gradient_map : sequence
As returned by `get_derivative_map`
"""
logging.trace("Working on parameter %s." % param)
pmaps = {}
# generate one template for each value of the parameter in question
# and store in pmaps
for param_value in test_vals:
hypo_maker.params[param].value = param_value
# make the template corresponding to the current value of the parameter
hypo_asimov_dist = hypo_maker.get_outputs(return_sum=True)
pmaps[param_value] = hypo_asimov_dist.nominal_values['total']
gradient_map = get_derivative_map(hypo_maps=pmaps, )
return pmaps, gradient_map
def saveNonDict(key, data, fh):
logging.trace(" key: %s is of type: %s"%(key,type(data)))
if type(data) in [np.ndarray,list]:
logging.trace(" >>saving to dataset: %s"%key)
fh.create_dataset(key,data=data)
elif type(data) in [float,int,bool]:
logging.trace(" >>saving '%s' to dataset: %s"%(str(data),key))
fh.create_dataset(key,data=data)
elif type(data) == str:
logging.trace(" >>saving '%s' to dataset: %s"%(data,key))
dtype = h5py.special_dtype(vlen=str)
fh.create_dataset(key,data=data,dtype=dtype)
elif data is None:
# NOTE: we convert it to the boolean false here, since
# I can't find a good way to store the 'None' type in h5py
data = False
logging.trace(" >>saving '%s' to dataset: %s"%(data,key))
fh.create_dataset(key,data=data)
else:
raise ValueError("Key: '%s' is unrecognized type: '%s'"%(key,type(data)))
return
def get_fisher_matrix(hypo_maker, test_vals, counter):
"""Compute Fisher matrices at fiducial hypothesis given data.
"""
from pisa.utils.pull_method import get_gradients
hypo_params = hypo_maker.params.free
#fisher = {'total': {}}
fid_hypo_asimov_dist = hypo_maker.get_outputs(return_sum=True)
counter += 1
pmaps = {'total': {}}
gradient_maps = {'total': {}}
for pname in hypo_params.names:
logging.trace("Computing binwise gradients for parameter '%s'." %
pname)
tpm, gm = get_gradients(
param=pname,
hypo_maker=hypo_maker,
test_vals=test_vals[pname],
)
counter += len(test_vals[pname])
# the maps corresponding to variations of
# a single param are not flattened
pmaps['total'][pname] = tpm
# these are flattened, which is also what the
# method below assumes
gradient_maps['total'][pname] = gm
# hypo param values are not at nominal anymore,
# but we don't use their values here
fisher, nonempty = build_fisher_matrix(
gradient_hist_flat_d=gradient_maps['total'],
fiducial_hist=fid_hypo_asimov_dist,
fiducial_params=hypo_params)
return fisher, gradient_maps, fid_hypo_asimov_dist, nonempty
def get_bg_hist(self, bg_data=None):
"""Histogram the bg data unless using real data, in which case load
the bg hist from disk cache."""
this_hash = hash_obj(
[self.reco_binning.hash, self.output_str, 'bg_hist'],
full_hash=self.full_hash)
if bg_data is None:
if self.bg_hist_hash == this_hash:
logging.trace('Loading bg hist from mem cache')
return self._bg_hist
if this_hash in self.disk_cache:
logging.debug('Loading bg hist from disk cache.')
bg_hist = self.disk_cache[this_hash]
else:
raise ValueError(
'bg hist object with correct hash not found in disk_cache')
else:
this_hash = hash_obj([this_hash, self.fit_hash],
full_hash=self.full_hash)
if self.bg_hist_hash == this_hash:
logging.trace('Loading bg hist from mem cache')
return self._bg_hist
bg_hist = self._histogram(events=bg_data,
binning=self.reco_binning,
weights=bg_data['pisa_weight'],
errors=True,
name='background',
tex=r'\rm{background}')
if self.disk_cache is not None:
if this_hash not in self.disk_cache:
logging.debug('Caching bg hist to disk.')
self.disk_cache[this_hash] = bg_hist
self.bg_hist_hash = this_hash
self._bg_hist = bg_hist
return bg_hist
def get_flux_maps(flux_service, ebins, czbins, nue_numu_ratio, energy_scale, **kwargs):
'''
Get a set of flux maps for the different primaries.
\params:
* flux_service -
* ebins/czbins - energy/coszenith bins to calculate flux
* nue_numu_ratio - systematic to be a proxy for the realistic
Flux_nue/Flux_numu and Flux_nuebar/Flux_numubar ratios,
keeping both the total flux from neutrinos and antineutrinos
constant. The adjusted ratios are given by
"nue_numu_ratio * original ratio".
* energy_scale - factor to scale energy bin centers by
'''
#Be verbose on input
params = get_params()
report_params(params, units = [''])
#Initialize return dict
maps = {'params': params}
for prim in primaries:
#Get the flux for this primary
maps[prim] = {'ebins': ebins,
'czbins': czbins,
'map': flux_service.get_flux(ebins*energy_scale,czbins,prim)}
#be a bit verbose
logging.trace("Total flux of %s is %u [s^-1 m^-2]"%
(prim,maps[prim]['map'].sum()))
# now scale the nue(bar) / numu(bar) flux ratios, keeping the total
# flux (nue + numu, nue_bar + numu_bar) constant, or return unscaled maps:
return apply_nue_numu_ratio(maps, nue_numu_ratio) if nue_numu_ratio != 1.0 else maps
def test_MutableMultiFileIterator():
"""Unit test for class `MutableMultiFileIterator`"""
import shutil
import tempfile
prefixes = ['a', 'b', 'c']
file_len = 4
reference_lines = [
# start in file a
'a0', 'a1',
# switch to file b after second line of a
'b0', 'b1',
# switch to file c after second line of b
'c0', 'c1', 'c2', 'c3',
# switch back to b after exhausting c
'b2', 'b3',
# switch back to a after exhausting b
'a2', 'a3'
]
tempdir = tempfile.mkdtemp()
try:
# Create test files
paths = [join(tempdir, prefix) for prefix in prefixes]
for prefix, path in zip(prefixes, paths):
with open(path, 'w') as f:
for i in range(file_len):
f.write('%s%d\n' % (prefix, i))
logging.trace(path)
actual_lines = []
with open(paths[0]) as fp:
file_iter = MutableMultiFileIterator(fp=fp, fpname=paths[0])
remaining_paths = paths[1:]
for record in file_iter:
actual_lines.append(record['line'].strip())
logging.trace(str(record))
if record['line'][1:].strip() == '1':
if remaining_paths:
path = remaining_paths.pop(0)
file_iter.switch_to_file(fpname=path)
else:
for l in str(file_iter.location).split('\n'):
logging.trace(l)
except:
shutil.rmtree(tempdir)
raise
if actual_lines != reference_lines:
raise ValueError('<< FAIL : test_MutableMultiFileIterator >>')
logging.info('<< PASS : test_MutableMultiFileIterator >>')
def _combine_xs(self, flavintgroup):
"""Combine all cross sections specified by the flavints in
`flavintgroup`. All CC and NC interactions are separately grouped
together and averaged, then the average of each interaction type
is added to the other.
If CC and NC interactions are present, they *must* be from the same
flavor(s). I.e., it doesn't make sense (and so causes an exception) if
you combine numu CC with numubar NC. It does make sense if you combine
numu and numubar CC with numu and numubar NC, though, and this is
allowed.
Notes
-----
Does not yet implement *Ngen/spectrum-weighted* averages, which are
necessary when combining cross sections of disparate flavor/interaction
types from different Monte Carlo simulation runs.
"""
flavintgroup = NuFlavIntGroup(flavintgroup)
# Trivial case: nothing to combine
if len(flavintgroup.flavints) == 1:
return self[flavintgroup.flavints[0]]
cc_flavints = flavintgroup.cc_flavints
nc_flavints = flavintgroup.nc_flavints
if cc_flavints and nc_flavints:
assert flavintgroup.cc_flavs == flavintgroup.nc_flavs, \
'Combining CC and NC but CC flavors do not match NC flavors'
cc_avg_xs = 0
if cc_flavints:
logging.trace('cc_flavints = %s' % (cc_flavints,))
cc_avg_xs = np.sum([self[k] for k in cc_flavints], axis=0) \
/ len(cc_flavints)
nc_avg_xs = 0
if nc_flavints:
logging.trace('nc_flavints = %s' % (nc_flavints,))
nc_avg_xs = np.sum([self[k] for k in nc_flavints], axis=0) \
/ len(nc_flavints)
tot_xs = cc_avg_xs + nc_avg_xs
logging.trace('mean(tot_xs) = %s' % (np.mean(tot_xs),))
return tot_xs
def find_resource(filename, fail = True):
'''
Try to find the resource given by directory/filename. Will first check if
filename is an absolute path, then relative to the $PISA
environment variable if set. Otherwise will look in the resources directory
of the pisa installation. Will return the file handle or throw an Exception
if the file is not found.
'''
#First check for absolute path
fpath = os.path.expanduser(os.path.expandvars(filename))
logging.trace("Checking if %s is a file..."%fpath)
if os.path.isfile(fpath):
logging.debug('Found %s'%(fpath))
return fpath
#Next check if $PISA is set in environment
logging.trace("Checking environment for $PISA...")
if 'PISA' in os.environ:
rpath = os.path.expanduser(os.path.expandvars(os.environ['PISA']))
logging.debug('Searching resource path PISA=%s'%rpath)
fpath = os.path.join(rpath,filename)
if os.path.isfile(fpath):
logging.debug('Found %s at %s'%(filename,fpath))
return fpath
#Not in the resource path, so look inside the package
logging.trace('Searching package resources...')
fpath = resource_filename(__name__,filename)
if os.path.isfile(fpath):
logging.debug('Found %s at %s'%(filename,fpath))
return fpath
#Nowhere to be found
if fail:
raise IOError('Could not find resource "%s"'%filename)
else:
logging.debug('Could not find resource "%s"'%filename)
return None
def test_nsi_parameterization():
"""Unit test for Hvac-like NSI parameterization."""
rand = np.random.RandomState(0)
alpha1, alpha2, deltansi = rand.rand(3) * 2. * np.pi
phi12, phi13, phi23 = rand.rand(3) * 2*np.pi - np.pi
eps_max_abs = 10.0
eps_scale, eps_prime = rand.rand(2) * 2 * eps_max_abs - eps_max_abs
nsi_params = VacuumLikeNSIParams()
nsi_params.eps_scale = eps_scale
nsi_params.eps_prime = eps_prime
nsi_params.phi12 = phi12
nsi_params.phi13 = phi13
nsi_params.phi23 = phi23
nsi_params.alpha1 = alpha1
nsi_params.alpha2 = alpha2
nsi_params.deltansi = deltansi
logging.trace('Checking agreement between numerical & analytical NSI matrix...')
eps_mat_numerical = nsi_params.eps_matrix
eps_mat_analytical = nsi_params.eps_matrix_analytical
try:
close = np.isclose(eps_mat_numerical, eps_mat_analytical, **ALLCLOSE_KW)
if not np.all(close):
logging.debug(
"Numerical NSI matrix:\n%s",
np.array2string(eps_mat_numerical, **ARY2STR_KW)
)
logging.debug(
"Analytical expansion (by hand):\n%s",
np.array2string(eps_mat_analytical, **ARY2STR_KW)
)
raise ValueError(
'Evaluating analytical expressions for NSI matrix elements'
' does not give agreement with numerical calculation!'
' Elementwise agreement:\n%s'
% close
)
except ValueError as err:
logging.warning(
"%s\nThis is expected."
" Going ahead with numerical calculation for now.", err
)
logging.trace('Now checking agreement with sympy calculation...')
eps_mat_sympy = nsi_sympy_mat_mult(
eps_scale_val=eps_scale,
eps_prime_val=eps_prime,
phi12_val=phi12,
phi13_val=phi13,
phi23_val=phi23,
alpha1_val=alpha1,
alpha2_val=alpha2,
deltansi_val=deltansi
)
logging.trace('ALLCLOSE_KW = {}'.format(ALLCLOSE_KW))
close = np.isclose(eps_mat_numerical, eps_mat_sympy, **ALLCLOSE_KW)
if not np.all(close):
logging.error(
'Numerical NSI matrix:\n%s',
np.array2string(eps_mat_numerical, **ARY2STR_KW)
)
logging.error(
'Sympy NSI matrix:\n%s', np.array2string(eps_mat_sympy, **ARY2STR_KW)
)
raise ValueError(
'Sympy and numerical calculations disagree! Elementwise agreement:\n'
'%s' % close
)
def single_kernel_set(self, e_true, cz_true, e_reco, cz_reco):
"""Construct a 4D kernel set from MC events using VBWKDE.
Given a set of MC events and each of their {energy{true, reco},
coszen{true, reco}}, generate a 4D NumPy array that maps a 2D true-flux
histogram onto the corresponding 2D reco-flux histogram.
The resulting 4D array can be indexed logically using
kernel4d[e_true_i, cz_true_j][e_reco_k, cz_reco_l]
where the 4 indices point from a single MC-true histogram bin (i,j) to
a single reco histogram bin (k,l).
Binning of both MC-true and reco histograms is the same and is given by
the values in self.ebins and self.czbins which define the bin *edges*
(not the bin centers; hence, len(self.ebins) is one greater than the
number of bins, etc.).
NOTE: Actual limits in energy used to group events into a single "true"
bin may be extended beyond the bin edges defined by self.ebins in order
to gather enough events to successfully apply VBWKDE.
Parameters
----------
e_true : sequence
MC-true neutrino energies, one per event
cz_true : sequence
MC-true neutrino coszen, one per event
e_reco : sequence
Reconstructed neutrino energies, one per event
cz_reco : sequence
Reconstructed neutrino coszen, one per event
Returns
-------
kernel4d : 4D array of float
Mapping from the number of events in each bin of the 2D
MC-true-events histogram to the number of events reconstructed in
each bin of the 2D reconstructed-events histogram. Dimensions are
len(self.ebins)-1 x len(self.czbins)-1 x len(self.ebins)-1 x
len(self.czbins)-1
since ebins and czbins define the histograms' bin edges.
"""
OVERFIT_FACTOR = 1.0
assert np.min(np.diff(self.ebins)) > 0, \
"Energy bin edges not monotonically increasing."
assert np.min(np.diff(self.czbins)) > 0, \
"coszen bin edges not monotonically increasing."
# NOTE: below defines bin centers on linear scale; other logic
# in this method assumes this to be the case, so
# **DO NOT USE** utils.utils.get_bin_centers in this method, which
# may return logarithmically-defined centers instead.
ebin_edges = np.array(self.ebins)
left_ebin_edges = ebin_edges[0:-1]
right_ebin_edges = ebin_edges[1:]
ebin_centers = (left_ebin_edges+right_ebin_edges)/2.0
n_ebins = len(ebin_centers)
czbin_edges = np.array(self.czbins)
left_czbin_edges = czbin_edges[0:-1]
right_czbin_edges = czbin_edges[1:]
czbin_centers = (left_czbin_edges+right_czbin_edges)/2.0
n_czbins = len(czbin_centers)
n_events = len(e_true)
if self.MIN_NUM_EVENTS > n_events:
self.MIN_NUM_EVENTS = n_events
if self.TGT_NUM_EVENTS > n_events:
self.TGT_NUM_EVENTS = n_events
# Object with which to store the 4D kernels: np 4D array
kernel4d = np.zeros((n_ebins, n_czbins, n_ebins, n_czbins))
# Object with which to store the 2D "aggregate_map": the total number
# of events reconstructed into a given (E, CZ) bin, used for sanity
# checks
aggregate_map = np.zeros((n_ebins, n_czbins))
for ebin_n in range(n_ebins):
ebin_min = left_ebin_edges[ebin_n]
ebin_max = right_ebin_edges[ebin_n]
ebin_mid = (ebin_min+ebin_max)/2.0
ebin_wid = ebin_max-ebin_min
logging.trace(
' processing true-energy bin_n=' + str(ebin_n) + ' of ' +
str(n_ebins-1) + ', E_{nu,true} in ' +
'[' + str(ebin_min) + ', ' + str(ebin_max) + '] ...'
)
# Absolute distance from these events' re-centered reco energies to
# the center of this energy bin; sort in ascending-distance order
abs_enu_dist = sorted(np.abs(e_true - ebin_mid))
# Grab the distance the number-"TGT_NUM_EVENTS" event is from the
# bin center
tgt_thresh_enu_dist = abs_enu_dist[self.TGT_NUM_EVENTS-1]
# Grab the distance the number-"MIN_NUM_EVENTS" event is from the
# bin center
min_thresh_enu_dist = abs_enu_dist[self.MIN_NUM_EVENTS-1]
# TODO: revisit the below algorithm with proper testing
# Make threshold distance (which is half the total width) no more
# than 4x the true-energy-bin width in order to capture the
# "target" number of points (TGT_NUM_EVENTS) but no less than half
# the bin width (i.e., the bin should be at least be as wide as the
# pre-defined bin width).
#
# HOWEVER, allow the threshold distance (bin half-width) to expand
# to as much as 4x the original bin full-width in order to capture
# the "minimum" number of points (MIN_NUM_EVENTS).
thresh_enu_dist = \
max(min(max(tgt_thresh_enu_dist, ebin_wid/2),
4*ebin_wid),
min_thresh_enu_dist)
# Grab all events within the threshold distance
in_ebin_ind = np.where(abs_enu_dist <= thresh_enu_dist)[0]
n_in_bin = len(in_ebin_ind)
# Extract just the neutrino-energy/coszen error columns' values for
# succinctness
enu_err = e_reco[in_ebin_ind] - e_true[in_ebin_ind]
cz_err = cz_reco[in_ebin_ind] - cz_true[in_ebin_ind]
#==================================================================
# Neutrino energy resolutions
#==================================================================
dmin = min(enu_err)
dmax = max(enu_err)
drange = dmax-dmin
e_lowerlim = min(self.ENERGY_RANGE[0]-ebin_mid*1.5, dmin-drange*0.5)
e_upperlim = max((np.max(ebin_edges)-ebin_mid)*1.5, dmax+drange*0.5)
egy_kde_lims = np.array([e_lowerlim, e_upperlim])
# Use at least min_num_pts points and at most the next-highest
# integer-power-of-two that allows for at least 10 points in the
# smallest energy bin
min_num_pts = 2**12
min_bin_width = np.min(ebin_edges)
min_pts_smallest_bin = 10.0
kde_range = np.diff(egy_kde_lims)
num_pts0 = kde_range/(min_bin_width/min_pts_smallest_bin)
kde_num_pts = int(max(2**10, 2**np.ceil(np.log2(num_pts0))))
logging.debug(
' Nevts=' + str(n_in_bin) + ' taken from [' +
str(ebin_mid-thresh_enu_dist) + ', ' +
str(ebin_mid+thresh_enu_dist) + ']' + ', KDE lims=' +
str(kde_range) + ', KDE_N: ' + str(kde_num_pts)
)
# Compute variable-bandwidth KDEs
enu_bw, enu_mesh, enu_pdf = kde.vbw_kde(
data = enu_err,
overfit_factor = OVERFIT_FACTOR,
MIN = egy_kde_lims[0],
MAX = egy_kde_lims[1],
N = kde_num_pts
)
if np.min(enu_pdf) < 0:
# Only issue warning if the most-negative value is negative
# beyond specified acceptable-numerical-precision threshold
# (EPSILON)
if np.min(enu_pdf) <= -self.EPSILON:
logging.warn(
"np.min(enu_pdf) < 0: Minimum value is " +
str(np.min(enu_pdf)) +
"; forcing all negative values to 0."
)
# Otherwise, just quietly clip any negative values at 0
enu_pdf = np.clip(a=enu_pdf, a_min=0, a_max=np.inf)
assert np.min(enu_pdf) >= 0, str(np.min(enu_pdf))
# Re-center distribution at the center of the energy bin for which
# errors were computed
offset_enu_mesh = enu_mesh+ebin_mid
offset_enu_pdf = enu_pdf
# Get reference area under the PDF, for checking after interpolated
# values are added.
#
# NOTE There should be NO normalization because any events lost due
# to cutting off tails outside the binned region are actually going
# to be lost, and so should penalize the total area.
int_val0 = np.trapz(y=offset_enu_pdf,
x=offset_enu_mesh)
# Create linear interpolator for the PDF
interp = interpolate.interp1d(
x = offset_enu_mesh,
y = offset_enu_pdf,
kind = 'linear',
copy = True,
bounds_error = True,
fill_value = np.nan
)
# Insert all bin edges' exact locations into the mesh (For accurate
# accounting of area in each bin, must include values out to bin
# edges)
edge_locs = [be for be in
np.concatenate((left_ebin_edges, right_ebin_edges))
if not(be in offset_enu_mesh)]
edge_locs.sort()
edge_pdfs = interp(edge_locs)
insert_ind = np.searchsorted(offset_enu_mesh, edge_locs)
offset_enu_mesh = np.insert(offset_enu_mesh, insert_ind, edge_locs)
offset_enu_pdf = np.insert(offset_enu_pdf, insert_ind, edge_pdfs)
int_val = np.trapz(y=offset_enu_pdf, x=offset_enu_mesh)
assert np.abs(int_val - int_val0) < self.EPSILON
# Chop off distribution at extrema of energy bins
valid_ind = np.where(
(offset_enu_mesh >= np.min(ebin_edges)) &
(offset_enu_mesh <= np.max(ebin_edges))
)[0]
offset_enu_mesh = offset_enu_mesh[valid_ind]
offset_enu_pdf = offset_enu_pdf[valid_ind]
# Check that there are no negative density values (after inserts)
assert np.min(offset_enu_pdf) > 0-self.EPSILON, \
str(np.min(offset_enu_pdf))
# Record the integrated area after removing parts outside binned
# range
tot_ebin_area0 = np.trapz(y=offset_enu_pdf,
x=offset_enu_mesh)
# Check that it integrates to <= 1, sanity check
assert tot_ebin_area0 < 1+self.EPSILON, str(tot_ebin_area0)
# Identify indices encapsulating the defined energy bins' ranges,
# and find the area of each bin
lbinds = np.searchsorted(offset_enu_mesh, left_ebin_edges)
rbinds = np.searchsorted(offset_enu_mesh, right_ebin_edges)
bininds = zip(lbinds, rbinds)
ebin_areas = [np.trapz(y=offset_enu_pdf[l:r+1],
x=offset_enu_mesh[l:r+1])
for (l, r) in bininds]
# Check that no bins have negative areas
assert np.min(ebin_areas) >= 0
# Sum the individual bins' areas
tot_ebin_area = np.sum(ebin_areas)
# Check that this total of all the bins is equal to the total area
# under the curve (i.e., make sure there is no overlap or gaps
# between bins)
assert np.abs(tot_ebin_area-tot_ebin_area0) < self.EPSILON, \
'tot_ebin_area=' + str(tot_ebin_area) + \
' should equal tot_ebin_area0=' + str(tot_ebin_area0)
#==================================================================
# Neutrino coszen resolutions
#==================================================================
dmin = min(cz_err)
dmax = max(cz_err)
drange = dmax-dmin
# NOTE the limits are 1 less than / 1 greater than the limits that
# the error will actually take on, so as to allow for any smooth
# roll-off at edges of data. The calculation of areas below
# captures all of the area, though, by reflecting bins defined in
# [-1, 1] about the points -1 and 1, thereby capturing any
# densities in the range [-3, +3]. This is not necessarily
# accurate, but it's better than throwing that info out entirely.
#
# NOTE also that since reco events as of now are only in range -1
# to 0, though, that there are "gaps" in the capture range, but
# this is due to densities being in the upper-hemisphere which we
# are intentionally ignoring, rather than the code here not taking
# them into account. Normalization is based upon *all* events,
# whether or not they fall within a bin specified above.
# Number of points in the mesh used for VBWKDE; must be large
# enough to capture fast changes in the data but the larger the
# number, the longer it takes to compute the densities at all the
# points. Here, just choosing a fixed number regardless of the data
# or binning
N_cz_mesh = 2**10
# Data range for VBWKDE to consider
cz_gaus_kde_min = -3
cz_gaus_kde_max = +2
cz_gaus_kde_failed = False
previous_fail = False
for n in xrange(3):
# TODO: only catch specific exception
try:
cz_bw, cz_mesh, cz_pdf = kde.vbw_kde(
data = cz_err,
overfit_factor = OVERFIT_FACTOR,
MIN = cz_gaus_kde_min,
MAX = cz_gaus_kde_max,
N = N_cz_mesh
)
except:
cz_gaus_kde_failed = True
if n == 0:
logging.trace('(cz vbwkde ')
logging.trace('fail, ')
# If failure occurred in vbw_kde, expand the data range it
# takes into account; this usually helps
cz_gaus_kde_min -= 1
cz_gaus_kde_max += 1
else:
if cz_gaus_kde_failed:
previous_fail = True
logging.trace('success!')
cz_gaus_kde_failed = False
finally:
if previous_fail:
logging.trace(')')
previous_fail = False
if not cz_gaus_kde_failed:
break
if cz_gaus_kde_failed:
logging.warn('Failed to fit VBWKDE!')
continue
if np.min(cz_pdf) < 0:
logging.warn("np.min(cz_pdf) < 0: Minimum value is " +
str(np.min(cz_pdf)) +
"; forcing all negative values to 0.")
np.clip(a=cz_mesh, a_min=0, a_max=np.inf)
assert np.min(cz_pdf) >= -self.EPSILON, \
str(np.min(cz_pdf))
for czbin_n in range(n_czbins):
czbin_mid = czbin_centers[czbin_n]
# Re-center distribution at the center of the current cz bin
offset_cz_mesh = cz_mesh + czbin_mid
# Create interpolation object, used to fill in bin edge values
interp = interpolate.interp1d(
x = offset_cz_mesh,
y = cz_pdf,
kind = 'linear',
copy = True,
bounds_error = False,
fill_value = 0
)
# Figure out where all bin edges lie in this re-centered
# distribution (some bins may be repeated since bins in [-1,0]
# and err in [-2,1]:
#
# 1. Find limits of mesh values..
mmin = offset_cz_mesh[0]
mmax = offset_cz_mesh[-1]
# 2. Map all bin edges into the full mesh-value range,
# reflecting about -1 and +1. If the reflected edge is outside
# the mesh range, use the exceeded limit of the mesh range as
# the bin edge instead.
#
# This maps every bin edge {i} to 3 new edges, indexed
# new_edges[i][{0,1,2}]. Bins are formed by adjacent indices
# and same-subindices, so what started as, e.g., bin 3 now is
# described by (left, right) edges at
# (new_edges[3][0], new_edges[4][0]),
# (new_edges[3][1], new_edges[4][1]), and
# (new_edges[3][2], new_edges[4][2]).
# NOTE / TODO: It's tempting to dynamically set the number of
# reflections to minimize computation time, but I think it
# breaks the code. Just set to a reasonably large number for
# now and accept the performance penalty. ALSO: if you change
# the parity of the number of reflections, the code below that
# has either (wrap_n % 2 == 0) or (wrap_n+1 % 2 == 0) must be
# swapped!!!
n_left_reflections = 4
n_right_reflections = 4
new_czbin_edges = []
for edge in czbin_edges:
edges_refl_left = []
for n in xrange(n_left_reflections):
edge_refl_left = reflect1d(edge, -1-(2*n))
if edge_refl_left < mmin:
edge_refl_left = mmin
edges_refl_left.append(edge_refl_left)
edges_refl_right = []
for n in xrange(n_right_reflections):
edge_refl_right = reflect1d(edge, +1+(2*n))
if edge_refl_right > mmax:
edge_refl_right = mmax
edges_refl_right.append(edge_refl_right)
# Include all left-reflected versions of this bin edge, in
# increasing-x order + this bin edge + right-reflected
# versions of this bin edge
new_czbin_edges.append(edges_refl_left[::-1] + [edge]
+ edges_refl_right)
# Record all unique bin edges
edge_locs = set()
[edge_locs.update(edges) for edges in new_czbin_edges]
# Throw away bin edges that are already in the mesh
[edge_locs.remove(edge) for edge in list(edge_locs)
if edge in offset_cz_mesh]
# Make into sorted list
edge_locs = sorted(edge_locs)
# Record the total area under the curve
int_val0 = np.trapz(y=cz_pdf, x=offset_cz_mesh)
# Insert the missing bin edge locations & pdf-values into
# the mesh & pdf, respectively
edge_pdfs = interp(edge_locs)
insert_ind = np.searchsorted(offset_cz_mesh, edge_locs)
offset_cz_mesh = np.insert(offset_cz_mesh, insert_ind,
edge_locs)
offset_cz_pdf = np.insert(cz_pdf, insert_ind, edge_pdfs)
assert np.min(offset_cz_pdf) > -self.EPSILON
# Check that this total of all the bins is equal to the total
# area under the curve (i.e., check there is no overlap between
# or gaps between bins)
int_val = np.trapz(y=offset_cz_pdf, x=offset_cz_mesh)
assert np.abs(int_val-1) < self.EPSILON
# Renormalize if it's not exactly 1
if int_val != 1.0:
offset_cz_pdf = offset_cz_pdf / int_val
# Add up the area in the bin and areas that are "reflected"
# into this bin
new_czbin_edges = np.array(new_czbin_edges)
czbin_areas = np.zeros(np.shape(new_czbin_edges)[0]-1)
for wrap_n in range(np.shape(new_czbin_edges)[1]):
bin_edge_inds = np.searchsorted(offset_cz_mesh,
new_czbin_edges[:,wrap_n])
lbinds = bin_edge_inds[0:-1]
rbinds = bin_edge_inds[1:]
# Make sure indices that appear first are less than indices
# that appear second in a pair of bin indices
if (wrap_n+1) % 2 == 0:
bininds = zip(rbinds, lbinds)
else:
bininds = zip(lbinds, rbinds)
tmp_areas = []
for (binind_left_edge, binind_right_edge) in bininds:
if binind_left_edge == binind_right_edge:
tmp_areas.append(0)
continue
this_bin_area = np.array(np.trapz(
y=offset_cz_pdf[binind_left_edge:binind_right_edge+1],
x=offset_cz_mesh[binind_left_edge:binind_right_edge+1]
))
tmp_areas.append(this_bin_area)
czbin_areas += np.array(tmp_areas)
assert np.min(czbin_areas) > -self.EPSILON
tot_czbin_area = np.sum(czbin_areas)
assert tot_czbin_area < int_val + self.EPSILON
kernel4d[ebin_n, czbin_n] = np.outer(ebin_areas, czbin_areas)
assert (np.sum(kernel4d[ebin_n, czbin_n]) -
tot_ebin_area*tot_czbin_area) < self.EPSILON
check_areas = kernel4d.sum(axis=(2,3))
assert np.max(check_areas) < 1 + self.EPSILON, str(np.max(check_areas))
assert np.min(check_areas) > 0 - self.EPSILON, str(np.min(check_areas))
return kernel4d
def store_recursively(fhandle, node, path=None, node_hashes=None):
if path is None:
path = []
if node_hashes is None:
node_hashes = {}
full_path = '/' + '/'.join(path)
if isinstance(node, dict):
logging.trace(" creating Group `%s`" % full_path)
try:
fhandle.create_group(full_path)
except ValueError:
pass
for key in sorted(node.iterkeys()):
key_str = str(key)
if not isinstance(key, str):
logging.warn('Stringifying key `' + key_str +
'`for use as name in HDF5 file')
val = node[key]
new_path = path + [key_str]
store_recursively(fhandle=fhandle, node=val, path=new_path,
node_hashes=node_hashes)
else:
# Check for existing node
node_hash = utils.hash_obj(node)
if node_hash in node_hashes:
logging.trace(" creating hardlink for Dataset: `%s` -> `%s`" %
(full_path, node_hashes[node_hash]))
# Hardlink the matching existing dataset
fhandle[full_path] = fhandle[node_hashes[node_hash]]
return
# For now, convert None to np.nan since h5py appears to not handle None
if node is None:
node = np.nan
logging.warn(" encountered `None` at node `%s`; converting to"
" np.nan" % full_path)
# "Scalar datasets don't support chunk/filter options". Shuffling
# is a good idea otherwise since subsequent compression will
# generally benefit; shuffling requires chunking. Compression is
# not done here since it is slow.
if np.isscalar(node):
shuffle = False
chunks = None
else:
shuffle = True
chunks = True
# Store the node_hash for linking to later if this is more than
# a scalar datatype. Assumed that "None" has
node_hashes[node_hash] = full_path
# TODO: Treat strings as follows? Would this break compatibility
# with pytables/Pandas? What are benefits? Leaving out for now.
# if isinstance(node, basestr):
# dtype = h5py.special_dtype(vlen=str)
# fh.create_dataset(k,data=v,dtype=dtype)
logging.trace(" creating dataset at node `%s`" % full_path)
try:
fhandle.create_dataset(name=full_path, data=node,
chunks=chunks, compression=None,
shuffle=shuffle, fletcher32=False)
except TypeError:
try:
shuffle = False
chunks = None
fhandle.create_dataset(name=full_path, data=node,
chunks=chunks, compression=None,
shuffle=shuffle, fletcher32=False)
except:
logging.error(' full_path: ' + full_path)
logging.error(' chunks : ' + str(chunks))
logging.error(' shuffle : ' + str(shuffle))
logging.error(' node : ' + str(node))
raise
def switch_to_file(self, fp=None, fpname=None):
"""Switch iterator to a new resource location to continue processing.
Parameters
----------
fp : None or file-like object
If `fp` is specified, this takes precedence over `fpname`.
fpname : None or string
Path of the file or resource to read from. This resource will be
located and opened if `fp` is None.
encoding
Argument is passed to the builtin ``open`` function for opening
the file.
"""
fpath = None
if fp is None:
assert fpname
resource = find_resource(fpname)
if isfile(resource):
fpath = abspath(expanduser(expandvars(resource)))
if fpath in self.fpaths_processed:
self._cleanup()
raise ValueError(
'Circular reference; already processed "%s" at path'
' "%s"' % (fpname, fpath))
else:
self._cleanup()
raise ValueError('`fpname` "%s" is not a file')
fp_ = c_open(fpath, encoding=None)
else:
fp_ = fp
if fpname is None:
if hasattr(fp_, 'name'):
fpname = fp_.name
else:
fpname = ''
try:
resource = find_resource(fpname)
except IOError:
pass
else:
if isfile(resource):
fpath = resource
if fp in self.fps_processed:
self._cleanup()
raise ValueError(
'Circular reference; already processed file pointer "%s"'
' at path "%s"' % (fp_, fpname))
if fpath is not None:
if fpath in self.fpaths_processed:
self._cleanup()
raise ValueError(
'Circular reference; already processed "%s" at path'
' "%s"' % (fpname, fpath))
self.fpaths_processed.append(fpath)
self.fps_processed.append(fp)
if fpath is not None:
self.fpaths_processed.append(fpath)
logging.trace('Switching to "%s" at path "%s"' % (fpname, fpath))
record = dict(fp=fp_, fpname=fpname, fpath=fpath, lineno=0, line='')
self._iter_stack.append(record)
def single_kernel_set(self, e_true, cz_true, e_reco, cz_reco,
flav, int_type, make_plots=False, out_dir=None):
"""Construct a 4D kernel set from MC events using VBWKDE.
Given a set of MC events and each of their {energy{true, reco},
coszen{true, reco}}, generate a 4D NumPy array that maps a 2D true-flux
histogram onto the corresponding 2D reco-flux histogram.
The resulting 4D array can be indexed logically using
kernel4d[e_true_i, cz_true_j][e_reco_k, cz_reco_l]
where the 4 indices point from a single MC-true histogram bin (i,j) to
a single reco histogram bin (k,l).
Binning of both MC-true and reco histograms is the same and is given by
the values in self.ebins and self.czbins which define the bin *edges*
(not the bin centers; hence, len(self.ebins) is one greater than the
number of bins, etc.).
NOTE: Actual limits in energy used to group events into a single "true"
bin may be extended beyond the bin edges defined by self.ebins in order
to gather enough events to successfully apply VBWKDE.
Parameters
----------
e_true : sequence
MC-true neutrino energies, one per event
cz_true : sequence
MC-true neutrino coszen, one per event
e_reco : sequence
Reconstructed neutrino energies, one per event
cz_reco : sequence
Reconstructed neutrino coszen, one per event
flav : str
int_type : str
make_plots : bool
out_dir : str or None
path to directory into which to save plots. ``None`` (default)
saves to PWD.
Returns
-------
kernel4d : 4D array of float
Mapping from the number of events in each bin of the 2D
MC-true-events histogram to the number of events reconstructed in
each bin of the 2D reconstructed-events histogram. Dimensions are
len(self.ebins)-1 x len(self.czbins)-1 x len(self.ebins)-1 x
len(self.czbins)-1
since ebins and czbins define the histograms' bin edges.
"""
OVERFIT_FACTOR = 1.0
if make_plots:
import matplotlib as mpl
import matplotlib.pyplot as plt
from matplotlib.backends.backend_pdf import PdfPages
from matplotlib.patches import Rectangle
plt.close(1)
plt.close(2)
plt.close(3)
def rugplot(a, y0, dy, ax, **kwargs):
return ax.plot([a,a], [y0, y0+dy], **kwargs)
plot_fname = '_'.join(['resolutions', 'vbwkde', flav, int_type]) + '.pdf'
if out_dir is not None:
plot_fname = os.path.join(out_dir, plot_fname)
TOP = 0.925
BOTTOM = 0.05
RIGHT = 0.97
LEFT = 0.07
HSPACE = 0.12
LABELPAD = 0.058
AXISBG = (0.5, 0.5, 0.5)
DARK_RED = (0.7, 0.0, 0.0)
HIST_PP = dict(
facecolor=(1,0.5,0.5), edgecolor=DARK_RED,
histtype='stepfilled', alpha=0.7, linewidth=2.0,
label=r'$\mathrm{Histogram}$'
)
N_HBINS = 25
DIFFUS_PP = dict(
color=(0.0, 0.0, 0.0), linestyle='-', marker=None, alpha=0.6,
linewidth=2.0, label=r'$\mathrm{VBWKDE}$'
)
RUG_PP = dict(color=(1.0, 1.0, 1.0), linewidth=0.4, alpha=0.5)
RUG_LAB =r'$\mathrm{Rug\,plot}$'
LEGFNTCOL = (1,1,1)
LEGFACECOL = (0.2,0.2,0.2)
GRIDCOL = (0.4, 0.4, 0.4)
pdfpgs = PdfPages(plot_fname)
assert np.min(np.diff(self.ebins)) > 0, \
"Energy bin edges not monotonically increasing."
assert np.min(np.diff(self.czbins)) > 0, \
"coszen bin edges not monotonically increasing."
# NOTE: below defines bin centers on linear scale; other logic
# in this method assumes this to be the case, so
# **DO NOT USE** utils.utils.get_bin_centers in this method, which
# may return logarithmically-defined centers instead.
ebin_edges = np.array(self.ebins)
left_ebin_edges = ebin_edges[0:-1]
right_ebin_edges = ebin_edges[1:]
ebin_centers = (left_ebin_edges+right_ebin_edges)/2.0
ebin_range = ebin_edges[-1] - ebin_edges[0]
n_ebins = len(ebin_centers)
czbin_edges = np.array(self.czbins)
left_czbin_edges = czbin_edges[0:-1]
right_czbin_edges = czbin_edges[1:]
czbin_centers = (left_czbin_edges+right_czbin_edges)/2.0
n_czbins = len(czbin_centers)
n_events = len(e_true)
if self.MIN_NUM_EVENTS > n_events:
self.MIN_NUM_EVENTS = n_events
if self.TGT_NUM_EVENTS > n_events:
self.TGT_NUM_EVENTS = n_events
# Object with which to store the 4D kernels: np 4D array
kernel4d = np.zeros((n_ebins, n_czbins, n_ebins, n_czbins))
# Object with which to store the 2D "aggregate_map": the total number
# of events reconstructed into a given (E, CZ) bin, used for sanity
# checks
aggregate_map = np.zeros((n_ebins, n_czbins))
for ebin_n in range(n_ebins):
ebin_min = left_ebin_edges[ebin_n]
ebin_max = right_ebin_edges[ebin_n]
ebin_mid = (ebin_min+ebin_max)/2.0
ebin_wid = ebin_max-ebin_min
logging.debug(
'Processing true-energy bin_n=' + format(ebin_n, 'd') + ' of ' +
format(n_ebins-1, 'd') + ', E_{nu,true} in ' +
'[' + format(ebin_min, '0.3f') + ', ' +
format(ebin_max, '0.3f') + '] ...'
)
# Absolute distance from these events' re-centered reco energies to
# the center of this energy bin; sort in ascending-distance order
abs_enu_dist = np.abs(e_true - ebin_mid)
sorted_abs_enu_dist = np.sort(abs_enu_dist)
# Grab the distance the number-"TGT_NUM_EVENTS" event is from the
# bin center
tgt_thresh_enu_dist = sorted_abs_enu_dist[self.TGT_NUM_EVENTS-1]
# Grab the distance the number-"MIN_NUM_EVENTS" event is from the
# bin center
min_thresh_enu_dist = sorted_abs_enu_dist[self.MIN_NUM_EVENTS-1]
# TODO: revisit the below algorithm with proper testing
# Make threshold distance (which is half the total width) no more
# than 4x the true-energy-bin width in order to capture the
# "target" number of points (TGT_NUM_EVENTS) but no less than half
# the bin width (i.e., the bin should be at least be as wide as the
# pre-defined bin width).
#
# HOWEVER, allow the threshold distance (bin half-width) to expand
# to as much as 4x the original bin full-width in order to capture
# the "minimum" number of points (MIN_NUM_EVENTS).
thresh_enu_dist = \
max(min(max(tgt_thresh_enu_dist, ebin_wid/2),
4*ebin_wid),
min_thresh_enu_dist)
# Grab all events within the threshold distance
in_ebin_ind = np.where(abs_enu_dist <= thresh_enu_dist)[0]
#print '** IN EBIN FIRST, LAST ENERGY:', e_reco[in_ebin_ind[0]], e_reco[in_ebin_ind[-1]]
n_in_bin = len(in_ebin_ind)
# Record lowest/highest energies that are included in the bin
actual_left_ebin_edge = min(ebin_min, min(e_true[in_ebin_ind])) #max(min(ebins), ebin_mid-thresh_enu_dist)
actual_right_ebin_edge = max(ebin_max, max(e_true[in_ebin_ind])) #(max(ebins), ebin_mid+thresh_enu_dist)
# Extract just the neutrino-energy/coszen error columns' values for
# succinctness
enu_err = e_reco[in_ebin_ind] - e_true[in_ebin_ind]
cz_err = cz_reco[in_ebin_ind] - cz_true[in_ebin_ind]
#==================================================================
# Neutrino energy resolutions
#==================================================================
dmin = min(enu_err)
dmax = max(enu_err)
drange = dmax-dmin
e_lowerlim = min(self.ENERGY_RANGE[0]-ebin_mid*1.5, dmin-drange*0.5)
e_upperlim = max((np.max(ebin_edges)-ebin_mid)*1.5, dmax+drange*0.5)
egy_kde_lims = np.array([e_lowerlim, e_upperlim])
# Use at least min_num_pts points and at most the next-highest
# integer-power-of-two that allows for at least 10 points in the
# smallest energy bin
min_num_pts = 2**12
min_bin_width = np.min(ebin_edges[1:]-ebin_edges[:-1])
min_pts_smallest_bin = 5.0
kde_range = np.diff(egy_kde_lims)
num_pts0 = kde_range/(min_bin_width/min_pts_smallest_bin)
kde_num_pts = int(max(min_num_pts, 2**np.ceil(np.log2(num_pts0))))
logging.debug(
' N_evts=' + str(n_in_bin) + ', taken from [' +
format(actual_left_ebin_edge, '0.3f') + ', ' +
format(actual_right_ebin_edge, '0.3f') + ']' + ', VBWKDE lims=' +
str(egy_kde_lims) + ', VBWKDE_N: ' + str(kde_num_pts)
)
# Compute variable-bandwidth KDEs
enu_bw, enu_mesh, enu_pdf = kde.vbw_kde(
data = enu_err,
overfit_factor = OVERFIT_FACTOR,
MIN = egy_kde_lims[0],
MAX = egy_kde_lims[1],
N = kde_num_pts
)
if np.min(enu_pdf) < 0:
# Only issue warning if the most-negative value is negative
# beyond specified acceptable-numerical-precision threshold
# (EPSILON)
if np.min(enu_pdf) <= -self.EPSILON:
logging.warn(
"np.min(enu_pdf) < 0: Minimum value is " +
str(np.min(enu_pdf)) +
"; forcing all negative values to 0."
)
# Otherwise, just quietly clip any negative values at 0
enu_pdf = np.clip(a=enu_pdf, a_min=0, a_max=np.inf)
assert np.min(enu_pdf) >= 0, str(np.min(enu_pdf))
# Re-center distribution at the center of the energy bin for which
# errors were computed
offset_enu_mesh = enu_mesh+ebin_mid
offset_enu_pdf = enu_pdf
# Get reference area under the PDF, for checking after interpolated
# values are added.
#
# NOTE There should be NO normalization because any events lost due
# to cutting off tails outside the binned region are actually going
# to be lost, and so should penalize the total area.
int_val0 = np.trapz(y=offset_enu_pdf,
x=offset_enu_mesh)
# Create linear interpolator for the PDF
interp = interpolate.interp1d(
x = offset_enu_mesh,
y = offset_enu_pdf,
kind = 'linear',
copy = True,
bounds_error = True,
fill_value = np.nan
)
# Insert all bin edges' exact locations into the mesh (For accurate
# accounting of area in each bin, must include values out to bin
# edges)
edge_locs = [be for be in
np.concatenate((left_ebin_edges, right_ebin_edges))
if not(be in offset_enu_mesh)]
edge_locs.sort()
edge_pdfs = interp(edge_locs)
insert_ind = np.searchsorted(offset_enu_mesh, edge_locs)
offset_enu_mesh = np.insert(offset_enu_mesh, insert_ind, edge_locs)
offset_enu_pdf = np.insert(offset_enu_pdf, insert_ind, edge_pdfs)
int_val = np.trapz(y=offset_enu_pdf, x=offset_enu_mesh)
assert np.abs(int_val - int_val0) < self.EPSILON
# Chop off distribution at extrema of energy bins
valid_ind = np.where(
(offset_enu_mesh >= np.min(ebin_edges)) &
(offset_enu_mesh <= np.max(ebin_edges))
)[0]
offset_enu_mesh = offset_enu_mesh[valid_ind]
offset_enu_pdf = offset_enu_pdf[valid_ind]
# Check that there are no negative density values (after inserts)
assert np.min(offset_enu_pdf) > 0-self.EPSILON, \
str(np.min(offset_enu_pdf))
# Record the integrated area after removing parts outside binned
# range
tot_ebin_area0 = np.trapz(y=offset_enu_pdf,
x=offset_enu_mesh)
# Check that it integrates to <= 1, sanity check
assert tot_ebin_area0 < 1+self.EPSILON, str(tot_ebin_area0)
# Identify indices encapsulating the defined energy bins' ranges,
# and find the area of each bin
lbinds = np.searchsorted(offset_enu_mesh, left_ebin_edges)
rbinds = np.searchsorted(offset_enu_mesh, right_ebin_edges)
bininds = zip(lbinds, rbinds)
ebin_areas = [np.trapz(y=offset_enu_pdf[l:r+1],
x=offset_enu_mesh[l:r+1])
for (l, r) in bininds]
# Check that no bins have negative areas
assert np.min(ebin_areas) >= 0
# Sum the individual bins' areas
tot_ebin_area = np.sum(ebin_areas)
# Check that this total of all the bins is equal to the total area
# under the curve (i.e., make sure there is no overlap or gaps
# between bins)
assert np.abs(tot_ebin_area-tot_ebin_area0) < self.EPSILON, \
'tot_ebin_area=' + str(tot_ebin_area) + \
' should equal tot_ebin_area0=' + str(tot_ebin_area0)
if make_plots:
fig1 = plt.figure(1, figsize=(8,10), dpi=90)
fig1.clf()
ax1 = fig1.add_subplot(211, axisbg=AXISBG)
# Retrieve region where VBWKDE lives
ml_ci = confInterval.MLConfInterval(x=enu_mesh, y=enu_pdf)
#for conf in np.logspace(np.log10(0.999), np.log10(0.95), 50):
# try:
# lb, ub, yopt, r = ml_ci.findCI_lin(conf=conf)
# except:
# pass
# else:
# break
#xlims = (min(-ebin_mid*1.5, lb),
# max(min(ub, 6*ebin_mid),2*ebin_mid))
lb, ub, yopt, r = ml_ci.findCI_lin(conf=0.98)
xlims = (lb, #min(-ebin_mid*1.5, lb),
max(min(ub, 6*ebin_mid),2*ebin_wid))
#xlims = (
# -ebin_wid*1.5,
# ebin_wid*1.5
#)
# min(ebin_mid*2, ebin_edges[-1]+(ebin_edges[-1]-ebin_edges[0])*0.1)
#)
# Histogram of events' reco error
hbins = np.linspace(dmin-0.02*drange, dmax+0.02*drange,
N_HBINS*np.round(drange/ebin_centers[ebin_n]))
hvals, hbins, hpatches = ax1.hist(enu_err,
bins=hbins,
normed=True,
**HIST_PP)
# Plot the VBWKDE
ax1.plot(enu_mesh, enu_pdf, **DIFFUS_PP)
axlims = ax1.axis('tight')
ax1.set_xlim(xlims)
ymax = axlims[3]*1.05
ax1.set_ylim(0, ymax)
# Grey-out regions outside binned region, so it's clear what
# part of tail(s) will be thrown away
width = -ebin_mid+ebin_edges[0]-xlims[0]
unbinned_region_tex = r'$\mathrm{Unbinned}$'
if width > 0:
ax1.add_patch(Rectangle((xlims[0],0), width, ymax, #zorder=-1,
alpha=0.30, facecolor=(0.0 ,0.0, 0.0), fill=True,
ec='none'))
ax1.text(xlims[0]+(xlims[1]-xlims[0])/40., ymax/10.,
unbinned_region_tex, fontsize=14, ha='left',
va='bottom', rotation=90, color='k')
width = xlims[1] - (ebin_edges[-1]-ebin_mid)
if width > 0:
ax1.add_patch(Rectangle((xlims[1]-width,0), width, ymax,
alpha=0.30, facecolor=(0, 0, 0),
fill=True, ec='none'))
ax1.text(xlims[1]-(xlims[1]-xlims[0])/40., ymax/10.,
unbinned_region_tex, fontsize=14, ha='right',
va='bottom', rotation=90, color='k')
# Rug plot of events' reco energy errors
ylim = ax1.get_ylim()
dy = ylim[1] - ylim[0]
ruglines = rugplot(enu_err, y0=ylim[1], dy=-dy/40., ax=ax1,
**RUG_PP)
ruglines[-1].set_label(RUG_LAB)
# Legend
leg_title_tex = r'$\mathrm{Normalized}\,E_\nu\mathrm{-err.\,distr.}$'
x1lab = ax1.set_xlabel(
r'$E_{\nu,\mathrm{reco}}-E_{\nu,\mathrm{true}}\;' +
r'(\mathrm{GeV})$', labelpad=LABELPAD
)
leg = ax1.legend(loc='upper right', title=leg_title_tex,
frameon=True, framealpha=0.8,
fancybox=True, bbox_to_anchor=[1,0.975])
# Other plot details
ax1.xaxis.set_label_coords(0.9, -LABELPAD)
ax1.xaxis.grid(color=GRIDCOL)
ax1.yaxis.grid(color=GRIDCOL)
leg.get_title().set_fontsize(16)
leg.get_title().set_color(LEGFNTCOL)
[t.set_color(LEGFNTCOL) for t in leg.get_texts()]
frame = leg.get_frame()
frame.set_facecolor(LEGFACECOL)
frame.set_edgecolor(None)
#==================================================================
# Neutrino coszen resolution for events in this energy bin
#==================================================================
dmin = min(cz_err)
dmax = max(cz_err)
drange = dmax-dmin
# NOTE the limits are 1 less than / 1 greater than the limits that
# the error will actually take on, so as to allow for any smooth
# roll-off at edges of data. The calculation of areas below
# captures all of the area, though, by reflecting bins defined in
# [-1, 1] about the points -1 and 1, thereby capturing any
# densities in the range [-3, +3]. This is not necessarily
# accurate, but it's better than throwing that info out entirely.
#
# NOTE also that since reco events as of now are only in range -1
# to 0, though, that there are "gaps" in the capture range, but
# this is due to densities being in the upper-hemisphere which we
# are intentionally ignoring, rather than the code here not taking
# them into account. Normalization is based upon *all* events,
# whether or not they fall within a bin specified above.
# Number of points in the mesh used for VBWKDE; must be large
# enough to capture fast changes in the data but the larger the
# number, the longer it takes to compute the densities at all the
# points. Here, just choosing a fixed number regardless of the data
# or binning
N_cz_mesh = 2**10
# Data range for VBWKDE to consider
cz_kde_min = -3
cz_kde_max = +2
cz_kde_failed = False
previous_fail = False
for n in xrange(3):
# TODO: only catch specific exception
try:
cz_bw, cz_mesh, cz_pdf = kde.vbw_kde(
data = cz_err,
overfit_factor = OVERFIT_FACTOR,
MIN = cz_kde_min,
MAX = cz_kde_max,
N = N_cz_mesh
)
except:
cz_kde_failed = True
if n == 0:
logging.trace('(cz vbwkde ')
logging.trace('fail, ')
# If failure occurred in vbw_kde, expand the data range it
# takes into account; this usually helps
cz_kde_min -= 1
cz_kde_max += 1
else:
if cz_kde_failed:
previous_fail = True
logging.trace('success!')
cz_kde_failed = False
finally:
if previous_fail:
logging.trace(')')
previous_fail = False
if not cz_kde_failed:
break
if cz_kde_failed:
logging.warn('Failed to fit VBWKDE!')
continue
if np.min(cz_pdf) < 0:
logging.warn("np.min(cz_pdf) < 0: Minimum value is " +
str(np.min(cz_pdf)) +
"; forcing all negative values to 0.")
np.clip(a=cz_mesh, a_min=0, a_max=np.inf)
assert np.min(cz_pdf) >= -self.EPSILON, \
str(np.min(cz_pdf))
# TODO: test and/or visualize the shifting & re-binning process
for czbin_n in range(n_czbins):
czbin_mid = czbin_centers[czbin_n]
# Re-center distribution at the center of the current cz bin
offset_cz_mesh = cz_mesh + czbin_mid
# Create interpolation object, used to fill in bin edge values
interp = interpolate.interp1d(
x = offset_cz_mesh,
y = cz_pdf,
kind = 'linear',
copy = True,
bounds_error = False,
fill_value = 0
)
# Figure out where all bin edges lie in this re-centered
# distribution (some bins may be repeated since bins in [-1,0]
# and err in [-2,1]:
#
# 1. Find limits of mesh values..
mmin = offset_cz_mesh[0]
mmax = offset_cz_mesh[-1]
# 2. Map all bin edges into the full mesh-value range,
# reflecting about -1 and +1. If the reflected edge is outside
# the mesh range, use the exceeded limit of the mesh range as
# the bin edge instead.
#
# This maps every bin edge {i} to 3 new edges, indexed
# new_edges[i][{0,1,2}]. Bins are formed by adjacent indices
# and same-subindices, so what started as, e.g., bin 3 now is
# described by (left, right) edges at
# (new_edges[3][0], new_edges[4][0]),
# (new_edges[3][1], new_edges[4][1]), and
# (new_edges[3][2], new_edges[4][2]).
# NOTE / TODO: It's tempting to dynamically set the number of
# reflections to minimize computation time, but I think it
# breaks the code. Just set to a reasonably large number for
# now and accept the performance penalty. ALSO: if you change
# the parity of the number of reflections, the code below that
# has either (wrap_n % 2 == 0) or (wrap_n+1 % 2 == 0) must be
# swapped!!!
n_left_reflections = 4
n_right_reflections = 4
new_czbin_edges = []
for edge in czbin_edges:
edges_refl_left = []
for n in xrange(n_left_reflections):
edge_refl_left = reflect1d(edge, -1-(2*n))
if edge_refl_left < mmin:
edge_refl_left = mmin
edges_refl_left.append(edge_refl_left)
edges_refl_right = []
for n in xrange(n_right_reflections):
edge_refl_right = reflect1d(edge, +1+(2*n))
if edge_refl_right > mmax:
edge_refl_right = mmax
edges_refl_right.append(edge_refl_right)
# Include all left-reflected versions of this bin edge, in
# increasing-x order + this bin edge + right-reflected
# versions of this bin edge
new_czbin_edges.append(edges_refl_left[::-1] + [edge]
+ edges_refl_right)
# Record all unique bin edges
edge_locs = set()
[edge_locs.update(edges) for edges in new_czbin_edges]
# Throw away bin edges that are already in the mesh
[edge_locs.remove(edge) for edge in list(edge_locs)
if edge in offset_cz_mesh]
# Make into sorted list
edge_locs = sorted(edge_locs)
# Record the total area under the curve
int_val0 = np.trapz(y=cz_pdf, x=offset_cz_mesh)
# Insert the missing bin edge locations & pdf-values into
# the mesh & pdf, respectively
edge_pdfs = interp(edge_locs)
insert_ind = np.searchsorted(offset_cz_mesh, edge_locs)
offset_cz_mesh = np.insert(offset_cz_mesh, insert_ind,
edge_locs)
offset_cz_pdf = np.insert(cz_pdf, insert_ind, edge_pdfs)
assert np.min(offset_cz_pdf) > -self.EPSILON
# Check that this total of all the bins is equal to the total
# area under the curve (i.e., check there is no overlap between
# or gaps between bins)
int_val = np.trapz(y=offset_cz_pdf, x=offset_cz_mesh)
assert np.abs(int_val-1) < self.EPSILON
# Renormalize if it's not exactly 1
if int_val != 1.0:
offset_cz_pdf = offset_cz_pdf / int_val
# Add up the area in the bin and areas that are "reflected"
# into this bin
new_czbin_edges = np.array(new_czbin_edges)
czbin_areas = np.zeros(np.shape(new_czbin_edges)[0]-1)
for wrap_n in range(np.shape(new_czbin_edges)[1]):
bin_edge_inds = np.searchsorted(offset_cz_mesh,
new_czbin_edges[:,wrap_n])
lbinds = bin_edge_inds[0:-1]
rbinds = bin_edge_inds[1:]
# Make sure indices that appear first are less than indices
# that appear second in a pair of bin indices
if (wrap_n+1) % 2 == 0:
bininds = zip(rbinds, lbinds)
else:
bininds = zip(lbinds, rbinds)
tmp_areas = []
for (binind_left_edge, binind_right_edge) in bininds:
if binind_left_edge == binind_right_edge:
tmp_areas.append(0)
continue
this_bin_area = np.array(np.trapz(
y=offset_cz_pdf[binind_left_edge:binind_right_edge+1],
x=offset_cz_mesh[binind_left_edge:binind_right_edge+1]
))
tmp_areas.append(this_bin_area)
czbin_areas += np.array(tmp_areas)
assert np.min(czbin_areas) > -self.EPSILON
tot_czbin_area = np.sum(czbin_areas)
assert tot_czbin_area < int_val + self.EPSILON
kernel4d[ebin_n, czbin_n] = np.outer(ebin_areas, czbin_areas)
assert (np.sum(kernel4d[ebin_n, czbin_n]) -
tot_ebin_area*tot_czbin_area) < self.EPSILON
if make_plots:
ax2 = fig1.add_subplot(212, axisbg=AXISBG)
hbins = np.linspace(dmin-0.02*drange, dmax+0.02*drange, N_HBINS*3)
hvals, hbins, hpatches = ax2.hist(cz_err, bins=hbins,
normed=True, **HIST_PP)
ax2.plot(cz_mesh, cz_pdf, **DIFFUS_PP)
fci = confInterval.MLConfInterval(x=cz_mesh,
y=cz_pdf)
lb, ub, yopt, r = fci.findCI_lin(conf=0.995)
axlims = ax2.axis('tight')
ax2.set_xlim(lb, ub)
ax2.set_ylim(0, axlims[3]*1.05)
ylim = ax2.get_ylim()
dy = ylim[1] - ylim[0]
ruglines = rugplot(cz_err, y0=ylim[1], dy=-dy/40., ax=ax2, **RUG_PP)
ruglines[-1].set_label(r'$\mathrm{Rug\,plot}$')
x2lab = ax2.set_xlabel(
r'$\cos\vartheta_{\mathrm{track,reco}}-\cos\vartheta_{\nu,\mathrm{true}}$',
labelpad=LABELPAD
)
ax2.xaxis.set_label_coords(0.9, -LABELPAD)
ax2.xaxis.grid(color=GRIDCOL)
ax2.yaxis.grid(color=GRIDCOL)
leg_title_tex = r'$\mathrm{Normalized}\,\cos\vartheta\mathrm{-err.\,distr.}$'
leg = ax2.legend(loc='upper right', title=leg_title_tex,
frameon=True, framealpha=0.8, fancybox=True,
bbox_to_anchor=[1,0.975])
leg.get_title().set_fontsize(16)
leg.get_title().set_color(LEGFNTCOL)
[t.set_color(LEGFNTCOL) for t in leg.get_texts()]
frame = leg.get_frame()
frame.set_facecolor(LEGFACECOL)
frame.set_edgecolor(None)
actual_bin_tex = ''
if (actual_left_ebin_edge != ebin_min) or (actual_right_ebin_edge != ebin_max):
actual_bin_tex = r'E_{\nu,\mathrm{true}}\in [' + \
format(actual_left_ebin_edge, '0.2f') + r',\,' + \
format(actual_right_ebin_edge, '0.2f') + r'] \mapsto '
stt = r'$\mathrm{Resolutions,\,' + flav_tex(flav) + r'\,' + \
int_tex(int_type) + r'}$' + '\n' + \
r'$' + actual_bin_tex + r'\mathrm{Bin}_{' + format(ebin_n, 'd') + r'}\equiv E_{\nu,\mathrm{true}}\in [' + format(ebin_min, '0.2f') + \
r',\,' + format(ebin_max, '0.2f') + r']\,\mathrm{GeV}' + \
r',\,N_\mathrm{events}=' + format(n_in_bin, 'd') + r'$'
fig1.subplots_adjust(top=TOP, bottom=BOTTOM, left=LEFT, right=RIGHT, hspace=HSPACE)
suptitle = fig1.suptitle(stt)
suptitle.set_fontsize(16)
suptitle.set_position((0.5,0.98))
fig1.savefig(pdfpgs, format='pdf')
check_areas = kernel4d.sum(axis=(2,3))
assert np.max(check_areas) < 1 + self.EPSILON, str(np.max(check_areas))
assert np.min(check_areas) > 0 - self.EPSILON, str(np.min(check_areas))
if make_plots:
fig2 = plt.figure(2, figsize=(8,10), dpi=90)
fig2.clf()
ax = fig2.add_subplot(111)
X, Y = np.meshgrid(range(n_czbins), range(n_ebins))
cm = mpl.cm.Paired_r
cm.set_over((1,1,1), 1)
cm.set_under((0,0,0), 1)
plt.pcolor(X, Y, check_areas, vmin=0+self.EPSILON, vmax=1.0,
shading='faceted', cmap=cm)
plt.colorbar(ticks=np.arange(0, 1.05, 0.05))
ax.grid(0)
ax.axis('tight')
ax.set_xlabel(r'$\cos\vartheta_\mathrm{true}\mathrm{\,bin\,num.}$')
ax.set_ylabel(r'$E_{\nu,\mathrm{true}}\mathrm{\,bin\,num.}$')
ax.set_title(r'$\mathrm{Fract\,of\,evts\,starting\,in\,each}\,(E_{\nu,\mathrm{true}},\,\cos\vartheta_\mathrm{true})\,\mathrm{bin\,that\,reco\,in\,bounds}$'+
'\n'+r'$\mathrm{None\,should\,be\,>1\,(shown\,white);\,no-event\,bins\,are\,black;\,avg.}=' + format(np.mean(check_areas),'0.3f') + r'$')
fig2.tight_layout()
fig2.savefig(pdfpgs, format='pdf')
check_areas2 = kernel4d.sum(axis=(0,1))
fig3 = plt.figure(2, figsize=(8,10), dpi=90)
fig3.clf()
ax = fig3.add_subplot(111)
X, Y = np.meshgrid(range(n_czbins), range(n_ebins))
cm = mpl.cm.Paired_r
cm.set_over((1,1,1), 1)
cm.set_under((0,0,0), 1)
plt.pcolor(X, Y, check_areas2, vmin=0+self.EPSILON,# vmax=1.0,
shading='faceted', cmap=cm)
plt.colorbar(ticks=np.arange(0, 0.1+np.ceil(10.*np.max(check_areas2))/10., 0.05))
ax.grid(0)
ax.axis('tight')
ax.set_xlabel(r'$\cos\vartheta_\mathrm{reco}\mathrm{\,bin\,num.}$')
ax.set_ylabel(r'$E_{\nu,\mathrm{reco}}\mathrm{\,bin\,num.}$')
ax.set_title(r'$\mathrm{Normed\,num\,events\,reconstructing\,into\,each}\,(E_{\nu,\mathrm{reco}},\,\cos\vartheta_\mathrm{reco})\,\mathrm{bin}$'+
'\n'+r'$\mathrm{No-event\,bins\,are\,black;\,avg.}=' + format(np.mean(check_areas2),'0.3f') + r'$')
fig3.tight_layout()
fig3.savefig(pdfpgs, format='pdf')
pdfpgs.close()
return kernel4d
# # author: Sebastian Boeser # [email protected] # # date: 2014-01-27 import os import sys import numpy as np from pisa.utils.log import logging # Try and get the much faster simplejson if we can try: import simplejson as json from simplejson import JSONDecodeError logging.trace("Using simplejson") except ImportError: import json as json # No DecodeError in default json, dummy one class JSONDecodeError(ValueError): pass logging.trace("Using json") def json_string(string): """Decode a json string""" return json.loads(string) def from_json(filename): """Open a file in JSON format an parse the content"""
def initialize_kernel(self):
'''
Initializes: 1) the grid_propagator class, 2) the device arrays
that will be passed to the propagateGrid() kernel and 3) the
kernel module.
'''
self.grid_prop = GridPropagator(self.earth_model,self.czcen_fine)
self.maxLayers = self.grid_prop.GetMaxLayers()
nczbins_fine = len(self.czcen_fine)
numLayers = np.zeros(nczbins_fine,dtype=np.int32)
densityInLayer = np.zeros((nczbins_fine*self.maxLayers),dtype=self.FTYPE)
distanceInLayer = np.zeros((nczbins_fine*self.maxLayers),dtype=self.FTYPE)
self.grid_prop.GetNumberOfLayers(numLayers)
self.grid_prop.GetDensityInLayer(densityInLayer)
self.grid_prop.GetDistanceInLayer(distanceInLayer)
# Copy all these earth info arrays to device:
self.d_numLayers = cuda.mem_alloc(numLayers.nbytes)
self.d_densityInLayer = cuda.mem_alloc(densityInLayer.nbytes)
self.d_distanceInLayer = cuda.mem_alloc(distanceInLayer.nbytes)
cuda.memcpy_htod(self.d_numLayers,numLayers)
cuda.memcpy_htod(self.d_densityInLayer,densityInLayer)
cuda.memcpy_htod(self.d_distanceInLayer,distanceInLayer)
self.d_ecen_fine = cuda.mem_alloc(self.ecen_fine.nbytes)
self.d_czcen_fine = cuda.mem_alloc(self.czcen_fine.nbytes)
#cuda.memcpy_htod(self.d_ecen_fine,self.ecen_fine)
cuda.memcpy_htod(self.d_czcen_fine,self.czcen_fine)
###############################################
###### DEFINE KERNEL
###############################################
kernel_template = """
#include "mosc.cu"
#include "mosc3.cu"
#include "utils.h"
#include <stdio.h>
__global__ void propagateGrid(double* d_smooth_maps,
double d_dm[3][3], double d_mix[3][3][2],
const double* const d_ecen_fine,
const double* const d_czcen_fine,
const int nebins_fine, const int nczbins_fine,
const int nebins, const int nczbins, const int maxLayers,
const int* const d_numberOfLayers,
const double* const d_densityInLayer,
const double* const d_distanceInLayer)
{
const int2 thread_2D_pos = make_int2(blockIdx.x*blockDim.x + threadIdx.x,
blockIdx.y*blockDim.y + threadIdx.y);
// ensure we don't access memory outside of bounds!
if(thread_2D_pos.x >= nczbins_fine || thread_2D_pos.y >= nebins_fine) return;
const int thread_1D_pos = thread_2D_pos.y*nczbins_fine + thread_2D_pos.x;
int eidx = thread_2D_pos.y;
int czidx = thread_2D_pos.x;
int kNuBar;
if(blockIdx.z == 0) kNuBar = 1;
else kNuBar=-1;
bool kUseMassEstates = false;
double TransitionMatrix[3][3][2];
double TransitionProduct[3][3][2];
double TransitionTemp[3][3][2];
double RawInputPsi[3][2];
double OutputPsi[3][2];
double Probability[3][3];
clear_complex_matrix( TransitionMatrix );
clear_complex_matrix( TransitionProduct );
clear_complex_matrix( TransitionTemp );
clear_probabilities( Probability );
int layers = *(d_numberOfLayers + czidx);
double energy = d_ecen_fine[eidx];
//double coszen = d_czcen_fine[czidx];
for( int i=0; i<layers; i++) {
double density = *(d_densityInLayer + czidx*maxLayers + i);
double distance = *(d_distanceInLayer + czidx*maxLayers + i);
get_transition_matrix( kNuBar,
energy,
density,
distance,
TransitionMatrix,
0.0,
d_mix,
d_dm);
if(i==0) { copy_complex_matrix(TransitionMatrix, TransitionProduct);
} else {
clear_complex_matrix( TransitionTemp );
multiply_complex_matrix( TransitionMatrix, TransitionProduct, TransitionTemp );
copy_complex_matrix( TransitionTemp, TransitionProduct );
}
} // end layer loop
// loop on neutrino types, and compute probability for neutrino i:
// We actually don't care about nutau -> anything since the flux there is zero!
for( unsigned i=0; i<2; i++) {
for ( unsigned j = 0; j < 3; j++ ) {
RawInputPsi[j][0] = 0.0;
RawInputPsi[j][1] = 0.0;
}
if( kUseMassEstates ) convert_from_mass_eigenstate(i+1,kNuBar,RawInputPsi,d_mix);
else RawInputPsi[i][0] = 1.0;
multiply_complex_matvec( TransitionProduct, RawInputPsi, OutputPsi );
Probability[i][0] +=OutputPsi[0][0]*OutputPsi[0][0]+OutputPsi[0][1]*OutputPsi[0][1];
Probability[i][1] +=OutputPsi[1][0]*OutputPsi[1][0]+OutputPsi[1][1]*OutputPsi[1][1];
Probability[i][2] +=OutputPsi[2][0]*OutputPsi[2][0]+OutputPsi[2][1]*OutputPsi[2][1];
}//end of neutrino loop
int efctr = nebins_fine/nebins;
int czfctr = nczbins_fine/nczbins;
int eidx_smooth = eidx/efctr;
int czidx_smooth = czidx/czfctr;
double scale = double(efctr*czfctr);
for (int i=0;i<2;i++) {
int iMap = 0;
if (kNuBar == 1) iMap = i*3;
else iMap = 6 + i*3;
for (unsigned to_nu=0; to_nu<3; to_nu++) {
int k = (iMap+to_nu);
double prob = Probability[i][to_nu];
atomicAdd((d_smooth_maps + k*nczbins*nebins + eidx_smooth*nczbins +
czidx_smooth),prob/scale);
}
}
}
"""
include_path = os.path.expandvars('$PISA/pisa/oscillations/grid_propagator/')
#cache_dir=os.path.expandvars('$PISA/pisa/oscillations/'+'.cache_dir')
logging.trace(" pycuda INC PATH: %s"%include_path)
#logging.trace(" pycuda cache_dir: %s"%cache_dir)
logging.trace(" pycuda FLAGS: %s"%pycuda.compiler.DEFAULT_NVCC_FLAGS)
self.module = SourceModule(kernel_template,
include_dirs=[include_path],
#cache_dir=cache_dir,
keep=True)
self.propGrid = self.module.get_function("propagateGrid")
return
