def main(return_outputs=False):
"""Main; call as script with `return_outputs=False` or interactively with
`return_outputs=True`"""
from pisa.utils.plotter import Plotter
args = parse_args()
set_verbosity(args.v)
plot_formats = []
if args.pdf:
plot_formats.append('pdf')
if args.png:
plot_formats.append('png')
distribution_maker = DistributionMaker(pipelines=args.pipeline) # pylint: disable=redefined-outer-name
if args.select is not None:
distribution_maker.select_params(args.select)
outputs = distribution_maker.get_outputs(return_sum=args.return_sum) # pylint: disable=redefined-outer-name
if args.outdir:
# TODO: unique filename: append hash (or hash per pipeline config)
fname = 'distribution_maker_outputs.json.bz2'
mkdir(args.outdir)
fpath = expand(os.path.join(args.outdir, fname))
to_file(outputs, fpath)
if args.outdir and plot_formats:
my_plotter = Plotter(outdir=args.outdir,
fmt=plot_formats,
log=False,
annotate=False)
for num, output in enumerate(outputs):
my_plotter.plot_2d_array(output, fname='dist_output_%d' % num)
if return_outputs:
return distribution_maker, outputs
def test_parse_pipeline_config():
"""Unit test for function `parse_pipeline_config`"""
from argparse import ArgumentParser
parser = ArgumentParser()
parser.add_argument('-p',
'--pipeline',
metavar='CONFIGFILE',
default='settings/pipeline/example.cfg',
help='File containing settings for the pipeline.')
parser.add_argument(
'-v',
action='count',
default=0,
help='Set verbosity level. Minimum is forced to level 1 (info)')
args = parser.parse_args()
args.v = max(1, args.v)
set_verbosity(args.v)
# Load via PISAConfigParser
config0 = PISAConfigParser()
config0.read(args.pipeline)
_ = parse_pipeline_config(config0)
# Load directly
config = parse_pipeline_config(args.pipeline)
logging.debug('Keys and values found in config:')
for key, vals in config.items():
logging.debug('%s: %s', key, vals)
logging.info('<< PASS : test_parse_pipeline_config >>')
def main(description=__doc__):
"""Script interface for `test_prob3numba` function"""
parser = ArgumentParser(description=description)
parser.add_argument("--ignore-fails", action="store_true")
parser.add_argument("--define-as-ref", action="store_true")
parser.add_argument("-v", action="count", default=Levels.WARN)
kwargs = vars(parser.parse_args())
set_verbosity(kwargs.pop("v"))
test_prob3numba(**kwargs)
def test_kde_bootstrapping(verbosity=Levels.WARN):
"""Unit test for the kde stage."""
set_verbosity(verbosity)
example_cfg = parse_pipeline_config("settings/pipeline/example.cfg")
# We have to remove containers with too few events, otherwise the KDE fails simply
# because too few distinct events are in one of the PID channels after bootstrapping.
example_cfg[("data", "simple_data_loader")]["output_names"] = [
"numu_cc",
"numubar_cc",
]
kde_stage_cfg = OrderedDict()
kde_stage_cfg["apply_mode"] = example_cfg[("utils", "hist")]["apply_mode"]
kde_stage_cfg["calc_mode"] = "events"
kde_stage_cfg["bootstrap"] = False
kde_stage_cfg["bootstrap_seed"] = 0
kde_stage_cfg["bootstrap_niter"] = 5
kde_pipe_cfg = deepcopy(example_cfg)
# Replace histogram stage with KDE stage
del kde_pipe_cfg[("utils", "hist")]
kde_pipe_cfg[("utils", "kde")] = kde_stage_cfg
# no errors in baseline since there is no bootstrapping enabled
kde_pipe_cfg["pipeline"]["output_key"] = "weights"
# get a baseline
dmaker = DistributionMaker([kde_pipe_cfg])
map_baseline = dmaker.get_outputs(return_sum=True)[0]
logging.debug(f"Baseline KDE'd map:\n{map_baseline}")
# Make sure that different seeds produce different maps, and that the same seed will
# produce the same map.
# We enable bootstrapping now, without re-loading everything, to save time.
dmaker.pipelines[0].output_key = ("weights", "errors")
dmaker.pipelines[0].stages[-1].bootstrap = True
map_seed0 = dmaker.get_outputs(return_sum=True)[0]
dmaker.pipelines[0].stages[-1].bootstrap_seed = 1
map_seed1 = dmaker.get_outputs(return_sum=True)[0]
logging.debug(f"Map with seed 0 is:\n{map_seed0}")
logging.debug(f"Map with seed 1 is:\n{map_seed1}")
assert not map_seed0 == map_seed1
dmaker.pipelines[0].stages[-1].bootstrap_seed = 0
map_seed0_reprod = dmaker.get_outputs(return_sum=True)[0]
assert map_seed0 == map_seed0_reprod
logging.info("<< PASS : kde_bootstrapping >>")
def test_kde_bootstrapping(verbosity=Levels.WARN):
"""Unit test for the kde stage."""
set_verbosity(verbosity)
test_cfg = deepcopy(TEST_CONFIGS.pipe_cfg)
test_cfg[("data", "toy_event_generator")] = deepcopy(
TEST_CONFIGS.event_generator_cfg
)
test_cfg[("aeff", "weight")] = deepcopy(TEST_CONFIGS.aeff_cfg)
test_cfg[("utils", "kde")] = deepcopy(TEST_CONFIGS.kde_cfg)
# get map, but without the linearization
test_cfg[("utils", "kde")]["linearize_log_dims"] = False
dmaker = DistributionMaker([test_cfg])
map_baseline_no_linearization = dmaker.get_outputs(return_sum=True)[0]
# get a baseline (with linearization, which we will use from here on out)
test_cfg[("utils", "kde")]["linearize_log_dims"] = True
dmaker = DistributionMaker([test_cfg])
map_baseline = dmaker.get_outputs(return_sum=True)[0]
logging.debug(f"Baseline KDE'd map:\n{map_baseline}")
# assert that linearization make a difference at all
total_no_lin = np.sum(map_baseline_no_linearization.nominal_values)
total_with_lin = np.sum(map_baseline.nominal_values)
assert not (total_no_lin == total_with_lin)
# but also that the difference isn't huge (< 5% difference in total bin count)
# --> This will fail if one forgets to *not* take the log when linearization
# is turned off, for example. In that case, most bins will be empty, because
# the binning would be lin while the KDE would be log.
assert np.abs(total_no_lin / total_with_lin - 1.0) < 0.05
# Make sure that different seeds produce different maps, and that the same seed will
# produce the same map.
# We enable bootstrapping now, without re-loading everything, to save time.
dmaker.pipelines[0].output_key = ("weights", "errors")
dmaker.pipelines[0].stages[-1].bootstrap = True
map_seed0 = dmaker.get_outputs(return_sum=True)[0]
dmaker.pipelines[0].stages[-1].bootstrap_seed = 1
map_seed1 = dmaker.get_outputs(return_sum=True)[0]
logging.debug(f"Map with seed 0 is:\n{map_seed0}")
logging.debug(f"Map with seed 1 is:\n{map_seed1}")
assert not map_seed0 == map_seed1
dmaker.pipelines[0].stages[-1].bootstrap_seed = 0
map_seed0_reprod = dmaker.get_outputs(return_sum=True)[0]
assert map_seed0 == map_seed0_reprod
logging.info("<< PASS : kde_bootstrapping >>")
def main():
"""Run `add_fluxes_to_file` function with arguments from command line"""
args = parse_args()
set_verbosity(args.v)
flux_table = load_2d_table(args.flux_file)
flux_file_bname, ext = splitext(basename(args.flux_file))
input_paths = []
for input_path in args.input:
if isdir(input_path):
for filename in listdir(input_path):
filepath = join(input_path, filename)
input_paths.append(filepath)
else:
input_paths += glob.glob(input_path)
input_paths = nsort(input_paths)
paths_to_process = []
basenames = []
for input_path in input_paths:
if isdir(input_path):
logging.debug('Path "%s" is a directory, skipping', input_path)
continue
firstpart, ext = splitext(input_path)
if ext.lstrip('.') not in HDF5_EXTS:
logging.debug('Path "%s" is not an HDF5 file, skipping',
input_path)
continue
bname = basename(firstpart)
if bname in basenames:
raise ValueError(
'Found files with duplicate basename "%s" (despite files'
' having different paths); resolve the ambiguous names and'
' re-run. Offending files are:\n "%s"\n "%s"' %
(bname, paths_to_process[basenames.index(bname)], input_path))
basenames.append(bname)
paths_to_process.append(input_path)
logging.info('Will process %d input file(s)...', len(paths_to_process))
for filepath in paths_to_process:
logging.info('Working on input file "%s"', filepath)
add_fluxes_to_file(data_file_path=filepath,
flux_table=flux_table,
flux_name='nominal',
outdir=args.outdir,
label=flux_file_bname)
def main():
"""Perform a hyperplane fit to discrete systematics sets."""
args = parse_args()
set_verbosity(args.v)
# Read in data and fit hyperplanes to it
input_data, fit_results = hyperplane(fit_cfg=args.fit_cfg)
# Save to disk
save_hyperplane_fits(input_data=input_data,
fit_results=fit_results,
outdir=args.outdir,
tag=args.tag)
def main(return_outputs=False):
"""Main; call as script with `return_outputs=False` or interactively with
`return_outputs=True`"""
from pisa.utils.plotter import Plotter
args = parse_args()
set_verbosity(args.v)
plot_formats = []
if args.pdf:
plot_formats.append('pdf')
if args.png:
plot_formats.append('png')
detectors = Detectors(args.pipeline,shared_params=args.shared_params)
Names = detectors.det_names
if args.select is not None:
detectors.select_params(args.select)
outputs = detectors.get_outputs(return_sum=args.return_sum)
#outputs = outputs[0].fluctuate(
# method='poisson', random_state=get_random_state([0, 0, 0]))
if args.outdir:
# TODO: unique filename: append hash (or hash per pipeline config)
fname = 'detectors_outputs.json.bz2'
mkdir(args.outdir)
fpath = expand(os.path.join(args.outdir, fname))
to_file(outputs, fpath)
if args.outdir and plot_formats:
my_plotter = Plotter(
outdir=args.outdir,
fmt=plot_formats, log=False,
annotate=False
)
for num, output in enumerate(outputs):
if args.return_sum:
my_plotter.plot_2d_array(
output,
fname=Names[num]
)
else:
for out in output:
my_plotter.plot_2d_array(
out,
fname=Names[num]
)
if return_outputs:
return detectors, outputs
def main():
"""Perform a hypersurface fit to discrete systematics sets."""
# Get args
args = parse_args()
set_verbosity(args.v)
# Read in data and fit hypersurfaces to it
hypersurfaces = create_hypersurfaces(fit_cfg=args.fit_cfg)
# Store as JSON
mkdir(args.outdir)
arbitrary_hypersurface = list(hypersurfaces.values())[0]
output_path = join( args.outdir, get_hypersurface_file_name(arbitrary_hypersurface, args.tag) )
to_file(hypersurfaces, output_path)
def main():
"""Script function to run tests on a cross section file"""
from argparse import ArgumentParser, ArgumentDefaultsHelpFormatter
set_verbosity(3)
parser = ArgumentParser(description='Test genie xsec service',
formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument('-x', '--xsec_file', type=str, required=True,
help='Input GENIE ROOT cross-section file')
parser.add_argument('-o', '--outdir', type=str, default='./',
help='Output directory for plots')
args = parser.parse_args()
# TODO(shivesh): tests
# test_XSec(args.xsec_file)
test_standard_plots(args.xsec_file, args.outdir+'/standard/')
test_per_e_plot(args.xsec_file, args.outdir+'/per_e/')
def parse_args():
"""Parse command line arguments"""
parser = ArgumentParser(
description='Print contents of a parsed config file',
formatter_class=ArgumentDefaultsHelpFormatter,
)
parser.add_argument(
'pipeline', metavar='CONFIGFILE',
nargs='?',
default='settings/pipeline/example.cfg',
help='Pipeline config file to parse',
)
parser.add_argument(
'-v',
action='count',
default=Levels.WARN,
help='Set verbosity level',
)
kwargs = vars(parser.parse_args())
set_verbosity(kwargs.pop('v'))
return kwargs
def main():
args = parse_args()
set_verbosity(args.verbose)
assert len(args.infiles[0]) == 2
assert len(args.subtitles) == 2
logging.info('Loading MapSet from files {0}'.format(args.infiles[0]))
o_mapset = MapSet.from_json(args.infiles[0][0])
t_mapset = MapSet.from_json(args.infiles[0][1])
assert len(o_mapset) == 1 and len(t_mapset) == 1
o_map = o_mapset.pop()
t_map = t_mapset.pop()
outfile = args.outdir + '/' + args.outname
make_plot(
maps=(o_map, t_map),
outfile=outfile,
logv=args.logv,
center_zero=args.center_zero,
vlabel=args.vlabel,
title=args.title,
sub_titles=args.subtitles,
)
mVac[0] -= delta
if mVac[2] == 0.:
mVac[2] += delta
dmVacVac[0, 0] = 0.
dmVacVac[1, 1] = 0.
dmVacVac[2, 2] = 0.
dmVacVac[0, 1] = mVac[0] - mVac[1]
dmVacVac[1, 0] = -dmVacVac[0, 1]
dmVacVac[0, 2] = mVac[0] - mVac[2]
dmVacVac[2, 0] = -dmVacVac[0, 2]
dmVacVac[1, 2] = mVac[1] - mVac[2]
dmVacVac[2, 1] = -dmVacVac[1, 2]
return dmVacVac
def test_pi_osc_params():
"""
# TODO: implement me!
"""
pass
if __name__ == '__main__':
from pisa import TARGET
from pisa.utils.log import set_verbosity, logging
assert TARGET == 'cpu', "Cannot test functions on GPU, set PISA_TARGET to 'cpu'"
set_verbosity(1)
test_pi_osc_params()
def parse_args(command, description):
"""Parse command line args.
Returns
-------
init_args_d : dict
"""
assert command in [discrete_hypo_test, inj_param_scan, systematics_tests]
parser = ArgumentParser(
description=description,
formatter_class=ArgumentDefaultsHelpFormatter,
)
parser.add_argument(
'-d',
'--logdir',
required=True,
metavar='DIR',
type=str,
help='Directory into which to store results and metadata.')
parser.add_argument('--min-settings',
type=str,
metavar='MINIMIZER_CFG',
default=None,
help='''Minimizer settings config file.''')
parser.add_argument(
'--min-method',
type=str,
default=None,
choices=('l-bfgs-b', 'slsqp'),
help='''Name of minimizer to use. Note that this takes precedence over
the minimizer method specified via the --min-settings config
file.''')
parser.add_argument(
'--min-opt',
type=str,
metavar='OPTION:VALUE',
nargs='+',
default=None,
help='''Minimizer option:value pair(s) (can specify multiple).
Values specified here override any of the same name in the config file
specified by --min-settings''')
parser.add_argument(
'--no-octant-check',
action='store_true',
help='''Disable fitting hypotheses in theta23 octant opposite initial
octant.''')
parser.add_argument(
'--ordering-check',
action='store_true',
help='''Fit both ordering hypotheses. This should only be flagged if
the ordering is NOT the discrete hypothesis being tested''')
parser.add_argument(
'--shared-params',
type=str,
default=None,
action='append',
help='''Shared parameters for multi detector analysis (repeat for
multiple). The values of these parameters are kept the same in all
detectors that contain the param.''')
if command == discrete_hypo_test:
# Data cannot be data for MC studies e.g. injected parameter scans so
# these arguments are redundant there.
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument(
'--data-is-data',
action='store_true',
help='''Data pipeline is based upon actual, measured data. The
naming scheme for stored results is chosen accordingly.''')
group.add_argument(
'--data-is-mc',
action='store_true',
help='''Data pipeline is based upon Monte Carlo simulation, and not
actual data. The naming scheme for stored results is chosen
accordingly. If this is selected, --fluctuate-data is forced off.'''
)
# For the MC tests (injected parameter scan, systematic tests etc.) you
# must have the same pipeline for h0, h1 and data. So this argument is
# instead replaced with a generic pipeline argument.
parser.add_argument(
'--h0-pipeline',
required=True,
type=str,
action='append',
metavar='PIPELINE_CFG',
help='''Settings for the generation of hypothesis h0
distributions; repeat this argument to specify multiple
pipelines.''')
else:
assert command in [inj_param_scan, systematics_tests]
parser.add_argument(
'--pipeline',
required=True,
type=str,
action='append',
metavar='PIPELINE_CFG',
help='''Settings for the generation of h0, h1 and data
distributions; repeat this argument to specify multiple
pipelines.''')
parser.add_argument(
'--h0-param-selections',
type=str,
default=None,
metavar='PARAM_SELECTOR_LIST',
help='''Comma-separated (no spaces) list of param selectors to apply to
hypothesis h0's distribution maker's pipelines.''')
parser.add_argument('--h0-name',
type=str,
metavar='NAME',
default=None,
help='''Name for hypothesis h0. E.g., "NO" for normal
ordering in the neutrino mass ordering analysis. Note that the name
here has no bearing on the actual process, so it's important that you
be careful to use a name that appropriately identifies the
hypothesis.''')
# For the MC tests (injected parameter scan, systematic tests etc.) you
# must have the same pipeline for h0, h1 and data. So this argument is
# hidden.
if command not in (inj_param_scan, systematics_tests):
parser.add_argument(
'--h1-pipeline',
type=str,
action='append',
default=None,
metavar='PIPELINE_CFG',
help='''Settings for the generation of hypothesis h1 distributions;
repeat this argument to specify multiple pipelines. If omitted, the
same settings as specified for --h0-pipeline are used to generate
hypothesis h1 distributions (and so you have to use the
--h1-param-selections argument to generate a hypotheses distinct
from hypothesis h0 but still use h0's distribution maker).''')
parser.add_argument(
'--h1-param-selections',
type=str,
default=None,
metavar='PARAM_SELECTOR_LIST',
help='''Comma-separated (no spaces) list of param selectors to apply to
hypothesis h1 distribution maker's pipelines.''')
parser.add_argument('--h1-name',
type=str,
metavar='NAME',
default=None,
help='''Name for hypothesis h1. E.g., "IO" for inverted
ordering in the neutrino mass ordering analysis. Note that the name
here has no bearing on the actual process, so it's important that you
be careful to use a name that appropriately identifies the
hypothesis.''')
# For the MC tests (injected parameter scan, systematic tests etc.) you
# must have the same pipeline for h0, h1 and data. So this argument is
# hidden.
if command not in (inj_param_scan, systematics_tests):
parser.add_argument(
'--data-pipeline',
type=str,
action='append',
default=None,
metavar='PIPELINE_CFG',
help='''Settings for the generation of "data" distributions; repeat
this argument to specify multiple pipelines. If omitted, the same
settings as specified for --h0-pipeline are used to generate data
distributions (i.e., data is assumed to come from hypothesis h0.'''
)
parser.add_argument(
'--data-param-selections',
type=str,
default=None,
metavar='PARAM_SELECTOR_LIST',
help='''Comma-separated list of param selectors to apply to the data
distribution maker's pipelines. If neither --data-pipeline nor
--data-param-selections are specified, *both* are copied from
--h0-pipeline and --h0-param-selections, respectively. However,
if --data-pipeline is specified while --data-param-selections is not,
then the param selections in the pipeline config file(s) specified are
used to produce data distributions.''')
parser.add_argument(
'--data-name',
type=str,
metavar='NAME',
default=None,
help='''Name for the data. E.g., "NO" for normal ordering in the
neutrino mass ordering analysis. Note that the name here has no bearing
on the actual process, so it's important that you be careful to use a
name that appropriately identifies the hypothesis.''')
# For the injected parameter scan and systematic studies, only the Asimov
# analysis should be used, so these arguments are not needed.
if command not in (inj_param_scan, systematics_tests):
parser.add_argument(
'--fluctuate-data',
action='store_true',
help='''Apply fluctuations to the data distribution. This should
*not* be set for analyzing "real" (measured) data, and it is common
to not use this feature even for Monte Carlo analysis. Note that if
this is not set, --num-data-trials and --data-start-ind are forced
to 1 and 0, respectively.''')
parser.add_argument(
'--fluctuate-fid',
action='store_true',
help='''Apply fluctuations to the fiducaial distributions. If this
flag is not set, --num-fid-trials and --fid-start-ind are forced to
1 and 0, respectively.''')
parser.add_argument(
'--metric',
type=str,
required=True,
metavar='METRIC',
action='append',
choices=sorted(ALL_METRICS),
help='''Name of metric(s) to use for optimizing the fit. Must be one of
%s. Repeat this argument if you want to use different metrics for
different detectors. If only one metric is specified, all detectors will
use the same. Otherwise you have to specify one metric for each detector
(even if two use the same) and pay attention to the order.''' %
(ALL_METRICS, ))
parser.add_argument(
'--other-metric',
type=str,
default=None,
metavar='METRIC',
action='append',
choices=['all'] + sorted(ALL_METRICS),
help='''Name of another metric to evaluate at the best-fit point. Must
be either 'all' or one of %s. Repeat this argument (or use 'all') to
specify multiple metrics.''' % (ALL_METRICS, ))
if command not in (inj_param_scan, systematics_tests):
parser.add_argument(
'--num-data-trials',
type=int,
default=1,
help='''When performing Monte Carlo analysis, set to > 1 to produce
multiple pseudodata distributions from the data distribution maker's
Asimov distribution. This is overridden if --fluctuate-data is not
set (since each data distribution will be identical if it is not
fluctuated). This is typically left at 1 (i.e., the Asimov
distribution is assumed to be representative.''')
parser.add_argument('--data-start-ind',
type=int,
default=0,
help='''Fluctated data set index.''')
parser.add_argument(
'--num-fid-trials',
type=int,
default=1,
help='''Number of fiducial pseudodata trials to run. In our
experience, it takes ~10^3-10^5 fiducial psuedodata trials to
achieve low uncertainties on the resulting significance, though
that exact number will vary based upon the details of an
analysis.''')
parser.add_argument('--fid-start-ind',
type=int,
default=0,
help='''Fluctated fiducial data index.''')
# A blind analysis only makes sense when the possibility of actually
# analysing data is available.
if command not in (inj_param_scan, systematics_tests):
parser.add_argument(
'--blind',
action='store_true',
help='''Blinded analysis. Do not show parameter values or store to
logfiles.''')
parser.add_argument(
'--allow-dirty',
action='store_true',
help='''Warning: Use with caution. (Allow for run despite dirty
repository.)''')
parser.add_argument(
'--allow-no-git-info',
action='store_true',
help='''*** DANGER! Use with extreme caution! (Allow for run despite
complete inability to track provenance of code.)''')
parser.add_argument(
'--no-min-history',
action='store_true',
help='''Do not store minimizer history (steps). This behavior is also
enforced if --blind is specified.''')
# Add in the arguments specific to the injected parameter scan.
if command == inj_param_scan:
parser.add_argument(
'--param_name',
type=str,
metavar='NAME',
required=True,
help='''Name of param to scan over. This must be in the config
files defined above. One exception is that you can define this as
`sin2theta23` and it will be interpreted not as theta23 values but
as the square of the sine of theta23 values instead.''')
parser.add_argument(
'--inj_vals',
type=str,
required=True,
help='''List of values to inject as true points in the parameter
defined above. Must be something that numpy can interpret. In this
script, numpy is imported as np so please use np in your string. An
example would be np.linspace(0.35,0.65,31).''')
parser.add_argument(
'--inj_units',
type=str,
required=True,
help='''A string to be able to deal with the units in the parameter
scan and make sure that they match those in the config files. Even
if the parameter is dimensionless this must be stated.''')
parser.add_argument(
'--use-inj-prior',
action='store_true',
help='''Generally, one should not use a prior on the parameter of
interest here since the Asimov analysis breaks down with the use of
non-central prior i.e. injecting a truth that differs from the
centre of the prior. Flag this to force the prior to be left on.'''
)
# Add in the arguments specific to the systematic tests.
if command == systematics_tests:
parser.add_argument(
'--inject_wrong',
action='store_true',
help='''Inject a parameter to some systematically wrong value.
This will be either +/- 1 sigma or +/- 10%% if such a definition
is impossible. By default this parameter will be fixed unless
the fit_wrong argument is also flagged.''')
parser.add_argument(
'--fit_wrong',
action='store_true',
help='''In the case of injecting a systematically wrong hypothesis
setting this argument will get the minimiser to try correct for it.
If inject_wrong is set to false then this must also be set to
false or else the script will fail.''')
parser.add_argument(
'--only_syst',
default=None,
type=str,
action='append',
metavar='PARAM_NAME',
help='''Specify the name of one of the systematics in the file to
run the test for this systematic. Repeat this argument to specify
multiple systematics. If none are provided, the test will be run
over all systematics in the pipeline.''')
parser.add_argument(
'--skip_baseline',
action='store_true',
help='''Skip the baseline systematic test i.e. the one where none
of them are fixed and/or modified. In most cases you will want this
for comparison but if you are only interested in the effect of
shifting certain systematics then this step can be skipped.''')
parser.add_argument(
'--pprint',
action='store_true',
help='''Live-updating one-line vew of metric and parameter values. (The
latter are not displayed if --blind is specified.)''')
parser.add_argument('-v',
action='count',
default=None,
help='set verbosity level')
args = parser.parse_args(sys.argv[2:])
assert args.min_settings is not None or args.min_method is not None
init_args_d = vars(args)
set_verbosity(init_args_d.pop('v'))
min_settings_from_file = init_args_d.pop('min_settings')
minimizer = init_args_d.pop('min_method')
min_opt = init_args_d.pop('min_opt')
# TODO: put this datastructure remnant from PISA 2 out of its misery...
minimizer_settings = dict(method=dict(value='', desc='no desc'),
options=dict(value=dict(), desc=dict()))
if min_settings_from_file is not None:
minimizer_settings.update(from_file(min_settings_from_file))
if minimizer is not None:
minimizer_settings['method'] = dict(value=minimizer, desc='no desc')
if min_opt is not None:
for opt_val_str in min_opt:
opt, val_str = [s.strip() for s in opt_val_str.split(':')]
try:
val = int(val_str)
except ValueError:
try:
val = float(val_str)
except ValueError:
val = val_str
minimizer_settings['options']['value'][opt] = val
minimizer_settings['options']['desc'][opt] = 'no desc'
init_args_d['minimizer_settings'] = minimizer_settings
init_args_d['check_octant'] = not init_args_d.pop('no_octant_check')
init_args_d['check_ordering'] = init_args_d.pop('ordering_check')
if command not in (inj_param_scan, systematics_tests):
init_args_d['data_is_data'] = not init_args_d.pop('data_is_mc')
else:
init_args_d['data_is_data'] = False
init_args_d['fluctuate_data'] = False
init_args_d['fluctuate_fid'] = False
init_args_d['store_minimizer_history'] = (
not init_args_d.pop('no_min_history'))
other_metrics = init_args_d.pop('other_metric')
if other_metrics is not None:
other_metrics = [s.strip().lower() for s in other_metrics]
if 'all' in other_metrics:
other_metrics = sorted(ALL_METRICS)
for m in init_args_d['metric']:
if m in other_metrics:
other_metrics.remove(m)
if not other_metrics:
other_metrics = None
else:
logging.info('Will evaluate other metrics %s', other_metrics)
init_args_d['other_metrics'] = other_metrics
return init_args_d
log.tprofile.debug('module %s, function %s: %.4f ms',
func.__module__, func.__name__,
(end_t - start_t) * 1000)
return profiled_func
def test_profile():
"""Unit tests for `profile` functional (decorator)"""
@profile
def get_number():
log.logging.trace('hello, i am get_number')
for x in xrange(500000):
yield x
@profile
def expensive_function():
log.logging.trace('hello, i am expensive fun')
for x in get_number():
_ = x ^ x ^ x
return 'some result!'
_ = expensive_function()
log.logging.info('<< ??? : test_profile >> inspect above outputs')
if __name__ == '__main__':
log.set_verbosity(2)
test_line_profile()
test_profile()
def main():
"""Main"""
args = parse_args()
args_d = vars(args)
set_verbosity(args_d.pop('v'))
make_toy_events(**args_d)
def run_unit_tests(path=PISA_PATH,
allow_missing=OPTIONAL_MODULES,
verbosity=Levels.WARN):
"""Run all tests found at `path` (or recursively below if `path` is a
directory).
Each module is imported and each test function is run initially with
`set_verbosity(verbosity)`, but if an exception is caught, the module is
re-imported or the test function is re-run with
`set_verbosity(Levels.TRACE)`, then the traceback from the (original)
exception emitted is displayed.
Parameters
----------
path : str
Path to file or directory
allow_missing : None or sequence of str
verbosity : int in pisa.utils.log.Levels
Raises
------
Exception
If any import or test fails not in `allow_missing`
"""
set_verbosity(verbosity)
logging.info("%sPlatform information:", PFX)
logging.info("%s HOSTNAME = %s", PFX, socket.gethostname())
logging.info("%s FQDN = %s", PFX, socket.getfqdn())
logging.info("%s OS = %s %s", PFX, platform.system(), platform.release())
for key, val in cpuinfo.get_cpu_info().items():
logging.info("%s %s = %s", PFX, key, val)
logging.info(PFX)
logging.info("%sModule versions:", PFX)
for module_name in REQUIRED_MODULES + OPTIONAL_MODULES:
try:
module = import_module(module_name)
except ImportError:
if module_name in REQUIRED_MODULES:
raise
ver = "optional module not installed or not import-able"
else:
if hasattr(module, "__version__"):
ver = module.__version__
else:
ver = "?"
logging.info("%s %s : %s", PFX, module_name, ver)
logging.info(PFX)
path = expand(path, absolute=True, resolve_symlinks=True)
if allow_missing is None:
allow_missing = []
elif isinstance(allow_missing, str):
allow_missing = [allow_missing]
tests = find_unit_tests(path)
module_pypaths_succeeded = []
module_pypaths_failed = []
module_pypaths_failed_ignored = []
test_pypaths_succeeded = []
test_pypaths_failed = []
test_pypaths_failed_ignored = []
for rel_file_path, test_func_names in tests.items():
pypath = ["pisa"] + rel_file_path[:-3].split("/")
parent_pypath = ".".join(pypath[:-1])
module_name = pypath[-1].replace(".", "_")
module_pypath = f"{parent_pypath}.{module_name}"
try:
set_verbosity(verbosity)
logging.info(PFX + f"importing {module_pypath}")
set_verbosity(Levels.WARN)
module = import_module(module_pypath, package=parent_pypath)
except Exception as err:
if (isinstance(err, ImportError) and hasattr(err, "name")
and err.name in allow_missing # pylint: disable=no-member
):
err_name = err.name # pylint: disable=no-member
module_pypaths_failed_ignored.append(module_pypath)
logging.warning(
f"{PFX}module {err_name} failed to import wile importing"
f" {module_pypath}, but ok to ignore")
continue
module_pypaths_failed.append(module_pypath)
set_verbosity(verbosity)
msg = f"<< FAILURE IMPORTING : {module_pypath} >>"
logging.error(PFX + "=" * len(msg))
logging.error(PFX + msg)
logging.error(PFX + "=" * len(msg))
# Reproduce the failure with full output
set_verbosity(Levels.TRACE)
try:
import_module(module_name, package=parent_pypath)
except Exception:
pass
set_verbosity(Levels.TRACE)
logging.exception(err)
set_verbosity(verbosity)
logging.error(PFX + "#" * len(msg))
continue
else:
module_pypaths_succeeded.append(module_pypath)
for test_func_name in test_func_names:
test_pypath = f"{module_pypath}.{test_func_name}"
try:
set_verbosity(verbosity)
logging.debug(PFX + f"getattr({module}, {test_func_name})")
set_verbosity(Levels.WARN)
test_func = getattr(module, test_func_name)
# Run the test function
set_verbosity(verbosity)
logging.info(PFX + f"{test_pypath}()")
set_verbosity(Levels.WARN)
test_func()
except Exception as err:
if (isinstance(err, ImportError) and hasattr(err, "name")
and err.name in allow_missing # pylint: disable=no-member
):
err_name = err.name # pylint: disable=no-member
test_pypaths_failed_ignored.append(module_pypath)
logging.warning(
PFX +
f"{test_pypath} failed because module {err_name} failed to"
+ f" load, but ok to ignore")
continue
test_pypaths_failed.append(test_pypath)
set_verbosity(verbosity)
msg = f"<< FAILURE RUNNING : {test_pypath} >>"
logging.error(PFX + "=" * len(msg))
logging.error(PFX + msg)
logging.error(PFX + "=" * len(msg))
# Reproduce the error with full output
set_verbosity(Levels.TRACE)
try:
test_func = getattr(module, test_func_name)
with np.printoptions(
precision=np.finfo(pisa.FTYPE).precision + 2,
floatmode="fixed",
sign=" ",
linewidth=200,
):
test_func()
except Exception:
pass
set_verbosity(Levels.TRACE)
logging.exception(err)
set_verbosity(verbosity)
logging.error(PFX + "#" * len(msg))
else:
test_pypaths_succeeded.append(test_pypath)
finally:
# remove references to the test function, e.g. to remove refs
# to pycuda / numba.cuda contexts so these can be closed
try:
del test_func
except NameError:
pass
# NOTE: Until we get all GPU code into Numba, need to unload pycuda
# and/or numba.cuda contexts before a module requiring the other one is
# to be imported.
# NOTE: the following causes a traceback to be emitted at the very end
# of the script, regardless of the exception catching here.
if (pisa.TARGET == "cuda" and pycuda is not None
and hasattr(pycuda, "autoinit")
and hasattr(pycuda.autoinit, "context")):
try:
pycuda.autoinit.context.detach()
except Exception:
pass
# Attempt to unload the imported module
# TODO: pipeline, etc. fail as isinstance(service, (Stage, PiStage)) is False
#if module_pypath in sys.modules and module_pypath != "pisa":
# del sys.modules[module_pypath]
#del module
# TODO: crashes program; subseqeunt calls in same shell crash(!?!?)
# if pisa.TARGET == 'cuda' and nbcuda is not None:
# try:
# nbcuda.close()
# except Exception:
# pass
# Summarize results
n_import_successes = len(module_pypaths_succeeded)
n_import_failures = len(module_pypaths_failed)
n_import_failures_ignored = len(module_pypaths_failed_ignored)
n_test_successes = len(test_pypaths_succeeded)
n_test_failures = len(test_pypaths_failed)
n_test_failures_ignored = len(test_pypaths_failed_ignored)
set_verbosity(verbosity)
logging.info(
PFX + f"<< IMPORT TESTS : {n_import_successes} imported,"
f" {n_import_failures} failed,"
f" {n_import_failures_ignored} failed to import but ok to ignore >>")
logging.info(PFX + f"<< UNIT TESTS : {n_test_successes} succeeded,"
f" {n_test_failures} failed,"
f" {n_test_failures_ignored} failed but ok to ignore >>")
# Exit with error if any failures (import or unit test)
if module_pypaths_failed or test_pypaths_failed:
msgs = []
if module_pypaths_failed:
msgs.append(
f"{n_import_failures} module(s) failed to import:\n " +
", ".join(module_pypaths_failed))
if test_pypaths_failed:
msgs.append(f"{n_test_failures} unit test(s) failed:\n " +
", ".join(test_pypaths_failed))
# Note the extra newlines before the exception to make it stand out;
# and newlines after the exception are due to the pycuda error message
# that is emitted when we call pycuda.autoinit.context.detach()
sys.stdout.flush()
sys.stderr.write("\n\n\n")
raise Exception("\n".join(msgs) + "\n\n\n")
pathlength = np.asarray(pathlength)
self._distance = pathlength
def test_Layers():
logging.info('Test layers calculation:')
layer = Layers('osc/PREM_4layer.dat')
layer.setElecFrac(0.4656, 0.4656, 0.4957)
cz = np.linspace(-1, 1, 1e5, dtype=FTYPE)
layer.calcLayers(cz)
logging.info('n_layers = %s' % layer.n_layers)
logging.info('density = %s' % layer.density)
logging.info('distance = %s' % layer.distance)
logging.info('Test path length calculation:')
layer = Layers(None)
cz = np.array([1., 0., -1.])
layer.calcPathLength(cz)
logging.info('coszen = %s' % cz)
logging.info('pathlengths = %s' % layer.distance)
logging.info('<< PASS : test_Layers >>')
if __name__ == '__main__':
set_verbosity(3)
test_Layers()
def main(return_outputs=False):
"""Run unit tests if `pipeline.py` is called as a script."""
from pisa.utils.plotter import Plotter
args = parse_args()
set_verbosity(args.v)
# Even if user specifies an integer on command line, it comes in as a
# string. Try to convert to int (e.g. if `'1'` is passed to indicate the
# second stage), and -- if successful -- use this as `args.only_stage`.
# Otherwise, the string value passed will be used (e.g. `'osc'` could be
# passed).
try:
only_stage_int = int(args.only_stage)
except (ValueError, TypeError):
pass
else:
args.only_stage = only_stage_int
if args.outdir:
mkdir(args.outdir)
else:
if args.pdf or args.png:
raise ValueError("No --outdir provided, so cannot save images.")
# Most basic parsing of the pipeline config (parsing only to this level
# allows for simple strings to be specified as args for updating)
bcp = PISAConfigParser()
bcp.read(args.pipeline)
# Update the config with any args specified on command line
if args.arg is not None:
for arg_list in args.arg:
if len(arg_list) < 2:
raise ValueError(
'Args must be formatted as: "section arg=val". Got "%s"'
" instead." % " ".join(arg_list))
section = arg_list[0]
remainder = " ".join(arg_list[1:])
eq_split = remainder.split("=")
newarg = eq_split[0].strip()
value = ("=".join(eq_split[1:])).strip()
logging.debug('Setting config section "%s" arg "%s" = "%s"',
section, newarg, value)
try:
bcp.set(section, newarg, value)
except NoSectionError:
logging.error(
'Invalid section "%s" specified. Must be one of %s',
section,
bcp.sections(),
)
raise
# Instantiate the pipeline
pipeline = Pipeline(bcp) # pylint: disable=redefined-outer-name
if args.select is not None:
pipeline.select_params(args.select, error_on_missing=True)
if args.only_stage is None:
stop_idx = args.stop_after_stage
try:
stop_idx = int(stop_idx)
except (TypeError, ValueError):
pass
if isinstance(stop_idx, str):
stop_idx = pipeline.index(stop_idx)
outputs = pipeline.get_outputs(idx=stop_idx) # pylint: disable=redefined-outer-name
if stop_idx is not None:
stop_idx += 1
indices = slice(0, stop_idx)
else:
assert args.stop_after_stage is None
idx = pipeline.index(args.only_stage)
stage = pipeline[idx]
indices = slice(idx, idx + 1)
# Create dummy inputs if necessary
inputs = None
if hasattr(stage, "input_binning"):
logging.warning(
"Stage requires input, so building dummy"
" inputs of random numbers, with random state set to the input"
" index according to alphabetical ordering of input names and"
" filled in alphabetical ordering of dimension names.")
input_maps = []
tmp = deepcopy(stage.input_binning)
alphabetical_binning = tmp.reorder_dimensions(sorted(tmp.names))
for input_num, input_name in enumerate(sorted(stage.input_names)):
# Create a new map with all 3's; name according to the input
hist = np.full(shape=alphabetical_binning.shape,
fill_value=3.0)
input_map = Map(name=input_name,
binning=alphabetical_binning,
hist=hist)
# Apply Poisson fluctuations to randomize the values in the map
input_map.fluctuate(method="poisson", random_state=input_num)
# Reorder dimensions according to user's original binning spec
input_map.reorder_dimensions(stage.input_binning)
input_maps.append(input_map)
inputs = MapSet(maps=input_maps, name="ones", hash=1)
outputs = stage.run(inputs=inputs)
for stage in pipeline[indices]:
if not args.outdir:
break
stg_svc = stage.stage_name + "__" + stage.service_name
fbase = os.path.join(args.outdir, stg_svc)
if args.intermediate or stage == pipeline[indices][-1]:
stage.outputs.to_json(fbase + "__output.json.bz2")
# also only plot if args intermediate or last stage
if args.intermediate or stage == pipeline[indices][-1]:
formats = OrderedDict(png=args.png, pdf=args.pdf)
if isinstance(stage.outputs, Data):
# TODO(shivesh): plots made here will use the most recent
# "pisa_weight" column and so all stages will have identical plots
# (one workaround is to turn on "memcache_deepcopy")
# TODO(shivesh): intermediate stages have no output binning
if stage.output_binning is None:
logging.debug("Skipping plot of intermediate stage %s",
stage)
continue
outputs = stage.outputs.histogram_set(
binning=stage.output_binning,
nu_weights_col="pisa_weight",
mu_weights_col="pisa_weight",
noise_weights_col="pisa_weight",
mapset_name=stg_svc,
errors=True,
)
try:
for fmt, enabled in formats.items():
if not enabled:
continue
my_plotter = Plotter(
stamp="Event rate",
outdir=args.outdir,
fmt=fmt,
log=False,
annotate=args.annotate,
)
my_plotter.ratio = True
my_plotter.plot_2d_array(outputs,
fname=stg_svc + "__output",
cmap="RdBu")
except ValueError as exc:
logging.error(
"Failed to save plot to format %s. See exception"
" message below",
fmt,
)
traceback.format_exc()
logging.exception(exc)
logging.warning("I can't go on, I'll go on.")
if return_outputs:
return pipeline, outputs
def main():
args = parse_args()
set_verbosity(args.v)
if args.plot:
import matplotlib as mpl
mpl.use('pdf')
import matplotlib.pyplot as plt
from pisa.utils.plotter import Plotter
cfg = from_file(args.fit_settings)
sys_list = cfg.get('general', 'sys_list').replace(' ', '').split(',')
stop_idx = cfg.getint('general', 'stop_after_stage')
for sys in sys_list:
# Parse info for given systematic
nominal = cfg.getfloat(sys, 'nominal')
degree = cfg.getint(sys, 'degree')
force_through_nominal = cfg.getboolean(sys, 'force_through_nominal')
runs = eval(cfg.get(sys, 'runs'))
#print "runs ", runs
smooth = cfg.get(sys, 'smooth')
x_values = np.array(sorted(runs))
# Build fit function
if force_through_nominal:
function = "lambda x, *p: np.polynomial.polynomial.polyval(x, [1.] + list(p))"
else:
function = "lambda x, *p: np.polynomial.polynomial.polyval(x, list(p))"
# Add free parameter for constant term
degree += 1
fit_fun = eval(function)
# Instantiate template maker
template_maker = Pipeline(args.template_settings)
if not args.set_param == '':
for one_set_param in args.set_param:
p_name, value = one_set_param.split("=")
#print "p_name,value= ", p_name, " ", value
value = parse_quantity(value)
value = value.n * value.units
param = template_maker.params[p_name]
#print "old ", p_name, "value = ", param.value
param.value = value
#print "new ", p_name, "value = ", param.value
template_maker.update_params(param)
inputs = {}
map_names = None
# Get sys templates
for run in runs:
for key, val in cfg.items('%s:%s'%(sys, run)):
if key.startswith('param.'):
_, pname = key.split('.')
param = template_maker.params[pname]
try:
value = parse_quantity(val)
param.value = value.n * value.units
except ValueError:
value = parse_string_literal(val)
param.value = value
param.set_nominal_to_current_value()
template_maker.update_params(param)
# Retreive maps
template = template_maker.get_outputs(idx=stop_idx)
if map_names is None: map_names = [m.name for m in template]
inputs[run] = {}
for m in template:
inputs[run][m.name] = m.hist
# Numpy acrobatics:
arrays = {}
for name in map_names:
arrays[name] = []
for x in x_values:
arrays[name].append(
inputs[x][name] / unp.nominal_values(inputs[nominal][name])
)
a = np.array(arrays[name])
arrays[name] = np.rollaxis(a, 0, len(a.shape))
# Shift to get deltas
x_values -= nominal
# Binning object (assuming they're all the same)
binning = template.maps[0].binning
shape = [d.num_bins for d in binning] + [degree]
shape_small = [d.num_bins for d in binning]
outputs = {}
errors = {}
for name in map_names:
# Now actualy perform some fits
outputs[name] = np.ones(shape)
errors[name] = np.ones(shape)
for idx in np.ndindex(*shape_small):
y_values = unp.nominal_values(arrays[name][idx])
y_sigma = unp.std_devs(arrays[name][idx])
if np.any(y_sigma):
popt, pcov = curve_fit(fit_fun, x_values, y_values,
sigma=y_sigma, p0=np.ones(degree))
else:
popt, pcov = curve_fit(fit_fun, x_values, y_values,
p0=np.ones(degree))
perr = np.sqrt(np.diag(pcov))
for k, p in enumerate(popt):
outputs[name][idx][k] = p
errors[name][idx][k] = perr[k]
# TODO(philippeller): the below block of code will fail
# Maybe plot
#if args.plot:
# fig_num = i + nx * j
# if fig_num == 0:
# fig = plt.figure(num=1, figsize=( 4*nx, 4*ny))
# subplot_idx = nx*(ny-1-j)+ i + 1
# plt.subplot(ny, nx, subplot_idx)
# #plt.snameter(x_values, y_values, color=plt_colors[name])
# plt.gca().errorbar(x_values, y_values, yerr=y_sigma,
# fmt='o', color=plt_colors[name],
# ecolor=plt_colors[name],
# mec=plt_colors[name])
# # Plot nominal point again in black
# plt.snameter([0.0], [1.0], color='k')
# f_values = fit_fun(x_values, *popt)
# fun_plot, = plt.plot(x_values, f_values,
# color=plt_colors[name])
# plt.ylim(np.min(unp.nominal_values(arrays[name]))*0.9,
# np.max(unp.nominal_values(arrays[name]))*1.1)
# if i > 0:
# plt.setp(plt.gca().get_yticklabels(), visible=False)
# if j > 0:
# plt.setp(plt.gca().get_xticklabels(), visible=False)
if smooth == 'gauss':
for name in map_names:
for d in range(degree):
outputs[name][...,d] = gaussian_filter(outputs[name][...,d],sigma=1)
if smooth == 'gauss_pid':
for name in map_names:
split_idx = binning.names.index('pid')
tot = len(binning)-1
for d in range(degree):
for p in range(len(binning['pid'])):
outputs[name][...,p,d] = gaussian_filter(
np.swapaxes(outputs[name], split_idx, tot)[...,p,d],
sigma=1
)
outputs[name] = np.swapaxes(outputs[name], split_idx, tot)
# Save the raw ones anyway
outputs['pname'] = sys
outputs['nominal'] = nominal
outputs['function'] = function
outputs['map_names'] = map_names
outputs['binning_hash'] = binning.hash
to_file(outputs, '%s/%s_sysfits_%s_%s.json'%(args.out_dir, sys,
args.tag, smooth))
if args.plot:
for d in range(degree):
maps = []
for name in map_names:
maps.append(Map(name='%s_raw'%name, hist=outputs[name][...,d],
binning=binning))
maps = MapSet(maps)
my_plotter = Plotter(
stamp='',
outdir=args.out_dir,
fmt='pdf',
log=False,
label=''
)
my_plotter.plot_2d_array(
maps,
fname='%s_%s_%s_%s'%(sys, args.tag, d, smooth),
)
def test_kde_stash(verbosity=Levels.WARN):
"""Unit test for the hist stashing feature.
Hist stashing can greatly speed up fits as long as the only free parameters
are in stages that work on the output histograms, rather than the individual
events. In particular, it should be strictly equivalent to either scale all weights
by a factor and then running the KDE, or to first calculate the KDE and then scale
all the bin counts by the same factor. This test ensures that the order of
operation really doesn't matter.
This should apply also to the errors, independent of whether the bootstrapping
method or the utils.set_variance stage was used to produce them.
"""
import pytest
from numpy.testing import assert_array_equal, assert_allclose
set_verbosity(verbosity)
def assert_correct_scaling(pipeline_cfg, fixed_errors=False):
"""Run the pipeline and assert that scaling by a factor of two is correct."""
dmaker = DistributionMaker([pipeline_cfg])
out = dmaker.get_outputs(return_sum="true")[0]
dmaker.pipelines[0].params.weight_scale = 2.0
out2 = dmaker.get_outputs(return_sum="true")[0]
if fixed_errors:
# this is special: We expect that the nominal counts are multiplied, but
# that hte errors stay fixed (applies to set_variance errors)
assert_array_equal(out.nominal_values * 2.0, out2.nominal_values)
assert_array_equal(out.std_devs, out2.std_devs)
else:
assert out * 2.0 == out2
## KDE without errors
# First aeff, then KDE
test_cfg = deepcopy(TEST_CONFIGS.pipe_cfg)
test_cfg[("data", "toy_event_generator")] = deepcopy(
TEST_CONFIGS.event_generator_cfg
)
test_cfg[("aeff", "weight")] = deepcopy(TEST_CONFIGS.aeff_cfg)
test_cfg[("utils", "kde")] = deepcopy(TEST_CONFIGS.kde_cfg)
assert_correct_scaling(test_cfg)
# First KDE, then aeff, with stashing
test_cfg = deepcopy(TEST_CONFIGS.pipe_cfg)
test_cfg[("data", "toy_event_generator")] = deepcopy(
TEST_CONFIGS.event_generator_cfg
)
test_cfg[("utils", "kde")] = deepcopy(TEST_CONFIGS.kde_cfg)
test_cfg[("aeff", "weight")] = deepcopy(TEST_CONFIGS.aeff_cfg)
# turn on stashing
test_cfg[("utils", "kde")]["stash_hists"] = True
# Change aeff calculation to binned mode (i.e. multiply bin counts)
test_cfg[("aeff", "weight")]["calc_mode"] = TEST_BINNING
test_cfg[("aeff", "weight")]["apply_mode"] = TEST_BINNING
assert_correct_scaling(test_cfg)
## KDE with bootstrap errors
# First aeff, then KDE with bootstrap
test_cfg = deepcopy(TEST_CONFIGS.pipe_cfg)
test_cfg[("data", "toy_event_generator")] = deepcopy(
TEST_CONFIGS.event_generator_cfg
)
test_cfg[("aeff", "weight")] = deepcopy(TEST_CONFIGS.aeff_cfg)
test_cfg[("utils", "kde")] = deepcopy(TEST_CONFIGS.kde_cfg)
# turn OFF stashing
test_cfg[("utils", "kde")]["stash_hists"] = False
# turn on bootstrapping
test_cfg[("utils", "kde")]["bootstrap"] = True
# return the errors
test_cfg["pipeline"]["output_key"] = ("weights", "errors")
assert_correct_scaling(test_cfg)
# First KDE with stashed hists and bootstrap, then aeff
test_cfg = deepcopy(TEST_CONFIGS.pipe_cfg)
test_cfg[("data", "toy_event_generator")] = deepcopy(
TEST_CONFIGS.event_generator_cfg
)
test_cfg[("utils", "kde")] = deepcopy(TEST_CONFIGS.kde_cfg)
test_cfg[("aeff", "weight")] = deepcopy(TEST_CONFIGS.aeff_cfg)
# turn on stashing
test_cfg[("utils", "kde")]["stash_hists"] = True
# turn on bootstrapping
test_cfg[("utils", "kde")]["bootstrap"] = True
# return the errors
test_cfg["pipeline"]["output_key"] = ("weights", "errors")
# need to change mode to binned
test_cfg[("aeff", "weight")]["calc_mode"] = TEST_BINNING
test_cfg[("aeff", "weight")]["apply_mode"] = TEST_BINNING
assert_correct_scaling(test_cfg)
## KDE with errors calculated using set_variance stage
# first aeff, then KDE and set_variance
test_cfg = deepcopy(TEST_CONFIGS.pipe_cfg)
test_cfg[("data", "toy_event_generator")] = deepcopy(
TEST_CONFIGS.event_generator_cfg
)
test_cfg[("aeff", "weight")] = deepcopy(TEST_CONFIGS.aeff_cfg)
test_cfg[("utils", "kde")] = deepcopy(TEST_CONFIGS.kde_cfg)
test_cfg[("utils", "set_variance")] = deepcopy(TEST_CONFIGS.set_variance_cfg)
# turn on stashing
test_cfg[("utils", "kde")]["stash_hists"] = False
# turn OFF bootstrapping
test_cfg[("utils", "kde")]["bootstrap"] = False
# return the errors
test_cfg["pipeline"]["output_key"] = ("weights", "errors")
# The set_variance stage only calculates errors the first time that the pipeline
# is evaluated, these errors are stored and re-instated on any sub-sequent
# evaluations. We expect therefore that only the nominal values scale.
assert_correct_scaling(test_cfg, fixed_errors=True)
# first KDE and set_variance, then aeff
test_cfg = deepcopy(TEST_CONFIGS.pipe_cfg)
test_cfg[("data", "toy_event_generator")] = deepcopy(
TEST_CONFIGS.event_generator_cfg
)
test_cfg[("utils", "kde")] = deepcopy(TEST_CONFIGS.kde_cfg)
test_cfg[("aeff", "weight")] = deepcopy(TEST_CONFIGS.aeff_cfg)
# It is still important that the `set_variance` stage is *last*.
test_cfg[("utils", "set_variance")] = deepcopy(TEST_CONFIGS.set_variance_cfg)
# turn on stashing
test_cfg[("utils", "kde")]["stash_hists"] = True
# turn OFF bootstrapping
test_cfg[("utils", "kde")]["bootstrap"] = False
# return the errors
test_cfg["pipeline"]["output_key"] = ("weights", "errors")
# need to change mode to binned
test_cfg[("aeff", "weight")]["calc_mode"] = TEST_BINNING
test_cfg[("aeff", "weight")]["apply_mode"] = TEST_BINNING
# We ensure that the behavior is the same as it has been when we were not stashing
# the histograms and used set_variance.
assert_correct_scaling(test_cfg, fixed_errors=True)
# Using the wrong order (not putting set_variance last)
test_cfg = deepcopy(TEST_CONFIGS.pipe_cfg)
test_cfg[("data", "toy_event_generator")] = deepcopy(
TEST_CONFIGS.event_generator_cfg
)
test_cfg[("utils", "kde")] = deepcopy(TEST_CONFIGS.kde_cfg)
# If set_variance is not the last stage, this breaks. The reason is a slightly
# silly design of set_variance. It should have been constructed such that the
# total normalization is always divided out, but it wasn't. The way it is
# constructed now, it is basically tuned by the scaling factor to work for the
# given livetime and breaks immediately when that changes.
test_cfg[("utils", "set_variance")] = deepcopy(TEST_CONFIGS.set_variance_cfg)
test_cfg[("aeff", "weight")] = deepcopy(TEST_CONFIGS.aeff_cfg)
# turn on stashing
test_cfg[("utils", "kde")]["stash_hists"] = True
# turn OFF bootstrapping
test_cfg[("utils", "kde")]["bootstrap"] = False
# return the errors
test_cfg["pipeline"]["output_key"] = ("weights", "errors")
# need to change mode to binned
test_cfg[("aeff", "weight")]["calc_mode"] = TEST_BINNING
test_cfg[("aeff", "weight")]["apply_mode"] = TEST_BINNING
# With the wrong order, this will fail.
# FIXME: If someone changes the behavior of set_variance in the future to be
# more robust, they are welcome to change this unit test.
with pytest.raises(AssertionError):
assert_correct_scaling(test_cfg, fixed_errors=True)
logging.info("<< PASS : kde_stash >>")
help="Save all the templates used to obtain the gradients.")
sselect.add_argument('-n','--no-save-templates', action='store_false',
default=False, dest='save_templates',
help="Do not save the templates for the different test points.")
parser.add_argument('-o','--outdir',type=str,default=os.getcwd(),metavar='DIR',
help="Output directory")
parser.add_argument('-v', '--verbose', action='count', default=None,
help='set verbosity level')
args = parser.parse_args()
# Set verbosity level
set_verbosity(args.verbose)
# Read the template settings
template_settings = from_json(args.template_settings)
# This file only contains the number of test points for each parameter (and perhaps eventually a non-linearity criterion)
grid_settings = from_json(args.grid_settings)
# Get the Fisher matrices for the desired hierarchy and fiducial settings
fisher_matrices = get_fisher_matrices(template_settings,grid_settings,args.inverted_truth,args.normal_truth,
args.dump_all_stages,args.save_templates,args.outdir)
# Fisher matrices are saved in any case
for data_tag in fisher_matrices:
fisher_basename = 'fisher_data_%s'%data_tag
#
# pisa tools and objects
#
from pisa.core.binning import OneDimBinning, MultiDimBinning
from pisa.core.map import Map, MapSet
from pisa.core.distribution_maker import DistributionMaker
from pisa.utils.config_parser import parse_pipeline_config
from pisa.core.param import Param, ParamSet
from pisa.analysis.analysis import Analysis
# debug tools
from pisa.utils.log import logging
from pisa.utils.profiler import line_profile
from pisa.utils.log import set_verbosity, Levels
set_verbosity(Levels.TRACE)
##################################################################################
STANDARD_CONFIG = os.environ['PISA'] + \
'/pisa/stages/data/super_simple_pipeline.cfg'
TRUE_MU = 20.
TRUE_SIGMA = 3.1
NBINS = 51
################################################################################
class ToyMCllhParam:
'''
Class defining the parameters of the Toy MC
def test_kde_histogramdd():
"""Unit tests for kde_histogramdd"""
from argparse import ArgumentParser
from shutil import rmtree
from tempfile import mkdtemp
from pisa import ureg
from pisa.core.map import Map, MapSet
from pisa.utils.log import logging, set_verbosity
from pisa.utils.plotter import Plotter
parser = ArgumentParser()
parser.add_argument("-v",
action="count",
default=None,
help="set verbosity level")
args = parser.parse_args()
set_verbosity(args.v)
temp_dir = mkdtemp()
try:
my_plotter = Plotter(
stamp="",
outdir=temp_dir,
fmt="pdf",
log=False,
annotate=False,
symmetric=False,
ratio=True,
)
b1 = OneDimBinning(name="coszen",
num_bins=20,
is_lin=True,
domain=[-1, 1],
tex=r"\cos(\theta)")
b2 = OneDimBinning(name="energy",
num_bins=10,
is_log=True,
domain=[1, 80] * ureg.GeV,
tex=r"E")
b3 = OneDimBinning(name="pid",
num_bins=2,
bin_edges=[0, 1, 2],
tex=r"pid")
binning = b1 * b2 * b3
# now let's generate some toy data
N = 100000
cz = np.random.normal(1, 1.2, N)
# cut away coszen outside -1, 1
cz = cz[(cz >= -1) & (cz <= 1)]
e = np.random.normal(30, 20, len(cz))
pid = np.random.uniform(0, 2, len(cz))
data = np.array([cz, e, pid]).T
# make numpy histogram for validation
bins = [unp.nominal_values(b.bin_edges) for b in binning]
raw_hist, _ = np.histogramdd(data, bins=bins)
# get KDE'ed histo
hist = kde_histogramdd(
data,
binning,
bw_method="silverman",
coszen_name="coszen",
oversample=10,
use_cuda=True,
stack_pid=True,
)
# put into mapsets and plot
m1 = Map(name="KDE", hist=hist, binning=binning)
m2 = Map(name="raw", hist=raw_hist, binning=binning)
with np.errstate(divide="ignore", invalid="ignore"):
m3 = m2 / m1
m3.name = "hist/KDE"
m3.tex = m3.name
m4 = m1 - m2
m4.name = "KDE - hist"
m4.tex = m4.name
ms = MapSet([m1, m2, m3, m4])
my_plotter.plot_2d_array(ms, fname="test_kde", cmap="summer")
except:
rmtree(temp_dir)
raise
else:
logging.warning("Inspect and manually clean up output(s) saved to %s" %
temp_dir)
def main():
"""main"""
args = parse_args()
kwargs = vars(args)
set_verbosity(kwargs.pop('v'))
test_example_pipelines(**kwargs)
tmp3 = Umat * tmp2
Hmat_sympy = Qrel * tmp3
# subtract constant * id
Hmat_sympy_minus_mumu = Hmat_sympy - Hmat_sympy[1, 1] * eye(3)
Hmat_sympy_minus_mumu[0, 0] = Hmat_sympy_minus_mumu[0, 0] - 1
eps_mat_sympy = Hmat_sympy_minus_mumu
# simplify
eps_mat_sympy_simpl = simplify(eps_mat_sympy)
# evaluate
eps_mat_sympy_eval = eps_mat_sympy_simpl.subs(
[(eps_scale, eps_scale_val), (eps_prime, eps_prime_val),
(phi12, phi12_val), (phi13, phi13_val), (phi23, phi23_val),
(alpha1, alpha1_val), (alpha2, alpha2_val),
(deltansi, deltansi_val)]
)
# real part
eps_mat_sympy_eval_re = re(eps_mat_sympy_eval)
# imaginary part
eps_mat_sympy_eval_im = im(eps_mat_sympy_eval)
# complex numpy array
return (
np.array(eps_mat_sympy_eval_re) + np.array(eps_mat_sympy_eval_im) * 1.j
).astype(CTYPE)
if __name__ == '__main__':
set_verbosity(Levels.INFO)
test_nsi_params()
test_nsi_parameterization()
choices=['H0', 'scan', 'feldman_cousins'],
default='H0',
help='''just run significance or whole scan''')
parser.add_argument('--range',
type=str,
default='np.linspace(0,2,11)*ureg.dimensionless',
help=''' scanning range''')
parser.add_argument('-f',
'--function',
type=str,
choices=['profile', 'fit'],
default='profile',
help='''what shpuld be executed''')
args = parser.parse_args()
set_verbosity(args.v)
if os.path.isfile(args.outfile):
print("Output file ", args.outfile,
" already existed, delete or remove it.")
else:
if args.blind:
assert (args.function == 'fit')
assert (args.pseudo_data == 'data')
if args.data_settings is None:
data_settings = args.template_settings
else:
data_settings = args.data_settings
data_maker = DistributionMaker(data_settings)
'--outfile',
dest='outfile',
metavar='FILE',
type=str,
action='store',
default="event_rate.json",
help='''file to store the output''')
parser.add_argument('-v',
'--verbose',
action='count',
default=None,
help='''set verbosity level''')
args = parser.parse_args()
#Set verbosity level
set_verbosity(args.verbose)
#Check binning
ebins, czbins = check_binning(args.osc_flux_maps)
logging.info("Defining aeff_service...")
if args.mc_mode:
logging.info(" Using effective area from EVENT DATA...")
aeff_service = AeffServiceMC(ebins,
czbins,
aeff_weight_file=args.weighted_aeff_file)
else:
logging.info(" Using effective area from PARAMETRIZATION...")
aeff_settings = from_json(find_resource(args.settings_file))
aeff_service = AeffServicePar(ebins, czbins, **aeff_settings)
def parse_args():
"""Parse command line arguments and return as a dict.
Returns
-------
kwargs
"""
parser = ArgumentParser(formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument(
'--template-settings',
metavar='CONFIGFILE',
required=True,
action='append',
help='''Settings for generating template distributions; repeat
this option to define multiple pipelines.''')
parser.add_argument(
'--steps',
type=int,
required=True,
help='''Provide a number of steps to scan the likelihood space over.'''
)
parser.add_argument(
'--hypo-param-selections',
type=str,
nargs='+',
required=False,
help='''Selection of params to use in order to generate the
hypothesised Asimov distributions.''')
parser.add_argument('--outdir',
required=True,
metavar='DIR',
type=str,
help='Directory into which to store results.')
parser.add_argument(
'--minimizer-settings',
type=str,
metavar='JSONFILE',
required=True,
help='''Settings related to the minimizer used in the LLR analysis.''')
parser.add_argument(
'--metric',
type=str,
choices=['llh', 'chi2', 'conv_llh', 'mod_chi2'],
required=True,
help='''Settings related to the minimizer used in the LLR analysis.''')
parser.add_argument(
'--debug-mode',
type=int,
choices=[0, 1, 2],
required=False,
default=1,
help='''How much information to keep in the output file. 0 for only
essentials for a physics analysis, 1 for more minimizer history, 2 for
whatever can be recorded.''')
parser.add_argument('-v',
action='count',
default=None,
help='set verbosity level')
kwargs = vars(parser.parse_args())
set_verbosity(kwargs.pop('v'))
return kwargs
B = numba.cuda.to_device(np.zeros((3, 3), dtype=FTYPE))
copy_matrix_guf(A, B)
test = B.copy_to_host()
ref = A.copy_to_host()
assert np.array_equal(test, ref), f"test:\n{test}\n!= ref:\n{ref}"
logging.info("<< PASS : test_copy_matrix >>")
# --------------------------------------------------------------------------- #
def parse_args():
"""parse command line args"""
parser = ArgumentParser()
parser.add_argument("-v", action="count", default=Levels.WARN, help="Verbosity")
args = parser.parse_args()
return vars(args)
if __name__ == "__main__":
set_verbosity(parse_args()["v"])
test_conjugate_transpose()
test_conjugate()
test_matrix_dot_matrix()
test_matrix_dot_vector()
test_clear_matrix()
test_copy_matrix()
'"%s", max. abs. fract. diff.: %s' %
(len(mus), impl, np.max(np.abs((test_unw / ref_unw - 1)))))
if not recursiveEquality(test_w, ref_w):
err_msgs.append(
'BAD RESULT (weighted), n_gaus=%d, implementation="%s"'
', max. abs. fract. diff.: %s' %
(len(mus), impl, np.max(np.abs((test_w / ref_w - 1)))))
if err_msgs:
for err_msg in err_msgs:
logging.error(err_msg)
raise ValueError('\n'.join(err_msgs))
tprofile.debug(
'gaussians() timings (unweighted) (Note:OMP_NUM_THREADS=%d; evaluated'
' at %.0e points)', OMP_NUM_THREADS, n_eval)
timings_str = ' '.join([format(t, '10d') for t in n_gaus])
tprofile.debug(' ' * 30 + 'Number of gaussians'.center(59))
tprofile.debug(' %15s %s', 'impl.', timings_str)
timings_str = ' '.join(['-' * 10 for t in n_gaus])
tprofile.debug(' %15s %s', '-' * 15, timings_str)
for impl in GAUS_IMPLEMENTATIONS:
# only report timings for unweighted case
timings_str = ' '.join([format(t[0], '10.3f') for t in timings[impl]])
tprofile.debug('Timings, %15s (ms): %s', impl, timings_str)
logging.info('<< PASS : test_gaussians >>')
if __name__ == "__main__":
set_verbosity(2)
test_gaussians()
