def add_fit(self, data, **kwargs):
"""
Add an individual FitResult. By default, the part of the fit that
contributed to the fitting is drawn solid, the remaining range
is dashed. Note that it is not possible
to add the same data twice, instead it will be redrawn with
the new arguments/style options provided.
Parameters
----------
data : FitResult
Added to the list of plotted elements.
kwargs
Keyword arguments passed to
:obj:`matplotlib.axes.Axes.plot`. Use to customise the
plots. If a `label` is set via `kwargs`, it will be added
as a note in the meta data. If `linestyle` is set, the
dashed plot of the region not contributing to the fit is
omitted.
"""
if not isinstance(data, FitResult):
log.exception("'data' needs to be of type FitResult")
raise ValueError
if not (self.type is None or self.type == 'correlation'):
log.exception(
"It is not possible to 'add_fit()' to " +
"an OutputHandler containing a time series\n" +
"\tHave you previously called 'add_ts()' on this handler?")
raise ValueError
self.type = 'correlation'
if self.xdata is None:
self.dt = data.dt
self.dtunit = data.dtunit
self.ax.set_xlabel('k [{}{}]'.format(data.dt, data.dtunit))
self.ax.set_ylabel('$r_{k}$')
self.ax.set_title('Correlation')
inds = self.set_xdata(data.steps, dt=data.dt, dtunit=data.dtunit)
# description for fallback
desc = str(data.desc)
# plot legend label
if 'label' in kwargs:
label = kwargs.get('label')
if label == '':
label = None
else:
# user wants custom label not intended to hide the legend
label = str(label)
else:
# user has not set anything, copy from desc if set
label = 'Fit ' + ut.math_from_doc(data.fitfunc, 0)
if desc != '':
label = desc + ' ' + label
# we dont support adding duplicates
oldcurves = []
if data in self.fits:
indfit = self.fits.index(data)
log.warning(
'Fit was already added ({})\n'.format(self.fitlabels[indfit]) +
'\tOverwriting with new style')
del self.fits[indfit]
del self.fitlabels[indfit]
oldcurves = self.fitcurves[indfit]
del self.fitcurves[indfit]
del self.fitkwargs[indfit]
self.fits.append(data)
self.fitlabels.append(label)
self.fitcurves.append(oldcurves)
self.fitkwargs.append(kwargs)
# refresh coefficients
for r in self.rks:
self._render_coefficients(r)
# refresh fits
for f in self.fits:
self._render_fit(f)
def overview(src, rks, fits, **kwargs):
"""
creates an A4 overview panel and returns the matplotlib figure element.
No Argument checks are done
"""
# ratios = np.ones(4)*.75
# ratios[3] = 0.25
ratios = None
# A4 in inches, should check rc params in the future
# matplotlib changes the figure size when modifying subplots
topshift = 0.925
fig, axes = plt.subplots(nrows=4,
figsize=(8.27, 11.69 * topshift),
gridspec_kw={"height_ratios": ratios})
# avoid huge file size for many trials due to separate layers.
# everything below 0 gets rastered to the same layer.
axes[0].set_rasterization_zorder(0)
# ------------------------------------------------------------------ #
# Time Series
# ------------------------------------------------------------------ #
tsout = OutputHandler(ax=axes[0])
tsout.add_ts(src, label='Trials')
if (src.shape[0] > 1):
try:
prevclr = plt.rcParams["axes.prop_cycle"].by_key()["color"][0]
except Exception:
prevclr = 'navy'
log.debug('Exception getting color cycle', exc_info=True)
tsout.add_ts(np.mean(src, axis=0), color=prevclr, label='Average')
else:
tsout.ax.legend().set_visible(False)
tsout.ax.set_title('Time Series (Input Data)')
tsout.ax.set_xlabel('t [{}{}]'.format(ut._printeger(rks[0].dt),
rks[0].dtunit))
# ------------------------------------------------------------------ #
# Mean Trial Activity
# ------------------------------------------------------------------ #
if (src.shape[0] > 1):
# average trial activites as function of trial number
taout = OutputHandler(rks[0].trialactivities, ax=axes[1])
try:
err1 = rks[0].trialactivities - np.sqrt(rks[0].trialvariances)
err2 = rks[0].trialactivities + np.sqrt(rks[0].trialvariances)
prevclr = plt.rcParams["axes.prop_cycle"].by_key()["color"][0]
taout.ax.fill_between(np.arange(1, rks[0].numtrials + 1),
err1,
err2,
color=prevclr,
alpha=0.2)
except Exception as e:
log.debug('Exception adding std deviation to plot', exc_info=True)
taout.ax.set_title('Mean Trial Activity and Std. Deviation')
taout.ax.set_xlabel('Trial i')
taout.ax.set_ylabel('$\\bar{A}_i$')
else:
# running average over the one trial to see if stays stationary
numsegs = kwargs.get(numsegs) if 'numsegs' in kwargs else 50
ravg = np.zeros(numsegs)
err1 = np.zeros(numsegs)
err2 = np.zeros(numsegs)
seglen = int(src.shape[1] / numsegs)
for s in range(numsegs):
temp = np.mean(src[0][s * seglen:(s + 1) * seglen])
ravg[s] = temp
stddev = np.sqrt(np.var(src[0][s * seglen:(s + 1) * seglen]))
err1[s] = temp - stddev
err2[s] = temp + stddev
taout = OutputHandler(ravg, ax=axes[1])
try:
prevclr = plt.rcParams["axes.prop_cycle"].by_key()["color"][0]
taout.ax.fill_between(np.arange(1, numsegs + 1),
err1,
err2,
color=prevclr,
alpha=0.2)
except Exception as e:
log.debug('Exception adding std deviation to plot', exc_info=True)
taout.ax.set_title(
'Average Activity and Stddev for {} Intervals'.format(numsegs))
taout.ax.set_xlabel('Interval i')
taout.ax.set_ylabel('$\\bar{A}_i$')
# ------------------------------------------------------------------ #
# Coefficients and Fit results
# ------------------------------------------------------------------ #
cout = OutputHandler(rks + fits, ax=axes[2])
fitcurves = []
fitlabels = []
for i, f in enumerate(cout.fits):
fitcurves.append(cout.fitcurves[i][0])
label = ut.math_from_doc(f.fitfunc, 5)
label += '\n\n$\\tau={:.2f}${}\n'.format(f.tau, f.dtunit)
if f.tauquantiles is not None:
label += '$[{:.2f}:{:.2f}]$\n\n' \
.format(f.tauquantiles[0], f.tauquantiles[-1])
else:
label += '\n\n'
label += '$m={:.5f}$\n'.format(f.mre)
if f.mrequantiles is not None:
label +='$[{:.5f}:{:.5f}]$' \
.format(f.mrequantiles[0], f.mrequantiles[-1])
else:
label += '\n'
fitlabels.append(label)
tempkwargs = {
# 'title': 'Fitresults',
'ncol': len(fitlabels),
'loc': 'upper center',
'mode': 'expand',
'frameon': True,
'markerfirst': True,
'fancybox': False,
# 'framealpha': 1,
'borderaxespad': 0,
'edgecolor': 'black',
# hide handles
'handlelength': 0,
'handletextpad': 0,
}
try:
axes[3].legend(fitcurves, fitlabels, **tempkwargs)
except Exception:
log.debug('Exception passed', exc_info=True)
del tempkwargs['edgecolor']
axes[3].legend(fitcurves, fitlabels, **tempkwargs)
# hide handles
for handle in axes[3].get_legend().legendHandles:
handle.set_visible(False)
# center text
for t in axes[3].get_legend().texts:
t.set_multialignment('center')
# apply stile and fill legend
axes[3].get_legend().get_frame().set_linewidth(0.5)
axes[3].axis('off')
axes[3].set_title('Fitresults\n[$12.5\\%$:$87.5\\%$]')
for a in axes:
a.xaxis.set_tick_params(width=0.5)
a.yaxis.set_tick_params(width=0.5)
for s in a.spines:
a.spines[s].set_linewidth(0.5)
fig.tight_layout(h_pad=2.0)
plt.subplots_adjust(top=topshift)
title = kwargs.get('title') if 'title' in kwargs else None
if (title is not None and title != ''):
fig.suptitle(title + '\n', fontsize=14)
if 'warning' in kwargs and kwargs.get('warning') is not None:
s = u'\u26A0 {}'.format(kwargs.get('warning'))
fig.text(.5,
.01,
s,
fontsize=13,
horizontalalignment='center',
color='red')
fig.text(.995,
.005,
'v{}'.format(__version__),
fontsize=8,
horizontalalignment='right',
color='silver')
return fig
def save_meta(self, fname=''):
"""
Saves only the details/source used to create the plot. It is
recommended to call this manually, if you decide to save
the plots yourself or when you want only the fit results.
Parameters
----------
fname : str, optional
Path where to save, without file extension. Defaults to "./mre"
"""
if not isinstance(fname, str): fname = str(fname)
if fname == '': fname = './mre'
# try creating enclosing dir if not existing
tempdir = os.path.abspath(os.path.expanduser(fname + "/../"))
os.makedirs(tempdir, exist_ok=True)
fname = os.path.expanduser(fname)
log.info('Saving meta to {}.tsv'.format(fname))
# fits
hdr = 'Mr. Estimator v{}\n'.format(__version__)
try:
for fdx, fit in enumerate(self.fits):
hdr += '{}\n'.format('-' * 72)
hdr += 'legendlabel: ' + str(self.fitlabels[fdx]) + '\n'
hdr += '{}\n'.format('-' * 72)
if fit.desc != '':
hdr += 'description: ' + str(fit.desc) + '\n'
hdr += 'm = {}\ntau = {} [{}]\n' \
.format(fit.mre, fit.tau, fit.dtunit)
if fit.quantiles is not None:
hdr += 'quantiles | tau [{}] | m:\n'.format(fit.dtunit)
for i, q in enumerate(fit.quantiles):
hdr += '{:6.3f} | '.format(fit.quantiles[i])
hdr += '{:8.3f} | '.format(fit.tauquantiles[i])
hdr += '{:8.8f}\n'.format(fit.mrequantiles[i])
hdr += '\n'
hdr += 'fitrange: {} <= k <= {} [{}{}]\n'.format(
fit.steps[0], fit.steps[-1], ut._printeger(fit.dt),
fit.dtunit)
hdr += 'function: ' + ut.math_from_doc(fit.fitfunc) + '\n'
# hdr += '\twith parameters:\n'
parname = list(inspect.signature(fit.fitfunc).parameters)[1:]
parlen = len(max(parname, key=len))
for pdx, par in enumerate(self.fits[fdx].popt):
unit = ''
if parname[pdx] == 'nu':
unit += '[1/{}]'.format(fit.dtunit)
elif parname[pdx].find('tau') != -1:
unit += '[{}]'.format(fit.dtunit)
hdr += '\t{: <{width}}'.format(parname[pdx] + ' ' + unit,
width=parlen + 5 +
len(fit.dtunit))
hdr += ' = {}\n'.format(par)
hdr += '\n'
except Exception as e:
log.debug('Exception passed', exc_info=True)
# rks / ts
labels = ''
dat = []
if self.ydata is not None and len(self.ydata) != 0:
hdr += '{}\n'.format('-' * 72)
hdr += 'Data\n'
hdr += '{}\n'.format('-' * 72)
labels += '1_' + self.xlabel
for ldx, label in enumerate(self.ylabels):
labels += '\t' + str(ldx + 2) + '_' + label
labels = labels.replace(' ', '_')
dat = np.vstack((self.xdata, np.asarray(self.ydata)))
np.savetxt(fname + '.tsv',
np.transpose(dat),
delimiter='\t',
header=hdr + labels)
def overview(src, rks, fits, **kwargs):
"""
creates an A4 overview panel and returns the matplotlib figure element.
No Argument checks are done
"""
ratios = np.ones(5)
ratios[4] = 0.0001
# ratios=None
# A5 in inches, should check rc params in the future
# matplotlib changes the figure size when modifying subplots
fig, axes = plt.subplots(nrows=5,
figsize=(5.8, 8.3),
gridspec_kw={"height_ratios": ratios})
# avoid huge file size for many trials due to separate layers.
# everything below 0 gets rastered to the same layer.
axes[0].set_rasterization_zorder(0)
# ------------------------------------------------------------------ #
# Time Series
# ------------------------------------------------------------------ #
tsout = OutputHandler(ax=axes[0])
tsout.add_ts(src, label="Trials")
if src.shape[0] > 1:
try:
prevclr = plt.rcParams["axes.prop_cycle"].by_key()["color"][0]
except Exception:
prevclr = "navy"
log.debug("Exception getting color cycle", exc_info=True)
tsout.add_ts(np.mean(src, axis=0), color=prevclr, label="Average")
else:
tsout.ax.legend().set_visible(False)
tsout.ax.set_title("Time Series", fontweight="bold", loc="center")
tsout.ax.set_title("(Input Data)",
fontsize="medium",
color="#646464",
loc="right")
tsout.ax.set_xlabel("t [{}{}]".format(
ut._printeger(rks[0].dt) + " " if rks[0].dt != 1 else "",
rks[0].dtunit))
# ------------------------------------------------------------------ #
# Mean Trial Activity
# ------------------------------------------------------------------ #
if src.shape[0] > 1:
# average trial activites as function of trial number
taout = OutputHandler(rks[0].trialactivities, ax=axes[1])
try:
err1 = rks[0].trialactivities - np.sqrt(rks[0].trialvariances)
err2 = rks[0].trialactivities + np.sqrt(rks[0].trialvariances)
prevclr = plt.rcParams["axes.prop_cycle"].by_key()["color"][0]
taout.ax.fill_between(np.arange(1, rks[0].numtrials + 1),
err1,
err2,
color=prevclr,
alpha=0.2)
except Exception as e:
log.debug("Exception adding std deviation to plot", exc_info=True)
taout.ax.set_title("Mean Trial Activity and Std. Deviation",
fontweight="bold")
taout.ax.set_xlabel("Trial i")
taout.ax.set_ylabel("$\\bar{A}_i$")
else:
# running average over the one trial to see if stays stationary
numsegs = kwargs.get(numsegs) if "numsegs" in kwargs else 50
ravg = np.zeros(numsegs)
err1 = np.zeros(numsegs)
err2 = np.zeros(numsegs)
seglen = int(src.shape[1] / numsegs)
for s in range(numsegs):
temp = np.mean(src[0][s * seglen:(s + 1) * seglen])
ravg[s] = temp
stddev = np.sqrt(np.var(src[0][s * seglen:(s + 1) * seglen]))
err1[s] = temp - stddev
err2[s] = temp + stddev
taout = OutputHandler(ravg, ax=axes[1])
try:
prevclr = plt.rcParams["axes.prop_cycle"].by_key()["color"][0]
taout.ax.fill_between(np.arange(1, numsegs + 1),
err1,
err2,
color=prevclr,
alpha=0.2)
except Exception as e:
log.debug("Exception adding std deviation to plot", exc_info=True)
taout.ax.set_title(
"Average Activity and Stddev for {} Intervals".format(numsegs),
fontweight="bold",
)
taout.ax.set_xlabel("Interval i")
taout.ax.set_ylabel("$\\bar{A}_i$")
# ------------------------------------------------------------------ #
# Coefficients and Fit results
# ------------------------------------------------------------------ #
cout = OutputHandler(rks + fits, ax=axes[2])
fitcurves = []
fitlabels = []
for i, f in enumerate(cout.fits):
fitcurves.append(cout.fitcurves[i][0])
label = ut.math_from_doc(f.fitfunc, 5)
label += "\n\n$\\tau={:.2f}${}\n".format(f.tau, f.dtunit)
if f.tauquantiles is not None:
label += "$[{:.2f}:{:.2f}]$\n\n".format(f.tauquantiles[0],
f.tauquantiles[-1])
else:
label += "\n\n"
label += "$m={:.5f}$\n".format(f.mre)
if f.mrequantiles is not None:
label += "$[{:.5f}:{:.5f}]$".format(f.mrequantiles[0],
f.mrequantiles[-1])
else:
label += "\n"
fitlabels.append(label)
tempkwargs = {
# 'title': 'Fitresults',
"ncol": len(fitlabels),
"loc": "upper center",
"mode": "expand",
"frameon": True,
"markerfirst": True,
"fancybox": False,
# 'framealpha': 1,
"borderaxespad": 0,
"edgecolor": "black",
# hide handles
"handlelength": 0,
"handletextpad": 0,
}
try:
axes[3].legend(fitcurves, fitlabels, **tempkwargs)
except Exception:
log.debug("Exception passed", exc_info=True)
del tempkwargs["edgecolor"]
axes[3].legend(fitcurves, fitlabels, **tempkwargs)
# hide handles
for handle in axes[3].get_legend().legendHandles:
handle.set_visible(False)
# center text
for t in axes[3].get_legend().texts:
t.set_multialignment("center")
# apply stile and fill legend
axes[3].get_legend().get_frame().set_linewidth(0.5)
axes[3].axis("off")
axes[3].set_title(
"Fitresults",
fontweight="bold",
loc="center",
)
axes[3].set_title(
" (with CI: [$12.5\\%:87.5\\%$])",
color="#646464",
fontsize="medium",
loc="right",
)
for a in axes:
a.xaxis.set_tick_params(width=0.5)
a.yaxis.set_tick_params(width=0.5)
for s in a.spines:
a.spines[s].set_linewidth(0.5)
# dummy axes for version and warnings
axes[4].axis("off")
fig.tight_layout()
plt.subplots_adjust(hspace=0.8, top=0.95, bottom=0.0, left=0.1, right=0.99)
title = kwargs.get("title") if "title" in kwargs else None
if title is not None and title != "":
fig.suptitle(title, fontsize=14)
plt.subplots_adjust(top=0.91)
if "warning" in kwargs and kwargs.get("warning") is not None:
s = "\u26A0 {}".format(kwargs.get("warning"))
fig.text(0.5,
0.01,
s,
fontsize=13,
horizontalalignment="center",
color="red")
fig.text(
0.995,
0.005,
"v{}".format(__version__),
fontsize="small",
horizontalalignment="right",
color="#646464",
)
return fig
def save_meta(self, fname=""):
"""
Saves only the details/source used to create the plot. It is
recommended to call this manually, if you decide to save
the plots yourself or when you want only the fit results.
Parameters
----------
fname : str, optional
Path where to save, without file extension. Defaults to "./mre"
"""
if not isinstance(fname, str):
fname = str(fname)
if fname == "":
fname = "./mre"
# try creating enclosing dir if not existing
tempdir = os.path.abspath(os.path.expanduser(fname + "/../"))
os.makedirs(tempdir, exist_ok=True)
fname = os.path.expanduser(fname)
log.info("Saving meta to {}.tsv".format(fname))
# fits
hdr = "Mr. Estimator v{}\n".format(__version__)
try:
for fdx, fit in enumerate(self.fits):
hdr += "{}\n".format("-" * 72)
hdr += "legendlabel: " + str(self.fitlabels[fdx]) + "\n"
hdr += "{}\n".format("-" * 72)
if fit.desc != "":
hdr += "description: " + str(fit.desc) + "\n"
hdr += "m = {}\ntau = {} [{}]\n".format(
fit.mre, fit.tau, fit.dtunit)
if fit.quantiles is not None:
hdr += "quantiles | tau [{}] | m:\n".format(fit.dtunit)
for i, q in enumerate(fit.quantiles):
hdr += "{:6.3f} | ".format(fit.quantiles[i])
hdr += "{:8.3f} | ".format(fit.tauquantiles[i])
hdr += "{:8.8f}\n".format(fit.mrequantiles[i])
hdr += "\n"
hdr += "fitrange: {} <= k <= {} [{} {}]\n".format(
fit.steps[0], fit.steps[-1], ut._printeger(fit.dt),
fit.dtunit)
hdr += "function: " + ut.math_from_doc(fit.fitfunc) + "\n"
# hdr += '\twith parameters:\n'
parname = list(inspect.signature(fit.fitfunc).parameters)[1:]
parlen = len(max(parname, key=len))
for pdx, par in enumerate(self.fits[fdx].popt):
unit = ""
if parname[pdx] == "nu":
unit += "[1/{}]".format(fit.dtunit)
elif parname[pdx].find("tau") != -1:
unit += "[{}]".format(fit.dtunit)
hdr += "\t{: <{width}}".format(parname[pdx] + " " + unit,
width=parlen + 5 +
len(fit.dtunit))
hdr += " = {}\n".format(par)
hdr += "\n"
except Exception as e:
log.debug("Exception passed", exc_info=True)
# rks / ts
labels = ""
dat = []
if self.ydata is not None and len(self.ydata) != 0:
hdr += "{}\n".format("-" * 72)
hdr += "Data\n"
hdr += "{}\n".format("-" * 72)
labels += "1_" + self.xlabel
for ldx, label in enumerate(self.ylabels):
labels += "\t" + str(ldx + 2) + "_" + label
labels = labels.replace(" ", "_")
dat = np.vstack((self.xdata, np.asarray(self.ydata)))
np.savetxt(fname + ".tsv",
np.transpose(dat),
delimiter="\t",
header=hdr + labels)
def fit(data,
fitfunc=f_exponential_offset,
steps=None,
fitpars=None,
fitbnds=None,
maxfev=None,
ignoreweights=True,
numboot=0,
quantiles=None,
seed=101,
desc=None,
description=None):
"""
Estimate the Multistep Regression Estimator by fitting the provided
correlation coefficients :math:`r_k`. The fit is performed using
:func:`scipy.optimize.curve_fit` and can optionally be provided with
(multiple) starting fitparameters and bounds.
Parameters
----------
data: CoefficientResult or ~numpy.array
Correlation coefficients to fit. Ideally, provide this as
:class:`CoefficientResult` as obtained from
:func:`coefficients`. If arrays are provided,
the function tries to match the data.
fitfunc : callable, optional
The model function, f(x, …).
Directly passed to `curve_fit()`:
It must take the independent variable as
the first argument and the parameters to fit as separate remaining
arguments.
Default is :obj:`f_exponential_offset`.
Other builtin options are :obj:`f_exponential` and
:obj:`f_complex`.
steps : ~numpy.array, optional
Specify the steps :math:`k` for which to fit (think fitrange).
If an array of length two is provided, e.g.
``steps=(minstep, maxstep)``, all enclosed values present in the
provdied `data`, including `minstep` and `maxstep` will be used.
Arrays larger than two are assumed to contain a manual choice of
steps and those that are also present in `data` will be used.
Strides other than one are possible.
Ignored if `data` is not passed as CoefficientResult.
Default: all values given in `data` are included in the fit.
Other Parameters
----------------
fitpars : ~numpy.ndarray, optional
The starting parameters for the fit. If the provided array is two
dimensional, multiple fits are performed and the one with the
smallest sum of squares of residuals is returned.
fitbounds : ~numpy.ndarray, optional
Lower and upper bounds for each parameter handed to the fitting
routine. Provide as numpy array of the form
``[[lowpar1, lowpar2, ...], [uppar1, uppar2, ...]]``
numboot : int, optional
Number of bootstrap samples to compute errors from. Default is 0
seed : int, None or 'random', optional
If `numboot` is not zero, provide a seed for the random number
generator. If ``seed=None``, seeding will be skipped.
Per default, the rng is (re)seeded everytime `fit()` is called so
that every repeated call returns the same error estimates.
quantiles: list, optional
If `numboot` is not zero, provide the quantiles to return
(between 0 and 1). See :obj:`numpy.quantile`.
Defaults are ``[.125, .25, .4, .5, .6, .75, .875]``
maxfev : int, optional
Maximum iterations for the fit.
description : str, optional
Provide a custom description.
Returns
-------
: :class:`FitResult`
The output is grouped and can be accessed
using its attributes (listed below).
"""
# ------------------------------------------------------------------ #
# Check arguments and prepare
# ------------------------------------------------------------------ #
log.debug('fit()')
if (ut._log_locals):
log.debug('Locals: {}'.format(locals()))
fitfunc = fitfunc_check(fitfunc)
# check input data type
if isinstance(data, CoefficientResult):
log.debug('Coefficients given in default format')
src = data
srcerrs = data.stderrs
dt = data.dt
dtunit = data.dtunit
else:
try:
log.info("Given data is no CoefficientResult. Guessing format")
dt = 1
dtunit = 'ms'
srcerrs = None
data = np.asarray(data)
if len(data.shape) == 1:
log.debug('1d array, assuming this to be coefficients')
if steps is not None and len(steps) == len(data):
log.debug("using steps provided in 'steps'")
tempsteps = np.copy(steps)
else:
log.debug("using linear steps starting at 1")
tempsteps = np.arange(1, len(data) + 1)
src = CoefficientResult(coefficients=data, steps=tempsteps)
elif len(data.shape) == 2:
if data.shape[0] > data.shape[1]: data = np.transpose(data)
if data.shape[0] == 1:
log.debug('nested 1d array, assuming coefficients')
if steps is not None and len(steps) == len(data[0]):
log.debug("using steps provided in 'steps'")
tempsteps = np.copy(steps)
else:
log.debug("using steps linear steps starting at 1")
tempsteps = np.arange(1, len(data[0]) + 1)
src = CoefficientResult(coefficients=data[0],
steps=tempsteps)
elif data.shape[0] == 2:
log.debug('2d array, assuming this to be ' +
'steps and coefficients')
tempsteps = data[0]
src = CoefficientResult(coefficients=data[1],
steps=tempsteps)
else:
raise TypeError
except Exception as e:
log.exception('Provided data has no compatible format')
raise
# check that input coefficients do not contain nans or infs
if not np.isfinite(src.coefficients).all():
log.exception(
"Provided coefficients contain elements that are not finite. " +
"Fits would not converge.\n" +
"One can use `np.isfinite(data.coefficients)` to find problematic elements."
)
raise ValueError
# check steps
if steps is None:
steps = (None, None)
try:
steps = np.array(steps)
assert len(steps.shape) == 1
except Exception as e:
log.exception('Please provide steps as ' +
'steps=(minstep, maxstep) or as one dimensional numpy ' +
'array containing all desired step values')
raise ValueError from e
if len(steps) == 2:
minstep = src.steps[0] # default: use what is in the given data
maxstep = src.steps[-1]
if steps[0] is not None:
minstep = steps[0]
if steps[1] is not None:
maxstep = steps[1]
if minstep > maxstep or minstep < 1:
log.debug('minstep={} is invalid, setting to 1'.format(minstep))
minstep = 1
if maxstep > src.steps[-1] or maxstep < minstep:
log.debug('maxstep={} is invalid'.format(maxstep))
maxstep = src.steps[-1]
log.debug('Adjusting maxstep to {}'.format(maxstep))
steps = np.arange(minstep, maxstep + 1, dtype=int)
log.debug('Checking steps between {} and {}'.format(minstep, maxstep))
else:
if (steps < 1).any():
log.exception('All provided steps must be >= 1')
raise ValueError
steps = np.asarray(steps, dtype=int)
log.debug('Using provided custom steps')
# make sure this is data, no pointer, so we dont overwrite anything
stepinds, _ = ut._intersecting_index(src.steps, steps)
srcsteps = np.copy(src.steps[stepinds])
if desc is not None and description is None:
description = str(desc)
if description is None:
try:
# this only works when data is a coefficient result
description = data.description
except Exception as e:
log.debug('Exception passed', exc_info=True)
else:
description = str(description)
# ignoreweights, new default
if ignoreweights:
srcerrs = None
else:
# make sure srcerrs are not all equal and select right indices
try:
srcerrs = srcerrs[stepinds]
if (srcerrs == srcerrs[0]).all():
srcerrs = None
except:
srcerrs = None
if fitfunc not in [f_exponential, f_exponential_offset, f_complex]:
log.info('Custom fitfunction specified {}'.format(fitfunc))
fitpars = fitpars_check(fitpars, fitfunc)
# should implement fitbnds_check(bnds, fitfunc)
if fitbnds is None: fitbnds = default_fitbnds(fitfunc)
# logging this should not cause an actual exception. ugly, needs rework
try:
if fitbnds is None:
bnds = np.array([-np.inf, np.inf])
log.info('Unbound fit to {}'.format(ut.math_from_doc(fitfunc)))
log.debug('kmin = {}, kmax = {}'.format(srcsteps[0], srcsteps[-1]))
ic = list(inspect.signature(fitfunc).parameters)[1:]
ic = ('{} = {:.3f}'.format(a, b) for a, b in zip(ic, fitpars[0]))
log.debug('Starting parameters: ' + ', '.join(ic))
else:
bnds = fitbnds
log.info('Bounded fit to {}'.format(ut.math_from_doc(fitfunc)))
log.debug('kmin = {}, kmax = {}'.format(srcsteps[0], srcsteps[-1]))
ic = list(inspect.signature(fitfunc).parameters)[1:]
ic = ('{0:<6} = {1:8.3f} in ({2:9.4f}, {3:9.4f})'.format(
a, b, c, d) for a, b, c, d in zip(ic, fitpars[0], fitbnds[
0, :], fitbnds[1, :]))
log.debug('First parameters:\n' + '\n'.join(ic))
except Exception as e:
log.debug('Exception when logging fitpars', exc_info=True)
if (fitpars.shape[0] > 1):
log.debug('Repeating fit with {} sets of initial parameters:'.format(
fitpars.shape[0]))
# ------------------------------------------------------------------ #
# Fit via scipy.curve_fit
# ------------------------------------------------------------------ #
# fitpars: 2d ndarray
# fitbnds: matching scipy.curve_fit: [lowerbndslist, upperbndslist]
maxfev = 100 * (len(fitpars[0]) + 1) if maxfev is None else int(maxfev)
def fitloop(ftcoefficients, ftmaxfev, fitlog=True):
ssresmin = np.inf
fulpopt = None
fulpcov = None
if len(fitpars) != 1 and fitlog:
log.info('Fitting with {} different start values'.format(
len(fitpars)))
for idx, pars in enumerate(tqdm(fitpars, disable=(not fitlog))):
try:
popt, pcov = scipy.optimize.curve_fit(fitfunc,
srcsteps * dt,
ftcoefficients,
p0=pars,
bounds=bnds,
maxfev=ftmaxfev,
sigma=srcerrs)
residuals = ftcoefficients - fitfunc(srcsteps * dt, *popt)
ssres = np.sum(residuals**2)
except Exception as e:
ssres = np.inf
popt = None
pcov = None
if fitlog:
log.debug('Fit %d did not converge. Ignoring this fit',
idx + 1)
log.debug('Exception passed', exc_info=True)
if ssres < ssresmin:
ssresmin = ssres
fulpopt = popt
fulpcov = pcov
if fitlog:
pass
# log.info('Finished %d fit(s)', len(fitpars))
return fulpopt, fulpcov, ssresmin
fulpopt, fulpcov, ssresmin = fitloop(src.coefficients[stepinds],
int(maxfev))
if fulpopt is None:
if maxfev > 10000:
pass
else:
log.warning('No fit converged after {} '.format(maxfev) +
'iterations. Increasing to 10000')
maxfev = 10000
fulpopt, fulpcov, ssresmin = fitloop(src.coefficients[stepinds],
int(maxfev))
# avoid crashing scripts if no fit converged, return np.nan result
if fulpopt is None:
log.exception('No fit converged afer %d iterations', maxfev)
try:
if description is None:
description = '(fit failed)'
else:
description = str(description) + ' (fit failed)'
except Exception as e:
log.debug('Exception passed', exc_info=True)
return FitResult(tau=np.nan,
mre=np.nan,
fitfunc=fitfunc,
steps=steps,
dt=dt,
dtunit=dtunit,
description=description)
try:
rsquared = 0.0
sstot = np.sum((src.coefficients[stepinds] -
np.mean(src.coefficients[stepinds]))**2)
rsquared = 1.0 - (ssresmin / sstot)
# adjusted rsquared to consider parameter number
rsquared = 1.0 - (1.0 - rsquared) * \
(len(stepinds) -1)/(len(stepinds) -1 - len(fulpopt))
except Exception as e:
log.debug('Exception passed when estimating rsquared', exc_info=True)
# ------------------------------------------------------------------ #
# Bootstrapping
# ------------------------------------------------------------------ #
taustderr = None
mrestderr = None
tauquantiles = None
mrequantiles = None
if src.numboot <= 1:
log.debug('Fitting of bootstrapsamples can only be done if ' +
"coefficients() was called with sufficient trials and " +
"bootstrapsamples were created by specifying 'numboot'")
elif fitfunc == f_linear:
log.warning('Bootstrap is not suppored for the f_linear fitfunction')
elif src.numboot > 1:
if numboot > src.numboot:
log.debug(
"The provided data does not contain enough " +
"bootstrapsamples (%d) to do the requested " +
"'numboot=%d' fits.\nCall 'coefficeints()' and 'fit()' " +
"with the same 'numboot' argument to avoid this.", src.numboot,
numboot)
numboot = src.numboot
if numboot == 0:
log.debug("'numboot=0' skipping bootstrapping")
else:
log.info('Bootstrapping {} replicas ({} fits each)'.format(
numboot, len(fitpars)))
log.debug('fit() seeding to {}'.format(seed))
if seed is None:
pass
elif seed == 'random':
np.random.seed(None)
else:
np.random.seed(seed)
bstau = np.full(numboot + 1, np.nan)
bsmre = np.full(numboot + 1, np.nan)
# use scipy default maxfev for errors
maxfev = 100 * (len(fitpars[0]) + 1)
for tdx in tqdm(range(numboot)):
bspopt, bspcov, bsres = fitloop(
src.bootstrapcrs[tdx].coefficients[stepinds], int(maxfev),
False)
try:
bstau[tdx] = bspopt[0]
bsmre[tdx] = np.exp(-1 * dt / bspopt[0])
except TypeError:
log.debug('Exception passed', exc_info=True)
bstau[tdx] = np.nan
bsmre[tdx] = np.nan
# log.info('{} Bootstrap replicas done'.format(numboot))
# add source sample?
bstau[-1] = fulpopt[0]
bsmre[-1] = np.exp(-1 * dt / fulpopt[0])
taustderr = np.sqrt(np.nanvar(bstau, ddof=1))
mrestderr = np.sqrt(np.nanvar(bsmre, ddof=1))
if quantiles is None:
quantiles = np.array([.125, .25, .4, .5, .6, .75, .875])
else:
quantiles = np.array(quantiles)
tauquantiles = np.nanpercentile(bstau, quantiles * 100.)
mrequantiles = np.nanpercentile(bsmre, quantiles * 100.)
tau = fulpopt[0]
mre = np.exp(-1 * dt / fulpopt[0])
if fitfunc == f_linear:
tau = None
mre = None
fulres = FitResult(tau=tau,
mre=mre,
fitfunc=fitfunc,
taustderr=taustderr,
mrestderr=mrestderr,
tauquantiles=tauquantiles,
mrequantiles=mrequantiles,
quantiles=quantiles,
popt=fulpopt,
pcov=fulpcov,
ssres=ssresmin,
rsquared=rsquared,
steps=steps,
dt=dt,
dtunit=dtunit,
description=description)
# ------------------------------------------------------------------ #
# consistency
# ------------------------------------------------------------------ #
log.info(
'Finished fitting ' +
'{} to {},\nmre = {}, tau = {}{}, ssres = {:.5f}'.format(
'the data' if description is None else "'" + description +
"'", fitfunc.__name__, ut._prerror(fulres.mre, fulres.mrestderr),
ut._prerror(fulres.tau, fulres.taustderr, 2,
2), fulres.dtunit, fulres.ssres))
if fulres.tau is None:
return fulres
try:
if src.method == 'trialseparated':
if fulres.tau > 0.1 * (src.triallen * dt):
log.warning(
"The obtained autocorrelationtime " +
"(tau~{:.0f}{}) ".format(fulres.tau, dtunit) +
"is larger than 10% of the trial length " +
"({:.0f}{}).".format(src.triallen*dt) +
("\nThe 'stationarymean' method might be more suitable." if \
src.numtrials > 1 else "")
)
except:
log.debug('Exception passed', exc_info=True)
try:
if src.method == 'stationarymean':
if fulres.tau > (src.triallen * dt):
log.warning("The obtained autocorrelationtime " +
"(tau~{:.0f}{}) ".format(fulres.tau, dtunit) +
"is larger than the trial length " +
"({:.0f}{}).".format(src.triallen * dt, dtunit) +
"\nDon't trust this estimate!")
except:
log.debug('Exception passed', exc_info=True)
# this was really just some back of the envelope suggestion.
# if fulres.tau >= 0.75*(steps[-1]*dt):
# log.warning('The obtained autocorrelationtime is large compared '+
# 'to the fitrange:\n' +
# "tmin~{:.0f}{}, tmax~{:.0f}{}, tau~{:.0f}{}\n"
# .format(steps[0]*dt, dtunit, steps[-1]*dt, dtunit, fulres.tau, dtunit) +
# 'Consider fitting with a larger \'maxstep\'')
# if fulres.tau <= 0.05*(steps[-1]*dt) or fulres.tau <= steps[0]*dt:
# log.warning('The obtained autocorrelationtime is small compared '+
# "to the fitrange:\n" +
# "tmin~{:.0f}{}, tmax~{:.0f}{}, tau~{:.0f}{}\n"
# .format(steps[0]*dt, dtunit, steps[-1]*dt, dtunit, fulres.tau, dtunit) +
# "Consider fitting with smaller 'minstep' and 'maxstep'")
if fitfunc is f_complex:
# check for amplitudes A>B, A>C, A>O
# tau, A, O, tauosc, B, gamma, nu, taugs, C
try:
if fulpopt[1] <= fulpopt[4] or fulpopt[1] <= fulpopt[8]:
log.warning(
'The amplitude of the exponential decay is ' +
'smaller than corrections: A=%f B=%f C=%f', fulpopt[1],
fulpopt[4], fulpopt[8])
except:
log.debug('Exception passed', exc_info=True)
return fulres