def minimiseChiSquared(self):
"""
Minimises chi-squared for a given object method and determines
the minimising parameters: m and c.
"""
# Initialises a Minuit object for the dataset and calculates the minimim
# chi-squared, associated parameters, and errors.
m = Minuit(self.getChiSquared,
print_level=0,
pedantic=False,
errordef=1)
fmin, param = m.migrad()
m.minos()
# Outputs the calculated parameters to the console.
m.print_param()
# Generates a plot for the varying of the parameter 'm'.
plt.figure()
m.draw_profile('m')
c, fa = m.profile('m')
plt.show()
# Generates a plot for the varying of the parameter 'c'.
plt.figure()
m.draw_profile('c')
c, fa = m.profile('c')
plt.show()
def minimise(self, task):
if task == 2:
m = Minuit(self.log_partial,
F=0.5,
tau1=1.0,
tau2=2.0,
limit_F=(0, 1),
limit_tau1=(0.000001, 10),
limit_tau2=(0.000001, 10),
errordef=0.5,
pedantic=False)
if task == 3:
m = Minuit(self.log_full,
F=0.5,
tau1=1.0,
tau2=2.0,
limit_F=(0, 1),
limit_tau1=(0.000001, 10),
limit_tau2=(0.000001, 10),
errordef=0.5,
pedantic=False)
fmin, param = m.migrad() # Runs the minimiser.
#m.minos() # Calculates the errors.
#print(m.values)
print(m.errors)
self.param = m.values
self.param_no = len(m.values)
# The rest of this method is a specific plotting procedure for Minuit.
plt.figure()
m.draw_profile('F')
plt.figure()
m.draw_profile('tau1')
plt.figure()
m.draw_profile('tau2')
plt.show()
#or you can get the gridded data
x, y, g, r = m.mncontour_grid('x','z', nsigma=3) # r is the raw data
pcolormesh(x,y,g)
colorbar()
# <codecell>
#1D value Scan
x,y = m.profile('x',subtract_min=True);
plot(x,y) #if you have matplotlib
# <codecell>
#we also provide convenience wrapper for drawing it
m.draw_profile('x');
# <codecell>
#2d contour NOT minos contour
x,y,z = m.contour('x','y',subtract_min=True)
cs = contour(x,y,z)
clabel(cs)
# <codecell>
#or a convenience wrapper
m.draw_contour('x','z');
# <markdowncell>
# <codecell>
m.migrad();
# <codecell>
ulh.show(m, parts=True)
# <codecell>
m.print_matrix()
# <codecell>
m.draw_profile('mass');#not exactly minos profile just a simple scan
# <codecell>
m.draw_contour('mass','f_0', show_sigma=False);
#not exactly minos contour though just a 2d scan
#Matt Bellis already signed up for this task.;
# <markdowncell>
# ####Note on complex PDF
# There is nothing preventing you from doing something like this:
# ```
# def mypdf(x,mass, gamma, m, c, f_0):
# return brietwigner(x, mass, gamma) + f_0*(m*x+c)/normalization
# ulh=UnbinnedLH(mypdf, data)
def find_mle_2sin(self,
x,
x2,
drive_freq=0,
fsamp=5000,
bandwidth=50,
noise_rms=0,
noise_rms2=0,
plot=False,
suppress_print=True,
**kwargs):
"""
The function is fitting the data with a sine template using iminuit.
The fitting is done after applying a bandpass filter.
:param suppress_print: suppress all printing
:param noise_rms, noise_rms2: std of the white gaussian noise
:param plot: plot the data and its fft
:param bandwidth: bandwidth for butter filter [Hz]
:param fsamp: sampling rate [1/sec]
:param x, x2: two 1-dim. position datasets (time domain)
:param drive_freq: drive frequency of the response
:return: estimated values, chi_square
"""
if not suppress_print:
print('Data overall time: ', len(x) / fsamp, ' sec.')
if noise_rms != 0:
self.noise_sigma = noise_rms
self.noise_sigma2 = noise_rms2
# apply a bandpass filter to data and store data in the correct place for the minimization
self.data_x = np.arange(0, len(x)) / fsamp
start = time.time()
if drive_freq != 0:
if not suppress_print:
print('Bandpass filter ON. Bandwidth: ', bandwidth, 'Hz')
b, a = signal.butter(3, [
2. * (drive_freq - bandwidth / 2.) / fsamp, 2. *
(drive_freq + bandwidth / 2.) / fsamp
],
btype='bandpass')
self.data_y = signal.filtfilt(b, a, x)
self.data_y2 = signal.filtfilt(b, a, x2)
else:
self.data_y = x
self.data_y2 = x2
end = time.time()
if not suppress_print:
print('bandpass time: ', end - start)
# we create an instance of Minuit and pass the function to minimize
mimuit_minimizer = Minuit(self.least_squares_2sines, **kwargs)
start = time.time()
mimuit_minimizer.migrad(ncall=50000)
end = time.time()
if not suppress_print:
print('minimization time: ', end - start)
print(mimuit_minimizer.get_param_states())
if plot:
_, ax = plt.subplots(1, 2, figsize=(9.5, 4))
# ax[0].scatter(self.data_x, self.data_y)
# fft = np.abs(np.fft.rfft(x)) ** 2
# freq = np.fft.rfftfreq(len(x), d=1. / fsamp)
# ax[0].set(title='raw data')
plt.subplot(121)
mimuit_minimizer.draw_profile('A', subtract_min=True)
plt.subplot(122)
mimuit_minimizer.draw_profile('A2', subtract_min=True)
plt.show()
if not suppress_print:
print('reduced chi2: ',
mimuit_minimizer.fval / (2 * len(self.data_y) - 4))
return mimuit_minimizer
# <codecell>
m.merrors
# <markdowncell>
# ###Contour and Profile(Scan)
# <codecell>
m.draw_mncontour('x','y') # you can get the raw value using m.mncontour or m.mncontour_grid;
# <codecell>
m.draw_profile('x') # this is 1d evaluation not minos scan;
# <markdowncell>
# ####Initial value, Limit, initial error, fixing
# <codecell>
m = Minuit(f, x=2, y=4, fix_y=True, limit_z=(-10,10), error_z=0.1)
# <codecell>
m.migrad();
# <markdowncell>
def shift_param(p1,
p2,
t1,
t2,
delta_t=0,
dt_lim=(-20, 20),
v=1,
spline_points=1e7,
eval_width=None,
k=3,
auto=False,
useminos=True,
imincall=1e4,
bins=1e3,
return_delta=False,
show=False,
ext=2):
"""
.. _shift_param:
Return values of `p1` and `p2` shifted by `delta_t`.
Shift a parameter :math:`p_1(t_1)` wrt. a second parameter
:math:`p_2(t_2)` by a time step :math:`\Delta t`. The shift can
also be done automatically to find best fit by using a minimizer
based on 'SEAL Minuit'(`iminuit`) with interpolated values.
Parameters
----------
p1 : 1D numpy.ndarray
Parameter to be shifted by `delta_t`
p2 : 1D numpy.ndarray
parameter to shift `p1` with respect to.
t1 : 1D numpy.ndarray of datetime.datetime objects
time values of `p1`. Should have same length as `p1`.
t2 : 1D numpy.ndarray of datetime.datetime objects
time values of `p2`. Should have same length as `p2`.
delta_t : int,float, optional
time shift in seconds to shift `p1` by. If used together with the
argument `auto=True`, this value will be used as a first guess to
the best fit. It can then be set to `None` if `dt_lim` is set.
A middle value will then be assumed (default ``0``).
dt_lim : int/float list_like of length 2 or int/float.
Maximum and minimum value of `delta_t` in seconds allowed for
minimizing algorithm. If `dt_lim` is a number, symmetry round
`delta_t` will be assumed, eg ``[delta_t-dt_lim,delta_t+dt_lim]``.
`int` or `float` must be non-negative. If ``dt_lim is None`` it
will be set to ``dt_lim=(delta_t - ((1-eval_ratio)/2)*abs(delta_t),
delta_t + ((1-eval_ratio)/2)*abs(delta_t))``, where
``eval_ratio=eval_width/len(p1)`` (default ``(-20,20)``).
v : int, optional
Verbosity level of function. ``0 <= v <= 2`` (default ``1``).
spline_points : int, optional
Number of points used to make a spline fit of p1 with. Number
will be reduced if p1 has fewer points. Float values will be
truncated (default ``1e7``).
eval_width : int, optional
Number of points in time to compare `p1` and `p2` values,
centered around the value of ``t1+delta_t``. Number will be
reduced by increasing span of dt_lim to accommodate for all
possible values of `delta_t`. If set to ``eval_width=None`` a
width corresponding to 60% of the length of p2 will be used
(default ``None``).
k : int, optional
Degree of the smoothing spline. Must be ``1 <= k <= 5``.
auto : bool, optional
Use minimizer to find best fit for `delta_t` (default ``False``).
useminos : bool
If ``auto=True``,run
`minos <http://iminuit.readthedocs.org/en/latest/api.html#iminuit.Minuit.minos>`_
(default ``True``)
imincall : int
If ``auto=True``, number of calls to migrad/minos. Float values
will be truncated (default ``1e4``)
bins : int
If `auto=True`, number of bins for profile of solution space (if
no solution is found from initial `delta_t`, divide dt_lim into
`bins`, and find best solution out of these). Also applicable for
when visualizing profile using ``show=True``. Float values will
be truncated (default ``1e3``).
return_delta : bool
return `delta_t` as output (default ``False``).
show : False
show solution profile in a plot (see iminuit `draw_profile
<http://iminuit.readthedocs.org/en/latest/api.html#iminuit.Minuit.draw_profile>`_
)(default ``False``).
ext : int
handling of values outside interpolation region:
- extrapolation = 0
- set to zero = 1
- raise error = 2
- set to constant(equal to boundary) = 3
Returns
-------
a tuple of numpy.ndarray's ``(p1,p2,t1+delta_t,t2)`` are returned.
Only values with temporal overlap are returned. Output will be of
equal length. If `return_delta=True`, a tuple
``(p1,p2,t1+delta_t,t2,delta_t)`` will be returned, with delta_t as
float.
Raises
------
ValueError
- if length's are incompatible
- if `eval_width`>length of p2
- if neither `delta_t` nor `dt_lim` are provided.
- if ``delta_t=None`` and `dt_lim` is a number.
- if `dt_lim` is negative
IndexError
if `dt_lim` has length less than 2.
Notes
-----
This function assumes uniform sampling rate, and may not give
desired results if this is not the case. As minimizing functions
can be non-trivial, some tweaking of arguments may be necessary
to get optimal results.
See also
--------
align_param, where_overlap
"""
import numexpr as ne
from iminuit import Minuit
t2 = t2.copy()
t1 = t1.copy()
if not auto:
if isinstance(delta_t, (int, float, dt.timedelta)):
return _shift_param(p1, p2, t1, t2, delta_t)
else:
raise ValueError("Invalid timeshift delta_t provided")
spline_points = int(spline_points)
imincall = int(imincall)
bins = int(bins)
len1, len2 = len(t1), len(t2)
if len(p1) != len(t1) or len(p2) != len(t2):
auxiliary.logger.error(
'incompatible lengths of parameter to time array:'+\
' p1:{}!=t1:{} or p2:{}!=t2:{}'
.format(len(p1),len(t1),len(p2),len(t2)) )
raise ValueError
is_dt_dt = False
if isinstance(delta_t, dt.timedelta):
delta_t = delta_t.total_seconds()
is_dt_dt = True
no_dt = False
if delta_t is None:
no_dt = True
if eval_width is not None:
if eval_width <= 0 or eval_width > len2:
auxiliary.logger.error(
"eval_width must be a positive integer less"+\
" than the length of p2, you provided {}.".format(eval_width))
raise ValueError
eval_ratio = eval_width / len1
if eval_ratio < 0.1:
low_ratio = True
else:
eval_width = int(0.6 * len1)
eval_ratio = eval_width / len1
if hasattr(dt_lim, '__iter__'):
if len(dt_lim) < 2:
auxiliary.logger.error("dt_lim needs to be of length 2")
raise IndexError
for i in range(2):
if dt_lim[i] is None:
if no_dt:
auxiliary.logger.error(
"If no delta_t is provided dt_lim tuple must be provided"
)
raise ValueError
if isinstance(dt_lim[i], dt.timedelta):
dt_lim[i] = dt_lim[i].total_seconds()
dt_low, dt_high = dt_lim[:2]
if not no_dt:
if delta_t < dt_low or delta_t > dt_high:
dt_low = delta_t - ((1 - eval_ratio) / 2) * abs(delta_t)
dt_high = delta_t + ((1 - eval_ratio) / 2) * abs(delta_t)
if v > 0:
auxiliary.logger.info(
"dt_lim has been changed from "+\
"{} to {} due to specified delta_t being outside range."
.format(dt_lim,(dt_low,dt_high)))
dt_lim = (dt_low, dt_high)
else: #assume number: max deviation from delta_t symmetrically
if dt_lim is not None:
if dt_lim <= 0:
auxiliary.logger.error(
"Float value dt_lim:{} out of bounds.".format(dt_lim)+\
" Needs to be positive or tuple of numbers or timedelta objects.")
raise ValueError
dt_lim = (delta_t - dt_lim, delta_t + dt_lim)
else:
if no_dt:
auxiliary.logger.error(
"If no delta_t is provided dt_lim tuple must be provided")
raise ValueError
dt_lim = (delta_t - ((1 - eval_ratio) / 2) * abs(delta_t),
delta_t + ((1 - eval_ratio) / 2) * abs(delta_t))
step = (t1[1] - t1[0]).total_seconds() #one time step in seconds
def check_eval_width(spline_points,
dt_lim,
eval_width,
eval_ratio,
tol=1e-3):
#subset of p1 for splining
if len1 > spline_points:
l_spl = len(p1[(l1 - spline_points) // 2:(l1 + spline_points) //
2])
else:
l_spl = len1
if no_dt:
#for slicing purposes define delta_t
delta_t_ = np.mean(dt_lim[:2])
else:
delta_t_ = delta_t
edge_dt_low = abs(dt_lim[1] - delta_t_)
edge_dt_high = abs(dt_lim[0] - delta_t_)
#dt_sec values converted to steps(rounded up):
edge_dt_low = int(edge_dt_low // step +
bool(abs(edge_dt_low % step) > tol))
edge_dt_high = int(edge_dt_high // step +
bool(abs(edge_dt_high % step) > tol))
auxiliary.logger.debug(
"Step size is {} seconds. Edges need {}".format(step,edge_dt_low)+\
"+{} steps to accommodate dt_lim values".format(edge_dt_high))
is_valid = True
if len2 < (eval_width + edge_dt_low + edge_dt_high):
eval_width = len2 - (edge_dt_low + edge_dt_high)
eval_ratio = eval_width / len1
auxiliary.logger.debug(
"eval_width adjusted to {} due to edge requirements".format(
eval_width))
if eval_width < 1:
auxiliary.logger.error(
'\n\t'.join((
'too few evaluation points. Consider reducing'+\
' spline_points or the span of dt_lim.'),
'Number of spline points: {}'.format(l_spl)+\
'dt_lim: {}'.format(dt_lim)))
is_valid = False
if auxiliary._is_interactive():
print("Insert new values"+\
" (If none specified, default values will be chosen")
l_spl_tmp = input("spline points[{}]: ".format(l_spl))
dt_lim0_tmp = input("dt_lim lower[{}]: ".format(dt_lim[0]))
dt_lim1_tmp = input("dt_lim upper[{}]: ".format(dt_lim[1]))
try:
if l_spl_tmp.strip():
l_spl = int(l_spl_tmp)
if dt_lim0_tmp.strip():
dt_lim[0] = float(dt_lim0_tmp)
if dt_lim1_tmp.strip():
dt_lim[1] = float(dt_lim1_tmp)
if not (l_spl_tmp.strip() or \
dt_lim0_tmp.strip() or dt_lim1_tmp.strip()):
raise ValueError
except KeyboardInterrupt:
auxiliary.logger.error(
"KeyboardInterrupt - aborting...")
raise
except Exception:
auxiliary.logger.error(
"No input variables or incorrect input variables")
raise
else:
raise ValueError(
"too few evaluation points."+\
" Consider reducing spline_points or the span of dt_lim")
if eval_width < 10 and v:
auxiliary.logger.warning(
'Only {} points used to evaluate goodness of fit.'.format(
eval_width))
return l_spl, dt_lim, edge_dt_low, edge_dt_high, eval_width, eval_ratio, is_valid
check_vars = spline_points, dt_lim, eval_width, eval_ratio, False
while True:
check_vars = check_eval_width(*check_vars[:-1])
if check_vars[-1]:
l_spl = check_vars[0]
dt_lim = check_vars[1]
edge_dt_low = check_vars[2]
edge_dt_high = check_vars[3]
eval_width = check_vars[4]
eval_ratio = check_vars[5]
delta_t = (dt_lim[0] + dt_lim[1]) / 2
break
p1_p = p1[(len1 - l_spl) // 2:(len1 + l_spl) // 2]
t1_p = t1[(len1 - l_spl) // 2:(len1 + l_spl) // 2]
auxiliary.logger.debug(
"number of spline points changed from '{}' to '{}'".format(
spline_points, l_spl))
t0 = t1_p[0] #arbitrarily chosen reference time
t_sec1 = auxiliary._to_sec_v(t1_p - t0)
p1_spl_f = InterpolatedUnivariateSpline(t_sec1, p1_p, k=k, ext=ext)
#ext: 0=extrapolate;1=0;2=valueerror,3=const
use_width = eval_width + edge_dt_low + edge_dt_high
eval_start = len2 // 2 - (use_width // 2 - edge_dt_low) - 1
eval_end = len2 // 2 + (use_width // 2 - edge_dt_high) #-1
if eval_start < 0:
eval_start = 0
t_sec2 = auxiliary._to_sec_v(t2[eval_start:eval_end] - t0)
fixing_vars = (('Length of array splined', l_spl), ('Length of eval array',
len(t_sec2)),
('eval_width', eval_width), ('eval_ratio', eval_ratio),
('dt_lim', dt_lim), ('low_dt buf. len', edge_dt_low),
('upp_dt buf. len', edge_dt_high),
('initial delta_t guess', delta_t), ('dt set', not no_dt),
('eval_start index', eval_start), ('eval_end index+1',
eval_end))
auxiliary.logger.debug(
'some variables related to the fitting listed:\n\t' + '\n\t'.join(
('{:20s}:{:20s}'.format(fixing_vars[i][0], str(fixing_vars[i][1]))
for i in range(len(fixing_vars)))))
last_vars = [delta_t, None]
def p_chisq_r(dt_candidate):
p1_fit = p1_spl_f(t_sec2 - dt_candidate)
p2_slice = p2[eval_start:eval_end]
chisq = ne.evaluate('sum((p2_slice-p1_fit)**2)')
last_vars[:] = dt_candidate, chisq
return chisq
m = Minuit(p_chisq_r,
dt_candidate=delta_t,
limit_dt_candidate=dt_lim,
print_level=bool(v >= 2),
error_dt_candidate=auxiliary._from_timedelta(t1[1] - t1[0]) /
10,
errordef=1)
MAXTRIES = 100
tries = 0
def minuit_fail_warning():
auxiliary.logger.warning(
"Unsuccessfil run of minimization algorithm, last recorded variables"+\
" are delta_t: {:.4f}, chisq: {}.".format(*last_vars)+\
" Aborted function 'shift_param'\nCheck that limits are appropriate,"+\
" or adjust the ext parameter(currently ext={},".format(ext)+\
" extrapolation=0,zero=1,raise error=2,constant=3)"+\
" to handle values outside specified region")
try:
mout = m.migrad(ncall=imincall)
while mout[0]['is_above_max_edm'] and not \
mout[0]['has_reached_call_limit']:
tries += 1
mout = m.migrad(ncall=imincall)
auxiliary.logger.debug(
"migrad did not converge; retrying. last vars:"+\
" delta_t={:.4f} chisq:{}".format(*last_vars))
if tries == MAXTRIES:
raise ValueError('Maximum number of tries reached')
except ValueError:
minuit_fail_warning()
return
if no_dt:
try:
prof = m.mnprofile('dt_candidate', bins=bins, bound=dt_lim)
except ValueError:
minuit_fail_warning()
delta_t = prof[0][np.argmin(prof[1])]
if useminos:
try:
mout = m.minos(maxcall=imincall)['dt_candidate']
except ValueError:
minuit_fail_warning()
return
delta_t = mout['min']
is_valid = mout['is_valid']
else:
delta_t = mout[1][0].value
is_valid = mout[0]['is_valid']
if not is_valid:
auxiliary.logger.info(
"Validity of solution is questionable; iminuit output dump: {}".
format(mout))
elif mout['at_lower_limit']:
auxiliary.logger.info(
"delta_t converged to solution near lower limit, consider rerunning"+\
" with new limits")
elif mout['at_upper_limit']:
auxiliary.logger.info(
"delta_t converged to solution near upper limit, consider rerunning"+\
" with new limits")
if v:
auxiliary.logger.info('output delta_t: {}'.format(delta_t))
if show:
try:
mout = m.draw_profile('dt_candidate', bins=bins, bound=dt_lim)
except ValueError:
minuit_fail_warning()
if delta_t is None:
return
if return_delta:
p1, p2, t1, t2 = _shift_param(p1, p2, t1, t2, delta_t)
if is_dt_dt:
delta_t = dt.timedelta(seconds=delta_t)
return p1, p2, t1, t2, delta_t
else:
return _shift_param(p1, p2, t1, t2, delta_t)
def test_n2logLf_TEB():
K2uK = 1e12
clscale = K2uK * 1.0
cls_fid = get_spectrum_camb(lmax, isDl=False) * clscale
cls_syn = get_spectrum_camb(
lmax, tau=0.0522, As=2.092e-9, r=0.01, isDl=False) * clscale
map_syn = hp.synfast(cls_syn, nside=nside, new=True)
cls_est = hp.anafast(map_syn, lmax=lmax)
cls_ana = get_spectrum_camb(lmax, isDl=False) * clscale
Cls_ana = lh.cls2Cls(cls_ana)
Cls_est = lh.cls2Cls(cls_est)
inv_Cls_fid, det_Cls_fid = lh.invdet_fid(cls_fid)
n2logLf = n2logL_approx_TEB(Cls_ana, Cls_est, inv_Cls_fid, det_Cls_fid)
print('Likelihood for scale %e = %e' % (clscale, n2logLf))
def fit_minuit_1(tau, As, r):
cls_ana = get_spectrum_camb(lmax=lmax, tau=tau, As=As, r=r,
isDl=False) * clscale
Cls_ana = lh.cls2Cls(cls_ana)
lk = n2logL_approx_TEB(Cls_ana, Cls_est, inv_Cls_fid, det_Cls_fid)
print('tau = %e, As = %e, r = %e, lk = %e' % (tau, As, r, lk))
return lk
tau0 = 0.0522
tau_limit = (0.02, 0.08)
As0 = 2.1e-9
As_limit = (1.5e-9, 2.5e-9)
r0 = 0.01
r_limit = (0.0, 0.4)
m = Minuit(fit_minuit_1,
tau=tau0,
As=As0,
r=r0,
limit_tau=tau_limit,
limit_As=As_limit,
limit_r=r_limit)
st = time.time()
res = m.migrad()
print('Elapsed time for migrad: %fs' % (time.time() - st))
st = time.time()
res = m.hesse()
print('Elapsed time for hesse: %fs' % (time.time() - st))
st = time.time()
#res = m.minos()
print('Elapsed time for minos: %fs' % (time.time() - st))
plt.figure()
m.draw_profile('tau')
plt.figure()
m.draw_profile('As')
plt.figure()
m.draw_profile('r')
tau_min = m.values['tau']
tau_err = m.errors['tau']
cls_min = get_spectrum_camb(lmax=lmax, tau=tau_min, isDl=False) * clscale
cls_upp = get_spectrum_camb(lmax=lmax, tau=tau_min + tau_err,
isDl=False) * clscale
cls_low = get_spectrum_camb(lmax=lmax, tau=tau_min - tau_err,
isDl=False) * clscale
ell = np.arange(len(cls_est[0]))
plt.figure()
plt.loglog(ell, cl2dl(cls_est[:3].T), '*')
plt.loglog(ell, cl2dl(cls_syn[:3].T), '--', linewidth=1.0)
plt.loglog(ell, cl2dl(cls_min[:3].T), '-', linewidth=2.0)
plt.loglog(ell, cl2dl(cls_upp[:3].T), '--', linewidth=0.5)
plt.loglog(ell, cl2dl(cls_low[:3].T), '--', linewidth=0.5)
pprint(res)
plt.show()
def shift_param(p1,p2,t1,t2,delta_t=0,dt_lim=(-20,20),v=1,spline_points=1e7,eval_width=None,k=3,auto=False,useminos=True,imincall=1e4,bins=1e3,return_delta=False,show=False,ext=2):
"""
.. _shift_param:
Return values of `p1` and `p2` shifted by `delta_t`.
Shift a parameter :math:`p_1(t_1)` wrt. a second parameter
:math:`p_2(t_2)` by a time step :math:`\Delta t`. The shift can
also be done automatically to find best fit by using a minimizer
based on 'SEAL Minuit'(`iminuit`) with interpolated values.
Parameters
----------
p1 : 1D numpy.ndarray
Parameter to be shifted by `delta_t`
p2 : 1D numpy.ndarray
parameter to shift `p1` with respect to.
t1 : 1D numpy.ndarray of datetime.datetime objects
time values of `p1`. Should have same length as `p1`.
t2 : 1D numpy.ndarray of datetime.datetime objects
time values of `p2`. Should have same length as `p2`.
delta_t : int,float, optional
time shift in seconds to shift `p1` by. If used together with the
argument `auto=True`, this value will be used as a first guess to
the best fit. It can then be set to `None` if `dt_lim` is set.
A middle value will then be assumed (default ``0``).
dt_lim : int/float list_like of length 2 or int/float.
Maximum and minimum value of `delta_t` in seconds allowed for
minimizing algorithm. If `dt_lim` is a number, symmetry round
`delta_t` will be assumed, eg ``[delta_t-dt_lim,delta_t+dt_lim]``.
`int` or `float` must be non-negative. If ``dt_lim is None`` it
will be set to ``dt_lim=(delta_t - ((1-eval_ratio)/2)*abs(delta_t),
delta_t + ((1-eval_ratio)/2)*abs(delta_t))``, where
``eval_ratio=eval_width/len(p1)`` (default ``(-20,20)``).
v : int, optional
Verbosity level of function. ``0 <= v <= 2`` (default ``1``).
spline_points : int, optional
Number of points used to make a spline fit of p1 with. Number
will be reduced if p1 has fewer points. Float values will be
truncated (default ``1e7``).
eval_width : int, optional
Number of points in time to compare `p1` and `p2` values,
centered around the value of ``t1+delta_t``. Number will be
reduced by increasing span of dt_lim to accommodate for all
possible values of `delta_t`. If set to ``eval_width=None`` a
width corresponding to 60% of the length of p2 will be used
(default ``None``).
k : int, optional
Degree of the smoothing spline. Must be ``1 <= k <= 5``.
auto : bool, optional
Use minimizer to find best fit for `delta_t` (default ``False``).
useminos : bool
If ``auto=True``,run
`minos <http://iminuit.readthedocs.org/en/latest/api.html#iminuit.Minuit.minos>`_
(default ``True``)
imincall : int
If ``auto=True``, number of calls to migrad/minos. Float values
will be truncated (default ``1e4``)
bins : int
If `auto=True`, number of bins for profile of solution space (if
no solution is found from initial `delta_t`, divide dt_lim into
`bins`, and find best solution out of these). Also applicable for
when visualizing profile using ``show=True``. Float values will
be truncated (default ``1e3``).
return_delta : bool
return `delta_t` as output (default ``False``).
show : False
show solution profile in a plot (see iminuit `draw_profile
<http://iminuit.readthedocs.org/en/latest/api.html#iminuit.Minuit.draw_profile>`_
)(default ``False``).
ext : int
handling of values outside interpolation region:
- extrapolation = 0
- set to zero = 1
- raise error = 2
- set to constant(equal to boundary) = 3
Returns
-------
a tuple of numpy.ndarray's ``(p1,p2,t1+delta_t,t2)`` are returned.
Only values with temporal overlap are returned. Output will be of
equal length. If `return_delta=True`, a tuple
``(p1,p2,t1+delta_t,t2,delta_t)`` will be returned, with delta_t as
float.
Raises
------
ValueError
- if length's are incompatible
- if `eval_width`>length of p2
- if neither `delta_t` nor `dt_lim` are provided.
- if ``delta_t=None`` and `dt_lim` is a number.
- if `dt_lim` is negative
IndexError
if `dt_lim` has length less than 2.
Notes
-----
This function assumes uniform sampling rate, and may not give
desired results if this is not the case. As minimizing functions
can be non-trivial, some tweaking of arguments may be necessary
to get optimal results.
See also
--------
align_param, where_overlap
"""
import numexpr as ne
from iminuit import Minuit
t2=t2.copy()
t1=t1.copy()
if not auto:
if isinstance(delta_t,(int,float,dt.timedelta)):
return _shift_param(p1,p2,t1,t2,delta_t)
else:
raise ValueError("Invalid timeshift delta_t provided")
spline_points=int(spline_points)
imincall=int(imincall)
bins=int(bins)
len1,len2=len(t1),len(t2)
if len(p1)!=len(t1) or len(p2)!=len(t2):
auxiliary.logger.error(
'incompatible lengths of parameter to time array:'+\
' p1:{}!=t1:{} or p2:{}!=t2:{}'
.format(len(p1),len(t1),len(p2),len(t2)) )
raise ValueError
is_dt_dt=False
if isinstance(delta_t,dt.timedelta):
delta_t=delta_t.total_seconds()
is_dt_dt=True
no_dt=False
if delta_t is None:
no_dt=True
if eval_width is not None:
if eval_width<=0 or eval_width>len2:
auxiliary.logger.error(
"eval_width must be a positive integer less"+\
" than the length of p2, you provided {}.".format(eval_width))
raise ValueError
eval_ratio=eval_width/len1
if eval_ratio<0.1:
low_ratio=True
else:
eval_width=int(0.6*len1)
eval_ratio=eval_width/len1
if hasattr(dt_lim,'__iter__'):
if len(dt_lim)<2:
auxiliary.logger.error("dt_lim needs to be of length 2")
raise IndexError
for i in range(2):
if dt_lim[i] is None:
if no_dt:
auxiliary.logger.error(
"If no delta_t is provided dt_lim tuple must be provided")
raise ValueError
if isinstance(dt_lim[i],dt.timedelta):
dt_lim[i]=dt_lim[i].total_seconds()
dt_low,dt_high=dt_lim[:2]
if not no_dt:
if delta_t<dt_low or delta_t>dt_high:
dt_low=delta_t - ((1-eval_ratio)/2)*abs(delta_t)
dt_high=delta_t + ((1-eval_ratio)/2)*abs(delta_t)
if v>0:
auxiliary.logger.info(
"dt_lim has been changed from "+\
"{} to {} due to specified delta_t being outside range."
.format(dt_lim,(dt_low,dt_high)))
dt_lim=(dt_low,dt_high)
else:#assume number: max deviation from delta_t symmetrically
if dt_lim is not None:
if dt_lim<=0:
auxiliary.logger.error(
"Float value dt_lim:{} out of bounds.".format(dt_lim)+\
" Needs to be positive or tuple of numbers or timedelta objects.")
raise ValueError
dt_lim=(delta_t-dt_lim,delta_t+dt_lim)
else:
if no_dt:
auxiliary.logger.error(
"If no delta_t is provided dt_lim tuple must be provided")
raise ValueError
dt_lim=(delta_t - ((1-eval_ratio)/2)*abs(delta_t),
delta_t + ((1-eval_ratio)/2)*abs(delta_t))
step=(t1[1]-t1[0]).total_seconds()#one time step in seconds
def check_eval_width(spline_points,dt_lim,eval_width,eval_ratio,tol=1e-3):
#subset of p1 for splining
if len1>spline_points:
l_spl=len(p1[(l1-spline_points)//2:(l1+spline_points)//2])
else:
l_spl=len1
if no_dt:
#for slicing purposes define delta_t
delta_t_=np.mean(dt_lim[:2])
else:
delta_t_=delta_t
edge_dt_low =abs(dt_lim[1]-delta_t_)
edge_dt_high=abs(dt_lim[0]-delta_t_)
#dt_sec values converted to steps(rounded up):
edge_dt_low =int(edge_dt_low //step + bool(abs(edge_dt_low %step)>tol))
edge_dt_high=int(edge_dt_high//step + bool(abs(edge_dt_high%step)>tol))
auxiliary.logger.debug(
"Step size is {} seconds. Edges need {}".format(step,edge_dt_low)+\
"+{} steps to accommodate dt_lim values".format(edge_dt_high))
is_valid=True
if len2<(eval_width+edge_dt_low+edge_dt_high):
eval_width=len2-(edge_dt_low+edge_dt_high)
eval_ratio=eval_width/len1
auxiliary.logger.debug(
"eval_width adjusted to {} due to edge requirements"
.format(eval_width))
if eval_width<1:
auxiliary.logger.error(
'\n\t'.join((
'too few evaluation points. Consider reducing'+\
' spline_points or the span of dt_lim.'),
'Number of spline points: {}'.format(l_spl)+\
'dt_lim: {}'.format(dt_lim)))
is_valid=False
if auxiliary._is_interactive():
print("Insert new values"+\
" (If none specified, default values will be chosen")
l_spl_tmp=input("spline points[{}]: ".format(l_spl))
dt_lim0_tmp=input("dt_lim lower[{}]: ".format(dt_lim[0]))
dt_lim1_tmp=input("dt_lim upper[{}]: ".format(dt_lim[1]))
try:
if l_spl_tmp.strip():
l_spl = int(l_spl_tmp)
if dt_lim0_tmp.strip():
dt_lim[0]=float(dt_lim0_tmp)
if dt_lim1_tmp.strip():
dt_lim[1]=float(dt_lim1_tmp)
if not (l_spl_tmp.strip() or \
dt_lim0_tmp.strip() or dt_lim1_tmp.strip()):
raise ValueError
except KeyboardInterrupt:
auxiliary.logger.error("KeyboardInterrupt - aborting...")
raise
except Exception:
auxiliary.logger.error(
"No input variables or incorrect input variables")
raise
else:
raise ValueError(
"too few evaluation points."+\
" Consider reducing spline_points or the span of dt_lim")
if eval_width<10 and v:
auxiliary.logger.warning(
'Only {} points used to evaluate goodness of fit.'
.format(eval_width))
return l_spl,dt_lim,edge_dt_low,edge_dt_high,eval_width,eval_ratio,is_valid
check_vars=spline_points,dt_lim,eval_width,eval_ratio,False
while True:
check_vars=check_eval_width(*check_vars[:-1])
if check_vars[-1]:
l_spl = check_vars[0]
dt_lim = check_vars[1]
edge_dt_low = check_vars[2]
edge_dt_high= check_vars[3]
eval_width = check_vars[4]
eval_ratio = check_vars[5]
delta_t=(dt_lim[0]+dt_lim[1])/2
break
p1_p=p1[(len1-l_spl)//2:(len1+l_spl)//2]
t1_p=t1[(len1-l_spl)//2:(len1+l_spl)//2]
auxiliary.logger.debug(
"number of spline points changed from '{}' to '{}'"
.format(spline_points,l_spl))
t0=t1_p[0]#arbitrarily chosen reference time
t_sec1=auxiliary._to_sec_v(t1_p-t0)
p1_spl_f=InterpolatedUnivariateSpline(t_sec1,p1_p,k=k,ext=ext)
#ext: 0=extrapolate;1=0;2=valueerror,3=const
use_width=eval_width+edge_dt_low+edge_dt_high
eval_start=len2//2-(use_width//2 - edge_dt_low) -1
eval_end =len2//2+(use_width//2 - edge_dt_high) #-1
if eval_start<0:
eval_start=0
t_sec2=auxiliary._to_sec_v(t2[eval_start:eval_end]-t0)
fixing_vars = (
('Length of array splined',l_spl),
('Length of eval array',len(t_sec2)),
('eval_width',eval_width),('eval_ratio',eval_ratio),
('dt_lim',dt_lim),
('low_dt buf. len',edge_dt_low),
('upp_dt buf. len',edge_dt_high),
('initial delta_t guess',delta_t),
('dt set',not no_dt),
('eval_start index',eval_start),
('eval_end index+1',eval_end)
)
auxiliary.logger.debug(
'some variables related to the fitting listed:\n\t'+'\n\t'
.join(('{:20s}:{:20s}'.format(fixing_vars[i][0],str(fixing_vars[i][1]))
for i in range(len(fixing_vars)))))
last_vars=[delta_t,None]
def p_chisq_r(dt_candidate):
p1_fit=p1_spl_f(t_sec2-dt_candidate)
p2_slice=p2[eval_start:eval_end]
chisq=ne.evaluate('sum((p2_slice-p1_fit)**2)')
last_vars[:]=dt_candidate,chisq
return chisq
m = Minuit(p_chisq_r,
dt_candidate=delta_t,
limit_dt_candidate=dt_lim,
print_level=bool(v>=2),
error_dt_candidate=auxiliary._from_timedelta(t1[1]-t1[0])/10,
errordef=1)
MAXTRIES=100
tries=0
def minuit_fail_warning():
auxiliary.logger.warning(
"Unsuccessfil run of minimization algorithm, last recorded variables"+\
" are delta_t: {:.4f}, chisq: {}.".format(*last_vars)+\
" Aborted function 'shift_param'\nCheck that limits are appropriate,"+\
" or adjust the ext parameter(currently ext={},".format(ext)+\
" extrapolation=0,zero=1,raise error=2,constant=3)"+\
" to handle values outside specified region")
try:
mout=m.migrad(ncall=imincall)
while mout[0]['is_above_max_edm'] and not \
mout[0]['has_reached_call_limit']:
tries+=1
mout=m.migrad(ncall=imincall)
auxiliary.logger.debug(
"migrad did not converge; retrying. last vars:"+\
" delta_t={:.4f} chisq:{}".format(*last_vars))
if tries==MAXTRIES:
raise ValueError('Maximum number of tries reached')
except ValueError:
minuit_fail_warning()
return
if no_dt:
try:
prof=m.mnprofile('dt_candidate',bins=bins,bound=dt_lim)
except ValueError:
minuit_fail_warning()
delta_t=prof[0][np.argmin(prof[1])]
if useminos:
try:
mout=m.minos(maxcall=imincall)['dt_candidate']
except ValueError:
minuit_fail_warning()
return
delta_t=mout['min']
is_valid=mout['is_valid']
else:
delta_t=mout[1][0].value
is_valid=mout[0]['is_valid']
if not is_valid:
auxiliary.logger.info(
"Validity of solution is questionable; iminuit output dump: {}"
.format(mout))
elif mout['at_lower_limit']:
auxiliary.logger.info(
"delta_t converged to solution near lower limit, consider rerunning"+\
" with new limits")
elif mout['at_upper_limit']:
auxiliary.logger.info(
"delta_t converged to solution near upper limit, consider rerunning"+\
" with new limits")
if v:
auxiliary.logger.info('output delta_t: {}'.format(delta_t))
if show:
try:
mout=m.draw_profile('dt_candidate',bins=bins,bound=dt_lim)
except ValueError:
minuit_fail_warning()
if delta_t is None:
return
if return_delta:
p1,p2,t1,t2=_shift_param(p1,p2,t1,t2,delta_t)
if is_dt_dt:
delta_t=dt.timedelta(seconds=delta_t)
return p1,p2,t1,t2,delta_t
else:
return _shift_param(p1,p2,t1,t2,delta_t)
