def move_elements(self, other, indices):
"""Remove elements with given indices, and append to another object.
NB: indices is boolean array, not real indices!"""
# Move elements with given indices (boolean array)
# to another LagrangianArray
# NB: scalars and 1D arrays are converted to ndarrays and concatenated
self_len = len(self)
other_len = len(other)
for var in self.variables:
self_var = getattr(self, var)
other_var = getattr(other, var)
if (not isinstance(self_var, np.ndarray) and
not isinstance(other_var, np.ndarray)) and \
(other_var == self_var):
continue # Equal scalars - we do nothing
# Copy elements to other
self_var = np.atleast_1d(self_var)
other_var = np.atleast_1d(other_var)
if len(self_var) < self_len: # Convert scalar to array
self_var = self_var*np.ones(self_len)
if len(other_var) < other_len: # Convert scalar to aray
other_var = other_var*np.ones(other_len)
if len(self_var) > 0:
setattr(other, var, np.concatenate((other_var,
self_var[indices])))
else:
setattr(other, var, self_var[indices])
setattr(self, var, self_var[~indices]) # Remove from self
def CVXOPT_SOCP_Solver(p, solverName):
if solverName == 'native_CVXOPT_SOCP_Solver': solverName = None
cvxopt_solvers.options['maxiters'] = p.maxIter
cvxopt_solvers.options['feastol'] = p.contol
cvxopt_solvers.options['abstol'] = p.ftol
if p.iprint <= 0:
cvxopt_solvers.options['show_progress'] = False
cvxopt_solvers.options['LPX_K_MSGLEV'] = 0
cvxopt_solvers.options['MSK_IPAR_LOG'] = 0
xBounds2Matrix(p)
#FIXME: if problem is search for MAXIMUM, not MINIMUM!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
f = copy(p.f).reshape(-1,1)
# CVXOPT has some problems with x0 so currently I decided to avoid using the one
Gq, hq = [], []
C, d, q, s = p.C, p.d, p.q, p.s
for i in range(len(q)):
Gq.append(Matrix(Vstack((-atleast_1d(q[i]),-atleast_1d(C[i]) if not isspmatrix(C[i]) else C[i]))))
hq.append(matrix(hstack((atleast_1d(s[i]), atleast_1d(d[i]))), tc='d'))
sol = cvxopt_solvers.socp(Matrix(p.f), Gl=Matrix(p.A), hl = Matrix(p.b), Gq=Gq, hq=hq, A=Matrix(p.Aeq), b=Matrix(p.beq), solver=solverName)
p.msg = sol['status']
if p.msg == 'optimal' : p.istop = SOLVED_WITH_UNIMPLEMENTED_OR_UNKNOWN_REASON
else: p.istop = -100
if sol['x'] is not None:
p.xf = asarray(sol['x']).flatten()
p.ff = sum(p.dotmult(p.f, p.xf))
else:
p.ff = nan
p.xf = nan*ones(p.n)
def param_maq(kaq, z, c, npor, top):
# Computes the parameters for a ModelBase from input for a maq model
kaq = np.atleast_1d(kaq).astype('d')
z = np.atleast_1d(z).astype('d')
c = np.atleast_1d(c).astype('d')
npor = np.atleast_1d(npor).astype('d')
if top == 'conf':
Naq = len(z) / 2
ltype = np.array( list( (Naq-1) * 'al' + 'a' ) )
else: # leaky layer on top
Naq = (len(z) - 1) / 2
ltype = np.array( list( Naq * 'la' ) )
if len(kaq) == 1: kaq = kaq * np.ones(Naq)
assert len(kaq) == Naq, 'Error: length of kaq needs to be 1 or' + str(Naq)
H = z[:-1] - z[1:]
assert np.all(H >= 0), 'Error: Not all layers thicknesses are non-negative' + str(H)
if top == 'conf':
if len(c) == 1: c = c * np.ones(Naq - 1)
if len(npor) == 1: npor = npor * np.ones(2 * Naq - 1)
assert len(c) == Naq-1, 'Error: Length of c needs to be 1 or' + str(Naq-1)
assert len(npor) == 2 * Naq - 1, 'Error: Length of npor needs to be 1 or' + str(2*Naq-1)
Haq = H[::2]
c = np.hstack((1e100,c))
else: # leaky layer on top
if len(c) == 1: c = c * np.ones(Naq)
if len(npor) == 1: npor = npor * np.ones(2 * Naq)
assert len(c) == Naq, 'Error: Length of c needs to be 1 or' + str(Naq)
assert len(npor) == 2 * Naq, 'Error: Length of npor needs to be 1 or' + str(2*Naq)
Haq = H[1::2]
return kaq, Haq, c, npor, ltype
def _check(self, lblev=37.0,
blev=9596.3, brlev=9500.0, brsvd1=9800.0,
bhlev=0.35, bhrlev=0.31, brsvd2=0.39,
dim=None):
lbvc = 65
lbcode = _lbcode(0) # unused
stash = STASH(1, 1, 1) # unused
coords_and_dims, factories = _convert_vertical_coords(
lbcode=lbcode, lbvc=lbvc, blev=blev, lblev=lblev, stash=stash,
bhlev=bhlev, bhrlev=bhrlev, brsvd1=brsvd1, brsvd2=brsvd2,
brlev=brlev, dim=dim)
expect_coords_and_dims = [
(DimCoord(lblev,
standard_name='model_level_number',
attributes={'positive': 'up'}), dim)]
brlev = np.atleast_1d(brlev)
brsvd1 = np.atleast_1d(brsvd1)
expect_coords_and_dims.append(
(DimCoord(blev,
long_name='level_height', units='m',
bounds=np.vstack((brlev, brsvd1)).T,
attributes={'positive': 'up'}), dim))
bhrlev = np.atleast_1d(bhrlev)
brsvd2 = np.atleast_1d(brsvd2)
expect_coords_and_dims.append(
(AuxCoord(bhlev,
long_name='sigma',
bounds=np.vstack((bhrlev, brsvd2)).T), dim))
expect_factories = [(HybridHeightFactory,
[{'long_name': 'level_height'},
{'long_name': 'sigma'},
Reference('orography')])]
self.assertCoordsAndDimsListsMatch(coords_and_dims,
expect_coords_and_dims)
self.assertEqual(factories, expect_factories)
def mm_to_galvo(self, x, y):
""" Given one or many points in mm space, map them to galvo space.
e.g.,
>>> Printer.mm_to_galvo(0, 0) # -> galvo ticks for middle of build area.
>>> Printer.mm_to_galvo([[0, 1, 2], [0, 0, 0]]) # -> A three-segment line along the x axis.
The returned array is 2xN, where N is the number of source points
"""
xshape = np.shape(x)
if self._grid_table is None:
grid = np.array(self.read_grid_table())
assert grid.shape == (5, 5, 2)
pts_mm = np.linspace(-64, 64, 5) # Grid positions in mm
# Interpolators for X and Y values (mm to galvo ticks)
fit_x = scipy.interpolate.interp2d(pts_mm, pts_mm, grid[:,:,0])
fit_y = scipy.interpolate.interp2d(pts_mm, pts_mm, grid[:,:,1])
self._grid_table = (fit_x, fit_y)
if np.shape(x) != np.shape(y):
raise TypeError('x and y shapes must match. Got x.shape: {}, y.shape: {}'.format(np.shape(x), np.shape(y)))
x = np.atleast_1d(x)
y = np.atleast_1d(y)
x_ = [self._grid_table[0](a, b) for a, b in zip(x, y)]
y_ = [self._grid_table[1](a, b) for a, b in zip(x, y)]
result = np.hstack([x_, y_]).T
if xshape == (): # If it's called with scalars, return a flat result.
return result.flatten()
return result
def _read_and_shape_grid_data(k, dirname):
if k in _grid_special_mapping:
fname, sl, ndim_expected = _grid_special_mapping[k]
else:
fname = k
sl = None
ndim_expected = None
data = None
try:
data = _read_mds(os.path.join(dirname, fname), force_dict=False)
except IOError:
try:
data = _read_mds(os.path.join(dirname, fname.upper()), force_dict=False)
except IOError:
warnings.warn("Couldn't load grid variable " + k)
if data is not None:
if sl is not None:
# have to reslice and reshape the data
if data.ndim != ndim_expected:
# if there is only one vertical level, some variables
# are squeeze at the mds level and need to get a dimension back
if data.ndim == 2 and ndim_expected == 3:
data.shape = (1,) + data.shape
else:
raise ValueError("Don't know how to handle data shape")
# now apply the slice
data = np.atleast_1d(data[sl])
else:
data = np.atleast_1d(data.squeeze())
return data
def _make_segment(self,x,y,threshold=None):
if threshold is None:
threshold = self._segment_threshold
x,y=np.atleast_1d(x),np.atleast_1d(y)
d2 = np.sqrt((np.roll(x,1)-x)**2+(np.roll(y,1)-y)**2)
w=np.where(d2 > threshold)[0]
#w=w[w!=0]
xx=[]
yy=[]
if len(w) == 1:
x=np.roll(x,-w[0])
y=np.roll(y,-w[0])
xx.append(x)
yy.append(y)
elif len(w) >= 2:
xx.append(x[0:w[0]])
yy.append(y[0:w[0]])
for i in xrange(len(w)-1):
xx.append(x[w[i]:w[i+1]])
yy.append(y[w[i]:w[i+1]])
xx.append(x[w[-1]:])
yy.append(y[w[-1]:])
else:
xx.append(x)
yy.append(y)
return xx,yy
def magnetic_correction(theta, u, v):
"""
Description:
This function corrects velocity profiles for the magnetic variation
(declination) at the measurement location. This calculation is used by
several different data products (e.g. VELPROF, WINDAVG) from multiple
instrument classes. The magnetic declination is obtained from the 2010
World Magnetic Model (WMM2010) provided by NOAA (see wmm_declination).
Implemented by:
2013-04-10: Christopher Wingard. Initial code.
2014-02-05: Christopher Wingadr. Converted to generic_function from
original implementation under adcp_functions/adcp_magvar.
Usage:
u_cor, v_cor = magnetic_correction(theta, u, v)
where
u_cor = eastward velocity profiles, in earth coordinates, with
the correction for magnetic variation applied.
v_cor = northward velocity profiles, in earth coordinates,
with the correction for magnetic variation applied.
theta = magnetic variation based on location (latitude, longitude and
altitude) and date [degrees]
u = uncorrected eastward velocity profiles in Earth coordinates
v = uncorrected northward velocity profiles in Earth coordinates
References:
OOI (2012). Data Product Specification for Velocity Profile and Echo
Intensity. Document Control Number 1341-00750.
https://alfresco.oceanobservatories.org/ (See: Company Home >> OOI
>> Controlled >> 1000 System Level >>
1341-00750_Data_Product_SPEC_VELPROF_OOI.pdf)
OOI (2013). Data Product Specification for Turbulent Velocity Profile
and Echo Intensity. Document Control Number 1341-00760.
https://alfresco.oceanobservatories.org/ (See: Company Home >> OOI
>> Controlled >> 1000 System Level >>
1341-00760_Data_Product_SPEC_VELPROF_OOI.pdf)
"""
theta_rad = np.radians(theta)
cosT = np.cos(theta_rad)
sinT = np.sin(theta_rad)
M = np.array([
[cosT, sinT],
[-1*sinT, cosT]
])
u = np.atleast_1d(u)
v = np.atleast_1d(v)
cor = np.dot(M, np.array([u, v]))
return cor[0], cor[1]
def xy2lonlat(self, x, y):
"""Calculate x,y in own projection from given lon,lat (scalars/arrays).
"""
if self.projected is True:
if self.proj.is_latlong():
return x, y
else:
if 'ob_tran' in self.proj4:
logging.info('NB: Converting degrees to radians ' +
'due to ob_tran srs')
x = np.radians(np.array(x))
y = np.radians(np.array(y))
return self.proj(x, y, inverse=True)
else:
np.seterr(invalid='ignore') # Disable warnings for nan-values
y = np.atleast_1d(np.array(y))
x = np.atleast_1d(np.array(x))
# NB: mask coordinates outside domain
x[x < self.xmin] = np.nan
x[x > self.xmax] = np.nan
y[y < self.ymin] = np.nan
y[y < self.ymin] = np.nan
lon = map_coordinates(self.lon, [y, x], order=1,
cval=np.nan, mode='nearest')
lat = map_coordinates(self.lat, [y, x], order=1,
cval=np.nan, mode='nearest')
return (lon, lat)
def __call__(self,x,y,dx=0,dy=0):
""" Interpolate the function.
Parameters
----------
x : 1D array
y : 1D array
The points to interpolate.
dx : int >= 0, < kx
dy : int >= 0, < ky
The order of partial derivatives in x and y, respectively.
Returns
-------
z : 2D array with shape (len(y), len(x))
The interpolated values.
"""
x = atleast_1d(x)
y = atleast_1d(y)
z = fitpack.bisplev(x, y, self.tck, dx, dy)
z = atleast_2d(z)
z = transpose(z)
if len(z)==1:
z = z[0]
return array(z)
def record_values(self, updateTimeOnly=False):
'''Records the current variable values of the space craft into a private list (to be plotted later)
'''
# Saves variable values!
# Append the time variable:
# O: FIX THESE
self.__values["time"].append(self.__time)
self.__values["cellid"].append(self.__cellid)
self.__values["xcoordinates"].append(self.__coordinates[0])
self.__values["ycoordinates"].append(self.__coordinates[1])
self.__values["zcoordinates"].append(self.__coordinates[2])
self.__values["timeindexes"].append(self.__time_index)
#self.__timevariable.append(self.__time)
#self.__cellidlist.append(self.__cellid)
#self.__coordinateslist.append(self.__coordinates)
# Set other variables if updateTimeOnly is false:
if updateTimeOnly == False:
for i in np.atleast_1d(self.__variables):
# Append the value at spacecraft's position:
self.__values[i].append(np.array(self.__vlsvfile.read_variable(i, self.__cellid)))
else:
for i in np.atleast_1d(self.__variables):
# Append the value at spacecraft's position:
# Note: Appends the latest value again so if list is [[5,2], [0, 3]] then after appending it looks like this: [[5,2,2], [0,3,3]]
self.__values[i].append(np.array(self.__values[i][len(self.__values[i])-1]))
def isnumeric(dat):
"""
isnumeric - Determine whether input is numeric
Syntax
tf = isnumeric(A)
Description
tf = isnumeric(A) returns logical 1 (true) if A is a numeric array and logical 0 (false)
otherwise. For example, sparse arrays and double-precision arrays are numeric, while strings,
cell arrays, and structure arrays and logicals are not.
Examples
Given the following cell array,
C{1,1} = pi; % double
C{1,2} = 'John Doe'; % char array
C{1,3} = 2 + 4i; % complex double
C{1,4} = ispc; % logical
C{1,5} = magic(3) % double array
C =
[3.1416] 'John Doe' [2.0000+ 4.0000i] [1][3x3 double]
isnumeric shows that all but C{1,2} and C{1,4} are numeric arrays.
for k = 1:5
x(k) = isnumeric(C{1,k});
end
x
x =
1 0 1 0 1
"""
return np.array([np.atleast_1d(d).dtype.kind in NUMERIC_KINDS for d in np.nditer(np.atleast_1d(dat))]).astype('int8')
def __call__(self,x,y,dx=0,dy=0):
"""Interpolate the function.
Parameters
----------
x : 1D array
x-coordinates of the mesh on which to interpolate.
y : 1D array
y-coordinates of the mesh on which to interpolate.
dx : int >= 0, < kx
Order of partial derivatives in x.
dy : int >= 0, < ky
Order of partial derivatives in y.
Returns
-------
z : 2D array with shape (len(y), len(x))
The interpolated values.
"""
x = atleast_1d(x)
y = atleast_1d(y)
z = fitpack.bisplev(x, y, self.tck, dx, dy)
z = atleast_2d(z)
z = transpose(z)
if len(z)==1:
z = z[0]
return array(z)
def frac_yr_to_jd(year, gregorian=True):
"""Convert a date in the Julian or Gregorian fractional year to the
Julian Day Number (Meeus 7.1).
Arguments:
- `year` : (int, float) year
Keywords:
- `gregorian` : (bool, default=True) If True, use Gregorian calendar,
else use Julian calendar
Returns:
- (float)
"""
year = np.atleast_1d(year)
day = np.atleast_1d(0.0).astype(np.float64)
year, day = list(map(np.array, np.broadcast_arrays(year, day)))
# For float years abuse the day variable
fyear = year - year.astype('i')
mask = fyear > 0
if np.any(mask):
year = year.astype('i')
days_in_year = cal_to_jd(year[mask] + 1) - cal_to_jd(year[mask])
day[mask] = days_in_year*fyear[mask]
return _scalar_if_one(cal_to_jd(year) + day)
return _scalar_if_one(cal_to_jd(year))
def logit(prop, max_events=None):
"""Convert proportion (expressed in the range [0, 1]) to logit.
Parameters
----------
prop : float | array-like
the occurrence proportion.
max_events : int | array-like | None
the number of events used to calculate ``prop``. Used in a correction
factor for cases when ``prop`` is 0 or 1, to prevent returning ``inf``.
If ``None``, no correction is done, and ``inf`` or ``-inf`` may result.
Returns
-------
lgt : ``numpy.ndarray``, with shape matching ``numpy.array(prop).shape``.
"""
prop = np.atleast_1d(prop).astype(float)
if np.any([prop > 1, prop < 0]):
raise ValueError('Proportions must be in the range [0, 1].')
if max_events is not None:
# add equivalent of half an event to 0s, and subtract same from 1s
max_events = np.atleast_1d(max_events) * np.ones_like(prop)
corr_factor = 0.5 / max_events
for loc in zip(*np.where(prop == 0)):
prop[loc] = corr_factor[loc]
for loc in zip(*np.where(prop == 1)):
prop[loc] = 1 - corr_factor[loc]
return np.log(prop / (np.ones_like(prop) - prop))
def cal_to_day_of_year(year, mon, day, gregorian=True):
"""Convert a date in the Julian or Gregorian calendars to day of the year
(Meeus 7.1).
Arguments:
- `year` : (int) year
- `mon` : (int) month
- `day` : (int) day
Keywords:
- `gregorian` : (bool, default=True) If True, use Gregorian calendar,
else use Julian calendar
Return:
- day number : 1 = Jan 1...365 (or 366 for leap years) = Dec 31.
"""
year = np.atleast_1d(year).astype(np.int64)
mon = np.atleast_1d(mon).astype(np.int64)
day = np.atleast_1d(day).astype(np.int64)
year, mon, day = np.broadcast_arrays(year, mon, day)
K = np.ones_like(year)
K[:] = 2
K[np.atleast_1d(is_leap_year(year, gregorian))] = 1
return _scalar_if_one(
(275 * mon / 9.0).astype(np.int64) -
(K * ((mon + 9) / 12.0).astype(np.int64)) + day - 30)
def day_of_year_to_cal(year, N, gregorian=True):
"""Convert a day of year number to a month and day in the Julian or
Gregorian calendars.
Arguments:
- `year` : year
- `N` : day of year, 1..365 (or 366 for leap years)
Keywords:
- `gregorian` : If True, use Gregorian calendar, else use Julian calendar
(default: True)
Return:
- (month, day) : (tuple)
"""
year = np.atleast_1d(year)
N = np.atleast_1d(N)
year, N = np.broadcast_arrays(year, N)
K = np.ones_like(N)
K[:] = 2
K[np.atleast_1d(is_leap_year(year, gregorian))] = 1
mon = (9 * (K + N) / 275.0 + 0.98).astype(np.int64)
mon[N < 32] = 1
day = (N - (275 * mon / 9.0).astype(np.int64) +
K * ((mon + 9) / 12.0).astype(np.int64) + 30).astype(np.int64)
return _scalar_if_one(mon), _scalar_if_one(day)
def _get_entries(fid, evoked_node):
"""Helper to get all evoked entries"""
comments = list()
aspect_kinds = list()
for ev in evoked_node:
for k in range(ev['nent']):
my_kind = ev['directory'][k].kind
pos = ev['directory'][k].pos
if my_kind == FIFF.FIFF_COMMENT:
tag = read_tag(fid, pos)
comments.append(tag.data)
my_aspect = dir_tree_find(ev, FIFF.FIFFB_ASPECT)[0]
for k in range(my_aspect['nent']):
my_kind = my_aspect['directory'][k].kind
pos = my_aspect['directory'][k].pos
if my_kind == FIFF.FIFF_ASPECT_KIND:
tag = read_tag(fid, pos)
aspect_kinds.append(int(tag.data))
comments = np.atleast_1d(comments)
aspect_kinds = np.atleast_1d(aspect_kinds)
if len(comments) != len(aspect_kinds) or len(comments) == 0:
fid.close()
raise ValueError('Dataset names in FIF file '
'could not be found.')
t = [aspect_rev.get(str(a), 'Unknown') for a in aspect_kinds]
t = ['"' + c + '" (' + tt + ')' for tt, c in zip(t, comments)]
t = ' ' + '\n '.join(t)
return comments, aspect_kinds, t
def writeEmissionSpectrumFile(wavelengths, intensities, fname, n_rv=10000):
'''
Helper function to write Excitation and Emission spectrum in file
for the mmpraytracer
Parameters
----------
wavelengths : list like
Wavelengths of the spectrum.
intensities : list like
Intensities of the spectrum.
fname : str
Filepath to save the spectrum.
n_rv : int
Number of random variables the spectrum is divided. Default 10000.
'''
intensities = np.atleast_1d(intensities)
wavelengths = np.atleast_1d(wavelengths)
# Invert emission spectrum
intensEMone = intensities / np.trapz(intensities, x=wavelengths)
rv = np.linspace(0, 1, n_rv)
cumEM = integrate.cumtrapz(intensEMone, wavelengths, initial=0)
invCumEM = sc.invertNiceFunction(wavelengths, cumEM, rv)
np.savetxt(fname, invCumEM)
def hyp0f1(v, z):
r"""Confluent hypergeometric limit function 0F1.
Parameters
----------
v, z : array_like
Input values.
Returns
-------
hyp0f1 : ndarray
The confluent hypergeometric limit function.
Notes
-----
This function is defined as:
.. math:: _0F_1(v,z) = \sum_{k=0}^{\inf}\frac{z^k}{(v)_k k!}.
It's also the limit as q -> infinity of ``1F1(q;v;z/q)``, and satisfies
the differential equation :math:``f''(z) + vf'(z) = f(z)`.
"""
v = atleast_1d(v)
z = atleast_1d(z)
v, z = np.broadcast_arrays(v, z)
arg = 2 * sqrt(abs(z))
old_err = np.seterr(all='ignore') # for z=0, a<1 and num=inf, next lines
num = where(z.real >= 0, iv(v - 1, arg), jv(v - 1, arg))
den = abs(z)**((v - 1.0) / 2)
num *= gamma(v)
np.seterr(**old_err)
num[z == 0] = 1
den[z == 0] = 1
return num / den
def check_gradient(fcn, Dfcn, x0, args=(), col_deriv=0):
"""Perform a simple check on the gradient for correctness.
"""
x = atleast_1d(x0)
n = len(x)
x = x.reshape((n,))
fvec = atleast_1d(fcn(x, *args))
m = len(fvec)
fvec = fvec.reshape((m,))
ldfjac = m
fjac = atleast_1d(Dfcn(x, *args))
fjac = fjac.reshape((m, n))
if col_deriv == 0:
fjac = transpose(fjac)
xp = zeros((n,), float)
err = zeros((m,), float)
fvecp = None
_minpack._chkder(m, n, x, fvec, fjac, ldfjac, xp, fvecp, 1, err)
fvecp = atleast_1d(fcn(xp, *args))
fvecp = fvecp.reshape((m,))
_minpack._chkder(m, n, x, fvec, fjac, ldfjac, xp, fvecp, 2, err)
good = (product(greater(err, 0.5), axis=0))
return (good, err)
def test_prs_bottilt_ccmp(self):
"""
Test prs_bottilt_ccmp function.
Values based on those described in DPS as available on Alfresco:
OOI (2012). Data Product Specification for Seafloor High-Resolution
Tilt. Document Control Number 1341-00060.
https://alfresco.oceanobservatories.org/ (See: Company Home >> OOI
>> Controlled >> 1000 System Level >>
1341-00060_Data_Product_SPEC_BOTTILT_OOI.pdf)
Note, DPS does not specify a test for this function. Values from CCMP
lookup table in DPS used to create this test by the function and
test_function author.
Implemented by Christopher Wingard, July 2013
"""
# set known inputs
scmp = np.atleast_1d([row[2] for row in self.lily])
snum = np.atleast_1d([row[3] for row in self.lily])
# set known output for the corrected compass direction
ccmp = np.atleast_1d([row[4] for row in self.lily])
# calculate the corrected compass direction
out = prsfunc.prs_bottilt_ccmp(scmp, snum)
# How'd we do?
np.testing.assert_array_equal(out, ccmp)
def __init__(self, wsize, tstep, n_coefs): # noqa: D102
self.wsize = np.atleast_1d(wsize)
self.tstep = np.atleast_1d(tstep)
self.n_coefs = np.atleast_1d(n_coefs)
self.n_dicts = len(tstep)
self.n_freqs = wsize // 2 + 1
self.n_steps = self.n_coefs // self.n_freqs
def coord_transform(x, y, z, affine):
""" Convert the x, y, z coordinates from one image space to another
space.
Parameters
----------
x : number or ndarray
The x coordinates in the input space
y : number or ndarray
The y coordinates in the input space
z : number or ndarray
The z coordinates in the input space
affine : 2D 4x4 ndarray
affine that maps from input to output space.
Returns
-------
x : number or ndarray
The x coordinates in the output space
y : number or ndarray
The y coordinates in the output space
z : number or ndarray
The z coordinates in the output space
Warning: The x, y and z have their Talairach ordering, not 3D
numy image ordering.
"""
coords = np.c_[np.atleast_1d(x).flat,
np.atleast_1d(y).flat,
np.atleast_1d(z).flat,
np.ones_like(np.atleast_1d(z).flat)].T
x, y, z, _ = np.dot(affine, coords)
return x.squeeze(), y.squeeze(), z.squeeze()
def _compute_model(self, pset):
"""Computes a model and inserts results into the Mongo collection."""
nBands = fsps.driver.get_n_bands()
nLambda = fsps.driver.get_n_lambda()
nAges = fsps.driver.get_n_ages()
fsps.driver.comp_sp(pset['dust_type'], pset['zmet'], pset['sfh'],
pset['tau'], pset['const'], pset['fburst'], pset['tburst'],
pset['dust_tesc'], pset['dust1'], pset['dust2'],
pset['dust_clumps'], pset['frac_nodust'], pset['dust_index'],
pset['mwr'], pset['wgp1'], pset['wgp2'], pset['wgp3'],
pset['duste_gamma'], pset['duste_umin'], pset['duste_qpah'],
pset['tage'])
if pset['tage'] == 0.:
# SFH over all ages is returned
mags = fsps.driver.get_csp_mags(nBands, nAges)
specs = fsps.driver.get_csp_specs(nLambda, nAges)
age, mass, lbol, sfr, dust_mass = fsps.driver.get_csp_stats(nAges)
else:
# get only a single age, stored in first age bin
# arrays must be re-formated to appear like one-age versions of
# the outputs from get_csp_mags, etc.
mags = fsps.driver.get_csp_mags_at_age(1, nBands)
specs = fsps.driver.get_csp_specs_at_age(1, nLambda)
age, mass, lbol, sfr, dust_mass \
= fsps.driver.get_csp_stats_at_age(1)
age = np.atleast_1d(age)
mass = np.atleast_1d(mass)
lbol = np.atleast_1d(lbol)
sfr = np.atleast_1d(sfr)
dust_mass = np.atleast_1d(dust_mass)
mags = np.atleast_2d(mags)
specs = np.atleast_2d(specs)
dataArray = self._splice_mag_spec_arrays(age, mass, lbol, sfr,
dust_mass, mags, specs, nLambda)
self._insert_model(pset.name, dataArray)
def _make_tuples(self, key):
print('Populating', key)
spikes = np.vstack([s.squeeze() for s in (preprocess.Spikes.RateTrace() & key).fetch('rate_trace')])
s = spikes.sum(axis=0)
nans = np.isnan(s)
key['leading_nans'] = int(nans[0])
key['trailing_nans'] = int(nans[1])
t = (preprocess.Sync() & key).fetch1('frame_times') # does not need to be unique
flip_first = (vis.Trial() * preprocess.Sync().proj('psy_id', trial_idx='first_trial') & key).fetch1('flip_times')
flip_last = (vis.Trial() * preprocess.Sync().proj('psy_id', trial_idx='last_trial') & key).fetch1('flip_times')
# (vis.Trial() * preprocess.Sync() & 'trial_idx between first_trial and last_trial')
fro = np.atleast_1d(flip_first.squeeze())[0]
to = np.atleast_1d(flip_last.squeeze())[
-1] # not necessarily where the stimulus stopped, just last presentation
idx_fro = np.argmin(np.abs(t - fro))
idx_to = np.argmin(np.abs(t - to)) + 1
key['stimulus_nans'] = int(np.any(nans[idx_fro:idx_to]))
if np.any(nans):
key['nan_idx'] = nans
key['stimulus_start'] = idx_fro + 1
key['stimulus_end'] = idx_to
self.insert1(key)
def dataqc_gradienttest_wrapper(dat, x, ddatdx, mindx, startdat, toldat, strict_validation=False):
if is_none(ddatdx) or is_fill(ddatdx) or is_none(mindx) or is_fill(mindx) or is_none(startdat) or is_fill(startdat) or is_none(toldat) or is_fill(toldat):
out = np.empty(dat.shape, dtype=np.int8)
out.fill(-99)
return out
outqc = dataqc_gradienttest(dat, x, [-np.atleast_1d(ddatdx)[-1], np.atleast_1d(ddatdx)[-1]], np.atleast_1d(mindx)[-1], np.atleast_1d(startdat)[-1], np.atleast_1d(toldat)[-1], strict_validation=strict_validation)
return outqc
def headalongline(self, x, y, t, layers=None):
"""Head along line or curve
Parameters
----------
x : array
x values of line
y : array
y values of line
t : list or array
times for which grid is returned
layers : integer, list or array, optional
layers for which grid is returned
Returns
-------
h : array size `nlayers, ntimes, nx`
"""
xg = np.atleast_1d(x)
yg = np.atleast_1d(y)
if layers is None:
Nlayers = self.aq.find_aquifer_data(xg[0], yg[0]).naq
else:
Nlayers = len(np.atleast_1d(layers))
nx = len(xg)
if len(yg) == 1:
yg = yg * np.ones(nx)
t = np.atleast_1d(t)
h = np.zeros( (Nlayers,len(t),nx) )
for i in range(nx):
h[:,:,i] = self.head(xg[i], yg[i], t, layers)
return h
def __new__(cls, val1, val2, scale, precision,
in_subfmt, out_subfmt, from_jd=False):
"""
Use __new__ instead of __init__ to output a class instance that
is the same as the class of the first Time object in the list.
"""
val1_0 = val1.flat[0]
if not (isinstance(val1_0, Time) and all(type(val) is type(val1_0)
for val in val1.flat)):
raise TypeError('Input values for {0} class must all be same '
'astropy Time type.'.format(cls.name))
if scale is None:
scale = val1_0.scale
if val1.shape:
vals = [getattr(val, scale)._time for val in val1]
jd1 = np.concatenate([np.atleast_1d(val.jd1) for val in vals])
jd2 = np.concatenate([np.atleast_1d(val.jd2) for val in vals])
else:
val = getattr(val1_0, scale)._time
jd1, jd2 = val.jd1, val.jd2
OutTimeFormat = val1_0._time.__class__
self = OutTimeFormat(jd1, jd2, scale, precision, in_subfmt, out_subfmt,
from_jd=True)
return self
def test_prs_bottilt_tmag(self):
"""
Test prs_bottilt_tmag function.
Note, DPS specifies a test data set that does not vary the X-tilt and
Y-tilt inputs to this function. They are 330 and -330 microradians,
respectively, for the entirety of the dataset. Test values below were
created by function and test_function author to ensure function
operates as expected over a range of potential values.
OOI (2012). Data Product Specification for Seafloor High-Resolution
Tilt. Document Control Number 1341-00060.
https://alfresco.oceanobservatories.org/ (See: Company Home >> OOI
>> Controlled >> 1000 System Level >>
1341-00060_Data_Product_SPEC_BOTTILT_OOI.pdf)
Implemented by Christopher Wingard, July 2013
"""
# set inputs
xtilt = np.atleast_1d([row[0] for row in self.lily])
ytilt = np.atleast_1d([row[1] for row in self.lily])
# set known output for the tilt magnitude
tmag = np.atleast_1d([row[5] for row in self.lily])
# calculate the tilt magnitude
out = prsfunc.prs_bottilt_tmag(xtilt, ytilt)
# How'd we do?
np.testing.assert_allclose(out, tmag, rtol=0.1, atol=0.1)
def permute_axes(array, axes, copy=False, order='C'):
"""Change the arrays axes order or combine multiple axes into one. Creates
a view if possible, but when axes are combined will usually return a copy.
Parameters
----------
array : array_like
Array for which to define new axes.
axes : iterable
Iterable giving for every new axis which old axis are combined into it.
np.newaxis/None can be used to insert a new axis. Elements must be
either ints or iterables of ints identifying each axis.
copy : bool
If True a copy is forced.
order : 'C', 'F', 'A' or 'K'
Whether new array should have C or Fortran order. See np.copy help for
details.
See Also
--------
swapaxes, rollaxis, reshape
Examples
--------
>>> a = np.arange(12).reshape(3,2,2)
Just reverse the axes order:
>>> permute_axes(a, (2, 1, 0))
array([[[ 0, 4, 8],
[ 2, 6, 10]],
[[ 1, 5, 9],
[ 3, 7, 11]]])
Combine axis 0 and 1 and put the last axis to the front:
>>> permute_axes(a, (-1, (0, 1)))
array([[ 0, 2, 4, 6, 8, 10],
[ 1, 3, 5, 7, 9, 11]])
Or going over the first two axes in different order:
>>> permute_axes(a, (-1, (1, 0)))
array([[ 0, 4, 8, 2, 6, 10],
[ 1, 5, 9, 3, 7, 11]])
"""
new_axes = []
for a in axes:
if a is None:
new_axes.append(None)
else:
a = np.atleast_1d(a)
if a.ndim > 1:
raise ValueError(
"All items of `axes` must be zero or one dimensional.")
new_axes.append(a) # array for slicing.
old_shape = np.asarray(array.shape)
old_strides = np.asarray(array.strides)
# Shape and strides for the copy operation:
tmp_shape = []
tmp_strides = []
final_shape = []
final_strides = [] # only used if no copy is needed.
check_axes = np.zeros(len(old_shape), dtype=bool)
must_copy = False
# create a reordered array first:
for ax, na in enumerate(new_axes):
if na is not None:
if np.any(check_axes[na]) or np.unique(na).shape != na.shape:
raise ValueError(
"All axis must at most occure once in the new array")
check_axes[na] = True
if len(na) != 0:
_shapes = old_shape[na]
_strides = old_strides[na]
tmp_shape.extend(_shapes)
tmp_strides.extend(_strides)
final_strides.append(_strides.min()) # smallest stride...
final_shape.append(_shapes.prod())
if not must_copy:
# If any of the strides do not fit together we must copy:
prev_stride = _strides[0]
for stride, shape in zip(_strides[1:], _shapes[1:]):
if shape == 1:
# 1 sized dimensions just do not matter, but they
# also do not matter for legality check.
continue
elif prev_stride != stride * shape:
must_copy = True
break
prev_stride = stride
continue # skip adding empty axis.
tmp_shape.append(1)
tmp_strides.append(0)
final_shape.append(1)
final_strides.append(0)
if not must_copy:
result = np.lib.stride_tricks.as_strided(array,
shape=final_shape,
strides=final_strides)
if copy:
return result.copy(order=order)
return result
# No need for explicite copy, as reshape must already create one since
# must_copy is True.
copy_from = np.lib.stride_tricks.as_strided(array,
shape=tmp_shape,
strides=tmp_strides)
return copy_from.reshape(final_shape, order=order)
def __init__(self,
data,
position,
size,
wcs=None,
mode='trim',
fill_value=np.nan,
copy=False):
if wcs is None:
wcs = getattr(data, 'wcs', None)
if isinstance(position, SkyCoord):
if wcs is None:
raise ValueError('wcs must be input if position is a '
'SkyCoord')
position = skycoord_to_pixel(position, wcs, mode='all') # (x, y)
if np.isscalar(size):
size = np.repeat(size, 2)
# special handling for a scalar Quantity
if isinstance(size, u.Quantity):
size = np.atleast_1d(size)
if len(size) == 1:
size = np.repeat(size, 2)
if len(size) > 2:
raise ValueError('size must have at most two elements')
shape = np.zeros(2).astype(int)
pixel_scales = None
# ``size`` can have a mixture of int and Quantity (and even units),
# so evaluate each axis separately
for axis, side in enumerate(size):
if not isinstance(side, u.Quantity):
shape[axis] = int(np.round(size[axis])) # pixels
else:
if side.unit == u.pixel:
shape[axis] = int(np.round(side.value))
elif side.unit.physical_type == 'angle':
if wcs is None:
raise ValueError('wcs must be input if any element '
'of size has angular units')
if pixel_scales is None:
pixel_scales = u.Quantity(proj_plane_pixel_scales(wcs),
wcs.wcs.cunit[axis])
shape[axis] = int(
np.round((side / pixel_scales[axis]).decompose()))
else:
raise ValueError('shape can contain Quantities with only '
'pixel or angular units')
data = np.asanyarray(data)
# reverse position because extract_array and overlap_slices
# use (y, x), but keep the input position
pos_yx = position[::-1]
cutout_data, input_position_cutout = extract_array(
data,
tuple(shape),
pos_yx,
mode=mode,
fill_value=fill_value,
return_position=True)
if copy:
cutout_data = np.copy(cutout_data)
self.data = cutout_data
self.input_position_cutout = input_position_cutout[::-1] # (x, y)
slices_original, slices_cutout = overlap_slices(data.shape,
shape,
pos_yx,
mode=mode)
self.slices_original = slices_original
self.slices_cutout = slices_cutout
self.shape = self.data.shape
self.input_position_original = position
self.shape_input = shape
((self.ymin_original, self.ymax_original),
(self.xmin_original, self.xmax_original)) = self.bbox_original
((self.ymin_cutout, self.ymax_cutout),
(self.xmin_cutout, self.xmax_cutout)) = self.bbox_cutout
# the true origin pixel of the cutout array, including any
# filled cutout values
self._origin_original_true = (self.origin_original[0] -
self.slices_cutout[1].start,
self.origin_original[1] -
self.slices_cutout[0].start)
if wcs is not None:
self.wcs = deepcopy(wcs)
self.wcs.wcs.crpix -= self._origin_original_true
self.wcs.array_shape = self.data.shape
if wcs.sip is not None:
self.wcs.sip = Sip(wcs.sip.a, wcs.sip.b, wcs.sip.ap,
wcs.sip.bp,
wcs.sip.crpix - self._origin_original_true)
else:
self.wcs = None
def constrain_I_i(Sf_vec, R0, gammatau):
S, f = Sf_vec
I_i = I_of_S(S, R0)
return np.atleast_1d(I_max_opt_of_S_i(S, f, R0, gammatau) - I_i)
def constrain_Scrit(Sf_vec, R0, gammatau):
S, f = Sf_vec
I_i = I_of_S(S, R0)
Scrit = (1 / R0) + 1e-5
return np.atleast_1d(S - gammatau * f * I_i - Scrit)
def fit(self, X, y, sample_weight=None):
"""Build a forest of trees from the training set (X, y).
Parameters
----------
X : array-like or sparse matrix of shape = [n_samples, n_features]
The training input samples. Internally, its dtype will be converted to
``dtype=np.float32``. If a sparse matrix is provided, it will be
converted into a sparse ``csc_matrix``.
y : array-like, shape = [n_samples] or [n_samples, n_outputs]
The target values (class labels in classification, real numbers in
regression).
sample_weight : array-like, shape = [n_samples] or None
Sample weights. If None, then samples are equally weighted. Splits
that would create child nodes with net zero or negative weight are
ignored while searching for a split in each node. In the case of
classification, splits are also ignored if they would result in any
single class carrying a negative weight in either child node.
Returns
-------
self : object
Returns self.
"""
# Validate or convert input data
X = check_array(X, accept_sparse="csc", dtype=DTYPE)
y = check_array(y, accept_sparse='csc', ensure_2d=False, dtype=None)
if issparse(X):
# Pre-sort indices to avoid that each individual tree of the
# ensemble sorts the indices.
X.sort_indices()
# Remap output
n_samples, self.n_features_ = X.shape
y = np.atleast_1d(y)
if y.ndim == 2 and y.shape[1] == 1:
warn(
"A column-vector y was passed when a 1d array was"
" expected. Please change the shape of y to "
"(n_samples,), for example using ravel().",
DataConversionWarning,
stacklevel=2)
if y.ndim == 1:
# reshape is necessary to preserve the data contiguity against vs
# [:, np.newaxis] that does not.
y = np.reshape(y, (-1, 1))
self.n_outputs_ = y.shape[1]
y, expanded_class_weight = self._validate_y_class_weight(y)
if getattr(y, "dtype", None) != DOUBLE or not y.flags.contiguous:
y = np.ascontiguousarray(y, dtype=DOUBLE)
if expanded_class_weight is not None:
if sample_weight is not None:
sample_weight = sample_weight * expanded_class_weight
else:
sample_weight = expanded_class_weight
# Check parameters
self._validate_estimator()
if not self.bootstrap:
raise ValueError(
"bootstrap=False is invalid for RandomForestSubsample")
if not 0.1 < self.target_imbalance_ratio <= 1.0:
raise ValueError(
"target_imbalance_ratio must be between 0.1 and 1.0")
if not self.bootstrap and self.oob_score:
raise ValueError("Out of bag estimation only available"
" if bootstrap=True")
random_state = check_random_state(self.random_state)
if not self.warm_start:
# Free allocated memory, if any
self.estimators_ = []
n_more_estimators = self.n_estimators - len(self.estimators_)
if n_more_estimators < 0:
raise ValueError('n_estimators=%d must be larger or equal to '
'len(estimators_)=%d when warm_start==True' %
(self.n_estimators, len(self.estimators_)))
elif n_more_estimators == 0:
warn("Warm-start fitting without increasing n_estimators does not "
"fit new trees.")
else:
if self.warm_start and len(self.estimators_) > 0:
# We draw from the random state to get the random state we
# would have got if we hadn't used a warm_start.
random_state.randint(MAX_INT, size=len(self.estimators_))
trees = []
for i in range(n_more_estimators):
tree = self._make_estimator(append=False,
random_state=random_state)
trees.append(tree)
# Parallel loop: we use the threading backend as the Cython code
# for fitting the trees is internally releasing the Python GIL
# making threading always more efficient than multiprocessing in
# that case.
trees = Parallel(
n_jobs=self.n_jobs, verbose=self.verbose,
backend="threading")(delayed(_parallel_build_trees)(
t,
self,
X,
y,
sample_weight,
i,
len(trees),
verbose=self.verbose,
class_weight=self.class_weight,
target_imbalance_ratio=self.target_imbalance_ratio)
for i, t in enumerate(trees))
# Collect newly grown trees
self.estimators_.extend(trees)
if self.oob_score:
self._set_oob_score(X, y)
# Decapsulate classes_ attributes
if hasattr(self, "classes_") and self.n_outputs_ == 1:
self.n_classes_ = self.n_classes_[0]
self.classes_ = self.classes_[0]
return self
reveal_type(np.full_like(B, i8, dtype=np.int64)) # E: numpy.ndarray reveal_type(np.ones(1)) # E: numpy.ndarray reveal_type(np.ones([1, 1, 1])) # E: numpy.ndarray reveal_type(np.full(1, i8)) # E: numpy.ndarray reveal_type(np.full([1, 1, 1], i8)) # E: numpy.ndarray reveal_type(np.indices([1, 2, 3])) # E: numpy.ndarray reveal_type(np.indices([1, 2, 3], sparse=True)) # E: tuple[numpy.ndarray] reveal_type(np.fromfunction(func, (3, 5))) # E: SubClass reveal_type(np.identity(10)) # E: numpy.ndarray reveal_type(np.atleast_1d(A)) # E: numpy.ndarray reveal_type(np.atleast_1d(C)) # E: numpy.ndarray reveal_type(np.atleast_1d(A, A)) # E: list[numpy.ndarray] reveal_type(np.atleast_1d(A, C)) # E: list[numpy.ndarray] reveal_type(np.atleast_1d(C, C)) # E: list[numpy.ndarray] reveal_type(np.atleast_2d(A)) # E: numpy.ndarray reveal_type(np.atleast_3d(A)) # E: numpy.ndarray reveal_type(np.vstack([A, A])) # E: numpy.ndarray reveal_type(np.vstack([A, C])) # E: numpy.ndarray reveal_type(np.vstack([C, C])) # E: numpy.ndarray reveal_type(np.hstack([A, A])) # E: numpy.ndarray
def detect_peaks(x:Sequence,
mph:Optional[Real]=None, mpd:int=1,
threshold:Real=0, left_threshold:Real=0, right_threshold:Real=0,
prominence:Optional[Real]=None, prominence_wlen:Optional[int]=None,
edge:Union[str,type(None)]='rising', kpsh:bool=False, valley:bool=False,
show:bool=False, ax=None,
verbose:int=0) -> np.ndarray:
"""
Detect peaks in data based on their amplitude and other features.
Parameters:
-----------
x: 1D array_like,
data
mph: positive number, optional,
abbr. for maximum (minimum) peak height,
detect peaks that are greater than minimum peak height (if parameter `valley` is False),
or peaks that are smaller than maximum peak height (if parameter `valley` is True)
mpd: positive integer, default 1,
abbr. for minimum peak distance,
detect peaks that are at least separated by minimum peak distance (in number of samples)
threshold: positive number, default 0,
detect peaks (valleys) that are greater (smaller) than `threshold`,
in relation to their neighbors within the range of `mpd`
left_threshold: positive number, default 0,
`threshold` that is restricted to the left
right_threshold: positive number, default 0,
`threshold` that is restricted to the left
prominence: positive number, optional,
threshold of prominence of the detected peaks (valleys)
prominence_wlen: positive int, optional,
the `wlen` parameter of the function `scipy.signal.peak_prominences`
edge: str or None, default 'rising',
can also be 'falling', 'both',
for a flat peak, keep only the rising edge ('rising'), only the falling edge ('falling'),
both edges ('both'), or don't detect a flat peak (None)
kpsh: bool, default False,
keep peaks with same height even if they are closer than `mpd`
valley: bool, default False,
if True (1), detect valleys (local minima) instead of peaks
show: bool, default False,
if True (1), plot data in matplotlib figure
ax: a matplotlib.axes.Axes instance, optional,
Returns:
--------
ind : 1D array_like
indeces of the peaks in `x`.
Notes:
------
The detection of valleys instead of peaks is performed internally by simply
negating the data: `ind_valleys = detect_peaks(-x)`
The function can handle NaN's
See this IPython Notebook [1]_.
References:
-----------
[1] http://nbviewer.ipython.org/github/demotu/BMC/blob/master/notebooks/DetectPeaks.ipynb
Examples
--------
>>> from detect_peaks import detect_peaks
>>> x = np.random.randn(100)
>>> x[60:81] = np.nan
>>> # detect all peaks and plot data
>>> ind = detect_peaks(x, show=True)
>>> print(ind)
>>> x = np.sin(2*np.pi*5*np.linspace(0, 1, 200)) + np.random.randn(200)/5
>>> # set minimum peak height = 0 and minimum peak distance = 20
>>> detect_peaks(x, mph=0, mpd=20, show=True)
>>> x = [0, 1, 0, 2, 0, 3, 0, 2, 0, 1, 0]
>>> # set minimum peak distance = 2
>>> detect_peaks(x, mpd=2, show=True)
>>> x = np.sin(2*np.pi*5*np.linspace(0, 1, 200)) + np.random.randn(200)/5
>>> # detection of valleys instead of peaks
>>> detect_peaks(x, mph=-1.2, mpd=20, valley=True, show=True)
>>> x = [0, 1, 1, 0, 1, 1, 0]
>>> # detect both edges
>>> detect_peaks(x, edge='both', show=True)
>>> x = [-2, 1, -2, 2, 1, 1, 3, 0]
>>> # set threshold = 2
>>> detect_peaks(x, threshold = 2, show=True)
Version history
---------------
'1.0.5':
The sign of `mph` is inverted if parameter `valley` is True
"""
data = deepcopy(x)
data = np.atleast_1d(data).astype('float64')
if data.size < 3:
return np.array([], dtype=int)
if valley:
data = -data
if mph is not None:
mph = -mph
# find indices of all peaks
dx = data[1:] - data[:-1] # equiv to np.diff()
# handle NaN's
indnan = np.where(np.isnan(data))[0]
if indnan.size:
data[indnan] = np.inf
dx[np.where(np.isnan(dx))[0]] = np.inf
ine, ire, ife = np.array([[], [], []], dtype=int)
if not edge:
ine = np.where((np.hstack((dx, 0)) < 0) & (np.hstack((0, dx)) > 0))[0]
else:
if edge.lower() in ['rising', 'both']:
ire = np.where((np.hstack((dx, 0)) <= 0) & (np.hstack((0, dx)) > 0))[0]
if edge.lower() in ['falling', 'both']:
ife = np.where((np.hstack((dx, 0)) < 0) & (np.hstack((0, dx)) >= 0))[0]
ind = np.unique(np.hstack((ine, ire, ife)))
if verbose >= 1:
print(f'before filtering by mpd = {mpd}, and threshold = {threshold}, ind = {ind.tolist()}')
print(f'additionally, left_threshold = {left_threshold}, right_threshold = {right_threshold}, length of data = {len(data)}')
# handle NaN's
if ind.size and indnan.size:
# NaN's and values close to NaN's cannot be peaks
ind = ind[np.in1d(ind, np.unique(np.hstack((indnan, indnan-1, indnan+1))), invert=True)]
if verbose >= 1:
print(f'after handling nan values, ind = {ind.tolist()}')
# peaks are only valid within [mpb, len(data)-mpb[
ind = np.array([pos for pos in ind if mpd<=pos<len(data)-mpd])
if verbose >= 1:
print(f'after fitering out elements too close to border by mpd = {mpd}, ind = {ind.tolist()}')
# first and last values of data cannot be peaks
# if ind.size and ind[0] == 0:
# ind = ind[1:]
# if ind.size and ind[-1] == data.size-1:
# ind = ind[:-1]
# remove peaks < minimum peak height
if ind.size and mph is not None:
ind = ind[data[ind] >= mph]
if verbose >= 1:
print(f'after filtering by mph = {mph}, ind = {ind.tolist()}')
# remove peaks - neighbors < threshold
_left_threshold = left_threshold if left_threshold > 0 else threshold
_right_threshold = right_threshold if right_threshold > 0 else threshold
if ind.size and (_left_threshold > 0 and _right_threshold > 0):
# dx = np.min(np.vstack([data[ind]-data[ind-1], data[ind]-data[ind+1]]), axis=0)
dx = np.max(np.vstack([data[ind]-data[ind+idx] for idx in range(-mpd, 0)]), axis=0)
ind = np.delete(ind, np.where(dx < _left_threshold)[0])
if verbose >= 2:
print(f'from left, dx = {dx.tolist()}')
print(f'after deleting those dx < _left_threshold = {_left_threshold}, ind = {ind.tolist()}')
dx = np.max(np.vstack([data[ind]-data[ind+idx] for idx in range(1, mpd+1)]), axis=0)
ind = np.delete(ind, np.where(dx < _right_threshold)[0])
if verbose >= 2:
print(f'from right, dx = {dx.tolist()}')
print(f'after deleting those dx < _right_threshold = {_right_threshold}, ind = {ind.tolist()}')
if verbose >= 1:
print(f'after filtering by threshold, ind = {ind.tolist()}')
# detect small peaks closer than minimum peak distance
if ind.size and mpd > 1:
ind = ind[np.argsort(data[ind])][::-1] # sort ind by peak height
idel = np.zeros(ind.size, dtype=bool)
for i in range(ind.size):
if not idel[i]:
# keep peaks with the same height if kpsh is True
idel = idel | (ind >= ind[i] - mpd) & (ind <= ind[i] + mpd) \
& (data[ind[i]] > data[ind] if kpsh else True)
idel[i] = 0 # Keep current peak
# remove the small peaks and sort back the indices by their occurrence
ind = np.sort(ind[~idel])
ind = np.array([item for item in ind if data[item]==np.max(data[item-mpd:item+mpd+1])])
if verbose >= 1:
print(f'after filtering by mpd, ind = {ind.tolist()}')
if prominence:
_p = peak_prominences(data, ind, prominence_wlen)[0]
ind = ind[np.where(_p >= prominence)[0]]
if verbose >= 1:
print(f'after filtering by prominence, ind = {ind.tolist()}')
if verbose >= 2:
print(f'with detailed prominence = {_p.tolist()}')
if show:
if indnan.size:
data[indnan] = np.nan
if valley:
data = -data
if mph is not None:
mph = -mph
_plot(data, mph, mpd, threshold, edge, valley, ax, ind)
return ind
def print_compress_drag(vehicle, analyses, filename='compress_drag.dat'):
"""This creates a file showing a breakdown of compressibility drag for the vehicle.
Assumptions:
None
Source:
N/A
Inputs:
vehicle.wings.main_wing.
sweeps.quarter_chord [-]
vehicle.wings.*.
tag <string>
thickness_to_chord [-]
vehicle.
tag <string>
reference_area [m^2]
analyses.configs.cruise.aerodynamics.settings Used in called function:
analyses.configs.cruise.aerodynamics.process.compute.drag.compressibility.wings.wing(state,settings,wing)
filename Sets file name to save (optional)
Outputs:
filename Saved file with name as above
Properties Used:
N/A
"""
# Unpack
sweep = vehicle.wings['main_wing'].sweeps.quarter_chord / Units.deg
t_c = vehicle.wings['main_wing'].thickness_to_chord
sref = vehicle.reference_area
settings = analyses.configs.cruise.aerodynamics.settings
# Define mach and CL vectors
mach_vec = np.linspace(0.45, 0.95, 11)
cl_vec = np.linspace(0.30, 0.80, 11)
# allocating array for each wing
cd_compress = Data()
for idw, wing in enumerate(vehicle.wings):
cd_compress[wing.tag] = np.zeros((len(mach_vec), len(cl_vec)))
cd_compress_tot = np.zeros_like(cd_compress.main_wing)
# Alocatting array necessary for the drag estimation method
state = Data()
state.conditions = SUAVE.Analyses.Mission.Segments.Conditions.Aerodynamics(
)
state.conditions.freestream.mach_number = mach_vec
# write header of file
fid = open(filename, 'w') # Open output file
fid.write('Output file with compressibility drag breakdown\n\n')
fid.write(' VEHICLE TAG : ' + vehicle.tag + '\n\n')
fid.write(' REFERENCE AREA ................ ' + str('%5.1f' % sref) +
' m2 ' + '\n')
fid.write(' MAIN WING SWEEP ............... ' + str('%5.1f' % sweep) +
' deg' + '\n')
fid.write(' MAIN WING THICKNESS RATIO ..... ' + str('%5.2f' % t_c) +
' ' + '\n')
fid.write(' \n')
fid.write(' TOTAL COMPRESSIBILITY DRAG \n')
fid.write(
str(
np.insert(
np.transpose(list(map('M={:5.3f} | '.format, (mach_vec)))), 0,
' CL | ')))
fid.write('\n')
# call aerodynamic method for each CL
for idcl, cl in enumerate(cl_vec):
state.conditions.aerodynamics.lift_breakdown.compressible_wings = np.atleast_1d(
cl)
# call method
for wing in vehicle.wings:
analyses.configs.cruise.aerodynamics.process.compute.drag.compressibility.wings.wing(
state, settings, wing)
# process output for print
drag_breakdown = state.conditions.aerodynamics.drag_breakdown.compressible
for wing in vehicle.wings:
cd_compress[wing.tag][:, idcl] = drag_breakdown[
wing.tag].compressibility_drag
cd_compress_tot[:, idcl] += drag_breakdown[
wing.tag].compressibility_drag
# print first the TOTAL COMPRESSIBILITY DRAG
fid.write(
str(
np.insert((np.transpose(
list(map('{:7.5f} | '.format,
(cd_compress_tot[:, idcl]))))), 0,
' {:5.3f} | '.format(cl))))
fid.write('\n')
fid.write(119 * '-')
# print results of other components
for wing in vehicle.wings:
fid.write('\n ' + wing.tag.upper() +
' ( t/c: {:4.3f} )'.format(wing.thickness_to_chord) + '\n')
fid.write(
str(
np.insert(
np.transpose(list(map('M={:5.3f} | '.format, (mach_vec)))),
0, ' CL | ')))
fid.write('\n')
for idcl, cl in enumerate(cl_vec):
fid.write(
str(
np.insert((np.transpose(
list(
map('{:7.5f} | '.format,
(cd_compress[wing.tag][:, idcl]))))), 0,
' {:5.3f} | '.format(cl))))
fid.write('\n')
fid.write(119 * '-')
# close file
fid.close
# Print timestamp
fid.write('\n\n' +
datetime.datetime.now().strftime(" %A, %d. %B %Y %I:%M:%S %p"))
#done!
return
def kalmanfilter(F, A, H, Q, R, y, X, xi10, ntrain, history=False):
"""
Returns the negative log-likelihood of y conditional on the information set
Assumes that the initial state and all innovations are multivariate
Gaussian.
Parameters
-----------
F : array-like
The (r x r) array holding the transition matrix for the hidden state.
A : array-like
The (nobs x k) array relating the predetermined variables to the
observed data.
H : array-like
The (nobs x r) array relating the hidden state vector to the
observed data.
Q : array-like
(r x r) variance/covariance matrix on the error term in the hidden
state transition.
R : array-like
(nobs x nobs) variance/covariance of the noise in the observation
equation.
y : array-like
The (nobs x 1) array holding the observed data.
X : array-like
The (nobs x k) array holding the predetermined variables data.
xi10 : array-like
Is the (r x 1) initial prior on the initial state vector.
ntrain : int
The number of training periods for the filter. This is the number of
observations that do not affect the likelihood.
Returns
-------
likelihood
The negative of the log likelihood
history or priors, history of posterior
If history is True.
Notes
-----
No input checking is done.
"""
# uses log of Hamilton 13.4.1
F = np.asarray(F)
H = np.atleast_2d(np.asarray(H))
n = H.shape[1] # remember that H gets transposed
y = np.asarray(y)
A = np.asarray(A)
X = np.asarray(X)
if y.ndim == 1: # note that Y is in rows for now
y = y[:, None]
nobs = y.shape[0]
xi10 = np.atleast_2d(np.asarray(xi10))
# if xi10.ndim == 1:
# xi10[:,None]
if history:
state_vector = [xi10]
Q = np.asarray(Q)
r = xi10.shape[0]
# Eq. 12.2.21, other version says P0 = Q
# p10 = np.dot(np.linalg.inv(np.eye(r**2)-np.kron(F,F)),Q.ravel('F'))
# p10 = np.reshape(P0, (r,r), order='F')
# Assume a fixed, known intial point and set P0 = Q
#TODO: this looks *slightly * different than Durbin-Koopman exact likelihood
# initialization p 112 unless I've misunderstood the notational translation.
p10 = Q
loglikelihood = 0
for i in range(nobs):
HTPHR = np.atleast_1d(np.squeeze(chain_dot(H.T, p10, H) + R))
# print HTPHR
# print HTPHR.ndim
# print HTPHR.shape
if HTPHR.ndim == 1:
HTPHRinv = 1. / HTPHR
else:
HTPHRinv = np.linalg.inv(HTPHR) # correct
# print A.T
# print X
# print H.T
# print xi10
# print y[i]
part1 = y[i] - np.dot(A.T, X) - np.dot(H.T, xi10) # correct
if i >= ntrain: # zero-index, but ntrain isn't
HTPHRdet = np.linalg.det(np.atleast_2d(HTPHR)) # correct
part2 = -.5 * chain_dot(part1.T, HTPHRinv, part1) # correct
#TODO: Need to test with ill-conditioned problem.
loglike_interm = (-n/2.) * np.log(2*np.pi) - .5*\
np.log(HTPHRdet) + part2
loglikelihood += loglike_interm
# 13.2.15 Update current state xi_t based on y
xi11 = xi10 + chain_dot(p10, H, HTPHRinv, part1)
# 13.2.16 MSE of that state
p11 = p10 - chain_dot(p10, H, HTPHRinv, H.T, p10)
# 13.2.17 Update forecast about xi_{t+1} based on our F
xi10 = np.dot(F, xi11)
if history:
state_vector.append(xi10)
# 13.2.21 Update the MSE of the forecast
p10 = chain_dot(F, p11, F.T) + Q
if not history:
return -loglikelihood
else:
return -loglikelihood, np.asarray(state_vector[:-1])
def rolling_window(array,
window=(0, ),
asteps=None,
wsteps=None,
axes=None,
intersperse=False):
"""Create a view of `array` which for every point gives the n-dimensional
neighbourhood of size window. New dimensions are added at the end of
`array` or after the corresponding original dimension.
Parameters
----------
array : array_like
Array to which the rolling window is applied.
window : int or tuple
Either a single integer to create a window of only the last axis or a
tuple to create it for the last len(window) axes. 0 can be used as a
to ignore a dimension in the window.
asteps : tuple
Aligned at the last axis, new steps for the original array, ie. for
creation of non-overlapping windows. (Equivalent to slicing result)
wsteps : int or tuple (same size as window)
steps for the added window dimensions. These can be 0 to repeat values
along the axis.
axes: int or tuple
If given, must have the same size as window. In this case window is
interpreted as the size in the dimension given by axes. IE. a window
of (2, 1) is equivalent to window=2 and axis=-2.
intersperse : bool
If True, the new dimensions are right after the corresponding original
dimension, instead of at the end of the array.
Returns
-------
A view on `array` which is smaller to fit the windows and has windows added
dimensions (0s not counting), ie. every point of `array` is an array of size
window.
Examples
--------
>>> a = np.arange(9).reshape(3,3)
>>> rolling_window(a, (2,2))
array([[[[0, 1],
[3, 4]],
[[1, 2],
[4, 5]]],
[[[3, 4],
[6, 7]],
[[4, 5],
[7, 8]]]])
Or to create non-overlapping windows, but only along the first dimension:
>>> rolling_window(a, (2,0), asteps=(2,1))
array([[[0, 3],
[1, 4],
[2, 5]]])
Note that the 0 is discared, so that the output dimension is 3:
>>> rolling_window(a, (2,0), asteps=(2,1)).shape
(1, 3, 2)
This is useful for example to calculate the maximum in all (overlapping)
2x2 submatrixes:
>>> rolling_window(a, (2,2)).max((2,3))
array([[4, 5],
[7, 8]])
Or delay embedding (3D embedding with delay 2):
>>> x = np.arange(10)
>>> rolling_window(x, 3, wsteps=2)
array([[0, 2, 4],
[1, 3, 5],
[2, 4, 6],
[3, 5, 7],
[4, 6, 8],
[5, 7, 9]])
"""
array = np.asarray(array)
orig_shape = np.asarray(array.shape)
window = np.atleast_1d(window).astype(int) # maybe crude to cast to int...
if axes is not None:
axes = np.atleast_1d(axes)
w = np.zeros(array.ndim, dtype=int)
for axis, size in zip(axes, window):
w[axis] = size
window = w
# Check if window is legal:
if window.ndim > 1:
raise ValueError("`window` must be one-dimensional.")
if np.any(window < 0):
raise ValueError("All elements of `window` must be larger then 1.")
if len(array.shape) < len(window):
raise ValueError(
"`window` length must be less or equal `array` dimension.")
_asteps = np.ones_like(orig_shape)
if asteps is not None:
asteps = np.atleast_1d(asteps)
if asteps.ndim != 1:
raise ValueError(
"`asteps` must be either a scalar or one dimensional.")
if len(asteps) > array.ndim:
raise ValueError(
"`asteps` cannot be longer then the `array` dimension.")
# does not enforce alignment, so that steps can be same as window too.
_asteps[-len(asteps):] = asteps
if np.any(asteps < 1):
raise ValueError("All elements of `asteps` must be larger then 1.")
asteps = _asteps
_wsteps = np.ones_like(window)
if wsteps is not None:
wsteps = np.atleast_1d(wsteps)
if wsteps.shape != window.shape:
raise ValueError("`wsteps` must have the same shape as `window`.")
if np.any(wsteps < 0):
raise ValueError("All elements of `wsteps` must be larger then 0.")
_wsteps[:] = wsteps
_wsteps[window ==
0] = 1 # make sure that steps are 1 for non-existing dims.
wsteps = _wsteps
# Check that the window would not be larger then the original:
if np.any(orig_shape[-len(window):] < window * wsteps):
raise ValueError(
"`window` * `wsteps` larger then `array` in at least one dimension."
)
new_shape = orig_shape # just renaming...
# For calculating the new shape 0s must act like 1s:
_window = window.copy()
_window[_window == 0] = 1
new_shape[-len(window):] += wsteps - _window * wsteps
new_shape = (new_shape + asteps - 1) // asteps
# make sure the new_shape is at least 1 in any "old" dimension (ie. steps
# is (too) large, but we do not care.
new_shape[new_shape < 1] = 1
shape = new_shape
strides = np.asarray(array.strides)
strides *= asteps
new_strides = array.strides[-len(window):] * wsteps
# The full new shape and strides:
if not intersperse:
new_shape = np.concatenate((shape, window))
new_strides = np.concatenate((strides, new_strides))
else:
_ = np.zeros_like(shape)
_[-len(window):] = window
_window = _.copy()
_[-len(window):] = new_strides
_new_strides = _
new_shape = np.zeros(len(shape) * 2, dtype=int)
new_strides = np.zeros(len(shape) * 2, dtype=int)
new_shape[::2] = shape
new_strides[::2] = strides
new_shape[1::2] = _window
new_strides[1::2] = _new_strides
new_strides = new_strides[new_shape != 0]
new_shape = new_shape[new_shape != 0]
return np.lib.stride_tricks.as_strided(array,
shape=new_shape,
strides=new_strides)
def _load_datasets(keys, locs, wavelengths, allow_missing=False):
'''
Load data from [<locs>] using <keys> as the columns.
Only loads data which has all the bands defined by
<wavelengths> (if necessary, e.g. for Rrs or bbp).
First key is assumed to be the x_data, remaining keys
(if any) are y_data.
- allow_missing=True will allow datasets which are missing bands
to be included in the returned data
Usage:
# Here, data/loc/Rrs.csv, data/loc/Rrs_wvl.csv, data/loc/bbp.csv,
# and data/chl.csv all exist, with the correct wavelengths available
# for Rrs and bbp (which is determined by Rrs_wvl.csv)
keys = ['Rrs', 'bbp', '../chl']
locs = 'data/loc'
wavelengths = [443, 483, 561, 655]
_load_datasets(keys, locs, wavelengths) # -> [Rrs443, Rrs483, Rrs561, Rrs665],
[bbp443, bbp483, bbp561, bbp655, chl],
{'bbp':slice(0,4), 'chl':slice(4,5)}
'''
def loadtxt(name, loc, required_wvl):
''' Error handling wrapper over np.loadtxt, with the addition of wavelength selection'''
dloc = Path(loc).joinpath(f'{name}.csv')
# TSS / TSM are synonymous
if 'tss' in name and not dloc.exists():
dloc = Path(loc).joinpath(f'{name.replace("tss","tsm")}.csv')
# CDOM is just an alias for a_cdom(443) or a_g(443)
if 'cdom' in name and not dloc.exists():
dloc = Path(loc).joinpath('ag.csv')
required_wvl = [443]
try:
required_wvl = np.array(required_wvl).flatten()
assert (dloc.exists()), (
f'Key {name} does not exist at {loc} ({dloc})')
data = np.loadtxt(
dloc,
delimiter=',',
dtype=float if name not in ['../Dataset', '../meta'] else str,
comments=None)
if len(data.shape) == 1: data = data[:, None]
if data.shape[1] > 1 and data.dtype.type is not np.str_:
# If we want to get all data, regardless of if bands are available...
if allow_missing:
new_data = [[np.nan] * len(data)] * len(required_wvl)
wvls = np.loadtxt(
Path(loc).joinpath(f'{dloc.stem}_wvl.csv'),
delimiter=',')[:, None]
idxs = np.abs(wvls - np.atleast_2d(required_wvl)).argmin(0)
valid = np.abs(wvls -
np.atleast_2d(required_wvl)).min(0) < 2
for j, (i, v) in enumerate(zip(idxs, valid)):
if v: new_data[j] = data[:, i]
data = np.array(new_data).T
else:
data = data[:, get_valid(dloc.stem, loc, required_wvl)]
if 'cdom' in name and dloc.stem == 'ag':
data = data[:,
find_wavelength(443, required_wvl)].flatten()[:,
None]
return data
except Exception as e:
if name not in ['Rrs']: # ['../chl', '../tss', '../cdom']:
if dloc.exists():
print(f'\n\tError fetching {name} from {loc}:\n{e}')
return np.array([]).reshape((0, 0))
raise e
def get_valid(name, loc, required_wvl, margin=2):
''' Dataset at <loc> must have all bands in <required_wvl> within <margin>nm '''
if 'HYPER' in str(loc): margin = 1
# First, validate all required wavelengths are within the margin of an available wavelength
wvls = np.loadtxt(Path(loc).joinpath(f'{name}_wvl.csv'),
delimiter=',')[:, None]
check = np.array(
[np.abs(wvls - w).min() <= margin for w in required_wvl])
assert (check.all()), '\n\t\t'.join([
f'{name} is missing {(~check).sum()} wavelengths:',
f'Needed {required_wvl}', f'Found {wvls.flatten()}',
f'Missing {required_wvl[~check]}', ''
])
# First, validate available wavelengths are within the margin of the required wavelengths
valid = np.array([True] * len(required_wvl))
if len(wvls) != len(required_wvl):
valid = np.abs(wvls - np.atleast_2d(required_wvl)).min(1) < margin
assert (valid.sum() == len(required_wvl)), [
wvls[valid].flatten(), required_wvl
]
# Then, ensure the order of the available wavelengths are the same as the required
if not all([w1 == w2 for w1, w2 in zip(wvls[valid], required_wvl)]):
valid = [np.abs(wvls.flatten() - w).argmin() for w in required_wvl]
assert (len(np.unique(valid)) == len(valid) ==
len(required_wvl)), [
valid, wvls[valid].flatten(), required_wvl
]
return valid
locs = [Path(loc).resolve() for loc in np.atleast_1d(locs)]
print('\n-------------------------')
print(f'Loading data for sensor {locs[0].parts[-1]}')
if allow_missing:
print('Allowing data regardless of whether all bands exist')
x_data = []
y_data = []
l_data = []
for loc in locs:
try:
loc_data = [loadtxt(key, loc, wavelengths) for key in keys]
print(
f'\tN={len(loc_data[0]):>5} | {loc.parts[-1]} / {loc.parts[-2]} ({[np.isfinite(ld).all(1).sum() for ld in loc_data[1:]]})'
)
assert (all([len(l) in [len(loc_data[0]), 0] for l in loc_data
])), dict(zip(keys, map(np.shape, loc_data)))
if all([
l.shape[1] == 0
for l in loc_data[(1 if len(loc_data) > 1 else 0):]
]):
print(f'Skipping dataset {loc}: missing all features')
continue
x_data += [loc_data.pop(0)]
y_data += [loc_data]
l_data += list(
zip([loc.parent.name] * len(x_data[-1]),
np.arange(len(x_data[-1]))))
except Exception as e:
# assert(0), e
# Allow invalid datasets if there are multiple to be fetched
print(f'\nError fetching {loc}:\n\t{e}')
if len(np.atleast_1d(locs)) == 1:
raise e
assert (len(x_data) > 0 or len(locs)
== 0), 'No datasets are valid with the given wavelengths'
assert (
all([x.shape[1] == x_data[0].shape[1] for x in x_data])
), f'Differing number of {keys[0]} wavelengths: {[x.shape for x in x_data]}'
# Determine the number of features each key should have
slices = []
for i, key in enumerate(keys[1:]):
shapes = [y[i].shape[1] for y in y_data]
slices.append(max(shapes))
for x, y in zip(x_data, y_data):
if y[i].shape[1] == 0:
y[i] = np.full((x.shape[0], max(shapes)), np.nan)
assert (all([y[i].shape[1] == y_data[0][i].shape[1] for y in y_data
])), f'{key} shape mismatch: {[y.shape for y in y_data]}'
# Drop any missing features
drop = []
for i, s in enumerate(slices):
if s == 0:
print(f'Dropping {keys[i+1]}: feature has no samples available')
drop.append(i)
slices = np.cumsum([0] +
[s for i, s in enumerate(slices) if i not in drop])
keys = [k for i, k in enumerate(keys[1:]) if i not in drop]
for y in y_data:
y = [z for i, z in enumerate(y) if i not in drop]
# Combine everything together
l_data = np.vstack(l_data)
x_data = np.vstack(x_data)
if len(keys) > 0:
y_data = np.vstack([np.hstack(y) for y in y_data])
assert (slices[-1] == y_data.shape[1]), [slices, y_data.shape]
assert (y_data.shape[0] == x_data.shape[0]), [
x_data.shape, y_data.shape
]
slices = {
k.replace('../', ''): slice(slices[i], s)
for i, (k, s) in enumerate(zip(keys, slices[1:]))
}
print(f'\tTotal prior to filtering: {len(x_data)}')
# Fit exponential function to ad and ag values, and eliminate samples with too much error
for product in ['ad', 'ag']:
if product in slices:
from .metrics import mdsa
from scipy.optimize import curve_fit
exponential = lambda x, a, b, c: a * np.exp(-b * x) + c
remove = np.zeros_like(y_data[:, 0]).astype(bool)
for i, sample in enumerate(y_data):
sample = sample[slices[product]]
assert (
len(sample) > 5
), f'Number of bands should be larger, when fitting exponential: {product}, {sample.shape}'
assert (
len(sample) == len(wavelengths)
), f'Sample size / wavelengths mismatch: {len(sample)} vs {len(wavelengths)}'
if np.all(np.isfinite(sample)) and np.min(sample) > -0.1:
try:
x = np.array(wavelengths) - np.min(wavelengths)
params, _ = curve_fit(exponential,
x,
sample,
bounds=((1e-3, 1e-3, 0),
(1e2, 1e0, 1e1)))
new_sample = exponential(x, *params)
# Should be < 10% error between original and fitted exponential
if mdsa(sample[None, :], new_sample[None, :]) < 10:
y_data[i, slices[product]] = new_sample
else:
remove[
i] = True # Exponential could be fit, but error was too high
except:
remove[
i] = True # Sample deviated so much from a smooth exponential decay that it could not be fit
# else: remove[i] = True # NaNs / negatives in the sample
# Don't actually drop them yet, in case we are fetching all samples regardless of nan composition
x_data[remove] = np.nan
y_data[remove] = np.nan
l_data[remove] = np.nan
if remove.sum():
print(
f'Removed {remove.sum()} / {len(remove)} samples due to poor quality {product} spectra'
)
assert ((~remove).sum(
)), f'All data removed due to {product} spectra quality...'
return x_data, y_data, slices, l_data
def fit(self, x, y, std_y=1.0):
"""Fit linear regression model to x-y data using the SVD.
Parameters
----------
x : array-like, shape (P, N)
Known input values as design matrix (one row per desired parameter)
y : array-like, shape (N,)
Known output measurements as sequence or numpy array
std_y : float or array-like, shape (N,), optional
Measurement error or uncertainty of `y` values, expressed as
standard deviation in units of `y`
Returns
-------
self : :class:`LinearLeastSquaresFit` object
Reference to self, to allow chaining of method calls
"""
x = np.atleast_2d(np.asarray(x))
y = np.atleast_1d(np.asarray(y))
# Convert uncertainty into array of shape (N,)
if np.isscalar(std_y):
std_y = np.tile(std_y, y.shape)
std_y = np.atleast_1d(np.asarray(std_y))
# Lower bound on uncertainty is determined by floating-point
# resolution (no upper bound)
np.clip(std_y,
max(np.mean(np.abs(y)), 1e-20) * np.finfo(y.dtype).eps,
np.inf,
out=std_y)
# Normalise uncertainty to avoid numerical blow-up
# (only relative uncertainty matters for parameter solution)
max_std_y = std_y.max()
std_y /= max_std_y
# Weight design matrix columns and output vector by `y` uncertainty
A = x / std_y[np.newaxis, :]
b = y / std_y
# Perform SVD on A, which is transpose of usual design matrix -
# let A^T = Ur S V^T to correspond with NRinC
# Shapes: A ~ PxN, b ~ N, V ~ PxP, s ~ P, S = diag(s) ~ PxP,
# "reduced U" Ur ~ NxP and Urt = Ur^T ~ PxN
V, s, Urt = np.linalg.svd(A, full_matrices=False)
# Set all "small" singular values below this relative cutoff equal to 0
s_cutoff = (len(x) * np.finfo(x.dtype).eps *
s[0] if self.rcond is None else self.rcond * s[0])
# Warn if the effective rank < P
# (i.e. some singular values are considered to be zero)
if np.any(s < s_cutoff):
warnings.warn('Least-squares fit may be poorly conditioned')
# Invert zero singular values to infinity, as we are actually
# interested in reciprocal of s, and zero singular values should be
# replaced by zero reciprocal values a la pseudo-inverse
s[s < s_cutoff] = np.inf
# Solve linear least-squares problem using SVD
# (see NRinC, 2nd ed, Eq. 15.4.17)
# In matrix form: p = V S^(-1) Ur^T b = Vs Ur^T b, where Vs = V S^(-1)
Vs = V / s[np.newaxis, :]
self.params = np.dot(Vs, np.dot(Urt, b))
# Also obtain covariance matrix of parameters
# (see NRinC, 2nd ed, Eq. 15.4.20)
# In matrix form: Cp = V S^(-2) V^T = Vs Vs^T
# (also rescaling with max std_y)
self.cov_params = np.dot(Vs, Vs.T) * (max_std_y**2)
return self
def get_detector_par(self, hdu, det):
"""
Return a DectectorContainer for the current image
Args:
hdu (`astropy.io.fits.HDUList`):
HDUList of the image of interest.
Ought to be the raw file, or else..
det (int):
Returns:
:class:`pypeit.images.detector_container.DetectorContainer`:
"""
# Binning
binning = self.get_meta_value(self.get_headarr(hdu), 'binning') # Could this be detector dependent??
# Detector 1
detector_dict1 = dict(
binning = binning,
det = 1,
dataext = 1,
specaxis = 0,
specflip = False,
spatflip = False,
platescale = 0.1185,
darkcurr = 4.19,
saturation = 65535., # ADU
nonlinear = 0.95, # Changed by JFH from 0.86 to 0.95
mincounts = -1e10,
numamplifiers = 1,
gain = np.atleast_1d(1.226),
ronoise = np.atleast_1d(2.570),
)
# Detector 2
detector_dict2 = detector_dict1.copy()
detector_dict2.update(dict(
det=2,
dataext=2,
darkcurr=3.46,
gain=np.atleast_1d(1.188),
ronoise=np.atleast_1d(2.491),
))
# Detector 3
detector_dict3 = detector_dict1.copy()
detector_dict3.update(dict(
det=3,
dataext=3,
darkcurr=4.03,
gain=np.atleast_1d(1.248),
ronoise=np.atleast_1d(2.618),
))
# Detector 4
detector_dict4 = detector_dict1.copy()
detector_dict4.update(dict(
det=4,
dataext=4,
darkcurr=3.80,
gain=np.atleast_1d(1.220),
ronoise=np.atleast_1d(2.557),
))
# Detector 5
detector_dict5 = detector_dict1.copy()
detector_dict5.update(dict(
det=5,
dataext=5,
darkcurr=4.71,
gain=np.atleast_1d(1.184),
ronoise=np.atleast_1d(2.482),
))
# Detector 6
detector_dict6 = detector_dict1.copy()
detector_dict6.update(dict(
det=6,
dataext=6,
darkcurr=4.28,
gain=np.atleast_1d(1.177),
ronoise=np.atleast_1d(2.469),
))
# Detector 7
detector_dict7 = detector_dict1.copy()
detector_dict7.update(dict(
det=7,
dataext=7,
darkcurr=3.33,
gain=np.atleast_1d(1.201),
ronoise=np.atleast_1d(2.518),
))
# Detector 8
detector_dict8 = detector_dict1.copy()
detector_dict8.update(dict(
det=8,
dataext=8,
darkcurr=3.69,
gain=np.atleast_1d(1.230),
ronoise=np.atleast_1d(2.580),
))
detectors = [detector_dict1, detector_dict2, detector_dict3, detector_dict4,
detector_dict5, detector_dict6, detector_dict7, detector_dict8]
# Return
return detector_container.DetectorContainer(**detectors[det-1])
def fit(
self,
X,
y,
metric='acc_metric',
feat_type=None,
dataset_name=None,
):
"""Fit *autosklearn* to given training set (X, y).
Parameters
----------
X : array-like or sparse matrix of shape = [n_samples, n_features]
The training input samples.
y : array-like, shape = [n_samples] or [n_samples, n_outputs]
The target classes.
metric : str, optional (default='acc_metric')
The metric to optimize for. Can be one of: ['acc_metric',
'auc_metric', 'bac_metric', 'f1_metric', 'pac_metric']. A
description of the metrics can be found in `the paper describing
the AutoML Challenge
<http://www.causality.inf.ethz.ch/AutoML/automl_ijcnn15.pdf>`_.
feat_type : list, optional (default=None)
List of str of `len(X.shape[1])` describing the attribute type.
Possible types are `Categorical` and `Numerical`. `Categorical`
attributes will be automatically One-Hot encoded.
dataset_name : str, optional (default=None)
Create nicer output. If None, a string will be determined by the
md5 hash of the dataset.
Returns
-------
self
"""
# Fit is supposed to be idempotent!
# But not if we use share_mode:
if not self._shared_mode:
self._delete_output_directories()
else:
# If this fails, it's likely that this is the first call to get
# the data manager
try:
D = self._backend.load_datamanager()
dataset_name = D.name
except IOError:
pass
self._create_output_directories()
y = np.atleast_1d(y)
if y.ndim == 1:
# reshape is necessary to preserve the data contiguity against vs
# [:, np.newaxis] that does not.
y = np.reshape(y, (-1, 1))
self._n_outputs = y.shape[1]
y = np.copy(y)
self._classes = []
self._n_classes = []
for k in six.moves.range(self._n_outputs):
classes_k, y[:, k] = np.unique(y[:, k], return_inverse=True)
self._classes.append(classes_k)
self._n_classes.append(classes_k.shape[0])
self._n_classes = np.array(self._n_classes, dtype=np.int)
if self._n_outputs > 1:
task = MULTILABEL_CLASSIFICATION
else:
if len(self._classes[0]) == 2:
task = BINARY_CLASSIFICATION
else:
task = MULTICLASS_CLASSIFICATION
# TODO: fix metafeatures calculation to allow this!
if y.shape[1] == 1:
y = y.flatten()
return super(AutoSklearnClassifier, self).fit(X, y, task, metric,
feat_type, dataset_name)
def decomposition(self, tmin=None, tmax=None, ytick_base=True, split=True,
figsize=(10, 8), axes=None, name=None,
return_warmup=False, min_ylim_diff=None, **kwargs):
"""Plot the decomposition of a time-series in the different stresses.
Parameters
----------
tmin: str or pandas.Timestamp, optional
tmax: str or pandas.Timestamp, optional
ytick_base: Boolean or float, optional
Make the ytick-base constant if True, set this base to float if
float.
split: bool, optional
Split the stresses in multiple stresses when possible. Default is
True.
axes: matplotlib.axes.Axes instance, optional
Matplotlib Axes instance to plot the figure on to.
figsize: tuple, optional
tuple of size 2 to determine the figure size in inches.
name: str, optional
Name to give the simulated time series in the legend.
return_warmup: bool, optional
Include the warmup period or not.
min_ylim_diff: float, optional
Float with the difference in the ylimits.
**kwargs: dict, optional
Optional arguments, passed on to the plt.subplots method.
Returns
-------
axes: list of matplotlib.axes.Axes
"""
o = self.ml.observations()
# determine the simulation
sim = self.ml.simulate(tmin=tmin, tmax=tmax,
return_warmup=return_warmup)
if name is not None:
sim.name = name
# determine the influence of the different stresses
contribs = self.ml.get_contributions(split=split, tmin=tmin, tmax=tmax,
return_warmup=return_warmup)
names = [s.name for s in contribs]
if self.ml.transform:
contrib = self.ml.get_transform_contribution(tmin=tmin, tmax=tmax)
contribs.append(contrib)
names.append(self.ml.transform.name)
# determine ylim for every graph, to scale the height
ylims = [(min([sim.min(), o[tmin:tmax].min()]),
max([sim.max(), o[tmin:tmax].max()]))]
for contrib in contribs:
hs = contrib[tmin:tmax]
if hs.empty:
if contrib.empty:
ylims.append((0.0, 0.0))
else:
ylims.append((contrib.min(), hs.max()))
else:
ylims.append((hs.min(), hs.max()))
if min_ylim_diff is not None:
for i, ylim in enumerate(ylims):
if np.diff(ylim) < min_ylim_diff:
ylims[i] = (np.mean(ylim) - min_ylim_diff / 2,
np.mean(ylim) + min_ylim_diff / 2)
# determine height ratios
height_ratios = _get_height_ratios(ylims)
nrows = len(contribs) + 1
if axes is None:
# open a new figure
gridspec_kw = {'height_ratios': height_ratios}
fig, axes = plt.subplots(nrows, sharex=True, figsize=figsize,
gridspec_kw=gridspec_kw, **kwargs)
axes = np.atleast_1d(axes)
o_label = o.name
set_axes_properties = True
else:
if len(axes) != nrows:
msg = 'Makes sure the number of axes equals the number of ' \
'series'
raise Exception(msg)
fig = axes[0].figure
o_label = ''
set_axes_properties = False
# plot simulation and observations in top graph
o_nu = self.ml.oseries.series.drop(o.index)
if not o_nu.empty:
# plot parts of the oseries that are not used in grey
o_nu.plot(linestyle='', marker='.', color='0.5', label='',
markersize=2, ax=axes[0], x_compat=True)
o.plot(linestyle='', marker='.', color='k', label=o_label,
markersize=3, ax=axes[0], x_compat=True)
sim.plot(ax=axes[0], x_compat=True)
if set_axes_properties:
axes[0].set_title('observations vs. simulation')
axes[0].set_ylim(ylims[0])
axes[0].grid(True)
axes[0].legend(ncol=3, frameon=False)
if ytick_base and set_axes_properties:
if isinstance(ytick_base, bool):
# determine the ytick-spacing of the top graph
yticks = axes[0].yaxis.get_ticklocs()
if len(yticks) > 1:
ytick_base = yticks[1] - yticks[0]
else:
ytick_base = None
axes[0].yaxis.set_major_locator(
MultipleLocator(base=ytick_base))
# plot the influence of the stresses
for i, contrib in enumerate(contribs):
ax = axes[i + 1]
contrib.plot(ax=ax, x_compat=True)
if set_axes_properties:
if ytick_base:
# set the ytick-spacing equal to the top graph
locator = MultipleLocator(base=ytick_base)
ax.yaxis.set_major_locator(locator)
ax.set_title(names[i])
ax.set_ylim(ylims[i + 1])
ax.grid(True)
ax.minorticks_off()
if set_axes_properties:
axes[0].set_xlim(tmin, tmax)
fig.tight_layout(pad=0.0)
return axes
def mask_to_pixel_coordinates(self, x=None, y=None, wave=None, order=1, filename=None,
corners=False):
r"""
Convert the mask coordinates in mm to pixel coordinates on the
DEIMOS detector.
If not already instantiated, the :attr:`slitmask`,
:attr:`grating`, :attr:`optical_model`, and :attr:`detector_map`
attributes are instantiated. If these are not instantiated, a
file must be provided. If no arguments are provided, the
function expects these attributes to be set and will output the
pixel coordinates for the centers of the slits in the
:attr:`slitmask` at the central wavelength of the
:attr:`grating`.
Method generally expected to be executed in one of two modes:
- Use the `filename` to read the slit mask and determine the
detector positions at the central wavelength.
- Specifically map the provided x, y, and wave values to the
detector.
If arrays are provided for both `x`, `y`, and `wave`, the
returned objects have the shape :math:`N_\lambda\times S_x`,
where :math:`S_x` is the shape of the x and y arrays.
Args:
x (array-like, optional):
The x coordinates in the slit mask in mm. Default is to
use the center of the slits in the :attr:`slitmask`.
y (array-like, optional):
The y coordinates in the slit mask in mm. Default is to
use the center of the slits in the :attr:`slitmask`.
wave (array-like, optional):
The wavelengths in angstroms for the propagated
coordinates. Default is to use the central wavelength
of the :attr:`grating`.
order (:obj:`int`, optional):
The grating order. Default is 1.
filename (:obj:`str`, optional):
The filename to use to (re)instantiate the
:attr:`slitmask` and :attr:`grating`. Default is to use
previously instantiated attributes.
corners (:obj:`bool`, optional):
Instead of using the centers of the slits in the
:attr:`slitmask`, return the detector pixel coordinates
for the corners of all slits.
Returns:
numpy.ndarray: Returns 5 arrays: (1-2) the x and y
coordinates in the image plane in mm, (3) the detector
(1-indexed) where the slit should land at the provided
wavelength(s), and (4-5) the pixel coordinates (1-indexed)
in the relevant detector.
Raises:
ValueError:
Raised if the user provides one but not both of the x
and y coordinates, if no coordinates are provided or
available within the :attr:`slitmask`, or if the
:attr:`grating` hasn't been defined and not file is
provided.
"""
# Cannot provide just one of x or y
if x is None and y is not None or x is not None and y is None:
raise ValueError('Must provide both x and y or neither to use slit mask.')
# Use the file to update the slitmask (if no x coordinates are
# provided) and the grating
if filename is not None:
if x is None and y is None:
# Reset the slit mask
self.get_slitmask(filename)
# Reset the grating
self.get_grating(filename)
# Check that any coordinates are available
if x is None and y is None and self.slitmask is None:
raise ValueError('No coordinates; Provide them directly or instantiate slit mask.')
# Make sure the coordinates are numpy arrays
_x = None if x is None else np.atleast_1d(x)
_y = None if y is None else np.atleast_1d(y)
if _x is None:
# Use all the slit centers or corners
_x = self.slitmask.corners[...,0].ravel() if corners else self.slitmask.center[:,0]
_y = self.slitmask.corners[...,1].ravel() if corners else self.slitmask.center[:,1]
# Check that the grating is defined
if self.grating is None:
raise ValueError('Must define a grating first; provide a file or use get_grating()')
# Instantiate the optical model or reset it grating
if self.optical_model is None:
self.optical_model = DEIMOSOpticalModel(self.grating)
else:
self.optical_model.reset_grating(self.grating)
# Instantiate the detector map, if necessary
self.get_detector_map()
# Compute the detector image plane coordinates (mm)
x_img, y_img = self.optical_model.mask_to_imaging_coordinates(_x, _y, wave=wave,
order=order)
# Reshape if computing the corner positions
if corners:
x_img = x_img.reshape(self.slitmask.corners.shape[:2])
y_img = y_img.reshape(self.slitmask.corners.shape[:2])
# Use the detector map to convert to the detector coordinates
return (x_img, y_img) + self.detector_map.ccd_coordinates(x_img, y_img)
def fit(self,
X,
y,
sample_mask=None,
X_argsorted=None,
check_input=True,
sample_weight=None):
"""Build a decision tree from the training set (X, y).
Parameters
----------
X : array-like, shape = [n_samples, n_features]
The training input samples. Use ``dtype=np.float32`` for maximum
efficiency.
y : array-like, shape = [n_samples] or [n_samples, n_outputs]
The target values (integers that correspond to classes in
classification, real numbers in regression).
Use ``dtype=np.float64`` and ``order='C'`` for maximum
efficiency.
sample_weight : array-like, shape = [n_samples] or None
Sample weights. If None, then samples are equally weighted. Splits
that would create child nodes with net zero or negative weight are
ignored while searching for a split in each node. In the case of
classification, splits are also ignored if they would result in any
single class carrying a negative weight in either child node.
check_input : boolean, (default=True)
Allow to bypass several input checking.
Don't use this parameter unless you know what you do.
Returns
-------
self : object
Returns self.
"""
random_state = check_random_state(self.random_state)
# Deprecations
if sample_mask is not None:
warn(
"The sample_mask parameter is deprecated as of version 0.14 "
"and will be removed in 0.16.", DeprecationWarning)
if X_argsorted is not None:
warn(
"The X_argsorted parameter is deprecated as of version 0.14 "
"and will be removed in 0.16.", DeprecationWarning)
# Convert data
if check_input:
X, = check_arrays(X,
dtype=DTYPE,
sparse_format="dense",
check_ccontiguous=True)
# Determine output settings
n_samples, self.n_features_ = X.shape
is_classification = isinstance(self, ClassifierMixin)
y = np.atleast_1d(y)
if y.ndim == 1:
# reshape is necessary to preserve the data contiguity against vs
# [:, np.newaxis] that does not.
y = np.reshape(y, (-1, 1))
self.n_outputs_ = y.shape[1]
if is_classification:
y = np.copy(y)
self.classes_ = []
self.n_classes_ = []
for k in xrange(self.n_outputs_):
classes_k, y[:, k] = unique(y[:, k], return_inverse=True)
self.classes_.append(classes_k)
self.n_classes_.append(classes_k.shape[0])
else:
self.classes_ = [None] * self.n_outputs_
self.n_classes_ = [1] * self.n_outputs_
self.n_classes_ = np.array(self.n_classes_, dtype=np.intp)
if getattr(y, "dtype", None) != DOUBLE or not y.flags.contiguous:
y = np.ascontiguousarray(y, dtype=DOUBLE)
# Check parameters
max_depth = (2**31) - 1 if self.max_depth is None else self.max_depth
if isinstance(self.max_features, six.string_types):
if self.max_features == "auto":
if is_classification:
max_features = max(1, int(np.sqrt(self.n_features_)))
else:
max_features = self.n_features_
elif self.max_features == "sqrt":
max_features = max(1, int(np.sqrt(self.n_features_)))
elif self.max_features == "log2":
max_features = max(1, int(np.log2(self.n_features_)))
else:
raise ValueError(
'Invalid value for max_features. Allowed string '
'values are "auto", "sqrt" or "log2".')
elif self.max_features is None:
max_features = self.n_features_
elif isinstance(self.max_features, (numbers.Integral, np.integer)):
max_features = self.max_features
else: # float
max_features = int(self.max_features * self.n_features_)
self.max_features_ = max_features
if len(y) != n_samples:
raise ValueError("Number of labels=%d does not match "
"number of samples=%d" % (len(y), n_samples))
if self.min_samples_split <= 0:
raise ValueError("min_samples_split must be greater than zero.")
if self.min_samples_leaf <= 0:
raise ValueError("min_samples_leaf must be greater than zero.")
if max_depth <= 0:
raise ValueError("max_depth must be greater than zero. ")
if not (0 < max_features <= self.n_features_):
raise ValueError("max_features must be in (0, n_features]")
if sample_weight is not None:
if (getattr(sample_weight, "dtype", None) != DOUBLE
or not sample_weight.flags.contiguous):
sample_weight = np.ascontiguousarray(sample_weight,
dtype=DOUBLE)
if len(sample_weight.shape) > 1:
raise ValueError("Sample weights array has more "
"than one dimension: %d" %
len(sample_weight.shape))
if len(sample_weight) != n_samples:
raise ValueError("Number of weights=%d does not match "
"number of samples=%d" %
(len(sample_weight), n_samples))
# Set min_samples_split sensibly
min_samples_split = max(self.min_samples_split,
2 * self.min_samples_leaf)
# Build tree
criterion = self.criterion
if not isinstance(criterion, Criterion):
if is_classification:
criterion = CRITERIA_CLF[self.criterion](self.n_outputs_,
self.n_classes_)
else:
criterion = CRITERIA_REG[self.criterion](self.n_outputs_)
splitter = self.splitter
if not isinstance(self.splitter, Splitter):
splitter = SPLITTERS[self.splitter](criterion, self.max_features_,
self.min_samples_leaf,
random_state)
self.tree_ = Tree(self.n_features_, self.n_classes_, self.n_outputs_,
splitter, max_depth, min_samples_split,
self.min_samples_leaf, random_state)
self.tree_.build(X, y, sample_weight=sample_weight)
if self.n_outputs_ == 1:
self.n_classes_ = self.n_classes_[0]
self.classes_ = self.classes_[0]
return self
def analyze(binObj,
task='skewer',
frange=None,
distort=True,
CenAlpha=None,
histbin=False,
statistic='mean',
suffix='temp',
overwrite=False,
skewer_index=None,
zq_cut=[0, 5],
parallel=False,
tt_bins=None,
verbose=True,
nboot=100,
calib_kwargs=None,
skewer_kwargs=None):
"""
Function to perform important operations on the binObj
Parameters:
binObj: An instance of the bin_class
task: one of ["data_points", "calibrate", "composite", "skewer"]
frange: the Lyman Alpha forest ranges used for the analysis
distort: warp the spectra to a common spectral index
CenAlpha: the common spectral index to warp to
histbin: perform histogram rebinninb
statistic: statistic to use when creating composites [task="composite"]
suffix: name of the file to write to
overwrite: overwrite the skewer in the LogLikes folder if duplicates
skewer_index: index of the skewers in the forest range (frange) to use
zq_cut: allows to perform a cut in the quasar redshift
parallel: whether to run the skewers in parallel
tt_bins: Bins in lyman alpha redshift to use for task="data_points"
calib_kwargs: additional keyword arguments for task="calibrate"
skewer_kwargs: additional keyword arguments for task="skewer"
"""
if frange is None:
frange = [1070, 1160]
lyInd = np.where((binObj.wl > frange[0]) & (binObj.wl < frange[1]))[0]
if skewer_index is None:
skewer_index = range(len(lyInd))
else:
skewer_index = np.atleast_1d(skewer_index)
outfile = task + '_' + suffix
if task == 'skewer' or task == 'data_points':
if verbose:
print('Total skewers available: {}, skewers analyzed in this '
'run: {}'.format(len(lyInd), len(skewer_index)))
myspec = binObj._flux[:, lyInd[skewer_index]]
myivar = binObj._ivar[:, lyInd[skewer_index]]
zMat = binObj._zAbs[:, lyInd[skewer_index]]
mywave = binObj.wl[lyInd[skewer_index]]
else:
myspec, myivar, zMat = binObj._flux, binObj._ivar, binObj._zAbs
mywave = binObj.wl
myz, myalpha = binObj._zq, binObj._alpha
# selecting according to quasar redshifts
zq_mask = (myz > zq_cut[0]) & (myz < zq_cut[1])
myspec = myspec[zq_mask]
myivar = myivar[zq_mask]
zMat = zMat[zq_mask]
myz, myalpha = myz[zq_mask], myalpha[zq_mask]
# B. DATA PREPROCESSING ---------------------------------------------------
if histbin:
# Histogram binning in parameter space
myp1, myp2 = binObj._par1, binObj._par2
myzbins = find_zbins(myz)
hInd = np.where((myz >= myzbins[0]) & (myz < myzbins[-1]))
# Modify the selection to choose only objects that fall in the
# zbins range
myz, myalpha = myz[hInd], myalpha[hInd]
myp1, myp2 = myp1[hInd], myp2[hInd]
myspec, myivar = myspec[hInd], myivar[hInd]
zMat = zMat[hInd]
if binObj._hWeights is None:
h_weights = hist_weights(myp1, myp2, myz, myzbins)
binObj._hWeights = h_weights
myivar = myivar * h_weights[:, None]
else:
myivar = myivar * binObj._hWeights[:, None]
if distort:
# Distort spectra in alpha space
outfile += '_distort'
if CenAlpha is None:
CenAlpha = np.median(myalpha)
distortMat = np.array([(mywave / 1450.)**ele
for ele in (CenAlpha - myalpha)])
myspec *= distortMat
myivar /= distortMat**2
if verbose:
print('All spectra distorted to alpha:', CenAlpha)
# C. CALIBRATION VS ESTIMATION --------------------------------------------
if task == "data_points":
print("Make sure that the reconstructed continuum has been run using "
"the same frange as that being used right now!")
# Data points for the transmission, using a continuum as the base
if binObj.continuum is None:
raise AttributeError("Set the reconstructed continuum for the"
"bin first!!!")
ivar_mask = (myivar > 0).flatten()
zLyAs = zMat.flatten()
zLyAs = zLyAs[ivar_mask]
# bin centers for the redshift-transmission plot
if tt_bins is None:
tt_bins = np.linspace(zLyAs.min(), zLyAs.max(), 40)
tt_cens = (tt_bins[1:] + tt_bins[:-1]) / 2.
# errors from t0-gamma fluctuations
# We are not going to use this in the paper !!!
tt_binned = np.zeros((len(binObj.continuum), len(tt_cens)))
for i in range(len(binObj.continuum)):
tt = (myspec / binObj.continuum[i]).flatten()
tt = tt[ivar_mask]
tt_binned[i] = binned_statistic(zLyAs,
tt,
statistic=np.mean,
bins=tt_bins).statistic
continuum = binObj.continuum.mean(0)
# estimates of the transmission central values - errors obtained
# using bootstrap as below
tt_cen = (myspec / continuum).flatten()
tt_cen = tt_cen[ivar_mask]
tt_data = binned_statistic(zLyAs,
tt_cen,
statistic=np.mean,
bins=tt_bins).statistic
# tt_std = binned_statistic(zLyAs, tt_cen, statistic=np.std,
# bins=tt_bins).statistic
# tt_counts = binned_statistic(zLyAs, None, statistic='count',
# bins=tt_bins).statistic
# errors from bootstrapping
print("Computing bootstrap samples of transmission")
tt_boot = np.zeros((nboot, len(tt_cens)))
for i in range(nboot):
np.random.seed()
ixs = np.random.randint(0, len(myivar), len(myivar))
sp_boot, iv_boot = myspec[ixs], myivar[ixs]
zz_boot = zMat[ixs]
ivar_mask = (iv_boot > 0).flatten()
zLyAs = zz_boot.flatten()
zLyAs = zLyAs[ivar_mask]
tt = (sp_boot / continuum).flatten()
tt = tt[ivar_mask]
tt_boot[i] = binned_statistic(zLyAs,
tt,
statistic=np.mean,
bins=tt_bins).statistic
# Save this to a file for future use -
# Use this for the analysis of figure 6 <--
data_full = np.array([tt_cens, tt_data, *tt_boot])
np.savetxt("data_points_" + binObj.name + ".dat", data_full)
return tt_cens, tt_data, tt_binned, tt_boot # , tt_std / np.sqrt(tt_counts)
if task == 'calibrate':
ixs = (myz > 1.6) & (myz < 4)
print('Number of spectra used for calibration are: %d' % ixs.sum())
rest_range = [[1280, 1290], [1320, 1330], [1345, 1360], [1440, 1480]]
# normalization range used
obs_min, obs_max = 4600, 4640
corrections.calibrate(binObj.wl, myspec[ixs], myivar[ixs], myz[ixs],
rest_range, obs_min, obs_max, binObj.name, True)
# D. COMPOSITE CREATION IF SPECIFIED --------------------------------------
if task == 'composite':
# Create composites using the spectra
# zbins = find_zbins(myz)
zbins = np.arange(2.1, 4.5, 0.05)
# comp_simple.compcompute(myspec, myivar, myz, mywave,
# zbins, statistic, outfile)
create_comp.create_comp(myspec, myivar, myz, mywave, zbins, outfile)
# E. LIKELIHOOD SKEWER ----------------------------------------------------
if task == 'skewer':
currDir = os.getcwd()
destDir = '../LogLikes' + '/Bin_' + outfile +\
str(frange[0]) + '_' + str(frange[1]) # <--
if not os.path.exists(destDir):
os.makedirs(destDir)
else:
if overwrite:
shutil.rmtree(destDir)
os.makedirs(destDir)
os.chdir(destDir)
start = timer()
# Do not plot graphs while in parallel
res = None
if parallel:
pass
# print('Running in parallel now!')
# myfunc_partial = partial(mcmc_skewer.mcmcSkewer, **skewer_kwargs)
# pool = Pool()
# res = pool.map(myfunc_partial,
# zip(np.array([zMat, myspec, myivar]).T, skewer_index))
# pool.close()
# pool.join()
# else:
# for j, ele in enumerate(skewer_index):
# res = mcmc_skewer.mcmcSkewer(
# [np.array([zMat[:, j], myspec[:, j], myivar[:, j]]).T, ele],
# **skewer_kwargs)
stop = timer()
print('Time elapsed:', stop - start)
os.chdir(currDir)
return mywave, res
def vec(x):
return np.atleast_1d(np.array(x, dtype=np.float32))
def halomod_power_spectrum(cosmo,
hmc,
k,
a,
prof,
prof_2pt=None,
prof2=None,
p_of_k_a=None,
normprof1=False,
normprof2=False,
get_1h=True,
get_2h=True):
""" Computes the halo model power spectrum for two
quantities defined by their respective halo profiles.
The halo model power spectrum for two profiles
:math:`u` and :math:`v` is:
.. math::
P_{u,v}(k,a) = I^2_0(k,a|u,v) +
I^1_1(k,a|u)\\,I^1_1(k,a|v)\\,P_{\\rm lin}(k,a)
where :math:`P_{\\rm lin}(k,a)` is the linear matter
power spectrum, :math:`I^1_1` is defined in the documentation
of :meth:`~HMCalculator.I_1_1`, and :math:`I^0_2` is defined
in the documentation of :meth:`~HMCalculator.I_0_2`.
Args:
cosmo (:class:`~pyccl.core.Cosmology`): a Cosmology object.
hmc (:class:`HMCalculator`): a halo model calculator.
k (float or array_like): comoving wavenumber in Mpc^-1.
a (float or array_like): scale factor.
prof (:class:`~pyccl.halos.profiles.HaloProfile`): halo
profile.
prof_2pt (:class:`~pyccl.halos.profiles_2pt.Profile2pt`):
a profile covariance object
returning the the two-point moment of the two profiles
being correlated. If `None`, the default second moment
will be used, corresponding to the products of the means
of both profiles.
prof2 (:class:`~pyccl.halos.profiles.HaloProfile`): a
second halo profile. If `None`, `prof` will be used as
`prof2`.
p_of_k_a (:class:`~pyccl.pk2d.Pk2D`): a `Pk2D` object to
be used as the linear matter power spectrum. If `None`,
the power spectrum stored within `cosmo` will be used.
normprof1 (bool): if `True`, this integral will be
normalized by :math:`I^0_1(k\\rightarrow 0,a|u)`
(see :meth:`~HMCalculator.I_0_1`), where
:math:`u` is the profile represented by `prof`.
normprof2 (bool): if `True`, this integral will be
normalized by :math:`I^0_1(k\\rightarrow 0,a|v)`
(see :meth:`~HMCalculator.I_0_1`), where
:math:`v` is the profile represented by `prof2`.
get_1h (bool): if `False`, the 1-halo term (i.e. the first
term in the first equation above) won't be computed.
get_2h (bool): if `False`, the 2-halo term (i.e. the second
term in the first equation above) won't be computed.
Returns:
float or array_like: integral values evaluated at each
combination of `k` and `a`. The shape of the output will
be `(N_a, N_k)` where `N_k` and `N_a` are the sizes of
`k` and `a` respectively. If `k` or `a` are scalars, the
corresponding dimension will be squeezed out on output.
"""
a_use = np.atleast_1d(a)
k_use = np.atleast_1d(k)
# Check inputs
if not isinstance(prof, HaloProfile):
raise TypeError("prof must be of type `HaloProfile`")
if (prof2 is not None) and (not isinstance(prof2, HaloProfile)):
raise TypeError("prof2 must be of type `HaloProfile` or `None`")
if prof_2pt is None:
prof_2pt = Profile2pt()
elif not isinstance(prof_2pt, Profile2pt):
raise TypeError("prof_2pt must be of type " "`Profile2pt` or `None`")
# Power spectrum
if isinstance(p_of_k_a, Pk2D):
def pkf(sf):
return p_of_k_a.eval(k_use, sf, cosmo)
elif (p_of_k_a is None) or (p_of_k_a == 'linear'):
def pkf(sf):
return linear_matter_power(cosmo, k_use, sf)
elif p_of_k_a == 'nonlinear':
def pkf(sf):
return nonlin_matter_power(cosmo, k_use, sf)
else:
raise TypeError("p_of_k_a must be `None`, \'linear\', "
"\'nonlinear\' or a `Pk2D` object")
na = len(a_use)
nk = len(k_use)
out = np.zeros([na, nk])
for ia, aa in enumerate(a_use):
# Compute first profile normalization
if normprof1:
norm1 = hmc.profile_norm(cosmo, aa, prof)
else:
norm1 = 1
# Compute second profile normalization
if prof2 is None:
norm2 = norm1
else:
if normprof2:
norm2 = hmc.profile_norm(cosmo, aa, prof2)
else:
norm2 = 1
norm = norm1 * norm2
if get_2h:
# Compute first bias factor
i11_1 = hmc.I_1_1(cosmo, k_use, aa, prof)
# Compute second bias factor
if prof2 is None:
i11_2 = i11_1
else:
i11_2 = hmc.I_1_1(cosmo, k_use, aa, prof2)
# Compute 2-halo power spectrum
pk_2h = pkf(aa) * i11_1 * i11_2
else:
pk_2h = 0.
if get_1h:
pk_1h = hmc.I_0_2(cosmo, k_use, aa, prof, prof_2pt, prof2)
else:
pk_1h = 0.
# Total power spectrum
out[ia, :] = (pk_1h + pk_2h) * norm
if np.ndim(a) == 0:
out = np.squeeze(out, axis=0)
if np.ndim(k) == 0:
out = np.squeeze(out, axis=-1)
return out
def __call__(self, x, *args, **kwds):
return super(Gradient,
self).__call__(np.atleast_1d(x).ravel(), *args,
**kwds).squeeze()
def normalized(a, axis=-1, order=2):
l2 = np.atleast_1d(np.linalg.norm(a, order, axis))
l2[l2 == 0] = 1
return a / np.expand_dims(l2, axis)
def fit(self,
X,
y,
sample_mask=None,
X_argsorted=None,
check_input=True,
sample_weight=None):
"""Build a decision tree from the training set (X, y).
Parameters
----------
X : array-like, shape = [n_samples, n_features]
The training input samples. Use ``dtype=np.float32``
and ``order='F'`` for maximum efficiency.
y : array-like, shape = [n_samples] or [n_samples, n_outputs]
The target values (integers that correspond to classes in
classification, real numbers in regression).
Use ``dtype=np.float64`` and ``order='C'`` for maximum
efficiency.
sample_mask : array-like, shape = [n_samples], dtype = bool or None
A bit mask that encodes the rows of ``X`` that should be
used to build the decision tree. It can be used for bagging
without the need to create of copy of ``X``.
If None a mask will be created that includes all samples.
X_argsorted : array-like, shape = [n_samples, n_features] or None
Each column of ``X_argsorted`` holds the row indices of ``X``
sorted according to the value of the corresponding feature
in ascending order.
I.e. ``X[X_argsorted[i, k], k] <= X[X_argsorted[j, k], k]``
for each j > i.
If None, ``X_argsorted`` is computed internally.
The argument is supported to enable multiple decision trees
to share the data structure and to avoid re-computation in
tree ensembles. For maximum efficiency use dtype np.int32.
sample_weight : array-like, shape = [n_samples] or None
Sample weights. If None, then samples are equally weighted. Splits
that would create child nodes with net zero or negative weight are
ignored while searching for a split in each node. In the case of
classification, splits are also ignored if they would result in any
single class carrying a negative weight in either child node.
check_input : boolean, (default=True)
Allow to bypass several input checking.
Don't use this parameter unless you know what you do.
Returns
-------
self : object
Returns self.
"""
if check_input:
X, y = check_arrays(X, y)
random_state = check_random_state(self.random_state)
# Convert data
if (getattr(X, "dtype", None) != DTYPE or X.ndim != 2
or not X.flags.fortran):
X = array2d(X, dtype=DTYPE, order="F")
n_samples, self.n_features_ = X.shape
is_classification = isinstance(self, ClassifierMixin)
y = np.atleast_1d(y)
if y.ndim == 1:
# reshape is necessary to preserve the data contiguity against vs
# [:, np.newaxis] that does not.
y = np.reshape(y, (-1, 1))
self.n_outputs_ = y.shape[1]
if is_classification:
y = np.copy(y)
self.classes_ = []
self.n_classes_ = []
for k in xrange(self.n_outputs_):
unique = np.unique(y[:, k])
self.classes_.append(unique)
self.n_classes_.append(unique.shape[0])
y[:, k] = np.searchsorted(unique, y[:, k])
else:
self.classes_ = [None] * self.n_outputs_
self.n_classes_ = [1] * self.n_outputs_
if getattr(y, "dtype", None) != DOUBLE or not y.flags.contiguous:
y = np.ascontiguousarray(y, dtype=DOUBLE)
if is_classification:
criterion = CLASSIFICATION[self.criterion](self.n_outputs_,
self.n_classes_)
else:
criterion = REGRESSION[self.criterion](self.n_outputs_)
# Check parameters
max_depth = np.inf if self.max_depth is None else self.max_depth
if isinstance(self.max_features, basestring):
if self.max_features == "auto":
if is_classification:
max_features = max(1, int(np.sqrt(self.n_features_)))
else:
max_features = self.n_features_
elif self.max_features == "sqrt":
max_features = max(1, int(np.sqrt(self.n_features_)))
elif self.max_features == "log2":
max_features = max(1, int(np.log2(self.n_features_)))
else:
raise ValueError(
'Invalid value for max_features. Allowed string '
'values are "auto", "sqrt" or "log2".')
elif self.max_features is None:
max_features = self.n_features_
else:
max_features = self.max_features
if len(y) != n_samples:
raise ValueError("Number of labels=%d does not match "
"number of samples=%d" % (len(y), n_samples))
if self.min_samples_split <= 0:
raise ValueError("min_samples_split must be greater than zero.")
if self.min_samples_leaf <= 0:
raise ValueError("min_samples_leaf must be greater than zero.")
if max_depth <= 0:
raise ValueError("max_depth must be greater than zero. ")
if self.min_density < 0.0 or self.min_density > 1.0:
raise ValueError("min_density must be in [0, 1]")
if not (0 < max_features <= self.n_features_):
raise ValueError("max_features must be in (0, n_features]")
if sample_mask is not None:
sample_mask = np.asarray(sample_mask, dtype=np.bool)
if sample_mask.shape[0] != n_samples:
raise ValueError("Length of sample_mask=%d does not match "
"number of samples=%d" %
(sample_mask.shape[0], n_samples))
if sample_weight is not None:
if (getattr(sample_weight, "dtype", None) != DOUBLE
or not sample_weight.flags.contiguous):
sample_weight = np.ascontiguousarray(sample_weight,
dtype=DOUBLE)
if len(sample_weight.shape) > 1:
raise ValueError("Sample weights array has more "
"than one dimension: %d" %
len(sample_weight.shape))
if len(sample_weight) != n_samples:
raise ValueError("Number of weights=%d does not match "
"number of samples=%d" %
(len(sample_weight), n_samples))
if X_argsorted is not None:
X_argsorted = np.asarray(X_argsorted, dtype=np.int32, order='F')
if X_argsorted.shape != X.shape:
raise ValueError("Shape of X_argsorted does not match "
"the shape of X")
# Set min_samples_split sensibly
min_samples_split = max(self.min_samples_split,
2 * self.min_samples_leaf)
# Build tree
self.tree_ = _tree.Tree(self.n_features_, self.n_classes_,
self.n_outputs_, criterion, max_depth,
min_samples_split, self.min_samples_leaf,
self.min_density, max_features,
self.find_split_, random_state)
self.tree_.build(X,
y,
sample_weight=sample_weight,
sample_mask=sample_mask,
X_argsorted=X_argsorted)
if self.n_outputs_ == 1:
self.n_classes_ = self.n_classes_[0]
self.classes_ = self.classes_[0]
# Compute importances
if self.compute_importances:
self.feature_importances_ = \
self.tree_.compute_feature_importances()
return self
* run (run) int64 1
* lev (lev) float64 0.0
* lat (lat) float64 -90.0
* eof (eof) int64 1 2 3 4 5
Dimensions without coordinates: time
References
----------
.. bibliography:: ../bibs/eofs.bib
"""
# Parse input
# WARNING: Have to explicitly 'push_left' extra dimensions (even though just
# specifying nflat_left is enough to flattened the desired dimensions) or the new
# 'eof' dimension is not pushed to the left of the extra dimensions. Think this
# is not a bug but consistent with design.
axis_space = np.atleast_1d(axis_space)
np.atleast_1d(axis_time).item() # ensure time axis is 1D
axis_space[axis_space < 0] += data.ndim
if axis_time < 0:
axis_time += data.ndim
if np.any(axis_space == axis_time):
raise ValueError(f'Cannot have {axis_time=} same as {axis_space=}.')
axis_extra = tuple(set(range(data.ndim)) - {axis_time, *axis_space})
# Remove the mean and optionally standardize the data
data = data - data.mean(axis=axis_time, keepdims=True) # remove mean
if normalize:
data = data / data.std(axis=axis_time, keepdims=True)
# Next apply weights
ntime = data.shape[axis_time] # number timesteps
def mean_radial_velocity_vs_r(sample1,
velocities1,
rbins,
sample2=None,
velocities2=None,
period=None,
do_auto=True,
do_cross=True,
num_threads=1,
max_sample_size=int(1e6),
approx_cell1_size=None,
approx_cell2_size=None):
"""
Calculate the mean pairwise velocity, :math:`\\bar{v}_{12}(r)`.
Example calls to this function appear in the documentation below.
See the :ref:`mock_obs_pos_formatting` documentation page for
instructions on how to transform your coordinate position arrays into the
format accepted by the ``sample1`` and ``sample2`` arguments.
See also :ref:`galaxy_catalog_analysis_tutorial6`.
Parameters
----------
sample1 : array_like
N1pts x 3 numpy array containing the 3-D positions of points.
velocities1 : array_like
N1pts x 3 array containing the 3-D components of the velocities.
rbins : array_like
array of boundaries defining the real space radial bins in which pairs are
counted.
sample2 : array_like, optional
N2pts x 3 array containing the 3-D positions of points.
velocities2 : array_like, optional
N2pts x 3 array containing the 3-D components of the velocities.
period : array_like, optional
Length-3 array defining periodic boundary conditions. If only
one number, Lbox, is specified, period is assumed to be [Lbox, Lbox, Lbox].
do_auto : boolean, optional
caclulate the auto-pairwise velocities?
do_cross : boolean, optional
caclulate the cross-pairwise velocities?
num_threads : int, optional
number of threads to use in calculation. Default is 1. A string 'max' may be used
to indicate that the pair counters should use all available cores on the machine.
max_sample_size : int, optional
Defines maximum size of the sample that will be passed to the pair counter.
If sample size exeeds max_sample_size, the sample will be randomly down-sampled
such that the subsample is equal to max_sample_size.
approx_cell1_size : array_like, optional
Length-3 array serving as a guess for the optimal manner by how points
will be apportioned into subvolumes of the simulation box.
The optimum choice unavoidably depends on the specs of your machine.
Default choice is to use *max(rbins)* in each dimension,
which will return reasonable result performance for most use-cases.
Performance can vary sensitively with this parameter, so it is highly
recommended that you experiment with this parameter when carrying out
performance-critical calculations.
approx_cell2_size : array_like, optional
Analogous to ``approx_cell1_size``, but for `sample2`. See comments for
``approx_cell1_size`` for details.
Returns
-------
v_12 : numpy.array
*len(rbins)-1* length array containing the mean pairwise velocity,
:math:`\\bar{v}_{12}(r)`, computed in each of the bins defined by ``rbins``.
Notes
-----
The pairwise velocity, :math:`v_{12}(r)`, is defined as:
.. math::
v_{12}(r) = \\vec{v}_{\\rm 1, pec} \\cdot \\vec{r}_{12}-\\vec{v}_{\\rm 2, pec} \\cdot \\vec{r}_{12}
where :math:`\\vec{v}_{\\rm 1, pec}` is the peculiar velocity of object 1, and
:math:`\\vec{r}_{12}` is the radial vector connecting object 1 and 2.
:math:`\\bar{v}_{12}(r)` is the mean of that quantity calculated in radial bins.
Pairs and radial velocities are calculated using
`~halotools.mock_observables.pair_counters.velocity_marked_npairs`.
For radial separation bins in which there are zero pairs, function returns zero.
Examples
--------
For demonstration purposes we will work with
halos in the `~halotools.sim_manager.FakeSim`. Here we'll just demonstrate
basic usage, referring to :ref:`galaxy_catalog_analysis_tutorial6` for a
more detailed demo.
>>> from halotools.sim_manager import FakeSim
>>> halocat = FakeSim()
>>> x = halocat.halo_table['halo_x']
>>> y = halocat.halo_table['halo_y']
>>> z = halocat.halo_table['halo_z']
We transform our *x, y, z* points into the array shape used by the pair-counter by
taking the transpose of the result of `numpy.vstack`. This boilerplate transformation
is used throughout the `~halotools.mock_observables` sub-package:
>>> sample1 = np.vstack((x,y,z)).T
We will do the same to get a random set of velocities.
>>> vx = halocat.halo_table['halo_vx']
>>> vy = halocat.halo_table['halo_vy']
>>> vz = halocat.halo_table['halo_vz']
>>> velocities = np.vstack((vx,vy,vz)).T
>>> rbins = np.logspace(-1, 1, 10)
>>> v_12 = mean_radial_velocity_vs_r(sample1, velocities, rbins, period=halocat.Lbox)
See also
--------
:ref:`galaxy_catalog_analysis_tutorial6`
"""
function_args = (sample1, velocities1, sample2, velocities2, period,
do_auto, do_cross, num_threads, max_sample_size,
approx_cell1_size, approx_cell2_size)
sample1, velocities1, sample2, velocities2, period, do_auto, do_cross,\
num_threads, _sample1_is_sample2, PBCs = _pairwise_velocity_stats_process_args(*function_args)
rbins = np.atleast_1d(rbins)
#create marks for the marked pair counter.
marks1 = np.vstack((sample1.T, velocities1.T)).T
marks2 = np.vstack((sample2.T, velocities2.T)).T
def marked_pair_counts(sample1, sample2, rbins, period, num_threads,
do_auto, do_cross, marks1, marks2, weight_func_id,
_sample1_is_sample2, approx_cell1_size,
approx_cell2_size):
"""
Count velocity weighted data pairs.
"""
if do_auto is True:
D1D1, dummy, N1N1 = velocity_marked_npairs_3d(
sample1,
sample1,
rbins,
weights1=marks1,
weights2=marks1,
weight_func_id=weight_func_id,
period=period,
num_threads=num_threads,
approx_cell1_size=approx_cell1_size,
approx_cell2_size=approx_cell1_size)
D1D1 = np.diff(D1D1)
N1N1 = np.diff(N1N1)
else:
D1D1 = None
D2D2 = None
N1N1 = None
N2N2 = None
if _sample1_is_sample2:
D1D2 = D1D1
D2D2 = D1D1
N1N2 = N1N1
N2N2 = N1N1
else:
if do_cross is True:
D1D2, dummy, N1N2 = velocity_marked_npairs_3d(
sample1,
sample2,
rbins,
weights1=marks1,
weights2=marks2,
weight_func_id=weight_func_id,
period=period,
num_threads=num_threads,
approx_cell1_size=approx_cell1_size,
approx_cell2_size=approx_cell2_size)
D1D2 = np.diff(D1D2)
N1N2 = np.diff(N1N2)
else:
D1D2 = None
N1N2 = None
if do_auto is True:
D2D2, dummy, N2N2 = velocity_marked_npairs_3d(
sample2,
sample2,
rbins,
weights1=marks2,
weights2=marks2,
weight_func_id=weight_func_id,
period=period,
num_threads=num_threads,
approx_cell1_size=approx_cell2_size,
approx_cell2_size=approx_cell2_size)
D2D2 = np.diff(D2D2)
N2N2 = np.diff(N2N2)
else:
D2D2 = None
N2N2 = None
return D1D1, D1D2, D2D2, N1N1, N1N2, N2N2
#count the sum of radial velocities and number of pairs
weight_func_id = 11
V1V1, V1V2, V2V2, N1N1, N1N2, N2N2 =\
marked_pair_counts(sample1, sample2, rbins, period,
num_threads, do_auto, do_cross,
marks1, marks2, weight_func_id,
_sample1_is_sample2,
approx_cell1_size, approx_cell2_size)
#return results: the sum of radial velocities divided by the number of pairs
if _sample1_is_sample2:
M_11 = V1V1 / N1N1
return np.where(np.isfinite(M_11), M_11, 0.)
else:
if (do_auto is True) & (do_cross is True):
M_11 = V1V1 / N1N1
M_12 = V1V2 / N1N2
M_22 = V2V2 / N2N2
return (np.where(np.isfinite(M_11), M_11,
0.), np.where(np.isfinite(M_12), M_12, 0.),
np.where(np.isfinite(M_22), M_22, 0.))
elif do_cross is True:
M_12 = V1V2 / N1N2
return np.where(np.isfinite(M_12), M_12, 0.)
elif (do_auto is True):
M_11 = V1V1 / N1N1
M_22 = V2V2 / N2N2
return np.where(np.isfinite(M_11), M_11,
0.), np.where(np.isfinite(M_22), M_22, 0.)
def firwin(numtaps, cutoff, width=None, window='hamming', pass_zero=True,
scale=True, nyq=None, fs=None):
"""
FIR filter design using the window method.
This function computes the coefficients of a finite impulse response
filter. The filter will have linear phase; it will be Type I if
`numtaps` is odd and Type II if `numtaps` is even.
Type II filters always have zero response at the Nyquist frequency, so a
ValueError exception is raised if firwin is called with `numtaps` even and
having a passband whose right end is at the Nyquist frequency.
Parameters
----------
numtaps : int
Length of the filter (number of coefficients, i.e. the filter
order + 1). `numtaps` must be even if a passband includes the
Nyquist frequency.
cutoff : float or 1D array_like
Cutoff frequency of filter (expressed in the same units as `nyq`)
OR an array of cutoff frequencies (that is, band edges). In the
latter case, the frequencies in `cutoff` should be positive and
monotonically increasing between 0 and `nyq`. The values 0 and
`nyq` must not be included in `cutoff`.
width : float or None, optional
If `width` is not None, then assume it is the approximate width
of the transition region (expressed in the same units as `nyq`)
for use in Kaiser FIR filter design. In this case, the `window`
argument is ignored.
window : string or tuple of string and parameter values, optional
Desired window to use. See `scipy.signal.get_window` for a list
of windows and required parameters.
pass_zero : bool, optional
If True, the gain at the frequency 0 (i.e. the "DC gain") is 1.
Otherwise the DC gain is 0.
scale : bool, optional
Set to True to scale the coefficients so that the frequency
response is exactly unity at a certain frequency.
That frequency is either:
- 0 (DC) if the first passband starts at 0 (i.e. pass_zero
is True)
- `nyq` (the Nyquist frequency) if the first passband ends at
`nyq` (i.e the filter is a single band highpass filter);
center of first passband otherwise
nyq : float, optional
*Deprecated. Use `fs` instead.* This is the Nyquist frequency.
Each frequency in `cutoff` must be between 0 and `nyq`. Default
is 1.
fs : float, optional
The sampling frequency of the signal. Each frequency in `cutoff`
must be between 0 and ``fs/2``. Default is 2.
Returns
-------
h : (numtaps,) ndarray
Coefficients of length `numtaps` FIR filter.
Raises
------
ValueError
If any value in `cutoff` is less than or equal to 0 or greater
than or equal to ``fs/2``, if the values in `cutoff` are not strictly
monotonically increasing, or if `numtaps` is even but a passband
includes the Nyquist frequency.
See Also
--------
firwin2
firls
minimum_phase
remez
Examples
--------
Low-pass from 0 to f:
>>> from scipy import signal
>>> numtaps = 3
>>> f = 0.1
>>> signal.firwin(numtaps, f)
array([ 0.06799017, 0.86401967, 0.06799017])
Use a specific window function:
>>> signal.firwin(numtaps, f, window='nuttall')
array([ 3.56607041e-04, 9.99286786e-01, 3.56607041e-04])
High-pass ('stop' from 0 to f):
>>> signal.firwin(numtaps, f, pass_zero=False)
array([-0.00859313, 0.98281375, -0.00859313])
Band-pass:
>>> f1, f2 = 0.1, 0.2
>>> signal.firwin(numtaps, [f1, f2], pass_zero=False)
array([ 0.06301614, 0.88770441, 0.06301614])
Band-stop:
>>> signal.firwin(numtaps, [f1, f2])
array([-0.00801395, 1.0160279 , -0.00801395])
Multi-band (passbands are [0, f1], [f2, f3] and [f4, 1]):
>>> f3, f4 = 0.3, 0.4
>>> signal.firwin(numtaps, [f1, f2, f3, f4])
array([-0.01376344, 1.02752689, -0.01376344])
Multi-band (passbands are [f1, f2] and [f3,f4]):
>>> signal.firwin(numtaps, [f1, f2, f3, f4], pass_zero=False)
array([ 0.04890915, 0.91284326, 0.04890915])
"""
# The major enhancements to this function added in November 2010 were
# developed by Tom Krauss (see ticket #902).
nyq = 0.5 * _get_fs(fs, nyq)
cutoff = np.atleast_1d(cutoff) / float(nyq)
# Check for invalid input.
if cutoff.ndim > 1:
raise ValueError("The cutoff argument must be at most "
"one-dimensional.")
if cutoff.size == 0:
raise ValueError("At least one cutoff frequency must be given.")
if cutoff.min() <= 0 or cutoff.max() >= 1:
raise ValueError("Invalid cutoff frequency: frequencies must be "
"greater than 0 and less than fs/2.")
if np.any(np.diff(cutoff) <= 0):
raise ValueError("Invalid cutoff frequencies: the frequencies "
"must be strictly increasing.")
if width is not None:
# A width was given. Find the beta parameter of the Kaiser window
# and set `window`. This overrides the value of `window` passed in.
atten = kaiser_atten(numtaps, float(width) / nyq)
beta = kaiser_beta(atten)
window = ('kaiser', beta)
pass_nyquist = bool(cutoff.size & 1) ^ pass_zero
if pass_nyquist and numtaps % 2 == 0:
raise ValueError("A filter with an even number of coefficients must "
"have zero response at the Nyquist frequency.")
# Insert 0 and/or 1 at the ends of cutoff so that the length of cutoff
# is even, and each pair in cutoff corresponds to passband.
cutoff = np.hstack(([0.0] * pass_zero, cutoff, [1.0] * pass_nyquist))
# `bands` is a 2D array; each row gives the left and right edges of
# a passband.
bands = cutoff.reshape(-1, 2)
# Build up the coefficients.
alpha = 0.5 * (numtaps - 1)
m = np.arange(0, numtaps) - alpha
h = 0
for left, right in bands:
h += right * sinc(right * m)
h -= left * sinc(left * m)
# Get and apply the window function.
from .signaltools import get_window
win = get_window(window, numtaps, fftbins=False)
h *= win
# Now handle scaling if desired.
if scale:
# Get the first passband.
left, right = bands[0]
if left == 0:
scale_frequency = 0.0
elif right == 1:
scale_frequency = 1.0
else:
scale_frequency = 0.5 * (left + right)
c = np.cos(np.pi * m * scale_frequency)
s = np.sum(h * c)
h /= s
return h
def mesh_to_fluxin(flux_mesh,
flux_tag,
fluxin="fluxin.out",
reverse=False,
sub_voxel=False,
cell_fracs=None,
cell_mats=None,
print_progress=True):
"""This function creates an ALARA fluxin file from fluxes tagged on a PyNE
Mesh object. Fluxes are printed in the order of the flux_mesh.__iter__().
Parameters
----------
flux_mesh : PyNE Mesh object
Contains the mesh with fluxes tagged on each volume element.
flux_tag : string
The name of the tag of the flux mesh. Flux values for different energy
groups are assumed to be represented as vector tags.
fluxin : string
The name of the ALARA fluxin file to be output.
reverse : bool
If true, fluxes will be printed in the reverse order as they appear in
the flux vector tagged on the mesh.
sub_voxel: bool, optional
If true, sub-voxel r2s work flow will be sued. Flux of a voxel will
be duplicated c times. Where c is the cell numbers of that voxel.
cell_fracs : structured array, optional
The output from dagmc.discretize_geom(). A sorted, one dimensional
array, each entry containing the following fields:
:idx: int
The volume element index.
:cell: int
The geometry cell number.
:vol_frac: float
The volume fraction of the cell withing the mesh ve.
:rel_error: float
The relative error associated with the volume fraction.
The array must be sorted with respect to both idx and cell, with
cell changing fastest.
cell_mats : dict, optional
Maps geometry cell numbers to PyNE Material objects.
The cell_fracs and cell_mats are used only when sub_voxel=True.
If sub_voxel=False, neither cell_fracs nor cell_mats will be used.
print_progress : bool
Print the progress with progress bar. Set to False to turn off progress.
"""
tag_flux = flux_mesh.get_tag(flux_tag)
# find number of e_groups
e_groups = tag_flux[list(mesh_iterate(flux_mesh.mesh))[0]]
e_groups = np.atleast_1d(e_groups)
num_e_groups = len(e_groups)
# write flux data block by block
with open(fluxin, "w") as f:
if not sub_voxel:
bar = IfBar("Writing alara fluxin",
max=len(flux_mesh),
suffix='%(percent).1f%% - %(eta)ds',
show=print_progress)
for i, mat, ve in flux_mesh:
f.write(_output_flux_block(ve, tag_flux, reverse))
bar.next()
bar.finish()
else:
ves = list(flux_mesh.iter_ve())
bar = IfBar("Writing alara fluxin",
max=len(cell_fracs),
suffix='%(percent).1f%% - %(eta)ds',
show=print_progress)
for i, row in enumerate(cell_fracs):
if len(cell_mats[row['cell']].comp) != 0:
f.write(
_output_flux_block(ves[row['idx']], tag_flux, reverse))
bar.next()
bar.finish()
def initialise(self, pos="covar", random_state=None):
"""
Initialise the emcee walkers.
Parameters
----------
pos : str or np.ndarray
Method for initialising the emcee walkers. One of:
- 'covar', use the estimated covariance of the system.
- 'jitter', add a small amount of gaussian noise to each parameter
- 'prior', sample random locations from the prior using Latin
Hyper Cube.
- pos, an array that specifies a snapshot of the walkers. Has shape
`(nwalkers, ndim)`, or `(ntemps, nwalkers, ndim)` if parallel
tempering is employed. You can also provide a previously
created chain.
random_state : {int, `np.random.RandomState`, `np.random.Generator`}
If `random_state` is not specified the `~np.random.RandomState`
singleton is used.
If `random_state` is an int, a new ``RandomState`` instance is
used, seeded with random_state.
If `random_state` is already a ``RandomState`` or a ``Generator``
instance, then that object is used.
Specify `random_state` for repeatable initialisations.
"""
nwalkers = self._nwalkers
nvary = self.nvary
# acquire a random number generator
rng = check_random_state(random_state)
# account for parallel tempering
_ntemps = self._ntemps
# If you're not doing parallel tempering, temporarily set the number of
# temperatures to be created to 1, thereby producing initial positions
# of (1, nwalkers, nvary), this first dimension should be removed at
# the end of the method
if self._ntemps == -1:
_ntemps = 1
# position is specified with array (no parallel tempering)
if (
isinstance(pos, np.ndarray)
and self._ntemps == -1
and pos.shape == (nwalkers, nvary)
):
init_walkers = np.copy(pos)[np.newaxis]
# position is specified with array (with parallel tempering)
elif (
isinstance(pos, np.ndarray)
and self._ntemps > -1
and pos.shape == (_ntemps, nwalkers, nvary)
):
init_walkers = np.copy(pos)
# position is specified with existing chain
elif isinstance(pos, np.ndarray):
self.initialise_with_chain(pos)
return
# position is to be created from covariance matrix
elif pos == "covar":
p0 = np.array(self._varying_parameters)
cov = self.objective.covar()
init_walkers = rng.multivariate_normal(
np.atleast_1d(p0), np.atleast_2d(cov), size=(_ntemps, nwalkers)
)
# position is specified by jittering the parameters with gaussian noise
elif pos == "jitter":
var_arr = np.array(self._varying_parameters)
pos = 1 + rng.standard_normal((_ntemps, nwalkers, nvary)) * 1.0e-4
pos *= var_arr
init_walkers = pos
# use the prior to initialise position
elif pos == "prior":
arr = np.zeros((_ntemps, nwalkers, nvary))
LHC = LatinHypercube(nvary, seed=random_state)
samples = LHC.random(n=_ntemps * nwalkers).reshape(
_ntemps, nwalkers, nvary
)
for i, param in enumerate(self._varying_parameters):
# bounds are not a closed interval, just jitter it.
if (
isinstance(param.bounds, Interval)
and not param.bounds._closed_bounds
):
vals = (
1 + rng.standard_normal((_ntemps, nwalkers)) * 1.0e-1
)
vals *= param.value
arr[..., i] = vals
else:
sample_arr = samples[..., i]
transformed = param.bounds.invcdf(sample_arr)
arr[..., i] = transformed
init_walkers = arr
else:
raise RuntimeError(
"Didn't use any known method for " "CurveFitter.initialise"
)
# if you're not doing parallel tempering then remove the first
# dimension
if self._ntemps == -1:
init_walkers = init_walkers[0]
# now validate initialisation, ensuring all init pos have finite
# logpost
for i, param in enumerate(self._varying_parameters):
init_walkers[..., i] = param.valid(init_walkers[..., i])
rstate0 = None
if isinstance(rng, np.random.RandomState):
rstate0 = rng.get_state()
self._state = State(init_walkers, random_state=rstate0)
# finally reset the sampler to reset the chain
# you have to do this at the end, not at the start because resetting
# makes self.sampler.chain == None and the PTsampler creation doesn't
# work
self.sampler.reset()
def photon_source_hdf5_to_mesh(mesh,
filename,
tags,
sub_voxel=False,
cell_mats=None):
"""This function reads in an hdf5 file produced by photon_source_to_hdf5
and tags the requested data to the mesh of a PyNE Mesh object. Any
combinations of nuclides and decay times are allowed. The photon source
file is assumed to be in mesh.__iter__() order
Parameters
----------
mesh : PyNE Mesh
The object containing the PyMOAB instance to be tagged.
filename : str
The path of the hdf5 version of the photon source file.
tags: dict
A dictionary were the keys are tuples with two values. The first is a
string denoting an nuclide in any form that is understood by
pyne.nucname (e.g. '1001', 'U-235', '242Am') or 'TOTAL' for all
nuclides. The second is a string denoting the decay time as it appears
in the file (e.g. 'shutdown', '1 h' '3 d'). The values of the
dictionary are the requested tag names for the combination of nuclide
and decay time. For example if one wanted tags for the photon source
densities from U235 at shutdown and from all nuclides at 1 hour, the
dictionary could be:
tags = {('U-235', 'shutdown') : 'tag1', ('TOTAL', '1 h') : 'tag2'}
sub_voxel: bool, optional
If the sub_voxel is True, then the sub-voxel r2s will be used.
Then the photon_source will be interpreted as sub-voxel photon source.
cell_mats : dict, optional
cell_mats is required when sub_voxel is True.
Maps geometry cell numbers to PyNE Material objects.
"""
# find number of energy groups
with tb.open_file(filename) as h5f:
num_e_groups = len(h5f.root.data[0][3])
max_num_cells = 1
ve0 = next(mesh.iter_ve())
if sub_voxel:
num_vol_elements = len(mesh)
subvoxel_array = _get_subvoxel_array(mesh, cell_mats)
# get max_num_cells
max_num_cells = len(np.atleast_1d(mesh.cell_number[ve0]))
# create a dict of tag handles for all keys of the tags dict
tag_handles = {}
tag_size = num_e_groups * max_num_cells
for tag_name in tags.values():
mesh.tag(tag_name,
np.zeros(tag_size, dtype=float),
'nat_mesh',
size=tag_size,
dtype=float)
tag_handles[tag_name] = mesh.get_tag(tag_name)
decay_times = _read_h5_dt(filename)
# iterate through each requested nuclide/dectay time
for cond in tags.keys():
with tb.open_file(filename) as h5f:
# Convert nuclide to the form found in the ALARA phtn_src
# file, which is similar to the Serpent form. Note this form is
# different from the ALARA input nuclide form found in nucname.
if cond[0] != u"TOTAL":
nuc = serpent(cond[0]).lower()
else:
nuc = u"TOTAL"
# time match, convert string mathch to float mathch
dt = _find_dt(cond[1], decay_times)
# create of array of rows that match the nuclide/decay criteria
matched_data = h5f.root.data.read_where(
"(nuc == '{0}') & (time == '{1}')".format(nuc, dt))
if not sub_voxel:
idx = 0
for i, _, ve in mesh:
if matched_data[idx][0] == i:
tag_handles[tags[cond]][ve] = matched_data[idx][3]
idx += 1
else:
tag_handles[tags[cond]][ve] = [0] * num_e_groups
else:
temp_mesh_data = np.empty(shape=(num_vol_elements, max_num_cells,
num_e_groups),
dtype=float)
temp_mesh_data.fill(0.0)
for sve, subvoxel in enumerate(subvoxel_array):
temp_mesh_data[subvoxel['idx'], subvoxel['scid'], :] = \
matched_data[sve][3][:]
for i, _, ve in mesh:
tag_handles[tags[cond]][ve] = \
temp_mesh_data[i, :].reshape(max_num_cells * num_e_groups)
def _iv(A, k, ncv, tol, which, v0, maxiter, return_singular, solver,
random_state):
# input validation/standardization for `solver`
# out of order because it's needed for other parameters
solver = str(solver).lower()
solvers = {"arpack", "lobpcg", "propack"}
if solver not in solvers:
raise ValueError(f"solver must be one of {solvers}.")
# input validation/standardization for `A`
A = aslinearoperator(A) # this takes care of some input validation
if not (np.issubdtype(A.dtype, np.complexfloating)
or np.issubdtype(A.dtype, np.floating)):
message = "`A` must be of floating or complex floating data type."
raise ValueError(message)
if np.prod(A.shape) == 0:
message = "`A` must not be empty."
raise ValueError(message)
# input validation/standardization for `k`
kmax = min(A.shape) if solver == 'propack' else min(A.shape) - 1
if int(k) != k or not (0 < k <= kmax):
message = "`k` must be an integer satisfying `0 < k < min(A.shape)`."
raise ValueError(message)
k = int(k)
# input validation/standardization for `ncv`
if solver == "arpack" and ncv is not None:
if int(ncv) != ncv or not (k < ncv < min(A.shape)):
message = ("`ncv` must be an integer satisfying "
"`k < ncv < min(A.shape)`.")
raise ValueError(message)
ncv = int(ncv)
# input validation/standardization for `tol`
if tol < 0 or not np.isfinite(tol):
message = "`tol` must be a non-negative floating point value."
raise ValueError(message)
tol = float(tol)
# input validation/standardization for `which`
which = str(which).upper()
whichs = {'LM', 'SM'}
if which not in whichs:
raise ValueError(f"`which` must be in {whichs}.")
# input validation/standardization for `v0`
if v0 is not None:
v0 = np.atleast_1d(v0)
if not (np.issubdtype(v0.dtype, np.complexfloating)
or np.issubdtype(v0.dtype, np.floating)):
message = ("`v0` must be of floating or complex floating "
"data type.")
raise ValueError(message)
shape = (A.shape[0], ) if solver == 'propack' else (min(A.shape), )
if v0.shape != shape:
message = f"`v0` must have shape {shape}."
raise ValueError(message)
# input validation/standardization for `maxiter`
if maxiter is not None and (int(maxiter) != maxiter or maxiter <= 0):
message = "`maxiter` must be a positive integer."
raise ValueError(message)
maxiter = int(maxiter) if maxiter is not None else maxiter
# input validation/standardization for `return_singular_vectors`
# not going to be flexible with this; too complicated for little gain
rs_options = {True, False, "vh", "u"}
if return_singular not in rs_options:
raise ValueError(f"`return_singular_vectors` must be in {rs_options}.")
random_state = check_random_state(random_state)
return (A, k, ncv, tol, which, v0, maxiter, return_singular, solver,
random_state)