def __init__(self, parent, submodel):
"""Argument 'submodel' determines for which submodel the coefficients
should be extracted.
"""
self.parent = parent
self.submodel_idx = parent.get_submodel_index(submodel)
used = []
coef_used = []
for i in range(self.parent.size()):
if (sometrue(self.parent.beta[:,i,self.submodel_idx].ravel()<>0.0)) and \
sometrue(self.parent.coefmap[:,i,self.submodel_idx].ravel()>=0):
used.append(i)
coef_used = coef_used + where(
self.parent.coefmap_alt[:, self.submodel_idx] ==
i)[0].tolist()
eqs_used = []
for i in range(self.parent.nequations()):
## ::IMPT:: this if..then needs to be commented for the constant calibration of model estimation
## and uncomment it for the prediction (lccm_runner) process
if (sometrue(self.parent.beta[i,:,self.submodel_idx].ravel()<>0.0)) and \
sometrue(self.parent.coefmap[i,:,self.submodel_idx].ravel()>=0):
eqs_used.append(i)
self.used_variables_idx = array(
used) #index of variables that are used by this submodel
self.used_coef_idx = array(coef_used)
self.used_equations_idx = array(eqs_used)
self.other_info = {}
def __init__(self, parent, submodel):
"""Argument 'submodel' determines for which submodel the coefficients
should be extracted.
"""
self.parent = parent
self.submodel_idx = parent.get_submodel_index(submodel)
used = []
coef_used = []
for i in range(self.parent.size()):
if (sometrue(self.parent.beta[:,i,self.submodel_idx].ravel()<>0.0)) and \
sometrue(self.parent.coefmap[:,i,self.submodel_idx].ravel()>=0):
used.append(i)
coef_used = coef_used + where(
self.parent.coefmap_alt[:, self.submodel_idx] ==
i)[0].tolist()
eqs_used = []
for i in range(self.parent.nequations()):
# if (sometrue(self.parent.beta[i,:,self.submodel_idx].ravel()<>0.0)) and \
# sometrue(self.parent.coefmap[i,:,self.submodel_idx].ravel()>=0):
eqs_used.append(i)
self.used_variables_idx = array(
used) #index of variables that are used by this submodel
self.used_coef_idx = array(coef_used)
self.used_equations_idx = array(eqs_used)
self.other_measures = {}
self.other_info = {}
def __call__(self, value):
if len(np.shape(value)) == 0:
value = np.atleast_1d(value)
if np.sometrue(np.less(value,self.lower)) or \
np.sometrue(np.greater(value, self.upper)):
return -np.inf
return 0.0
def finterp(band, t, p, param, gen, extrap=False):
'''interpolate at time t and param p for param,gen combo.'''
load_data(band, param, gen)
if param == 'dm15':
f = dm15_flux[(band, gen)]
ef = dm15_eflux[(band, gen)]
else:
f = st_flux[(band, gen)]
ef = st_eflux[(band, gen)]
if len(num.shape(t)) == 0:
scalar = 1
else:
scalar = 0
t = num.atleast_1d(t)
# First the evaluation mtarix:
Z = num.atleast_2d(bisplev(t, p, f))[:, 0]
eZ = num.atleast_2d(bisplev(t, p, ef))[:, 0]
if not extrap:
mask = num.greater_equal(t, f[0][0]) * num.less_equal(t, f[0][-1])
mask = mask * num.greater(Z, 0)
Z = num.where(mask, Z, 1)
eZ = num.where(mask, eZ, -1)
else:
t1, t2 = get_t_lim(band, param, gen)
mask = num.logical_not(num.isnan(Z))
# extrapolate lower with t^2 law
if num.sometrue(num.less(t, t1)):
Tp = bisplev(t1, p, f, dx=1)
T = bisplev(t1, p, f)
eT = bisplev(t, p, ef)
t0 = t1 - 2 * T / Tp
a = T / (t1 - t0)**2
Z = num.where(num.less(t, t1), a * num.power(t - t0, 2), Z)
eZ = num.where(num.less(t, t1), eT, eZ)
mask = mask * num.greater(Z, 0) * num.greater(t, t0)
if num.sometrue(num.greater(t, t2)):
# extrapolate with m = a*(t-t2)+b
Tp = bisplev(t2, p, f, dx=1)
T = bisplev(t2, p, f)
eT = bisplev(t2, p, ef)
b = -2.5 * num.log10(T)
a = -2.5 / num.log(10) / T * Tp
f = num.power(10, -0.4 * (a * (t - t2) + b))
Z = num.where(num.greater(t, t2), f, Z)
eZ = num.where(num.greater(t, t2), eT, eZ)
Z = num.where(mask, Z, 1)
if scalar:
return Z[0], eZ[0], mask[0]
else:
return Z, eZ, mask
def finterp(band, t, p, param, gen, extrap=False):
'''interpolate at time t and param p for param,gen combo.'''
load_data(band,param,gen)
if param == 'dm15':
f = dm15_flux[(band,gen)]
ef = dm15_eflux[(band,gen)]
else:
f = st_flux[(band,gen)]
ef = st_eflux[(band,gen)]
if len(num.shape(t)) == 0:
scalar = 1
else:
scalar = 0
t = num.atleast_1d(t)
# First the evaluation mtarix:
Z = num.atleast_2d(bisplev(t, p, f))[:,0]
eZ = num.atleast_2d(bisplev(t, p, ef))[:,0]
if not extrap:
mask = num.greater_equal(t,f[0][0])*num.less_equal(t,f[0][-1])
mask = mask*num.greater(Z, 0)
Z = num.where(mask, Z, 1)
eZ = num.where(mask, eZ, -1)
else:
t1,t2 = get_t_lim(band, param, gen)
mask = -num.isnan(Z)
# extrapolate lower with t^2 law
if num.sometrue(num.less(t,t1)):
Tp = bisplev(t1, p, f, dx=1)
T = bisplev(t1, p, f)
eT = bisplev(t, p, ef)
t0 = t1 - 2*T/Tp; a = T/(t1-t0)**2
Z = num.where(num.less(t, t1), a*num.power(t-t0,2), Z)
eZ = num.where(num.less(t, t1), eT, eZ)
mask = mask*num.greater(Z,0)*num.greater(t, t0)
if num.sometrue(num.greater(t, t2)):
# extrapolate with m = a*(t-t2)+b
Tp = bisplev(t2, p, f, dx=1)
T = bisplev(t2, p, f)
eT = bisplev(t2, p, ef)
b = -2.5*num.log10(T)
a = -2.5/num.log(10)/T*Tp
f = num.power(10, -0.4*(a*(t-t2)+b))
Z = num.where(num.greater(t, t2), f, Z)
eZ = num.where(num.greater(t, t2), eT, eZ)
Z = num.where(mask, Z, 1)
if scalar:
return Z[0],eZ[0],mask[0]
else:
return Z,eZ,mask
def in_limits(lon, lat, limit):
limitin = recale_limits(limit)
lats = limitin[0]
lons = limitin[1]
late = limitin[2]
lone = limitin[3]
ind = []
lon = recale(lon, degrees=True)
# find out lon between S and E (start and end)
if lons < lone:
lon_flag = (lon >= lons) & (lon <= lone)
# find out lon outside S and E (between E and S)
else:
lon_flag = (lon >= lons) | (lon <= lone)
# nothing remains
if not np.sometrue(lon_flag):
print "* WARNING : No lon [", lons, ",", lone, "]"
return ind, lon_flag
# find lat constraints
lat_flag = (lat >= lats) & (lat <= late)
# nothing remains
if not np.sometrue(lat_flag):
print "* WARNING : No Lat [", lats, ",", late, "] with lon [", lons, ",", lone, "]"
return ind, lat_flag
#Construct flag and index arrays
if (len(lon_flag) == len(lat_flag)):
flag = lon_flag & lat_flag
ind = np.arange(len(lon)).compress(flag)
else:
flag = (lon_flag, lat_flag)
ind = (np.arange(len(lon)).compress(flag[0]),
np.arange(len(lat)).compress(flag[1]))
# compress indexes
# ind = ind.compress(flag)
return ind, flag
def _normalize_weights(self, w):
if (np.sometrue(w < 0.0)):
raise ValueError("Negative basket weights")
s = np.sum(w)
if (s == 0.0):
raise ValueError("Basket weights sum up to zero")
return np.divide(w, s)
def get_energy(self, positions):
"""Return the energy of the nearest local minimum."""
if np.sometrue(self.positions != positions):
self.positions = positions
self.atoms.set_positions(positions)
try:
if self.optimizer == QuasiNewton:
opt = self.optimizer(self.atoms,
logfile=self.optimizer_logfile)
elif self.optimizer.__name__ == "FIRE":
opt = self.optimizer(self.atoms,
maxmove = self.mss,
dt = 0.2, dtmax = 1.0,
logfile=self.optimizer_logfile)
else:
opt = self.optimizer(self.atoms,
logfile=self.optimizer_logfile,
maxstep=self.mss)
# opt = self.optimizer(self.atoms,
# logfile=self.optimizer_logfile,
# maxstep=self.mss)
opt.run(fmax=self.fmax)
self.num_localops += 1
# this caused an error when using QN local optimizer because
# energy is not an attriute of the class Hopping
#self.energy = self.atoms.get_potential_energy()
self.local_min_pos = self.atoms.get_positions()
except:
# Something went wrong.
# In GPAW the atoms are probably to near to each other.
return None
return self.atoms.get_potential_energy()
def get_energy(self, positions=None):
#print 'getting energy, pos: ', positions
if positions is None:
#print 'energy is: ', self.energy
return self.energy
"""Return the energy of the nearest local minimum."""
if np.sometrue(self.positions != positions):
self.positions = positions
self.atoms.set_positions(positions)
try:
opt = self.optimizer(self.atoms,
logfile=self.optimizer_logfile,
maxstep=self.mss)
opt.run(fmax=self.fmax)
self.energy = self.atoms.get_potential_energy()
#print 'energy good ', self.energy
self.local_optimizations += 1
except:
print 'Exception: ', sys.exc_info()[0]
traceback.print_exc()
# Something went wrong.
# In GPAW the atoms are probably to near to each other.
return None
#print 'old energy: ', self.energy
return self.energy
def map_screen(self, data_array):
""" map_screen(data_array) -> screen_array
Overrides AbstractMapper. Maps values from data space to screen space.
"""
# Ensure that data_array is actually an array.
if not isinstance(data_array, ndarray):
data_array = array(data_array, ndmin=1)
# First convert to a [0,1] space, then to the screen space.
if not self._cache_valid:
self._compute_scale()
if self._inter_scale == 0.0:
intermediate = data_array * 0.0
else:
try:
with np.errstate(invalid='ignore'):
mask = (data_array <= LOG_MINIMUM) | isnan(data_array)
if sometrue(mask):
data_array = array(data_array, copy=True, ndmin=1)
data_array[mask] = self.fill_value
intermediate = (log(data_array) -
self._inter_offset) / self._inter_scale
except ValueError:
intermediate = zeros(len(data_array))
result = intermediate * self._screen_scale + self._screen_offset
return result
def _update_selection(self):
""" Sets the selection datasource's metadata to a mask of all
the points selected
"""
if self.selection_datasource is None:
return
selected_mask = zeros(self.selection_datasource._data.shape,
dtype=numpy.bool)
data = self._get_data()
# Compose the selection mask from the cached selections first, then
# the active selection, taking into account the selection mode only
# for the active selection
for selection in self._previous_selections:
selected_mask |= points_in_polygon(
data, selection, False).astype(bool, copy=False)
active_selection = points_in_polygon(
data, self._active_selection, False).astype(bool, copy=False)
if self.selection_mode == 'exclude':
# XXX I think this should be "set difference"? - CJW
selected_mask |= active_selection
selected_mask = ~selected_mask
elif self.selection_mode == 'invert':
selected_mask ^= active_selection
else:
selected_mask |= active_selection
if sometrue(selected_mask != self.selection_datasource.metadata[self.metadata_name]):
self.selection_datasource.metadata[self.metadata_name] = selected_mask
self.selection_changed = True
def rotate(self, A):
"""
Compute the rotated stiffness matrix
"""
A = np.asarray(A)
# Is this a rotation matrix?
if np.sometrue(
np.abs(
np.dot(np.array(A), np.transpose(np.array(A))) -
np.eye(3, dtype=float)) > self.tol):
raise RuntimeError('Matrix *A* does not describe a rotation.')
C = []
for i, j in Voigt_notation:
for k, l in Voigt_notation:
h = 0.0
if i == j and k == l:
h += self.la
if i == k and j == l:
h += self.mu
if i == l and j == k:
h += self.mu
h += self.al * np.sum(A[i, :] * A[j, :] * A[k, :] * A[l, :])
C += [h]
self.C = np.asarray(C)
self.C.shape = (6, 6)
return self.C
def rotate_elastic_constants(C, A, tol=1e-6):
"""
Return rotated elastic moduli for a general crystal given the elastic
constant in Voigt notation.
Parameters
----------
C : array_like
6x6 matrix of elastic constants (Voigt notation).
A : array_like
3x3 rotation matrix.
Returns
-------
C : array
6x6 matrix of rotated elastic constants (Voigt notation).
"""
A = np.asarray(A)
# Is this a rotation matrix?
if np.sometrue(
np.abs(
np.dot(np.array(A), np.transpose(np.array(A))) -
np.eye(3, dtype=float)) > tol):
raise RuntimeError('Matrix *A* does not describe a rotation.')
# Rotate
return full_3x3x3x3_to_Voigt_6x6(
np.einsum('ia,jb,kc,ld,abcd->ijkl', A, A, A, A,
Voigt_6x6_to_full_3x3x3x3(C)))
def _update_selection(self):
""" Sets the selection datasource's 'selection' metadata element
to a mask of all the points selected
"""
if self.selection_datasource is None:
return
selected_mask = zeros(self.selection_datasource._data.shape, dtype=numpy.int32)
data = self._get_data()
# Compose the selection mask from the cached selections first, then
# the active selection, taking into account the selection mode only
# for the active selection
for selection in self._previous_selections:
selected_mask |= (points_in_polygon(data, selection, False))
if self.selection_mode == 'exclude':
selected_mask |= (points_in_polygon(data, self._active_selection, False))
selected_mask = 1 - selected_mask
elif self.selection_mode == 'invert':
selected_mask = -1 * (selected_mask -points_in_polygon(data, self._active_selection, False))
else:
selected_mask |= (points_in_polygon(data, self._active_selection, False))
if sometrue(selected_mask != self.selection_datasource.metadata['selection']):
self.selection_datasource.metadata['selection'] = selected_mask
self.selection_changed = True
return
def evaluate(self, target, force=False):
"""Evaluate interface consistency against target interface's trajectory
on specified conditions.
Optional force argument forces model to recompute its test trajectory,
e.g. because of a known change in model parameters, ics, etc.
"""
assert isinstance(target, ModelInterface), \
"Target argument must be another interface object"
if len(self.compatibleInterfaces) > 0 and \
target.__class__.__name__ not in self.compatibleInterfaces \
and not npy.sometrue([common.compareBaseClass(target, ctype) \
for ctype in self.compatibleInterfaces]):
raise ValueError("Target interface not of compatible type")
try:
self.conditions
except AttributeError:
self.setup_conditions(conditions, self.get_test_traj())
force = force or target.test_traj is None
if force:
# discard returned traj here (still accessible via target.test_traj)
target.get_test_traj(force=force)
self.prepare_conditions(target)
try:
result = self.conditions(target)
except KeyError:
raise KeyError("Condition evaluation failed")
return result
def region_encloses_grid_center(region, grid_centers):
poly = bbox_to_poly(region.bbox)
hits = measure.points_in_poly(grid_centers, poly)
if np.sometrue(hits) and (np.sum(hits) == 1):
return True, np.argwhere(hits).flatten().tolist()
else:
return False, None
def model(params=params,
vars=vars,
paramnames=paramnames,
filters=filters,
value=1.0):
# Set the parameters in the model
for i, param in enumerate(paramnames):
if debug:
print("setting ", param, " to ", params[i])
self.model.parameters[param] = params[i]
logp = 0
numpts = 0
for i, f in enumerate(filters):
mod, err, mask = self.model(f, self.sn.data[f].MJD)
m = mask * self.sn.data[f].mask
if not np.sometrue(m):
continue
numpts += np.sum(m)
tau = np.power(vars[i] + np.power(self.sn.data[f].e_mag, 2),
-1)
logp += pymc.normal_like(self.sn.data[f].mag[m], mod[m],
tau[m])
#if numpts < len(paramnames):
# return -np.inf
return logp
def _update_selection(self):
""" Sets the selection datasource's 'selection' metadata element
to a mask of all the points selected
"""
if self.selection_datasource is None:
return
selected_mask = zeros(self.selection_datasource._data.shape,
dtype=numpy.bool)
data = self._get_data()
# Compose the selection mask from the cached selections first, then
# the active selection, taking into account the selection mode only
# for the active selection
for selection in self._previous_selections:
selected_mask |= (points_in_polygon(data, selection, False))
if self.selection_mode == 'exclude':
selected_mask |= (points_in_polygon(data, self._active_selection,
False))
selected_mask = 1 - selected_mask
elif self.selection_mode == 'invert':
selected_mask = -1 * (selected_mask - points_in_polygon(
data, self._active_selection, False))
else:
selected_mask |= (points_in_polygon(data, self._active_selection,
False))
if sometrue(selected_mask !=
self.selection_datasource.metadata['selection']):
self.selection_datasource.metadata['selection'] = selected_mask
self.selection_changed = True
return
def rotate(self, A):
"""
Compute the rotated stiffness matrix
"""
A = np.asarray(A)
# Is this a rotation matrix?
if np.sometrue(np.abs(np.dot(np.array(A), np.transpose(np.array(A))) -
np.eye(3, dtype=float)) > self.tol):
raise RuntimeError('Matrix *A* does not describe a rotation.')
C = [ ]
for i, j in Voigt_notation:
for k, l in Voigt_notation:
h = 0.0
if i == j and k == l:
h += self.la
if i == k and j == l:
h += self.mu
if i == l and j == k:
h += self.mu
h += self.al*np.sum(A[i,:]*A[j,:]*A[k,:]*A[l,:])
C += [ h ]
self.C = np.asarray(C)
self.C.shape = (6, 6)
return self.C
def rotate_elastic_constants(C, A, tol=1e-6):
"""
Return rotated elastic moduli for a general crystal given the elastic
constant in Voigt notation.
Parameters
----------
C : array_like
6x6 matrix of elastic constants (Voigt notation).
A : array_like
3x3 rotation matrix.
Returns
-------
C : array
6x6 matrix of rotated elastic constants (Voigt notation).
"""
A = np.asarray(A)
# Is this a rotation matrix?
if np.sometrue(np.abs(np.dot(np.array(A), np.transpose(np.array(A))) -
np.eye(3, dtype=float)) > tol):
raise RuntimeError('Matrix *A* does not describe a rotation.')
# Rotate
return full_3x3x3x3_to_Voigt_6x6(np.einsum('ia,jb,kc,ld,abcd->ijkl',
A, A, A, A,
Voigt_6x6_to_full_3x3x3x3(C)))
def _update_selection(self):
""" Sets the selection datasource's metadata to a mask of all
the points selected
"""
if self.selection_datasource is None:
return
selected_mask = zeros(self.selection_datasource._data.shape,
dtype=numpy.bool)
data = self._get_data()
# Compose the selection mask from the cached selections first, then
# the active selection, taking into account the selection mode only
# for the active selection
for selection in self._previous_selections:
selected_mask |= points_in_polygon(
data, selection, False).astype(bool, copy=False)
active_selection = points_in_polygon(
data, self._active_selection, False).astype(bool, copy=False)
if self.selection_mode == 'exclude':
# XXX I think this should be "set difference"? - CJW
selected_mask |= active_selection
selected_mask = ~selected_mask
elif self.selection_mode == 'invert':
selected_mask ^= active_selection
else:
selected_mask |= active_selection
if sometrue(selected_mask != self.selection_datasource.metadata[self.metadata_name]):
self.selection_datasource.metadata[self.metadata_name] = selected_mask
self.selection_changed = True
def map_screen(self, data_array):
""" map_screen(data_array) -> screen_array
Overrides AbstractMapper. Maps values from data space to screen space.
"""
# Ensure that data_array is actually an array.
if not isinstance(data_array, ndarray):
data_array = array(data_array, ndmin=1)
# First convert to a [0,1] space, then to the screen space.
if not self._cache_valid:
self._compute_scale()
if self._inter_scale == 0.0:
intermediate = data_array*0.0
else:
try:
mask = (data_array <= LOG_MINIMUM) | isnan(data_array)
if sometrue(mask):
data_array = array(data_array, copy=True, ndmin=1)
data_array[mask] = self.fill_value
intermediate = (log(data_array) - self._inter_offset)/self._inter_scale
except ValueError:
intermediate = zeros(len(data_array))
result = intermediate * self._screen_scale + self._screen_offset
return result
def stabilize(self, max_steps=200):
"""Run the network until it stabilizes.
A network is considered stable if the output does not change
from one timestep to the next. (Note that depending on the
transfer function, potentials can still change by this
definition)
If parameter max_steps is supplied, it is the maximum number of
timesteps to run, by default 200.
Return the number of time steps used to stabilize, or None
if the network did not stabilize within the maxiumum steps.
"""
for x in xrange(max_steps):
# Make a copy that is not changed by calc_timestep
prev = list(self.output)
self.calc_timestep()
diff = numpy.subtract(prev, self.output)
# All must be 0
if not numpy.sometrue(diff):
self.log.info("Stabilized in %s timesteps", x)
return x
return None
def get_energy(self, positions, symbols):
"""Return the energy of the nearest local minimum."""
if np.sometrue(self.positions != positions) or self.swap_atoms:
self.positions = positions
self.atoms.set_positions(positions)
self.atoms.set_chemical_symbols(symbols)
try:
#Lei: enable 'FIRE' optimizer
if self.optimizer.__name__ == "FIRE":
opt = self.optimizer(self.atoms,
maxmove=1.0,
dt=0.2,
dtmax=1.0,
logfile=self.optimizer_logfile)
else:
opt = self.optimizer(self.atoms,
logfile=self.optimizer_logfile,
maxstep=self.mss)
# opt = self.optimizer(self.atoms,
# logfile=self.optimizer_logfile,
# maxstep=self.mss)
opt.run(fmax=self.fmax)
self.energy = self.atoms.get_potential_energy()
self.local_min_pos = self.atoms.get_positions()
except:
# Something went wrong.
# In GPAW the atoms are probably to near to each other.
return None
return self.energy
def find_initial_contour_pairs(self):
new_pairs = set()
for (low_point, high_point) in self.end_points:
if self.contour_pair_interpolation(low_point, high_point) is None:
(low_point, high_point) = (high_point, low_point)
assert self.contour_pair_interpolation(
low_point,
high_point) is not None, ("bad end points " + repr(
(low_point, high_point)))
while np.sometrue(np.abs(low_point - high_point) > 1):
mid_point = (low_point + high_point) // 2
if self.contour_pair_interpolation(low_point,
mid_point) is not None:
high_point = mid_point
else:
assert self.contour_pair_interpolation(
mid_point, high_point) is not None
low_point = mid_point
pair_to_interpolation = self.find_all_adjacent_contour_pairs(
low_point, True)
pair_to_interpolation.update(
self.find_all_adjacent_contour_pairs(high_point, False))
assert len(pair_to_interpolation) > 0
self.interpolated_contour_pairs.update(pair_to_interpolation)
new_pairs.update(set(pair_to_interpolation))
self.new_contour_pairs = new_pairs
def load_img(fname: str,
channel_dedup: bool = True,
mask: bool = False) -> np.ndarray:
"""
Load the image as a numpy array
If channel_dedup is True, we drop channel dims that are homogenous
"""
# https://stackoverflow.com/questions/3803888/how-to-load-png-images-with-4-channels
assert os.path.isfile(fname), f"Cannot find {fname}"
retval = cv2.imread(
fname, cv2.IMREAD_UNCHANGED) # Returns np.uint8 for normal images
assert len(
retval.shape) <= 3, f"Got image with anomalous dimensions: {fname}"
if not mask:
assert len(retval.shape) == 3
if channel_dedup and len(retval.shape) == 3:
drop_channels = []
for c_dim in range(retval.shape[2]):
if np.all(retval[:, :, c_dim] == retval[0, 0, c_dim]):
drop_channels.append(c_dim)
for c in drop_channels[::-1]:
retval = np.delete(retval, c, axis=2)
# Convert to binary mask
if mask:
retval = retval.astype(np.bool)
assert np.sometrue(retval), f"Got mask that is all False: {fname}"
assert len(retval.shape) == 2
return retval
def in_limits(lon, lat, limit):
limitin = recale_limits(limit)
lats = limitin[0]
lons = limitin[1]
late = limitin[2]
lone = limitin[3]
ind=[]
lon = recale(lon,degrees=True)
# find out lon between S and E (start and end)
if lons < lone:
lon_flag = (lon >= lons) & (lon <= lone)
# find out lon outside S and E (between E and S)
else:
lon_flag = (lon >= lons) | (lon <= lone)
# nothing remains
if not np.sometrue(lon_flag):
print "* WARNING : No lon [", lons, ",", lone, "]"
return ind, lon_flag
# find lat constraints
lat_flag = (lat >= lats) & (lat <= late)
# nothing remains
if not np.sometrue(lat_flag):
print "* WARNING : No Lat [", lats, ",", late, "] with lon [", lons, ",", lone, "]"
return ind, lat_flag
#Construct flag and index arrays
if (len(lon_flag) == len(lat_flag)) :
flag = lon_flag & lat_flag
ind = np.arange(len(lon)).compress(flag)
else :
flag = (lon_flag,lat_flag)
ind = (np.arange(len(lon)).compress(flag[0]),np.arange(len(lat)).compress(flag[1]))
# compress indexes
# ind = ind.compress(flag)
return ind, flag
def isbinstr(arg):
# supports unary + / - at front, and checks for usage of exponentials
# (using 'E' or 'e')
s = arg.lower()
try:
if s[0] in ['+', '-']:
s_rest = s[1:]
else:
s_rest = s
except IndexError:
return False
if '0' not in s_rest and '1' not in s_rest:
return False
pts = s.count('.')
exps = s.count('e')
pm = s_rest.count('+') + s_rest.count('-')
if pts > 1 or exps > 1 or pm > 1:
return False
if exps == 1:
exp_pos = s.find('e')
pre_exp = s[:exp_pos]
# must be numbers before and after the 'e'
if not np.sometrue([n in ('0', '1') for n in pre_exp]):
return False
if s[-1] == 'e':
# no chars after 'e'!
return False
if not np.sometrue([n in ('0','1','2','3','4','5','6','7','8','9') \
for n in s[exp_pos:]]):
return False
# check that any additional +/- occurs directly after 'e'
if pm == 1:
pm_pos = max([s_rest.find('+'), s_rest.find('-')])
if s_rest[pm_pos - 1] != 'e':
return False
e_rest = s_rest[pm_pos + 1:] # safe due to previous check
s_rest = s_rest[:pm_pos + 1]
else:
e_rest = s[exp_pos + 1:]
s_rest = s[:exp_pos + 1]
# only remaining chars in s after e and possible +/- are numbers
if '.' in e_rest:
return False
# cannot use additional +/- if not using exponent
if pm == 1 and exps == 0:
return False
return np.alltrue([n in ('0', '1', '.', 'e', '+', '-') for n in s_rest])
def makeMappingArray(N, data, gamma=1.0):
"""Create an *N* -element 1-d lookup table
*data* represented by a list of x,y0,y1 mapping correspondences.
Each element in this list represents how a value between 0 and 1
(inclusive) represented by x is mapped to a corresponding value
between 0 and 1 (inclusive). The two values of y are to allow
for discontinuous mapping functions (say as might be found in a
sawtooth) where y0 represents the value of y for values of x
<= to that given, and y1 is the value to be used for x > than
that given). The list must start with x=0, end with x=1, and
all values of x must be in increasing order. Values between
the given mapping points are determined by simple linear interpolation.
Alternatively, data can be a function mapping values between 0 - 1
to 0 - 1.
The function returns an array "result" where ``result[x*(N-1)]``
gives the closest value for values of x between 0 and 1.
"""
if callable(data):
xind = np.linspace(0, 1, N)**gamma
lut = np.clip(np.array(data(xind), dtype=np.float), 0, 1)
return lut
try:
adata = np.array(data)
except:
raise TypeError("data must be convertable to an array")
shape = adata.shape
if len(shape) != 2 and shape[1] != 3:
raise ValueError("data must be nx3 format")
x = adata[:,0]
y0 = adata[:,1]
y1 = adata[:,2]
if x[0] != 0. or x[-1] != 1.0:
raise ValueError(
"data mapping points must start with x=0. and end with x=1")
if np.sometrue(np.sort(x)-x):
raise ValueError(
"data mapping points must have x in increasing order")
# begin generation of lookup table
x = x * (N-1)
lut = np.zeros((N,), np.float)
xind = (N - 1) * np.linspace(0, 1, N)**gamma
ind = np.searchsorted(x, xind)[1:-1]
lut[1:-1] = ( ((xind[1:-1] - x[ind-1]) / (x[ind] - x[ind-1]))
* (y0[ind] - y1[ind-1]) + y1[ind-1])
lut[0] = y1[0]
lut[-1] = y0[-1]
# ensure that the lut is confined to values between 0 and 1 by clipping it
np.clip(lut, 0.0, 1.0)
#lut = where(lut > 1., 1., lut)
#lut = where(lut < 0., 0., lut)
return lut
def makeMappingArray(N, data, gamma=1.0):
"""Create an *N* -element 1-d lookup table
*data* represented by a list of x,y0,y1 mapping correspondences.
Each element in this list represents how a value between 0 and 1
(inclusive) represented by x is mapped to a corresponding value
between 0 and 1 (inclusive). The two values of y are to allow
for discontinuous mapping functions (say as might be found in a
sawtooth) where y0 represents the value of y for values of x
<= to that given, and y1 is the value to be used for x > than
that given). The list must start with x=0, end with x=1, and
all values of x must be in increasing order. Values between
the given mapping points are determined by simple linear interpolation.
Alternatively, data can be a function mapping values between 0 - 1
to 0 - 1.
The function returns an array "result" where ``result[x*(N-1)]``
gives the closest value for values of x between 0 and 1.
"""
if callable(data):
xind = np.linspace(0, 1, N)**gamma
lut = np.clip(np.array(data(xind), dtype=np.float), 0, 1)
return lut
try:
adata = np.array(data)
except:
raise TypeError("data must be convertable to an array")
shape = adata.shape
if len(shape) != 2 and shape[1] != 3:
raise ValueError("data must be nx3 format")
x = adata[:,0]
y0 = adata[:,1]
y1 = adata[:,2]
if x[0] != 0. or x[-1] != 1.0:
raise ValueError(
"data mapping points must start with x=0. and end with x=1")
if np.sometrue(np.sort(x)-x):
raise ValueError(
"data mapping points must have x in increasing order")
# begin generation of lookup table
x = x * (N-1)
lut = np.zeros((N,), np.float)
xind = (N - 1) * np.linspace(0, 1, N)**gamma
ind = np.searchsorted(x, xind)[1:-1]
lut[1:-1] = ( ((xind[1:-1] - x[ind-1]) / (x[ind] - x[ind-1]))
* (y0[ind] - y1[ind-1]) + y1[ind-1])
lut[0] = y1[0]
lut[-1] = y0[-1]
# ensure that the lut is confined to values between 0 and 1 by clipping it
np.clip(lut, 0.0, 1.0)
#lut = where(lut > 1., 1., lut)
#lut = where(lut < 0., 0., lut)
return lut
def populate_sheet_param(self):
sheets = [
s for s in topo.sim.objects(self.sheet_type).values() if sometrue([
isinstance(p, self.projection_type) for p in s.in_connections
])
]
self.plotgroup.params()['sheet'].objects = sheets
self.plotgroup.sheet = sheets[0] # CB: necessary?
def symmetric_f_measure(a):
'''
returns symmetrized version of F-measure
'''
a = np.asarray(a, dtype = 'float64').reshape(4)
assert np.all(a >= 0) and np.sometrue(a > 0.0)
TN, FN, FP, TP = tuple(a)
return 1.0 - 0.5 * (FN + FP) / (max([TN, TP]) + 0.5 * (FN + FP))
def proj_transform_vec_clip(vec, M):
vecw = np.dot(M, vec)
w = vecw[3]
txs, tys, tzs = vecw[0]/w, vecw[1]/w, vecw[2]/w
tis = (vecw[0] >= 0) * (vecw[0] <= 1) * (vecw[1] >= 0) * (vecw[1] <= 1)
if np.sometrue(tis):
tis = vecw[1] < 1
return txs, tys, tzs, tis
def populate_sheet_param(self):
sheets = [
s
for s in topo.sim.objects(self.sheet_type).values()
if sometrue([isinstance(p, self.projection_type) for p in s.in_connections])
]
self.plotgroup.params()["sheet"].objects = sheets
self.plotgroup.sheet = sheets[0] # CB: necessary?
def _setup(self):
'''Given the current set of params, setup the interpolator.'''
self.tck = splrep(self.x, self.y, 1.0 / self.ey, **self.pars)
# Check for NaN's in the tck.
if num.sometrue(num.isnan(self.tck[1])):
raise ValueError, "The Spline is invalid. It is possible the data are too noisy, or smoothing is too low. Try increasing 's' or fixing your errors"
self.setup = True
self.realization = None
def _get_cover_types(self, lct):
"""Return a list of landcover types present in the lct grid"""
x = []
max_type = int(maximum.reduce(ravel(lct)))
for itype in range(1, max_type + 1):
if sometrue(ravel(lct) == itype):
x = x + [itype]
return array(x)
def _get_cover_types(self, lct):
"""Return a list of landcover types present in the lct grid"""
x = []
max_type = int(maximum.reduce(ravel(lct)))
for itype in range(1, max_type+1):
if sometrue(ravel(lct) == itype):
x = x + [itype]
return array(x)
def _setup(self):
'''Given the current set of params, setup the interpolator.'''
self.tck = splrep(self.x, self.y, 1.0/self.ey, **self.pars)
# Check for NaN's in the tck.
if num.sometrue(num.isnan(self.tck[1])):
raise ValueError, "The Spline is invalid. It is possible the data are too noisy, or smoothing is too low. Try increasing 's' or fixing your errors"
self.setup = True
self.realization = None
def bin_ndarray(ndarray, new_shape, weights=None, operation=numpy.mean):
"""
Bins an ndarray in all axes based on the target shape, by summing
or averaging.
Parameters
----------
ndarray : array_like
the input array to re-bin
new_shape : tuple
the tuple holding the desired new shape
weights : array_like, optional
weights to multiply the input array by, before running the re-binning
operation,
Notes
-----
* Dimensions in `new_shape` must be integral factor smaller than the
old shape
* Number of output dimensions must match number of input dimensions.
* See https://gist.github.com/derricw/95eab740e1b08b78c03f
Examples
--------
>>> m = numpy.arange(0,100,1).reshape((10,10))
>>> n = bin_ndarray(m, new_shape=(5,5), operation=numpy.sum)
>>> print(n)
[[ 22 30 38 46 54]
[102 110 118 126 134]
[182 190 198 206 214]
[262 270 278 286 294]
[342 350 358 366 374]]
"""
if ndarray.shape == new_shape:
raise ValueError(
"why are we re-binning if the new shape equals the old shape?")
if ndarray.ndim != len(new_shape):
raise ValueError("Shape mismatch: {} -> {}".format(
ndarray.shape, new_shape))
if numpy.sometrue(numpy.mod(ndarray.shape, new_shape)):
args = (str(new_shape), str(ndarray.shape))
msg = "desired shape of %s must be integer factor smaller than the old shape %s" % args
raise ValueError(msg)
compression_pairs = [(d, c // d) for d, c in zip(new_shape, ndarray.shape)]
flattened = [l for p in compression_pairs for l in p]
ndarray = ndarray.reshape(flattened)
if weights is not None: weights = weights.reshape(flattened)
for i in range(len(new_shape)):
if weights is not None:
ndarray = operation(ndarray * weights, axis=-1 * (i + 1))
weights = numpy.sum(weights, axis=-1 * (i + 1))
ndarray /= weights
else:
ndarray = operation(ndarray, axis=-1 * (i + 1))
return ndarray
def get_non_zero_equations(self):
"""Return an equation index for equations with at least one non-zero coefficient."""
equations_index = self.get_equations_index()
coef_values = self.get_coefficient_values()
non_zero_eq = []
for i in equations_index:
if (sometrue(coef_values[i,:]<>0.0)):
non_zero_eq.append(i)
return array(non_zero_eq)
def proj_transform_vec_clip(vec, M):
vecw = np.dot(M, vec)
w = vecw[3]
# clip here..
txs, tys, tzs = vecw[0]/w, vecw[1]/w, vecw[2]/w
tis = (vecw[0] >= 0) * (vecw[0] <= 1) * (vecw[1] >= 0) * (vecw[1] <= 1)
if np.sometrue(tis):
tis = vecw[1] < 1
return txs, tys, tzs, tis
def polygamma(n, x):
"""Polygamma function which is the nth derivative of the digamma (psi)
function."""
n, x = asarray(n), asarray(x)
cond = (n==0)
fac2 = (-1.0)**(n+1) * gamma(n+1.0) * zeta(n+1,x)
if sometrue(cond,axis=0):
return where(cond, psi(x), fac2)
return fac2
def test_sample_noreplace(self):
start_time = time.time()
sample = sample_noreplace(self.all, self.size, return_index=True)
logger.log_status("sample_noreplace %s from %s items array in " % (self.size,self.n) + str(time.time() - start_time) + " sec")
self.assertEqual(sample.size, self.size, msg ="sample size not equal to size parameter")
assert isinstance(sample, ndarray), "sample is not of type ndarray"
assert 0 <= sample.min() <= self.n-1, "sampled elements not in between min and max of source array"
assert 0 <= sample.max() <= self.n-1, "sampled elements not in between min and max of source array"
assert not sometrue(find_duplicates(sample)), "there are duplicates in samples"
def polygamma(n, x):
"""Polygamma function which is the nth derivative of the digamma (psi)
function."""
n, x = asarray(n), asarray(x)
cond = (n==0)
fac2 = (-1.0)**(n+1) * gamma(n+1.0) * zeta(n+1,x)
if sometrue(cond,axis=0):
return where(cond, psi(x), fac2)
return fac2
def coarsen(self):
"""Return coarsened `GridDescriptor` object.
Reurned descriptor has 2x2x2 fewer grid points."""
if np.sometrue(self.N_c % 2):
raise ValueError('Grid %s not divisible by 2!' % self.N_c)
return self.new_descriptor(self.N_c // 2)
def calcOverlap(self,r1,rn):
if rn==None:
return False
rm=n.dot(self.o1,r1)
rn=n.dot(self.o2,rn)
eq=rm+rn
bm=n.greater(eq,0)
bv=n.sometrue(bm,0)
return not n.alltrue(bv)
def coarsen(self):
"""Return coarsened `GridDescriptor` object.
Reurned descriptor has 2x2x2 fewer grid points."""
if np.sometrue(self.N_c % 2):
raise ValueError('Grid %s not divisible by 2!' % self.N_c)
return self.new_descriptor(self.N_c // 2)
def rebin_factor( a, newshape ):
'''Rebin an array to a new shape.
newshape must be a factor of a.shape.
'''
assert len(a.shape) == len(newshape)
assert not np.sometrue(np.mod( a.shape, newshape ))
slices = [ slice(None,None, old/new) for old,new in zip(a.shape,newshape) ]
return a[slices]
def truncated_normal(center, sigma, size=(), rand=np.random):
# Return truncated normal
values = np.zeros(size, dtype=float)
x = rand.normal(center, sigma, size=size)
mask = (abs(x - center) > 2 * sigma)
while np.sometrue(mask):
x[mask] = rand.normal(center, sigma, size=mask.sum())
mask = (abs(x - center) > 2 * sigma)
return x
def bin_ndarray(ndarray, new_shape, weights=None, operation=numpy.mean):
"""
Bins an ndarray in all axes based on the target shape, by summing
or averaging.
Parameters
----------
ndarray : array_like
the input array to re-bin
new_shape : tuple
the tuple holding the desired new shape
weights : array_like, optional
weights to multiply the input array by, before running the re-binning
operation,
Notes
-----
* Dimensions in `new_shape` must be integral factor smaller than the
old shape
* Number of output dimensions must match number of input dimensions.
* See https://gist.github.com/derricw/95eab740e1b08b78c03f
Example
-------
>>> m = np.arange(0,100,1).reshape((10,10))
>>> n = bin_ndarray(m, new_shape=(5,5), operation=numpy.sum)
>>> print(n)
[[ 22 30 38 46 54]
[102 110 118 126 134]
[182 190 198 206 214]
[262 270 278 286 294]
[342 350 358 366 374]]
"""
if ndarray.shape == new_shape:
raise ValueError("why are we re-binning if the new shape equals the old shape?")
if ndarray.ndim != len(new_shape):
raise ValueError("Shape mismatch: {} -> {}".format(ndarray.shape, new_shape))
if numpy.sometrue(numpy.mod(ndarray.shape, new_shape)):
args = (str(new_shape), str(ndarray.shape))
msg = "desired shape of %s must be integer factor smaller than the old shape %s" %args
raise ValueError(msg)
compression_pairs = [(d, c//d) for d, c in zip(new_shape, ndarray.shape)]
flattened = [l for p in compression_pairs for l in p]
ndarray = ndarray.reshape(flattened)
if weights is not None: weights = weights.reshape(flattened)
for i in range(len(new_shape)):
if weights is not None:
ndarray = operation(ndarray*weights, axis=-1*(i+1))
weights = numpy.sum(weights, axis=-1*(i+1))
ndarray /= weights
else:
ndarray = operation(ndarray, axis=-1*(i+1))
return ndarray
def _make_mapping_array(self, n, data):
"""Creates an N-element 1-D lookup table
The *data* parameter is a list of x,y0,y1 mapping correspondences (which
can be lists or tuples), where all the items are values between 0 and 1,
inclusive. The items in the mapping are:
* x: a value being mapped
* y0: the value of y for values of x less than or equal to the given x value.
* y1: the value of y for values of x greater than the given x value.
The two values of y allow for discontinuous mapping functions (for
example, as might be found in a sawtooth function)
The list must start with x=0, end with x=1, and
all values of x must be in increasing order. Values between
the given mapping points are determined by simple linear interpolation.
The function returns an array "result" where result[x*(N-1)]
gives the closest value for values of x between 0 and 1.
"""
try:
adata = array(data)
except:
raise TypeError("data must be convertable to an array")
shape = adata.shape
if len(shape) != 2 and shape[1] != 3:
raise ValueError("data must be nx3 format")
x = adata[:,0]
y0 = adata[:,1]
y1 = adata[:,2]
if x[0] != 0. or x[-1] != 1.0:
raise ValueError(
"data mapping points must start with x=0. and end with x=1")
if sometrue(sort(x)-x):
raise ValueError(
"data mapping points must have x in increasing order")
# begin generation of lookup table
x = x * (n-1)
lut = zeros((n,), float32)
xind = arange(float32(n), dtype=float32)
ind = searchsorted(x, xind)[1:-1]
lut[1:-1] = ( divide(xind[1:-1] - take(x,ind-1),
take(x,ind)-take(x,ind-1) )
*(take(y0,ind)-take(y1,ind-1)) + take(y1,ind-1))
lut[0] = y1[0]
lut[-1] = y0[-1]
# ensure that the lut is confined to values between 0 and 1 by clipping it
lut = lut.clip(0, 1)
return lut
def cerca(self, nd, shift):
self.shift=shift
self.matrice.trasforma(1.0, shift)
m = min(4*nd, self.dim)
self.nsteps = m
self.allocaMemory()
vect_init = self.class4vect(self.dim)
# vect_init.set_value(0,1.0)
vect_init.set_all_random(1.0)
# vect_init.set_to_one()
k=0
nc=0
self.passeggia(k,m,vect_init)
while nc<nd :
#print " DIAGONALIZZAZIONE "
self.diago(k,m)
nc = self.converged(m)
if k and not numpy.sometrue(abs(self.beta[:k])>self.tol) :
break
if (nc+2*nd) >= m:
k=m-1
else:
k=nc+2*nd
#print "KKKKKKKKK " , k
self.ricipolla(k,m)
self.countdumpab+=1
#print " k,m , dim", k, m, self.dim
# return m # sentinell
self.passeggia(k,m,self.q[m])
if m==self.dim:
return m
else:
return k
def test_prob2dsample(self):
start_time = time.time()
sample = prob2dsample(self.all, self.sample_size, self.prob, return_index=True)
logger.log_status("prob2dsample (%s, %s) items array in " % self.sample_size + str(time.time() - start_time) + " sec")
self.assertEqual(sample.shape, self.sample_size, msg ="sample size not equal to sample size parameter")
assert isinstance(sample, ndarray), "sample is not of type ndarray"
assert 0 <= sample.min() <= self.n-1, "sampled elements not in between min and max of source array"
assert 0 <= sample.max() <= self.n-1, "sampled elements not in between min and max of source array"
assert all(not_equal(self.prob[sample], 0.0)), "elements with zero weight in the sample"
for i in range(sample.shape[0]):
assert not sometrue(find_duplicates(sample[i,:])), "there are duplicates in samples at row %s" % i
def neighbour_factor( a, newshape ):
'''Rebin an array to a new shape.
newshape must be a factor of a.shape
Uses nearest neighbour lookup.
'''
assert len(a.shape) == len(newshape)
assert not n.sometrue(n.mod( a.shape, newshape ))
slices = [ slice(None,None, old/new) \
for old,new in zip(a.shape,newshape) ]
return a[slices]
def odds_ratio(a, alpha = 1.0):
'''
takes tuple (TN, FN, FP, TP) or array
( TN FN
TP FP ) and returns odds_ratio
'''
#adding alpha to prevent zero_division
a = np.asarray(a, dtype = 'float64').reshape(4)
assert np.all(a >= 0) and np.sometrue(a > 0.0)
TN, FN, FP, TP = tuple(a)
return np.log((TP + alpha) * (TN + alpha) / ((FP + alpha) * (FN + alpha)))
def lnlike(p, varinfo, snobj, bands):
# first, assign all variables to the model:
for id,var in enumerate(varinfo['varlist']):
if varinfo[var]['fixed']:
if var in snobj.model.parameters:
snobj.model.parameters[var] = varinfo[var]['value']
else:
snobj.model.nparameters[var] = varinfo[var]['value']
else:
val = p[varinfo[var]['index']]
if var in snobj.model.parameters:
snobj.model.parameters[var] = val
else:
snobj.model.nparameters[var] = p[varinfo[var]['index']]
lp = 0
for band in bands:
mod,err,mask = snobj.model.__call__(band, snobj.data[band].MJD)
fitflux = varinfo['fitflux']
if fitflux:
if snobj.model.model_in_mags:
f = np.power(10, -0.4*(mod - snobj.data[band].filter.zp))
cov_f = np.power(f*err/1.0857,2)
else:
f = mod
cov_f = np.power(err, 2)
else:
if snobj.model.model_in_mags:
f = mod
cov_f = np.power(err, 2)
else:
f = -2.5*log10(mod) + snobj.data[band].filter.zp
cov_f = np.power(err/mod*1.0857,2)
m = mask*snobj.data[band].mask
if not np.sometrue(m):
# We're outside the support of the data
return -np.inf
N = sum(m)
X = snobj.data[band].flux[m] - f[m]
#if not np.alltrue(m):
# ids = np.nonzero(-m)[0]
# thiscovar = np.delete(np.delete(snobj.bcovar[band],ids,axis=0),
# ids, axis=1)
#else:
# thiscovar = snobj.bcovar[band]
#detcovar = np.linalg.det(thiscovar)
#invcovar = np.linalg.inv(thiscovar)
#lp = lp - 0.5*np.log(2*np.pi**N*detcovar) -\
# 0.5*np.dot(X, np.dot(invcovar,X))
denom = cov_f[m] + np.power(snobj.data[band].e_flux[m],2)
lp = lp - 0.5*np.sum(np.power(X,2)/denom + \
np.log(denom) + np.log(2*np.pi))
return lp
def coarsen(self):
"""Return coarsened `GridDescriptor` object.
Reurned descriptor has 2x2x2 fewer grid points."""
if np.sometrue(self.N_c % 2):
raise ValueError('Grid %s not divisible by 2!' % self.N_c)
gd = GridDescriptor(self.N_c // 2, self.cell_cv,
self.pbc_c, self.comm, self.parsize_c)
gd.use_fixed_bc = self.use_fixed_bc
return gd
