def spo_all_xr(observed: np.array,
modeled: np.array,
inf_as_na: bool = True,
decimals: int = 2):
result = calc_gof_stats(observed.ravel(), modeled.ravel(), inf_as_na,
decimals).to_xarray()
return result
def plot_with_gradient(y: np.array,
gradient: np.array,
ax=None,
x=None,
colormap="plasma",
color_bar=True,
normalise=False,
**kwargs):
"""
Uses template to show gradient of another variable
:param ax: axis of the plot
:param x: x data
:param y: y data
:param gradient: creates gradient on the plot
:param colormap: viridis/plasma/...
:param color_bar: add colour bar
:param normalise: boolean - 0-1 normalisation
:return:
"""
if x is None:
x = range(y.ravel().shape[0])
if len(gradient.shape) > 2:
raise Exception("Gradient is not 1D vector, but has {}".format(
len(gradient.shape)))
if len(gradient.shape) == 2:
gradient = gradient.ravel()
fig = None
if ax is None:
fig, ax = plt.subplots(1, 1)
fig.set_size_inches(18.5, 10.5)
points = np.array([x, y]).T.reshape(-1, 1, 2) # create gradient colour
segments = np.concatenate([points[:-2], points[1:-1], points[2:]], axis=1)
if normalise:
lc = LineCollection(segments,
cmap=colormap,
linewidth=4,
norm=Normalize(vmin=0, vmax=1))
else:
lc = LineCollection(segments, cmap=colormap, linewidth=4)
lc.set_array(gradient) # set up gradient
line = ax.add_collection(lc)
ax.autoscale()
if color_bar:
plt.colorbar(line, ax=ax)
if "title" in kwargs:
ax.set_title(kwargs["title"])
if fig is not None:
return fig, ax
return ax
def display_classification_areas(
model,
cords: np.array,
labels: np.array,
output_path: str = None
) -> None:
x_start = cords[:, 0].min() - 0.5
x_end = cords[:, 0].max() + 0.5
y_start = cords[:, 1].min() - 0.5
y_end = cords[:, 1].max() + 0.5
grid = np.mgrid[x_start:x_end:100j, y_start:y_end:100j]
grid_2d = grid.reshape(2, -1).T
x_grid, y_grid = grid
prediction = model.predict(grid_2d)
plt.style.use('dark_background')
plt.figure(figsize=(10, 10))
plt.axis('off')
plt.contourf(x_grid, y_grid, prediction.reshape(100, 100),
alpha=0.7, cmap=plt.cm.Spectral)
plt.scatter(cords[:, 0], cords[:, 1], c=labels.ravel(),
s=50, cmap=plt.cm.Spectral, edgecolors='white')
if output_path:
plt.savefig(output_path, bbox_inches='tight')
plt.show()
def nppow(a: np.array, b: Union[int, float, np.array]) -> np.array:
"""
evaluates a**b element-by-element
:param np.array a:
:param Union[int, float, np.array] b: if an array,
should have the same shape as `a`
:return: an array of the same shape as `a`
"""
mina = np.min(a)
if mina <= 0:
print_stars("All elements of a must be positive in nppow!")
sys.exit(1)
if isinstance(b, (int, float)):
return a**b
else:
if a.shape != b.shape:
print_stars(
"nppow: b is not a number or an array of the same shape as a!")
sys.exit(1)
avec = a.ravel()
bvec = b.ravel()
a_pow_b = avec**bvec
return a_pow_b.reshape(a.shape)
def similarity_measure_area_of_overlap(arr_fixed: np.array,
arr_to_transform: np.array,
values: np.array,
debug=False):
'''
Transforms `arr_to_transform` using affine transformation parameters in `values`, then returns the normalised mutual information between the result and `arr_fixed`.
Only the pixels covered by both arrays are considered when calculating the mutual information.
'''
assert arr_fixed.shape == arr_to_transform.shape
arr_transformed = transform_using_values(arr_to_transform,
values,
cval=float('-inf'))
arr_1 = arr_fixed.ravel()
arr_2 = arr_transformed.ravel()
arr_1_reduced = []
arr_2_reduced = []
assert len(arr_1) >= 3
if debug:
print("min = " + str(arr_to_transform.min()))
print(values)
for i in range(len(arr_1)):
if arr_2[i] >= arr_to_transform.min():
arr_1_reduced.append(arr_1[i])
arr_2_reduced.append(arr_2[i])
if debug:
print("Length = " + str(len(arr_2_reduced)))
if len(arr_1_reduced) < 3:
return 0
else:
sm = im.similarity_measure(np.array(arr_1_reduced),
np.array(arr_2_reduced),
measure="NMI")
return sm
def costFunction(
nnParams: np.array,
hiddenLayerSize: int,
X: np.array,
y: np.array,
lmbda: float = 0,
) -> float:
# Setup useful variables
(m, n, k, y) = setupVars(X, y)
(theta1, theta2) = reshapeParams(nnParams, n, hiddenLayerSize, k)
(a3, _) = forwardProp(theta1, theta2, X)
y_vec = y.ravel()
h_vec = a3.ravel()
J = ((-y_vec.dot(np.log(h_vec))) - ((1 - y_vec).dot(np.log(1 - h_vec)))) / m
if lmbda > 0:
multiplier = lmbda / (2 * m)
t1Reg = theta1[:, 1:].ravel()
t2Reg = theta2[:, 1:].ravel()
J += multiplier * (t1Reg.dot(t1Reg) + t2Reg.dot(t2Reg))
print(f"Current J: {J}")
return J
def der_nppow(a: np.array, b: Union[int, float, np.array]) -> np.array:
"""
evaluates the derivatives in a and b of element-by-element a**b
:param np.array a:
:param Union[int, float, np.array] b: if an array,
should have the same shape as `a`
:return: a pair of two arrays of the same shape as `a`
"""
mina = np.min(a)
if mina <= 0:
print_stars("All elements of a must be positive in der_nppow!")
sys.exit(1)
if isinstance(b, (int, float)):
a_pow_b = a**b
return (b * a_pow_b / a, a_pow_b * log(a))
else:
if a.shape != b.shape:
print_stars(
"nppow: b is not a number or an array of the same shape as a!")
sys.exit(1)
avec = a.ravel()
bvec = b.ravel()
a_pow_b = avec**bvec
der_wrt_a = a_pow_b * bvec / avec
der_wrt_b = a_pow_b * nplog(avec)
return (der_wrt_a.reshape(a.shape), der_wrt_b.reshape(a.shape))
def __filterLFP__(self, data: np.array, sample_rate: int):
from scipy.signal import filtfilt, firwin
if self.fs is None:
from ephysiopy import fs
self.fs = fs
if self.lfp_lowcut is None:
from ephysiopy import lfp_lowcut
self.lfp_lowcut = lfp_lowcut
if self.lfp_highcut is None:
from ephysiopy import lfp_highcut
self.lfp_highcut = lfp_highcut
nyq = sample_rate / 2.0
lowcut = self.lfp_lowcut / nyq
highcut = self.lfp_highcut / nyq
if highcut >= 1.0:
highcut = 1.0 - np.finfo(float).eps
if lowcut <= 0.0:
lowcut = np.finfo(float).eps
b = firwin(sample_rate + 1, [lowcut, highcut],
window="black",
pass_zero=False)
y = filtfilt(b, [1], data.ravel(), padtype="odd")
return y
def log_histogram(self, tag: str, data: np.array, step: int, num_bars: int = 30):
"""
Adds a histogram to log.
Parameters
----------
tag: str
data: np.array
Array of any shape.
step: int
num_bars: int
The number of bars if the resulting histogram.
"""
data = data.ravel()
min_ = data.min()
max_ = data.max()
sum_ = data.sum()
sum_sq = data.dot(data)
if min_ == max_:
num = 1
bucket_limit = [min_]
bucket = [len(data)]
else:
bucket, bucket_limit = np.histogram(data, num_bars)
num = len(bucket_limit)
bucket_limit = bucket_limit[1:]
hist = HistogramProto(min=min_, max=max_, sum=sum_, sum_squares=sum_sq, num=num,
bucket_limit=bucket_limit, bucket=bucket)
self._write_event(tag, step, histo=hist)
def _get_uncertainty(rho_surface: np.array, mask: np.array) -> sp.lil_matrix:
r_mat = rho_surface * 0.05
r_mat[np.logical_not(mask)] = 0.
N = mask.ravel().shape[0]
r_mat_sp = sp.lil_matrix((N, N))
r_mat_sp.setdiag(1. / (r_mat.ravel())**2)
return r_mat_sp.tocsr()
def grad_pass(self, y_grad: np.array):
x_grad = y_grad.ravel() @ self.sp_matrix.T
x_grad = x_grad.reshape(*self.x_view)
if self.padding[0]:
x_grad = x_grad[:, :, self.padding[0]:-self.padding[0]]
if self.padding[1]:
x_grad = x_grad[..., self.padding[1]:-self.padding[1]]
return x_grad
def numpy_to_matrix_block(jvm: JVMView, np_arr: np.array):
assert (np_arr.ndim <= 2)
rows = np_arr.shape[0]
cols = np_arr.shape[1] if np_arr.ndim == 2 else 1
if not isinstance(np_arr, np.ndarray):
np_arr = np.asarray(np_arr, dtype=np.float64)
if np_arr.dtype is np.dtype(np.int32):
arr = np_arr.ravel().astype(np.int32)
value_type = jvm.org.apache.sysds.common.Types.ValueType.INT32
elif np_arr.dtype is np.dtype(np.float32):
arr = np_arr.ravel().astype(np.float32)
value_type = jvm.org.apache.sysds.common.Types.ValueType.FP32
else:
arr = np_arr.ravel().astype(np.float64)
value_type = jvm.org.apache.sysds.common.Types.ValueType.FP64
buf = bytearray(arr.tostring())
convert_method = jvm.org.apache.sysds.runtime.compress.utils.Py4jConverterUtils.convertPy4JArrayToMB
return convert_method(buf, rows, cols, value_type)
def similarity_measure_using_neighbour_similarity(arr_moving: np.array,
arr_ref: np.array,
arr_ref_ns: np.array,
values: np.array,
debug=False,
max_groups=3):
'''
Transforms `arr_moving` using affine transformation parameters in `values`, then returns a similarity measure between the result and `arr_ref`.
The pixels in `arr_ref` are split into no more than `max_groups` groups of roughly equal size. Group assignment depends on the intensity of the corresponding pixel in `arr_ref_ns`.
For each group, the normalised mutual information between the corresponding pixels in `arr_ref` and the transformed array are calculated. These values are weighted by the average intensity of the corresponding pixels in `arr_ref_ns` and the weighted average is returned as the overall similarity measure. Low-uncertainty regions therefore have the greatest influence over the value returned.
'''
assert arr_moving.shape == arr_ref.shape
assert arr_ref_ns.shape == arr_ref.shape
arr_transformed = transform_using_values(arr_moving,
values,
cval=float('-inf'))
transform_mask = (arr_transformed >= arr_moving.min() - 1).astype(np.int32)
if transform_mask.max() != 1:
return 0
group_list = get_percentile_masks(arr_ref_ns, max_groups=max_groups)
percentile_masks = []
max_num_masks = len(group_list)
for i in range(max_num_masks):
current_group = group_list[i] * transform_mask
merge_with_previous_group = False
if i > 0:
current_group_size = current_group.sum()
previous_group_size = percentile_masks[-1].sum()
if previous_group_size < 4 or current_group_size < 4:
merge_with_previous_group = True
if merge_with_previous_group:
percentile_masks[-1] += current_group
else:
percentile_masks.append(current_group)
num_masks = len(percentile_masks)
percentile_averages = np.empty(num_masks)
similarity_measures = np.zeros(num_masks)
for i in range(num_masks):
mask = percentile_masks[i]
denominator = max(1, mask.sum())
percentile_averages[i] = (arr_ref_ns * mask).sum() / denominator
if mask.sum() >= 4:
arr_transformed_reduced = arr_transformed.ravel()[np.where(
mask.ravel() == 1)]
arr_ref_reduced = arr_ref.ravel()[np.where(mask.ravel() == 1)]
similarity_measures[i] = max(
0,
im.similarity_measure(np.array(arr_transformed_reduced),
np.array(arr_ref_reduced),
measure="NMI"))
denominator = max(1, percentile_averages.sum())
similarity_measures *= percentile_averages / denominator
sm = similarity_measures.sum()
assert sm >= 0
assert sm <= 1
return sm
def atm_temperature_array(alt: np.array,
alt_units: str = 'ft',
temperature_units: str = 'R') -> np.array:
"""Gets the temperature as a numpy array"""
alt = np.asarray(alt)
temp_rankine = np.array([
atm_temperature(alti, alt_units=alt_units, temperature_units='R')
for alti in alt.ravel()
]).reshape(alt.shape)
return temp_rankine * _rankine_to_temperature_units(temperature_units)
def cross_matrix(mat: np.array) -> np.array:
"""Calculate cross product matrix.
A[ij] = x_i * y_j - y_i * x_j
Args:
mat: A 2D array to take the cross product of.
Returns:
The cross matrix.
"""
skv = np.roll(np.roll(np.diag(mat.ravel()), 1, 1), -1, 0)
return skv - skv.T
def _create_seldon_data_def(array: np.array, ty: SeldonPayload):
datadef = {}
if ty == SeldonPayload.TENSOR:
datadef["tensor"] = {"shape": array.shape, "values": array.ravel().tolist()}
elif ty == SeldonPayload.NDARRAY:
datadef["ndarray"] = array.tolist()
elif ty == SeldonPayload.TFTENSOR:
raise NotImplementedError("Seldon payload %s not supported" % ty)
else:
raise Exception("Unknown Seldon payload %s" % ty)
return datadef
def display_2d_data_set(x: np.array, y: np.array, output_path: str = None) -> None:
plt.style.use('dark_background')
plt.figure(figsize=(10, 10))
plt.axis('off')
plt.scatter(x[:, 0], x[:, 1], c=y.ravel(), s=50,
cmap=plt.cm.Spectral, edgecolors='white')
if output_path:
plt.savefig(output_path, bbox_inches='tight')
plt.show()
def quant2(img: np.array, bits: int) -> np.array:
"""Quantize image by maximizing spread of intensities"""
pixels = list(sorted(img.ravel()))
n = 1 << bits
def f(x):
for i in range(n - 1):
index = (i + 1) * (len(pixels) - 1) // n
if x <= pixels[index]:
return i
return n - 1
return np.vectorize(f)(img)
def numpy_to_matrix_block(sds: 'SystemDSContext', np_arr: np.array):
"""Converts a given numpy array, to internal matrix block representation.
:param sds: The current systemds context.
:param np_arr: the numpy array to convert to matrixblock.
"""
assert (np_arr.ndim <=
2), "np_arr invalid, because it has more than 2 dimensions"
rows = np_arr.shape[0]
cols = np_arr.shape[1] if np_arr.ndim == 2 else 1
# If not numpy array then convert to numpy array
if not isinstance(np_arr, np.ndarray):
np_arr = np.asarray(np_arr, dtype=np.float64)
jvm: JVMView = sds.java_gateway.jvm
# flatten and prepare byte buffer.
if np_arr.dtype is np.dtype(np.uint8):
arr = np_arr.ravel()
value_type = jvm.org.apache.sysds.common.Types.ValueType.UINT8
elif np_arr.dtype is np.dtype(np.int32):
arr = np_arr.ravel()
value_type = jvm.org.apache.sysds.common.Types.ValueType.INT32
elif np_arr.dtype is np.dtype(np.float32):
arr = np_arr.ravel()
value_type = jvm.org.apache.sysds.common.Types.ValueType.FP32
else:
arr = np_arr.ravel().astype(np.float64)
value_type = jvm.org.apache.sysds.common.Types.ValueType.FP64
buf = bytearray(arr.tobytes())
# Send data to java.
try:
j_class: JavaClass = jvm.org.apache.sysds.runtime.util.Py4jConverterUtils
return j_class.convertPy4JArrayToMB(buf, rows, cols, value_type)
except Exception as e:
sds.exception_and_close(e)
def _histogram_colour_match(source: np.array, template: np.array):
"""
Adjust the pixel values of a grayscale image such that its histogram
matches that of a target image
Plagiarised from: http://stackoverflow.com/questions/32655686/histogram-matching-of-two-images-in-python-2-x
:param source: Image to transform; the histogram is computed over the flattened array
:type: np.ndarray
:param template: Template image; can have different dimensions to source
:type: np.ndarray
:return: the transformed output image
:rtype: np.ndarray
"""
oldshape = source.shape
source = source.ravel()
template = template.ravel()
# get the set of unique pixel values and their corresponding indices and
# counts
s_values, bin_idx, s_counts = np.unique(source,
return_inverse=True,
return_counts=True)
t_values, t_counts = np.unique(template, return_counts=True)
# take the cumsum of the counts and normalize by the number of pixels to
# get the empirical cumulative distribution functions for the source and
# template images (maps pixel value --> quantile)
s_quantiles = np.cumsum(s_counts).astype(np.float64)
s_quantiles /= s_quantiles[-1]
t_quantiles = np.cumsum(t_counts).astype(np.float64)
t_quantiles /= t_quantiles[-1]
# interpolate linearly to find the pixel values in the template image
# that correspond most closely to the quantiles in the source image
interp_t_values = np.interp(s_quantiles, t_quantiles, t_values)
return interp_t_values[bin_idx].reshape(oldshape)
def score(self,
actual: np.array,
predicted: np.array,
sample_weight: typing.Optional[np.array] = None,
labels: typing.Optional[np.array] = None) -> float:
if sample_weight is None:
sample_weight = np.ones(actual.shape[0])
predicted = predicted.ravel()
good_rows = predicted >= 0
if not good_rows.any():
return 30
delta = predicted[good_rows] - actual[good_rows]
sample_weight = sample_weight[good_rows]
loss = np.log1p(np.cosh(delta))
return np.sum(sample_weight * loss) / np.sum(sample_weight)
def numpy_to_matrix_block(jvm: JVMView, np_arr: np.array):
"""Converts a given numpy array, to internal matrix block representation.
:param jvm: The current JVM instance running systemds.
:param np_arr: the numpy array to convert to matrixblock.
"""
assert (np_arr.ndim <=
2), "np_arr invalid, because it has more than 2 dimensions"
rows = np_arr.shape[0]
cols = np_arr.shape[1] if np_arr.ndim == 2 else 1
if not isinstance(np_arr, np.ndarray):
np_arr = np.asarray(np_arr, dtype=np.float64)
if np_arr.dtype is np.dtype(np.int32):
arr = np_arr.ravel().astype(np.int32)
value_type = jvm.org.apache.sysds.common.Types.ValueType.INT32
elif np_arr.dtype is np.dtype(np.float32):
arr = np_arr.ravel().astype(np.float32)
value_type = jvm.org.apache.sysds.common.Types.ValueType.FP32
else:
arr = np_arr.ravel().astype(np.float64)
value_type = jvm.org.apache.sysds.common.Types.ValueType.FP64
buf = bytearray(arr.tostring())
convert_method = jvm.org.apache.sysds.runtime.util.Py4jConverterUtils.convertPy4JArrayToMB
return convert_method(buf, rows, cols, value_type)
def print_multilabel_balance(y: np.array):
"""
Calculates the minority class sample size for every label q in a multilabel
set
"""
if y.shape[-1] == 1:
n_classes = np.max(y.ravel()) + 1
y = target_to_onehot(y, n_classes=n_classes).reshape(-1, n_classes)
for q in range(y.shape[-1]):
yq = y[:, q]
num = np.sum(yq).round(2)
freq = np.round((num / len(yq)), 2)
print(f"Class: {q}\t Count: {num}\t Freq: {freq}".expandtabs(15))
def bpbounds_tri_x2y2z2(p: np.array,
A: np.array = A_tri_x2y2z2,
ice: np.array = ice_tri_x2y2z2) -> Dict:
prod = A @ p.ravel()
ivinequality = prod[ice == 0]
low = -1 * prod[ice == 1]
upp = prod[ice == -1]
inequality = min(ivinequality) >= 0
bplow = max(low)
bpupp = min(upp)
return {
"inequality": inequality,
"bplow": bplow,
"bpupp": bpupp,
"bplower": low,
"bpupper": upp
}
def __init__(
self,
X_train: np.array,
y_train: np.array,
num_gradient_updates: int = num_gradient_updates,
):
self.estimator = MLPRegressor(
activation='relu',
hidden_layer_sizes=(50, 50, 50),
learning_rate='adaptive',
verbose=False,
max_iter=num_gradient_updates,
tol=1e-6,
early_stopping=True,
)
self.scaler = StandardScaler()
X = self.scaler.fit_transform(X_train)
self.estimator.fit(X, y_train.ravel())
def bpbounds_tri_x2y2z3(p: np.array,
A: np.array = A_tri_x2y2z3,
ice: np.array = ice_tri_x2y2z3,
cons: np.array = cons_tri_x2y2z3) -> Dict:
prod = A @ p.ravel()
prod = prod + cons
ivinequality = prod[ice == 0]
low = prod[ice == -1]
upp = -1 * prod[ice == 1]
inequality = max(ivinequality) <= 0
bplow = max(low)
bpupp = min(upp)
return {
"inequality": inequality,
"bplow": bplow,
"bpupp": bpupp,
"bplower": low,
"bpupper": upp
}
def predict(self, x: np.array, y: np.array = None) -> dict:
"""
predict and evaluate
:param x: testing data
:param y: testing label
:return: the prediction (and the accuracy if test label is provided)
"""
xipred = np.matmul(self.theta, np.transpose(x))
for i in range(len(xipred)):
xipred[i] += self.t[i]
etapred = np.matmul(np.transpose(expit(xipred)), self.beta) + self.b
mupred = expit(etapred)
print(mupred)
ypred = np.ones(x.shape[0])
for i in range(len(ypred)):
if mupred[i] < 0.5:
ypred[i] = 0
if y is not None:
accuracy = 1 - np.mean(np.abs(y.ravel() - ypred.ravel()))
return {'ypred': ypred, 'accuracy': accuracy}
else:
return {'ypred': ypred}
def meltParams(theta1: np.array, theta2: np.array) -> np:
return np.concatenate((theta1.ravel(), theta2.ravel()))
def __init__(self, all_arrow_combs: np.array = ALL_ARROW_COMBS):
encoder = LabelEncoder().fit(all_arrow_combs.ravel())
super(LabelArrowEncoder, self).__init__(encoder=encoder)
def convert_array_to_1d_list(array: np.array) -> list:
return array.ravel().tolist()
