def spo_all_xr(observed: np.array, modeled: np.array, inf_as_na: bool = True, decimals: int = 2): result = calc_gof_stats(observed.ravel(), modeled.ravel(), inf_as_na, decimals).to_xarray() return result
def plot_with_gradient(y: np.array, gradient: np.array, ax=None, x=None, colormap="plasma", color_bar=True, normalise=False, **kwargs): """ Uses template to show gradient of another variable :param ax: axis of the plot :param x: x data :param y: y data :param gradient: creates gradient on the plot :param colormap: viridis/plasma/... :param color_bar: add colour bar :param normalise: boolean - 0-1 normalisation :return: """ if x is None: x = range(y.ravel().shape[0]) if len(gradient.shape) > 2: raise Exception("Gradient is not 1D vector, but has {}".format( len(gradient.shape))) if len(gradient.shape) == 2: gradient = gradient.ravel() fig = None if ax is None: fig, ax = plt.subplots(1, 1) fig.set_size_inches(18.5, 10.5) points = np.array([x, y]).T.reshape(-1, 1, 2) # create gradient colour segments = np.concatenate([points[:-2], points[1:-1], points[2:]], axis=1) if normalise: lc = LineCollection(segments, cmap=colormap, linewidth=4, norm=Normalize(vmin=0, vmax=1)) else: lc = LineCollection(segments, cmap=colormap, linewidth=4) lc.set_array(gradient) # set up gradient line = ax.add_collection(lc) ax.autoscale() if color_bar: plt.colorbar(line, ax=ax) if "title" in kwargs: ax.set_title(kwargs["title"]) if fig is not None: return fig, ax return ax
def display_classification_areas( model, cords: np.array, labels: np.array, output_path: str = None ) -> None: x_start = cords[:, 0].min() - 0.5 x_end = cords[:, 0].max() + 0.5 y_start = cords[:, 1].min() - 0.5 y_end = cords[:, 1].max() + 0.5 grid = np.mgrid[x_start:x_end:100j, y_start:y_end:100j] grid_2d = grid.reshape(2, -1).T x_grid, y_grid = grid prediction = model.predict(grid_2d) plt.style.use('dark_background') plt.figure(figsize=(10, 10)) plt.axis('off') plt.contourf(x_grid, y_grid, prediction.reshape(100, 100), alpha=0.7, cmap=plt.cm.Spectral) plt.scatter(cords[:, 0], cords[:, 1], c=labels.ravel(), s=50, cmap=plt.cm.Spectral, edgecolors='white') if output_path: plt.savefig(output_path, bbox_inches='tight') plt.show()
def nppow(a: np.array, b: Union[int, float, np.array]) -> np.array: """ evaluates a**b element-by-element :param np.array a: :param Union[int, float, np.array] b: if an array, should have the same shape as `a` :return: an array of the same shape as `a` """ mina = np.min(a) if mina <= 0: print_stars("All elements of a must be positive in nppow!") sys.exit(1) if isinstance(b, (int, float)): return a**b else: if a.shape != b.shape: print_stars( "nppow: b is not a number or an array of the same shape as a!") sys.exit(1) avec = a.ravel() bvec = b.ravel() a_pow_b = avec**bvec return a_pow_b.reshape(a.shape)
def similarity_measure_area_of_overlap(arr_fixed: np.array, arr_to_transform: np.array, values: np.array, debug=False): ''' Transforms `arr_to_transform` using affine transformation parameters in `values`, then returns the normalised mutual information between the result and `arr_fixed`. Only the pixels covered by both arrays are considered when calculating the mutual information. ''' assert arr_fixed.shape == arr_to_transform.shape arr_transformed = transform_using_values(arr_to_transform, values, cval=float('-inf')) arr_1 = arr_fixed.ravel() arr_2 = arr_transformed.ravel() arr_1_reduced = [] arr_2_reduced = [] assert len(arr_1) >= 3 if debug: print("min = " + str(arr_to_transform.min())) print(values) for i in range(len(arr_1)): if arr_2[i] >= arr_to_transform.min(): arr_1_reduced.append(arr_1[i]) arr_2_reduced.append(arr_2[i]) if debug: print("Length = " + str(len(arr_2_reduced))) if len(arr_1_reduced) < 3: return 0 else: sm = im.similarity_measure(np.array(arr_1_reduced), np.array(arr_2_reduced), measure="NMI") return sm
def costFunction( nnParams: np.array, hiddenLayerSize: int, X: np.array, y: np.array, lmbda: float = 0, ) -> float: # Setup useful variables (m, n, k, y) = setupVars(X, y) (theta1, theta2) = reshapeParams(nnParams, n, hiddenLayerSize, k) (a3, _) = forwardProp(theta1, theta2, X) y_vec = y.ravel() h_vec = a3.ravel() J = ((-y_vec.dot(np.log(h_vec))) - ((1 - y_vec).dot(np.log(1 - h_vec)))) / m if lmbda > 0: multiplier = lmbda / (2 * m) t1Reg = theta1[:, 1:].ravel() t2Reg = theta2[:, 1:].ravel() J += multiplier * (t1Reg.dot(t1Reg) + t2Reg.dot(t2Reg)) print(f"Current J: {J}") return J
def der_nppow(a: np.array, b: Union[int, float, np.array]) -> np.array: """ evaluates the derivatives in a and b of element-by-element a**b :param np.array a: :param Union[int, float, np.array] b: if an array, should have the same shape as `a` :return: a pair of two arrays of the same shape as `a` """ mina = np.min(a) if mina <= 0: print_stars("All elements of a must be positive in der_nppow!") sys.exit(1) if isinstance(b, (int, float)): a_pow_b = a**b return (b * a_pow_b / a, a_pow_b * log(a)) else: if a.shape != b.shape: print_stars( "nppow: b is not a number or an array of the same shape as a!") sys.exit(1) avec = a.ravel() bvec = b.ravel() a_pow_b = avec**bvec der_wrt_a = a_pow_b * bvec / avec der_wrt_b = a_pow_b * nplog(avec) return (der_wrt_a.reshape(a.shape), der_wrt_b.reshape(a.shape))
def __filterLFP__(self, data: np.array, sample_rate: int): from scipy.signal import filtfilt, firwin if self.fs is None: from ephysiopy import fs self.fs = fs if self.lfp_lowcut is None: from ephysiopy import lfp_lowcut self.lfp_lowcut = lfp_lowcut if self.lfp_highcut is None: from ephysiopy import lfp_highcut self.lfp_highcut = lfp_highcut nyq = sample_rate / 2.0 lowcut = self.lfp_lowcut / nyq highcut = self.lfp_highcut / nyq if highcut >= 1.0: highcut = 1.0 - np.finfo(float).eps if lowcut <= 0.0: lowcut = np.finfo(float).eps b = firwin(sample_rate + 1, [lowcut, highcut], window="black", pass_zero=False) y = filtfilt(b, [1], data.ravel(), padtype="odd") return y
def log_histogram(self, tag: str, data: np.array, step: int, num_bars: int = 30): """ Adds a histogram to log. Parameters ---------- tag: str data: np.array Array of any shape. step: int num_bars: int The number of bars if the resulting histogram. """ data = data.ravel() min_ = data.min() max_ = data.max() sum_ = data.sum() sum_sq = data.dot(data) if min_ == max_: num = 1 bucket_limit = [min_] bucket = [len(data)] else: bucket, bucket_limit = np.histogram(data, num_bars) num = len(bucket_limit) bucket_limit = bucket_limit[1:] hist = HistogramProto(min=min_, max=max_, sum=sum_, sum_squares=sum_sq, num=num, bucket_limit=bucket_limit, bucket=bucket) self._write_event(tag, step, histo=hist)
def _get_uncertainty(rho_surface: np.array, mask: np.array) -> sp.lil_matrix: r_mat = rho_surface * 0.05 r_mat[np.logical_not(mask)] = 0. N = mask.ravel().shape[0] r_mat_sp = sp.lil_matrix((N, N)) r_mat_sp.setdiag(1. / (r_mat.ravel())**2) return r_mat_sp.tocsr()
def grad_pass(self, y_grad: np.array): x_grad = y_grad.ravel() @ self.sp_matrix.T x_grad = x_grad.reshape(*self.x_view) if self.padding[0]: x_grad = x_grad[:, :, self.padding[0]:-self.padding[0]] if self.padding[1]: x_grad = x_grad[..., self.padding[1]:-self.padding[1]] return x_grad
def numpy_to_matrix_block(jvm: JVMView, np_arr: np.array): assert (np_arr.ndim <= 2) rows = np_arr.shape[0] cols = np_arr.shape[1] if np_arr.ndim == 2 else 1 if not isinstance(np_arr, np.ndarray): np_arr = np.asarray(np_arr, dtype=np.float64) if np_arr.dtype is np.dtype(np.int32): arr = np_arr.ravel().astype(np.int32) value_type = jvm.org.apache.sysds.common.Types.ValueType.INT32 elif np_arr.dtype is np.dtype(np.float32): arr = np_arr.ravel().astype(np.float32) value_type = jvm.org.apache.sysds.common.Types.ValueType.FP32 else: arr = np_arr.ravel().astype(np.float64) value_type = jvm.org.apache.sysds.common.Types.ValueType.FP64 buf = bytearray(arr.tostring()) convert_method = jvm.org.apache.sysds.runtime.compress.utils.Py4jConverterUtils.convertPy4JArrayToMB return convert_method(buf, rows, cols, value_type)
def similarity_measure_using_neighbour_similarity(arr_moving: np.array, arr_ref: np.array, arr_ref_ns: np.array, values: np.array, debug=False, max_groups=3): ''' Transforms `arr_moving` using affine transformation parameters in `values`, then returns a similarity measure between the result and `arr_ref`. The pixels in `arr_ref` are split into no more than `max_groups` groups of roughly equal size. Group assignment depends on the intensity of the corresponding pixel in `arr_ref_ns`. For each group, the normalised mutual information between the corresponding pixels in `arr_ref` and the transformed array are calculated. These values are weighted by the average intensity of the corresponding pixels in `arr_ref_ns` and the weighted average is returned as the overall similarity measure. Low-uncertainty regions therefore have the greatest influence over the value returned. ''' assert arr_moving.shape == arr_ref.shape assert arr_ref_ns.shape == arr_ref.shape arr_transformed = transform_using_values(arr_moving, values, cval=float('-inf')) transform_mask = (arr_transformed >= arr_moving.min() - 1).astype(np.int32) if transform_mask.max() != 1: return 0 group_list = get_percentile_masks(arr_ref_ns, max_groups=max_groups) percentile_masks = [] max_num_masks = len(group_list) for i in range(max_num_masks): current_group = group_list[i] * transform_mask merge_with_previous_group = False if i > 0: current_group_size = current_group.sum() previous_group_size = percentile_masks[-1].sum() if previous_group_size < 4 or current_group_size < 4: merge_with_previous_group = True if merge_with_previous_group: percentile_masks[-1] += current_group else: percentile_masks.append(current_group) num_masks = len(percentile_masks) percentile_averages = np.empty(num_masks) similarity_measures = np.zeros(num_masks) for i in range(num_masks): mask = percentile_masks[i] denominator = max(1, mask.sum()) percentile_averages[i] = (arr_ref_ns * mask).sum() / denominator if mask.sum() >= 4: arr_transformed_reduced = arr_transformed.ravel()[np.where( mask.ravel() == 1)] arr_ref_reduced = arr_ref.ravel()[np.where(mask.ravel() == 1)] similarity_measures[i] = max( 0, im.similarity_measure(np.array(arr_transformed_reduced), np.array(arr_ref_reduced), measure="NMI")) denominator = max(1, percentile_averages.sum()) similarity_measures *= percentile_averages / denominator sm = similarity_measures.sum() assert sm >= 0 assert sm <= 1 return sm
def atm_temperature_array(alt: np.array, alt_units: str = 'ft', temperature_units: str = 'R') -> np.array: """Gets the temperature as a numpy array""" alt = np.asarray(alt) temp_rankine = np.array([ atm_temperature(alti, alt_units=alt_units, temperature_units='R') for alti in alt.ravel() ]).reshape(alt.shape) return temp_rankine * _rankine_to_temperature_units(temperature_units)
def cross_matrix(mat: np.array) -> np.array: """Calculate cross product matrix. A[ij] = x_i * y_j - y_i * x_j Args: mat: A 2D array to take the cross product of. Returns: The cross matrix. """ skv = np.roll(np.roll(np.diag(mat.ravel()), 1, 1), -1, 0) return skv - skv.T
def _create_seldon_data_def(array: np.array, ty: SeldonPayload): datadef = {} if ty == SeldonPayload.TENSOR: datadef["tensor"] = {"shape": array.shape, "values": array.ravel().tolist()} elif ty == SeldonPayload.NDARRAY: datadef["ndarray"] = array.tolist() elif ty == SeldonPayload.TFTENSOR: raise NotImplementedError("Seldon payload %s not supported" % ty) else: raise Exception("Unknown Seldon payload %s" % ty) return datadef
def display_2d_data_set(x: np.array, y: np.array, output_path: str = None) -> None: plt.style.use('dark_background') plt.figure(figsize=(10, 10)) plt.axis('off') plt.scatter(x[:, 0], x[:, 1], c=y.ravel(), s=50, cmap=plt.cm.Spectral, edgecolors='white') if output_path: plt.savefig(output_path, bbox_inches='tight') plt.show()
def quant2(img: np.array, bits: int) -> np.array: """Quantize image by maximizing spread of intensities""" pixels = list(sorted(img.ravel())) n = 1 << bits def f(x): for i in range(n - 1): index = (i + 1) * (len(pixels) - 1) // n if x <= pixels[index]: return i return n - 1 return np.vectorize(f)(img)
def numpy_to_matrix_block(sds: 'SystemDSContext', np_arr: np.array): """Converts a given numpy array, to internal matrix block representation. :param sds: The current systemds context. :param np_arr: the numpy array to convert to matrixblock. """ assert (np_arr.ndim <= 2), "np_arr invalid, because it has more than 2 dimensions" rows = np_arr.shape[0] cols = np_arr.shape[1] if np_arr.ndim == 2 else 1 # If not numpy array then convert to numpy array if not isinstance(np_arr, np.ndarray): np_arr = np.asarray(np_arr, dtype=np.float64) jvm: JVMView = sds.java_gateway.jvm # flatten and prepare byte buffer. if np_arr.dtype is np.dtype(np.uint8): arr = np_arr.ravel() value_type = jvm.org.apache.sysds.common.Types.ValueType.UINT8 elif np_arr.dtype is np.dtype(np.int32): arr = np_arr.ravel() value_type = jvm.org.apache.sysds.common.Types.ValueType.INT32 elif np_arr.dtype is np.dtype(np.float32): arr = np_arr.ravel() value_type = jvm.org.apache.sysds.common.Types.ValueType.FP32 else: arr = np_arr.ravel().astype(np.float64) value_type = jvm.org.apache.sysds.common.Types.ValueType.FP64 buf = bytearray(arr.tobytes()) # Send data to java. try: j_class: JavaClass = jvm.org.apache.sysds.runtime.util.Py4jConverterUtils return j_class.convertPy4JArrayToMB(buf, rows, cols, value_type) except Exception as e: sds.exception_and_close(e)
def _histogram_colour_match(source: np.array, template: np.array): """ Adjust the pixel values of a grayscale image such that its histogram matches that of a target image Plagiarised from: http://stackoverflow.com/questions/32655686/histogram-matching-of-two-images-in-python-2-x :param source: Image to transform; the histogram is computed over the flattened array :type: np.ndarray :param template: Template image; can have different dimensions to source :type: np.ndarray :return: the transformed output image :rtype: np.ndarray """ oldshape = source.shape source = source.ravel() template = template.ravel() # get the set of unique pixel values and their corresponding indices and # counts s_values, bin_idx, s_counts = np.unique(source, return_inverse=True, return_counts=True) t_values, t_counts = np.unique(template, return_counts=True) # take the cumsum of the counts and normalize by the number of pixels to # get the empirical cumulative distribution functions for the source and # template images (maps pixel value --> quantile) s_quantiles = np.cumsum(s_counts).astype(np.float64) s_quantiles /= s_quantiles[-1] t_quantiles = np.cumsum(t_counts).astype(np.float64) t_quantiles /= t_quantiles[-1] # interpolate linearly to find the pixel values in the template image # that correspond most closely to the quantiles in the source image interp_t_values = np.interp(s_quantiles, t_quantiles, t_values) return interp_t_values[bin_idx].reshape(oldshape)
def score(self, actual: np.array, predicted: np.array, sample_weight: typing.Optional[np.array] = None, labels: typing.Optional[np.array] = None) -> float: if sample_weight is None: sample_weight = np.ones(actual.shape[0]) predicted = predicted.ravel() good_rows = predicted >= 0 if not good_rows.any(): return 30 delta = predicted[good_rows] - actual[good_rows] sample_weight = sample_weight[good_rows] loss = np.log1p(np.cosh(delta)) return np.sum(sample_weight * loss) / np.sum(sample_weight)
def numpy_to_matrix_block(jvm: JVMView, np_arr: np.array): """Converts a given numpy array, to internal matrix block representation. :param jvm: The current JVM instance running systemds. :param np_arr: the numpy array to convert to matrixblock. """ assert (np_arr.ndim <= 2), "np_arr invalid, because it has more than 2 dimensions" rows = np_arr.shape[0] cols = np_arr.shape[1] if np_arr.ndim == 2 else 1 if not isinstance(np_arr, np.ndarray): np_arr = np.asarray(np_arr, dtype=np.float64) if np_arr.dtype is np.dtype(np.int32): arr = np_arr.ravel().astype(np.int32) value_type = jvm.org.apache.sysds.common.Types.ValueType.INT32 elif np_arr.dtype is np.dtype(np.float32): arr = np_arr.ravel().astype(np.float32) value_type = jvm.org.apache.sysds.common.Types.ValueType.FP32 else: arr = np_arr.ravel().astype(np.float64) value_type = jvm.org.apache.sysds.common.Types.ValueType.FP64 buf = bytearray(arr.tostring()) convert_method = jvm.org.apache.sysds.runtime.util.Py4jConverterUtils.convertPy4JArrayToMB return convert_method(buf, rows, cols, value_type)
def print_multilabel_balance(y: np.array): """ Calculates the minority class sample size for every label q in a multilabel set """ if y.shape[-1] == 1: n_classes = np.max(y.ravel()) + 1 y = target_to_onehot(y, n_classes=n_classes).reshape(-1, n_classes) for q in range(y.shape[-1]): yq = y[:, q] num = np.sum(yq).round(2) freq = np.round((num / len(yq)), 2) print(f"Class: {q}\t Count: {num}\t Freq: {freq}".expandtabs(15))
def bpbounds_tri_x2y2z2(p: np.array, A: np.array = A_tri_x2y2z2, ice: np.array = ice_tri_x2y2z2) -> Dict: prod = A @ p.ravel() ivinequality = prod[ice == 0] low = -1 * prod[ice == 1] upp = prod[ice == -1] inequality = min(ivinequality) >= 0 bplow = max(low) bpupp = min(upp) return { "inequality": inequality, "bplow": bplow, "bpupp": bpupp, "bplower": low, "bpupper": upp }
def __init__( self, X_train: np.array, y_train: np.array, num_gradient_updates: int = num_gradient_updates, ): self.estimator = MLPRegressor( activation='relu', hidden_layer_sizes=(50, 50, 50), learning_rate='adaptive', verbose=False, max_iter=num_gradient_updates, tol=1e-6, early_stopping=True, ) self.scaler = StandardScaler() X = self.scaler.fit_transform(X_train) self.estimator.fit(X, y_train.ravel())
def bpbounds_tri_x2y2z3(p: np.array, A: np.array = A_tri_x2y2z3, ice: np.array = ice_tri_x2y2z3, cons: np.array = cons_tri_x2y2z3) -> Dict: prod = A @ p.ravel() prod = prod + cons ivinequality = prod[ice == 0] low = prod[ice == -1] upp = -1 * prod[ice == 1] inequality = max(ivinequality) <= 0 bplow = max(low) bpupp = min(upp) return { "inequality": inequality, "bplow": bplow, "bpupp": bpupp, "bplower": low, "bpupper": upp }
def predict(self, x: np.array, y: np.array = None) -> dict: """ predict and evaluate :param x: testing data :param y: testing label :return: the prediction (and the accuracy if test label is provided) """ xipred = np.matmul(self.theta, np.transpose(x)) for i in range(len(xipred)): xipred[i] += self.t[i] etapred = np.matmul(np.transpose(expit(xipred)), self.beta) + self.b mupred = expit(etapred) print(mupred) ypred = np.ones(x.shape[0]) for i in range(len(ypred)): if mupred[i] < 0.5: ypred[i] = 0 if y is not None: accuracy = 1 - np.mean(np.abs(y.ravel() - ypred.ravel())) return {'ypred': ypred, 'accuracy': accuracy} else: return {'ypred': ypred}
def meltParams(theta1: np.array, theta2: np.array) -> np: return np.concatenate((theta1.ravel(), theta2.ravel()))
def __init__(self, all_arrow_combs: np.array = ALL_ARROW_COMBS): encoder = LabelEncoder().fit(all_arrow_combs.ravel()) super(LabelArrowEncoder, self).__init__(encoder=encoder)
def convert_array_to_1d_list(array: np.array) -> list: return array.ravel().tolist()