コード例 #1
0
ファイル: ImageProcessor.py プロジェクト: shifvb/WasteBin
 def norm_image(self, arr: np.ndarray):
     """
     将一个numpy数组正则化(0~255),并转成np.uint8类型
     :param arr: 要处理的numpy数组
     :return: 值域在0~255之间的uint8数组
     """
     if not arr.min() == arr.max():
         arr = (arr - arr.min()) / (arr.max() - arr.min()) * 255
     return np.array(arr, dtype=np.uint8)
コード例 #2
0
ファイル: profile.py プロジェクト: jrkerns/pylinac
def stretch(array: np.ndarray, min: int=0, max: int=1, fill_dtype=None) -> np.array:
    """'Stretch' the profile to the fit a new min and max value and interpolate in between.
    From: http://www.labri.fr/perso/nrougier/teaching/numpy.100/  exercise #17

    Parameters
    ----------
    array: numpy.ndarray
        The numpy array to stretch.
    min : number
        The new minimum of the values.
    max : number
        The new maximum value.
    fill_dtype : numpy data type
        If None (default), the array will be stretched to the passed min and max.
        If a numpy data type (e.g. np.int16), the array will be stretched to fit the full range of values
        of that data type. If a value is given for this parameter, it overrides ``min`` and ``max``.
    """
    new_max = max
    new_min = min
    if fill_dtype is not None:
        try:
            di = np.iinfo(fill_dtype)
        except ValueError:
            di = np.finfo(fill_dtype)
        new_max = di.max
        new_min = di.min
    # perfectly normalize the array (0..1)
    stretched_array = (array - array.min())/(array.max() - array.min())
    # stretch normalized array to new max/min
    stretched_array *= new_max
    stretched_array += new_min
    return stretched_array.astype(array.dtype)
コード例 #3
0
ファイル: psf_fitting.py プロジェクト: bjd2385/dist-fitting
    def __call__(self, data: np.ndarray, learning_rate: float =1.0,
                 steps: int =1000, db: bool =True) -> List[float]:
        """ `Learn` the parameters of best fit for the given data and model """

        _min = data.min()
        _max = data.max()

        # scale amplitude to [0, 1]
        self.data = (data - _min) / (_max - _min)

        self.cubeX, self.cubeY = data.shape
        self.learning_rate = learning_rate
        self.steps = steps

        # perform the fit
        result = self.simplefit()

        # unscale amplitude of resultant
        result[0] = result[0] * (_max - _min) + _min

        result_as_list = result.tolist()

        self._counter += 1

        return result_as_list
コード例 #4
0
ファイル: exposureanalyzer.py プロジェクト: awacha/cct
def get_statistics(matrix: np.ndarray, masktotal: Union[np.ndarray, int, None],
                   mask: Union[np.ndarray, int, None] = None) -> Dict:
    """Calculate different statistics of a detector image, such as sum, max,
    center of gravity, etc."""
    assert (isinstance(matrix, np.ndarray))
    if mask is None:
        mask = 1
    if masktotal is None:
        masktotal = 1
    assert isinstance(masktotal, np.ndarray) or isinstance(masktotal, int)
    assert isinstance(mask, np.ndarray) or isinstance(mask, int)
    result = {}
    matrixorig = matrix
    for prefix, mask in [('total_', masktotal), ('', mask)]:
        matrix = matrixorig * mask
        x = np.arange(matrix.shape[0])
        y = np.arange(matrix.shape[1])
        result[prefix + 'sum'] = matrix.sum()
        result[prefix + 'max'] = matrix.max()
        result[prefix + 'beamx'] = (matrix * x[:, np.newaxis]).sum() / result[prefix + 'sum']
        result[prefix + 'beamy'] = (matrix * y[np.newaxis, :]).sum() / result[prefix + 'sum']
        result[prefix + 'sigmax'] = (
                                        (matrix * (x[:, np.newaxis] - result[prefix + 'beamx']) ** 2).sum() /
                                        result[prefix + 'sum']) ** 0.5
        result[prefix + 'sigmay'] = (
                                        (matrix * (y[np.newaxis, :] - result[prefix + 'beamy']) ** 2).sum() /
                                        result[prefix + 'sum']) ** 0.5
        result[prefix + 'sigma'] = (result[prefix + 'sigmax'] ** 2 + result[prefix + 'sigmay'] ** 2) ** 0.5
    return result
コード例 #5
0
ファイル: vega.py プロジェクト: janpipek/physt
def _create_cmap_scale(values_arr: np.ndarray, vega: dict, kwargs: dict):
    cmap = kwargs.pop("cmap", DEFAULT_PALETTE)
    cmap_min = float(kwargs.pop("cmap_min", values_arr.min()))
    cmap_max = float(kwargs.pop("cmap_max", values_arr.max()))

    # TODO: Apply cmap_normalize parameter

    vega["scales"].append(
        {
            "name": "color",
            "type": "sequential",
            "domain": [cmap_min, cmap_max],
            "range": {"scheme": cmap},
            "zero": False,
            "nice": False
        }
    )
コード例 #6
0
ファイル: sndfileio.py プロジェクト: gesellkammer/sndfileio
def sndwrite(samples:np.ndarray, sr:int, outfile:str, encoding:str='auto') -> None:
    """
    samples  --> Array-like. the actual samples, shape=(nframes, channels)
    sr       --> Sampling-rate
    outfile  --> The name of the outfile. the extension will determine
                 the file-format.
                 The formats supported depend on the available backends
                 Without additional backends, only uncompressed formats
                 are supported (wav, aif)
    encoding --> one of:
                 - 'auto' or None: the encoding is determined from the format
                                   given by the extension of outfile, and
                                   from the data
                 - 'pcm16'
                 - 'pcm24'
                 - 'pcm32'
                 - 'flt32'

                 NB: not all file formats support all encodings.
                     Throws a SndfileError if the format does not support
                     the given encoding

          If set to 'auto', an encoding will be selected based on the
          file-format and on the data. The bitdepth of the data is
          measured, and if the file-format supports it, it will be used.
          For bitdepths of 8, 16 and 24 bits, a PCM encoding will be used.
          For a bitdepth of 32 bits, a FLOAT encoding will be used,
          or the next lower supported encoding
    """
    if encoding in ('auto', None):
        encoding = _guessEncoding(samples, outfile)
    # normalize in the case where there would be clipping
    clipping = ((samples > 1).any() or (samples < -1).any())
    if encoding.startswith('pcm') and clipping:
        maxvalue = max(samples.max(), abs(samples.min()))
        samples = samples / maxvalue
    backend = _getWriteBackend(outfile, encoding)
    if not backend:
        raise SndfileError("No backend found to support the given format")
    logger.debug(f"sndwrite: using backend {backend.name}")
    return backend.write(samples, sr, outfile, encoding)
コード例 #7
0
def fit_spi(image: np.ndarray,
            beam: np.ndarray,
            freqs: np.ndarray,
            weights: np.ndarray,
            threshold: float,
            nthreads: int,
            pb_min: float,
            padding_frac: float,
            ref_freq: float,
            dest=sys.stdout):

    try:
        assert image.ndim == 3
        assert beam.ndim == 3
        assert image.shape == beam.shape
        assert image.shape[0] > 1
        assert freqs.size == image.shape[0]
        assert weights.size == image.shape[0]
    except Exception as e:
        raise e

    # beam cut off
    image = np.where(beam > pb_min, image, 0)

    nband = image.shape[0]

    # get pixels above threshold
    minimage = np.amin(image, axis=0)
    maskindices = np.argwhere(minimage > threshold)
    if not maskindices.size:
        raise ValueError("No components found above threshold. "
                         "Try lowering your threshold."
                         "Max of convolved model is %3.2e" % image.max())
    fitcube = image[:, maskindices[:, 0], maskindices[:, 1]].T
    beam_comps = beam[:, maskindices[:, 0], maskindices[:, 1]].T

    ncomps, _ = fitcube.shape
    fitcube = da.from_array(fitcube.astype(np.float64),
                            chunks=(ncomps//nthreads, nband))
    beam_comps = da.from_array(beam_comps.astype(np.float64),
                               chunks=(ncomps//nthreads, nband))
    weights = da.from_array(weights.astype(np.float64), chunks=(nband))
    freqsdask = da.from_array(freqs.astype(np.float64), chunks=(nband))

    print("Fitting %i components" % ncomps, file=dest)
    alpha, alpha_err, Iref, i0_err = \
        fit_spi_components(fitcube, weights, freqsdask, ref_freq,
                           beam=beam_comps).compute()
    print("Done. Writing output. \n", file=dest)

    alphamap = np.zeros(image[0].shape, dtype=image.dtype)
    alphamap[...] = np.nan
    alpha_err_map = np.zeros(image[0].shape, dtype=image.dtype)
    alpha_err_map[...] = np.nan
    i0map = np.zeros(image[0].shape, dtype=image.dtype)
    i0map[...] = np.nan
    i0_err_map = np.zeros(image[0].shape, dtype=image.dtype)
    i0_err_map[...] = np.nan
    alphamap[maskindices[:, 0], maskindices[:, 1]] = alpha
    alpha_err_map[maskindices[:, 0], maskindices[:, 1]] = alpha_err
    i0map[maskindices[:, 0], maskindices[:, 1]] = Iref
    i0_err_map[maskindices[:, 0], maskindices[:, 1]] = i0_err

    return alphamap, alpha_err_map, i0map, i0_err_map
コード例 #8
0
def Workflow_ctnnb1(
    struct_img: np.ndarray,
    rescale_ratio: float = -1,
    output_type: str = "default",
    output_path: Union[str, Path] = None,
    fn: Union[str, Path] = None,
    output_func=None,
):
    """
    classic segmentation workflow wrapper for structure CTNNB1

    Parameter:
    -----------
    struct_img: np.ndarray
        the 3D image to be segmented
    rescale_ratio: float
        an optional parameter to allow rescale the image before running the
        segmentation functions, default is no rescaling
    output_type: str
        select how to handle output. Currently, four types are supported:
        1. default: the result will be saved at output_path whose filename is
            original name without extention + "_struct_segmentaiton.tiff"
        2. array: the segmentation result will be simply returned as a numpy array
        3. array_with_contour: segmentation result will be returned together with
            the contour of the segmentation
        4. customize: pass in an extra output_func to do a special save. All the
            intermediate results, names of these results, the output_path, and the
            original filename (without extension) will be passed in to output_func.
    """
    ##########################################################################
    # PARAMETERS:
    #   note that these parameters are supposed to be fixed for the structure
    #   and work well accross different datasets

    intensity_norm_param = [4, 27]
    gaussian_smoothing_sigma = 1
    gaussian_smoothing_truncate_range = 3.0
    dot_2d_sigma = 1.5
    dot_2d_cutoff = 0.01
    minArea = 10
    ##########################################################################

    out_img_list = []
    out_name_list = []

    ###################
    # PRE_PROCESSING
    ###################
    # intenisty normalization (min/max)
    struct_img = intensity_normalization(struct_img,
                                         scaling_param=intensity_norm_param)

    out_img_list.append(struct_img.copy())
    out_name_list.append("im_norm")

    # rescale if needed
    if rescale_ratio > 0:
        struct_img = zoom(struct_img, (1, rescale_ratio, rescale_ratio),
                          order=2)

        struct_img = (struct_img - struct_img.min() +
                      1e-8) / (struct_img.max() - struct_img.min() + 1e-8)
        gaussian_smoothing_truncate_range = (
            gaussian_smoothing_truncate_range * rescale_ratio)

    # smoothing
    structure_img_smooth = image_smoothing_gaussian_3d(
        struct_img,
        sigma=gaussian_smoothing_sigma,
        truncate_range=gaussian_smoothing_truncate_range,
    )

    out_img_list.append(structure_img_smooth.copy())
    out_name_list.append("im_smooth")

    ###################
    # core algorithm
    ###################

    response = dot_slice_by_slice(structure_img_smooth, log_sigma=dot_2d_sigma)
    bw = response > dot_2d_cutoff

    ###################
    # POST-PROCESSING
    ###################
    seg = remove_small_objects(bw,
                               min_size=minArea,
                               connectivity=1,
                               in_place=False)

    # output
    seg = seg > 0
    seg = seg.astype(np.uint8)
    seg[seg > 0] = 255

    out_img_list.append(seg.copy())
    out_name_list.append("bw_final")

    if output_type == "default":
        # the default final output, simply save it to the output path
        save_segmentation(seg, False, Path(output_path), fn)
    elif output_type == "customize":
        # the hook for passing in a customized output function
        # use "out_img_list" and "out_name_list" in your hook to
        # customize your output functions
        output_func(out_img_list, out_name_list, Path(output_path), fn)
    elif output_type == "array":
        return seg
    elif output_type == "array_with_contour":
        return (seg, generate_segmentation_contour(seg))
    else:
        raise NotImplementedError("invalid output type: {output_type}")
コード例 #9
0
ファイル: profile.py プロジェクト: jrkerns/pylinac
def peak_detect(values: np.ndarray, threshold: Union[float, int]=None, min_distance: Union[float, int]=10,
                max_number: int=None, search_region: Tuple[float, float]=(0.0, 1.0),
                find_min_instead: bool=False) -> Tuple[np.ndarray, np.ndarray]:
    """Find the peaks or valleys of a 1D signal.

    Uses the difference (np.diff) in signal to find peaks. Current limitations include:
        1) Only for use in 1-D data; 2D may be possible with the gradient function.
        2) Will not detect peaks at the very edge of array (i.e. 0 or -1 index)

    Parameters
    ----------
    values : array-like
        Signal values to search for peaks within.
    threshold : int, float
        The value the peak must be above to be considered a peak. This removes "peaks"
        that are in a low-value region.
        If passed an int, the actual value is the threshold.
        E.g. when passed 15, any peak less with a value <15 is removed.
        If passed a float, it will threshold as a percent. Must be between 0 and 1.
        E.g. when passed 0.4, any peak <40% of the maximum value will be removed.
    min_distance : int, float
        If passed an int, parameter is the number of elements apart a peak must be from neighboring peaks.
        If passed a float, must be between 0 and 1 and represents the ratio of the profile to exclude.
        E.g. if passed 0.05 with a 1000-element profile, the minimum peak width will be 0.05*1000 = 50 elements.
    max_number : int
        Specify up to how many peaks will be returned. E.g. if 3 is passed in and 5 peaks are found, only the 3 largest
        peaks will be returned.
    find_min_instead : bool
        If False (default), peaks will be returned.
        If True, valleys will be returned.

    Returns
    -------
    max_vals : numpy.array
        The values of the peaks found.
    max_idxs : numpy.array
        The x-indices (locations) of the peaks.

    Raises
    ------
    ValueError
        If float not between 0 and 1 passed to threshold.
    """
    peak_vals = []  # a list to hold the y-values of the peaks. Will be converted to a numpy array
    peak_idxs = []  # ditto for x-values (index) of y data.

    if find_min_instead:
        values = -values

    """Limit search to search region"""
    left_end = search_region[0]
    if is_float_like(left_end):
        left_index = int(left_end*len(values))
    elif is_int_like(left_end):
        left_index = left_end
    else:
        raise ValueError(f"{left_end} must be a float or int")

    right_end = search_region[1]
    if is_float_like(right_end):
        right_index = int(right_end * len(values))
    elif is_int_like(right_end):
        right_index = right_end
    else:
        raise ValueError(f"{right_end} must be a float or int")

    # minimum peak spacing calc
    if isinstance(min_distance, float):
        if 0 > min_distance >= 1:
            raise ValueError("When min_peak_width is passed a float, value must be between 0 and 1")
        else:
            min_distance = int(min_distance * len(values))

    values = values[left_index:right_index]

    """Determine threshold value"""
    if isinstance(threshold, float) and threshold < 1:
        data_range = values.max() - values.min()
        threshold = threshold * data_range + values.min()
    elif isinstance(threshold, float) and threshold >= 1:
        raise ValueError("When threshold is passed a float, value must be less than 1")
    elif threshold is None:
        threshold = values.min()

    """Take difference"""
    values_diff = np.diff(values.astype(float))  # y and y_diff must be converted to signed type.

    """Find all potential peaks"""
    for idx in range(len(values_diff) - 1):
        # For each item of the diff array, check if:
        # 1) The y-value is above the threshold.
        # 2) The value of y_diff is positive (negative for valley search), it means the y-value changed upward.
        # 3) The next y_diff value is zero or negative (or positive for valley search); a positive-then-negative diff value means the value
        # is a peak of some kind. If the diff is zero it could be a flat peak, which still counts.

        # 1)
        if values[idx + 1] < threshold:
            continue

        y1_gradient = values_diff[idx] > 0
        y2_gradient = values_diff[idx + 1] <= 0

        # 2) & 3)
        if y1_gradient and y2_gradient:
            # If the next value isn't zero it's a single-pixel peak. Easy enough.
            if values_diff[idx + 1] != 0:
                peak_vals.append(values[idx + 1])
                peak_idxs.append(idx + 1 + left_index)
            # elif idx >= len(y_diff) - 1:
            #     pass
            # Else if the diff value is zero, it could be a flat peak, or it could keep going up; we don't know yet.
            else:
                # Continue on until we find the next nonzero diff value.
                try:
                    shift = 0
                    while values_diff[(idx + 1) + shift] == 0:
                        shift += 1
                        if (idx + 1 + shift) >= (len(values_diff) - 1):
                            break
                    # If the next diff is negative (or positive for min), we've found a peak. Also put the peak at the center of the flat
                    # region.
                    is_a_peak = values_diff[(idx + 1) + shift] < 0
                    if is_a_peak:
                        peak_vals.append(values[int((idx + 1) + np.round(shift / 2))])
                        peak_idxs.append((idx + 1 + left_index) + np.round(shift / 2))
                except IndexError:
                    pass

    # convert to numpy arrays
    peak_vals = np.array(peak_vals)
    peak_idxs = np.array(peak_idxs)

    """Enforce the min_peak_distance by removing smaller peaks."""
    # For each peak, determine if the next peak is within the min peak width range.
    index = 0
    while index < len(peak_idxs) - 1:

        # If the second peak is closer than min_peak_distance to the first peak, find the larger peak and remove the other one.
        if peak_idxs[index] > peak_idxs[index + 1] - min_distance:
            if peak_vals[index] > peak_vals[index + 1]:
                idx2del = index + 1
            else:
                idx2del = index
            peak_vals = np.delete(peak_vals, idx2del)
            peak_idxs = np.delete(peak_idxs, idx2del)
        else:
            index += 1

    """If Maximum Number passed, return only up to number given based on a sort of peak values."""
    if max_number is not None and len(peak_idxs) > max_number:
        sorted_peak_vals = peak_vals.argsort()  # sorts low to high
        peak_vals = peak_vals[sorted_peak_vals[-max_number:]]
        peak_idxs = peak_idxs[sorted_peak_vals[-max_number:]]

    # If we were looking for minimums, convert the values back to the original sign
    if find_min_instead:
        peak_vals = -peak_vals

    return peak_vals, peak_idxs
コード例 #10
0
def plot_shap_values(x: np.ndarray,
                     shap_values: np.ndarray,
                     val_list: List[str],
                     normalizing_dict: Dict[str, Dict[str, float]],
                     value_to_plot: str,
                     normalize_shap_plots: bool = True,
                     show: bool = False,
                     polished_value_name: Optional[str] = None,
                     pred_date: Optional[Tuple[int, int]] = None,
                     fig: Optional[Figure] = None) -> None:
    """Plots the denormalized values against their shap values, so that
    variations in the input features to the model can be compared to their effect
    on the model. For example plots, see notebooks/08_gt_recurrent_model.ipynb.
    Parameters:
    ----------
    x: np.array
        The input to a model for a single data instance
    shap_values: np.array
        The corresponding shap values (to x)
    val_list: list
        A list of the variable names, for axis labels
    normalizing_dict: dict
        The normalizing dict saved by the `Engineer`, so that the x array can be
        denormalized
    value_to_plot: str
        The specific input variable to plot. Must be in val_list
    normalize_shap_plots: bool = True
        If True, then the scale of the shap plots will be uniform across all
        variable plots (on an instance specific basis).
    show: bool = False
        If True, a plot of the variable `value_to_plot` against its shap values will be plotted.
    polished_value_name: Optional[str] = None
        If passed to the model, this is used instead of value_to_plot when labelling the axes.
    pred_month: Optional[Tuple[int, int]] = None
        If passed to the model, the x axis will contain actual months instead of the index.
        Note the tuple is [int_month, int_year]
    fig: Optional[Figure] = None
        The figure upon which to construct the plot. If None is passed, matplotlib will use
        plt.gcf() to get the figure
    """
    # first, lets isolate the lists
    idx = val_list.index(value_to_plot)

    x_val = x[:, idx]

    # we also want to denormalize
    for norm_var in normalizing_dict.keys():
        if value_to_plot.endswith(norm_var):
            x_val = (x_val * normalizing_dict[norm_var]['std']) + \
                normalizing_dict[norm_var]['mean']
            break

    shap_val = shap_values[:, idx]

    months = list(range(1, len(x_val) + 1))

    if pred_date is not None:
        int_months, int_years = [], []
        cur_month, cur_year = pred_date[0], pred_date[1]
        for i in range(1, len(x_val) + 1):
            cur_month = cur_month - 1
            if cur_month == 0:
                cur_month = 12
                cur_year -= 1
            int_months.append(cur_month)
            int_years.append(cur_year)
        str_dates = [
            f'{int2month[m]}{y}' for m, y in zip(int_months, int_years)
        ][::-1]

    host = host_subplot(111, axes_class=AA.Axes, figure=fig)
    plt.subplots_adjust(right=0.75)

    par1 = host.twinx()
    par1.axis["right"].toggle(all=True)

    if normalize_shap_plots:
        par1.set_ylim(shap_values.min(), shap_values.max())

    if polished_value_name is None:
        polished_value_name = value_to_plot

    host.set_xlabel("Months")
    host.set_ylabel(polished_value_name)
    par1.set_ylabel("Shap value")

    p1, = host.plot(months,
                    x_val,
                    label=polished_value_name,
                    linestyle='dashed')
    p2, = par1.plot(months, shap_val, label="Shap value")

    host.axis["left"].label.set_color(p1.get_color())
    par1.axis["right"].label.set_color(p2.get_color())

    host.legend(loc='lower left', framealpha=0.5)

    if pred_date is not None:
        modulo = (len(months) - 1) % 2
        host.set_xticks(months[modulo::2])
        host.set_xticklabels(str_dates[modulo::2])

    plt.draw()
    if show:
        plt.show()
コード例 #11
0
def softmax(logits: np.ndarray) -> np.ndarray:
    """Numerically stable softmax function"""
    pref = np.exp(logits - logits.max())
    return pref / pref.sum()
コード例 #12
0
    def saturation_vapour_pressure_goff_gratch(
            self, temperature: ndarray) -> ndarray:
        """
        Saturation Vapour pressure in a water vapour system calculated using
        the Goff-Gratch Equation (WMO standard method).

        Args:
            temperature:
                Temperature values in Kelvin. Valid from 173K to 373K

        Returns:
            Corresponding values of saturation vapour pressure for a pure
            water vapour system, in hPa.

        References:
            Numerical data and functional relationships in science and
            technology. New series. Group V. Volume 4. Meteorology.
            Subvolume b. Physical and chemical properties of the air, P35.
        """
        constants = {
            1: 10.79574,
            2: 5.028,
            3: 1.50475e-4,
            4: -8.2969,
            5: 0.42873e-3,
            6: 4.76955,
            7: 0.78614,
            8: -9.09685,
            9: 3.56654,
            10: 0.87682,
            11: 0.78614,
        }
        triple_pt = TRIPLE_PT_WATER

        # Values for which method is considered valid (see reference).
        # WetBulbTemperature.check_range(temperature.data, 173., 373.)
        if (temperature.max() > self.MAX_VALID_TEMPERATURE
                or temperature.min() < self.MIN_VALID_TEMPERATURE):
            msg = "Temperatures out of SVP table range: min {}, max {}"
            warnings.warn(msg.format(temperature.min(), temperature.max()))

        svp = temperature.copy()
        for cell in np.nditer(svp, op_flags=["readwrite"]):
            if cell > triple_pt:
                n0 = constants[1] * (1.0 - triple_pt / cell)
                n1 = constants[2] * np.log10(cell / triple_pt)
                n2 = constants[3] * (1.0 -
                                     np.power(10.0,
                                              (constants[4] *
                                               (cell / triple_pt - 1.0))))
                n3 = constants[5] * (np.power(10.0,
                                              (constants[6] *
                                               (1.0 - triple_pt / cell))) -
                                     1.0)
                log_es = n0 - n1 + n2 + n3 + constants[7]
                cell[...] = np.power(10.0, log_es)
            else:
                n0 = constants[8] * ((triple_pt / cell) - 1.0)
                n1 = constants[9] * np.log10(triple_pt / cell)
                n2 = constants[10] * (1.0 - (cell / triple_pt))
                log_es = n0 - n1 + n2 + constants[11]
                cell[...] = np.power(10.0, log_es)

        return svp
コード例 #13
0
ファイル: volume.py プロジェクト: erikmannerfelt/xdem
def hypsometric_binning(ddem: np.ndarray, ref_dem: np.ndarray, bins: Union[float, np.ndarray] = 50.0,
                        kind: str = "fixed", aggregation_function: Callable = np.median) -> pd.DataFrame:
    """
    Separate the dDEM in discrete elevation bins.
    The elevation bins will be calculated based on all ref_dem valid pixels.
    ddem may contain NaN/masked values over the same area, they will be excluded before the aggregation.

    It is assumed that the dDEM is calculated as 'ref_dem - dem' (not 'dem - ref_dem').

    :param ddem: The dDEM as a 2D or 1D array.
    :param ref_dem: The reference DEM as a 2D or 1D array.
    :param bins: The bin size, count, or array, depending on the binning method ('kind').
    :param kind: The kind of binning to do. Choices: ['fixed', 'count', 'quantile', 'custom'].
    :param aggregation_function: The function to aggregate the elevation values within a bin. Defaults to the median.

    :returns: A Pandas DataFrame with elevation bins and dDEM statistics.
    """
    assert ddem.shape == ref_dem.shape

    # Convert ddem mask into NaN
    ddem, _ = xdem.spatial_tools.get_array_and_mask(ddem)

    # Extract only the valid values, i.e. valid in ref_dem
    valid_mask = ~xdem.spatial_tools.get_mask(ref_dem)
    ddem = np.array(ddem[valid_mask])
    ref_dem = np.array(ref_dem.squeeze()[valid_mask])

    if isinstance(bins, np.ndarray):
        zbins = bins
    elif kind == "fixed":
        zbins = np.arange(ref_dem.min(), ref_dem.max() + bins + 1e-6, step=bins)  # +1e-6 in case min=max (1 point)
    elif kind == "count":
        # Make bins between mean_dem.min() and a little bit above mean_dem.max().
        # The bin count has to be bins + 1 because zbins[0] will be a "below min value" bin, which will be irrelevant.
        zbins = np.linspace(ref_dem.min(), ref_dem.max() + 1e-6 / bins, num=int(bins + 1))
    elif kind == "quantile":
        # Make the percentile steps. The bins + 1 is explained above.
        steps = np.linspace(0, 100, num=int(bins) + 1)
        zbins = np.fromiter(
            (np.percentile(ref_dem, step) for step in steps),
            dtype=float
        )
        # The uppermost bin needs to be a tiny amount larger than the highest value to include it.
        zbins[-1] += 1e-6
    elif kind == "custom":
        zbins = bins  # type: ignore
    else:
        raise ValueError(f"Invalid bin kind: {kind}. Choices: ['fixed', 'count', 'quantile', 'custom']")

    # Generate bins and get bin indices from the mean DEM
    indices = np.digitize(ref_dem, bins=zbins)

    # Calculate statistics for each bin.
    # If no values exist, all stats should be nans (except count with should be 0)
    # medians, means, stds, nmads = (np.zeros(shape=bins.shape[0] - 1, dtype=ddem.dtype) * np.nan, ) * 4
    values = np.zeros(shape=zbins.shape[0] - 1, dtype=ddem.dtype) * np.nan
    counts = np.zeros_like(values, dtype=int)
    for i in np.arange(indices.min(), indices.max() + 1):
        values_in_bin = ddem[indices == i]

        # Remove possible Nans
        values_in_bin = values_in_bin[np.isfinite(values_in_bin)]

        # Skip if no values are in the bin.
        if values_in_bin.shape[0] == 0:
            continue

        try:
            values[i - 1] = aggregation_function(values_in_bin)
            counts[i - 1] = values_in_bin.shape[0]
        except IndexError as exception:
            # If custom bins were added, i may exceed the bin range, which will be silently ignored.
            if kind == "custom" and "out of bounds" in str(exception):
                continue
            raise exception

    # Collect the results in a dataframe
    output = pd.DataFrame(
        index=pd.IntervalIndex.from_breaks(zbins),
        data=np.vstack([
            values, counts
        ]).T,
        columns=["value", "count"]
    )

    return output
コード例 #14
0
ファイル: plot_data.py プロジェクト: Mirantis/ceph-monitoring
def get_kde_img(vals: numpy.ndarray) -> str:
    fig, ax = pyplot.subplots(figsize=(6, 4), tight_layout=True)
    seaborn.kdeplot(vals, ax=ax, clip=(vals.min(), vals.max()))
    return get_img(fig)
コード例 #15
0
def normalize(x: np.ndarray):
    """Scale to 0-1
    """
    return (x - x.mean(axis=0)) / (x.max(axis=0) - x.min(axis=0))
コード例 #16
0
    def transform(self, v: np.ndarray,
                  f: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
        num_of_vertex = v.shape[0]
        num_of_face = f.shape[0]

        maxlen = (v.max(axis=0) - v.min(axis=0)).max()

        # load config
        theta = self.config[THETA]
        phi = self.config[PHI]
        rotation = rMat(theta, phi)
        translation = np.array([
            self.config[T_X], self.config[T_Y], self.config[T_Z]
        ]) * maxlen * MOVE_PORTION

        # load pivot
        norms = np.linalg.norm(v, axis=1)
        rank = np.argsort(norms)
        portion = 1 - 0.05 * self.config[PIVOTS]
        idx = math.floor(portion * num_of_vertex)
        idx = idx if idx != num_of_vertex else idx - 1
        pivot = rank[idx]

        # 1 move, -1 flow, 0 fix
        s = np.zeros(num_of_vertex)
        for idx, p in enumerate(v):
            dist = np.linalg.norm(p - v[pivot])
            if dist < maxlen * MOVE_PORTION:
                s[idx] = 1
            elif dist < maxlen * FLOW_PORTION:
                s[idx] = -1
            else:
                s[idx] = 0

        b = np.array([[
            t[0] for t in [(i, s[i]) for i in range(0, v.shape[0])]
            if t[1] >= 0
        ]]).T
        arap = igl.ARAP(v, f, 3, b)
        bc = np.zeros((b.size, 3))
        for i in range(b.shape[0]):
            bc[i] = v[b[i]]
            if s[b[i]] == 1:
                bc[i] = rotation @ bc[i] + translation
        vn = arap.solve(bc, v)
        """
        vcp = deepcopy(v)
        num_of_vertex = vcp.shape[0]

        maxlen = (vcp.max(axis=0) - vcp.min(axis=0)).max()

        # load config
        theta = self.config[THETA]
        phi = self.config[PHI]
        rotation = rMat(theta, phi)
        translation = np.array([self.config[T_X], self.config[T_Y], self.config[T_Z]]) * maxlen * MOVE_PORTION

        # load pivot
        pivot =  math.floor(num_of_vertex * self.config[PIVOTS])
        pivot = pivot if pivot < num_of_vertex else pivot - 1

        # 1 move, -1 flow, 0 fix
        s = np.zeros(num_of_vertex)
        for idx, p in enumerate(vcp):
            dist = np.linalg.norm(p - vcp[pivot])
            if dist < maxlen * MOVE_PORTION:
                s[idx] = 1
            elif dist < maxlen * FLOW_PORTION:
                s[idx] = -1
            else:
                s[idx] = 0

        b = np.array([[t[0] for t in [(i, s[i]) for i in range(0, vcp.shape[0])] if t[1] >= 0]]).T
        arap = igl.ARAP(vcp, f, 3, b)
        bc = np.zeros((b.size, 3))
        for i in range(b.shape[0]):
            bc[i] = vcp[b[i]]
            if s[b[i]] == 1:
                bc[i] = rotation @ bc[i] + translation
        vn = arap.solve(bc, vcp)
        """
        return vn, deepcopy(f)
コード例 #17
0
def create_2d_meshgrid(x: np.ndarray, y: np.ndarray, samples=100) -> Tuple[np.ndarray, np.ndarray]:
    dx = (x.max() - x.min()) / 5
    dy = (y.max() - y.min()) / 5
    return np.meshgrid(
        np.linspace(x.min() - dx, x.max() + dx, num=samples),
        np.linspace(y.min() - dy, y.max() + dy, num=samples))
コード例 #18
0
ファイル: chu_liu_edmonds.py プロジェクト: apmoore1/allennlp
def decode_mst(energy: numpy.ndarray,
               length: int,
               has_labels: bool = True) -> Tuple[numpy.ndarray, numpy.ndarray]:
    """
    Note: Counter to typical intuition, this function decodes the _maximum_
    spanning tree.

    Decode the optimal MST tree with the Chu-Liu-Edmonds algorithm for
    maximum spanning arborescences on graphs.

    Parameters
    ----------
    energy : ``numpy.ndarray``, required.
        A tensor with shape (num_labels, timesteps, timesteps)
        containing the energy of each edge. If has_labels is ``False``,
        the tensor should have shape (timesteps, timesteps) instead.
    length : ``int``, required.
        The length of this sequence, as the energy may have come
        from a padded batch.
    has_labels : ``bool``, optional, (default = True)
        Whether the graph has labels or not.
    """
    if has_labels and energy.ndim != 3:
        raise ConfigurationError("The dimension of the energy array is not equal to 3.")
    elif not has_labels and energy.ndim != 2:
        raise ConfigurationError("The dimension of the energy array is not equal to 2.")
    input_shape = energy.shape
    max_length = input_shape[-1]

    # Our energy matrix might have been batched -
    # here we clip it to contain only non padded tokens.
    if has_labels:
        energy = energy[:, :length, :length]
        # get best label for each edge.
        label_id_matrix = energy.argmax(axis=0)
        energy = energy.max(axis=0)
    else:
        energy = energy[:length, :length]
        label_id_matrix = None
    # get original score matrix
    original_score_matrix = energy
    # initialize score matrix to original score matrix
    score_matrix = numpy.array(original_score_matrix, copy=True)

    old_input = numpy.zeros([length, length], dtype=numpy.int32)
    old_output = numpy.zeros([length, length], dtype=numpy.int32)
    current_nodes = [True for _ in range(length)]
    representatives: List[Set[int]] = []

    for node1 in range(length):
        original_score_matrix[node1, node1] = 0.0
        score_matrix[node1, node1] = 0.0
        representatives.append({node1})

        for node2 in range(node1 + 1, length):
            old_input[node1, node2] = node1
            old_output[node1, node2] = node2

            old_input[node2, node1] = node2
            old_output[node2, node1] = node1

    final_edges: Dict[int, int] = {}

    # The main algorithm operates inplace.
    chu_liu_edmonds(length, score_matrix, current_nodes,
                    final_edges, old_input, old_output, representatives)

    heads = numpy.zeros([max_length], numpy.int32)
    if has_labels:
        head_type = numpy.ones([max_length], numpy.int32)
    else:
        head_type = None

    for child, parent in final_edges.items():
        heads[child] = parent
        if has_labels:
            head_type[child] = label_id_matrix[parent, child]

    return heads, head_type
コード例 #19
0
def plot_solitons(data: np.ndarray,
                  filename: str,
                  show: Optional[bool]=False,
                  write: Optional[bool]=False):
    """
    Plot (and animate) the solitons.

    :param data: 2d array of values
    :param filename: the savename of the file
    :param show: whether or not to show the plot
    :param write: Whether or not to save the plot

    :return peaks: contains a 2d np.ndarray for every frame
    :return solitons: soliton name to array of times and positions
    :return soliton_hash: soliton name to dictionary of times to positions

    TODO: need to auto config parameters:
        * gaussian filter parameter
        * threshold
        * find_peaks parameter
        * x and y limits
        * Frame intervals
    """
    # Initialize datapoints
    domain = np.arange(len(data))
    cdata = data[:, 0]   # First column

    # Smooth the data for more accuracy, only need for raw data
    print('\tSmoothing Data')
    sdata = scfilt.gaussian_filter1d(data, 0, axis=0)
    scdata = sdata[:, 0]

    # Establish figure
    print('\tInitializing Figure')
    fig = plt.figure()
    ax = fig.add_axes([0.1, 0.1, 0.8, 0.8])

    # Plot Raw Data
    line1, = ax.plot(domain, cdata, 'b-', alpha=0.5)
    # Plot Smoothed Data
    line, = ax.plot(domain, scdata, 'b-')

    vert_domain = np.arange(0, 10)
    ax.plot(len(data) * 0.25 * np.ones(len(vert_domain)), vert_domain, 'r-')

    # Set figure axes limits
    ax.set_ylim(0, data.max() + 0.1 * data.max())
    ax.set_xlim(0, domain.max())

    # Grab and plot peaks
    if input('\tgenerate new peaks? (y/n) ') == 'y':
        peaks = get_peaks(domain, sdata)
        np.save('{}.npy'.format(filename), peaks)
    else:
        peaks = np.load('{}.npy'.format(filename))

    # Identify Solitons
    solitons, soliton_hash = get_solitons(peaks)

    # Plot Initial Peaks
    points, = ax.plot(peaks[0][:, 0], peaks[0][:, 1],
                      linestyle='', marker='o', color='r')

    # Annotate Initial Solitons (set to (0, 0) if not present)
    annotations = []
    labels = list(solitons.keys())
    default_pos = (0, 0)
    for key in labels:
        value = soliton_hash[key]
        pos = default_pos
        if 0 in list(value.keys()):
            pos = (value[0][0], value[0][1])
        an = ax.annotate(r'${}$'.format(key),
                         xy=pos, size=20)
        annotations.append(an)

    # Cover up hidden annotations with little white box. Currently not working.
    rectangle = plt.Rectangle((0, 0), 250, 20, fc='w', ec='k', clip_on=True)
    ax.add_patch(rectangle)

    def animate(i: int):
        """ Animation function for plot """
        if i % 100 == 0:
            print(i, sep=' ')
        cdata = data[:, i]  # New datapoints
        scdata = sdata[:, i]  # New Smoothed datapoints
        line.set_ydata(scdata)
        line1.set_ydata(cdata)
        points.set_data(peaks[i][:, 0], peaks[i][:, 1])

        ax.set_title('t=kajsdfh')

        for j in range(len(labels)):
            key = labels[j]
            positions = soliton_hash[key]
            annotation = annotations[j]
            pos = default_pos
            if i in list(positions.keys()):
                pos = (positions[i][0], positions[i][1])
            annotation.set_position(pos)
        return line, annotations

    def init():
        """ Renders the first frame """
        return line, annotations

    ani = animation.FuncAnimation(fig, animate, np.arange(0, len(data[0])),
                                  init_func=init,
                                  interval=50)

    if write:
        print('Animating')
        ani.save('{}.mp4'.format(filename))
        # editor.VideoFileClip('anim.mp4')\
        #         .write_gif('anim.gif')
    if show:
        print('Animating')
        plt.show()

    return peaks, solitons, soliton_hash
コード例 #20
0
ファイル: logmart.py プロジェクト: scienceopen/airtools
def logmart(A: np.ndarray, b: np.ndarray,
            *,
            relax: float = 1.,
            x0: float = None,
            sigma: float = None,
            max_iter: int = 200) -> tuple:
    """
    Displays delta Chisquare.
    Program is stopped if Chisquare increases.
    A is NxM array
    b is Nx1 vector
    returns Mx1 vector

    relax	     user specified relaxation constant	(default is 20.)
    x0	     user specified initial guess (N vector)  (default is backproject y, i.e., y#A)
    max_iter	user specified max number of iterations (default is 20)

    AUTHOR:	Joshua Semeter
    LAST MODIFIED:	5-2015

      Simple test problem
    A = np.diag([5, 5, 5])
    x = np.array([1,2,3])
    b = A @ x
    """
# %% parameter check
    if b.ndim != 1:
        raise ValueError('y must be a column vector')
    if A.ndim != 2:
        raise ValueError('A must be a matrix')
    if A.shape[0] != b.size:
        raise ValueError('A and y number of rows must match')
    if not isinstance(relax, float):
        raise ValueError('relax is a scalar float')

    b = b.copy()  # needed to avoid modifying outside this function!
# %% set defaults
    if sigma is None:
        sigma = np.ones_like(b)

    if x0 is None:  # backproject
        x = A.T @ b / A.ravel().sum()
        xA = A @ x
        x = x * b.max() / xA.max()
    elif isinstance(x0, (float, int)) or x0.size == 1:  # replicate
        x = x0 * np.ones_like(b)
    else:
        x = x0
# %% make sure there are no 0's in y
    b[b <= 1e-8] = 1e-8
    # W=sigma;
    # W=linspace(1,0,size(A,1))';
    # W=rand(size(A,1),1);
    W = np.ones(A.shape[0])
    W = W / W.sum()

    i = 0
    done = False
    arg = ((A @ x - b)/sigma)**2.
    chi2 = np.sqrt(arg.sum())

# %%  iterate solution, plot estimated data (diag elems of x#A)
    while not done:
        i += 1
        xold = x
        xA = A @ x
        t = (1/xA).min()
        C = relax*t*(1.-(xA/b))
        x = x / (1 - x*(A.T @ (W*C)))
# %% monitor solution
        chiold = chi2
        chi2 = np.sqrt((((xA - b)/sigma)**2).sum())
        # dchi2=(chi2-chiold);
        done = ((chi2 > chiold) & (i > 2)) | (i == max_iter) | (chi2 < 0.7)
# %% plot
#        figure(9); clf; hold off;
#        Nest=reshape(x,69,83);
#        imagesc(Nest); caxis([0,1e11]);
#        set(gca,'YDir','normal'); set(gca,'XDir','normal');
#        pause(0.02)
    y_est = A @ xold

    return xold, y_est, chi2, i
コード例 #21
0
    def _logsum_rows(cls, x: np.ndarray) -> np.ndarray:

        x = np.asarray(x)
        m = x.max(axis=1)
        return np.log(np.exp(x - m[:, None]).sum(axis=1)) + m
コード例 #22
0
def plot_interp(time: datetime, grid: typing.Dict[str, np.ndarray], parm: np.ndarray, name: str, fg: "mplf.Figure", **kwargs):
    """

    xp:  eastward distance (rads.)
        should be interpreted as northward distance (in rads.).
        Irrespective of ordering of xg.theta, this will be monotonic increasing!!!

    zp: altitude (meters)

    y:  this is a mag colat. coordinate and is only used for defining
        grid in linspaces below, runs backward from north distance,
        hence the negative sign
    """

    cmap = None
    if name.startswith("J") or name == "Phitop":
        cmap = "bwr"
        vmax = abs(parm).max()
        vmin = -vmax
    elif name.startswith("v"):
        cmap = "bwr"
        vmax = 80.0
        vmin = -vmax
    elif name.startswith("T"):
        vmin = 0.0
        vmax = parm.max()
    else:
        vmin = vmax = None
    # %% SIZE OF SIMULATION
    lx1, lx2, lx3 = grid["lx"]
    inds1 = slice(2, lx1 + 2)
    inds2 = slice(2, lx2 + 2)
    inds3 = slice(2, lx3 + 2)
    # %% SIZE OF PLOT GRID THAT WE ARE INTERPOLATING ONTO
    meantheta = grid["theta"].mean()
    # this is a mag colat. coordinate and is only used for defining grid in linspaces below
    # runs backward from north distance, hence the negative sign
    # [radians]
    y = -1 * (grid["theta"] - meantheta)
    # eastward distance [radians]
    x = grid["x2"][inds2] / R_EARTH / math.sin(meantheta)
    # altitude [meters]
    z = grid["alt"] / 1e3

    # arbitrary output plot resolution
    lxp = 500
    lyp = 500
    lzp = 500

    # eastward distance [meters]
    xp = np.linspace(x.min(), x.max(), lxp) * R_EARTH * math.sin(meantheta)
    # northward distance [meters]
    yp = np.linspace(y.min(), y.max(), lyp) * R_EARTH
    # upward distance [meters]
    zp = np.linspace(z.min(), z.max(), lzp) * 1e3
    # %% INTERPOLATE ONTO PLOTTING GRID
    if grid["lx"][2] == 1:  # alt./lon. slice
        ax = fg.gca()
        ax.set_title(f"{name}: {time.isoformat()}  {gitrev()}")
        # meridional meshgrid, this defines the grid for plotting
        # slice expects the first dim. to be "y" ("z" in the 2D case)
        # %% CONVERT ANGULAR COORDINATES TO MLAT,MLON
        i = np.argsort(xp)  # FIXME: this was in Matlab code--what is its purpose?

        if name == "rayleighs":
            f = interp.interp1d(grid["x2"][inds2], parm, axis=1, bounds_error=False)
            # hack for pcolormesh to put labels in center of pixel
            wl = kwargs["wavelength"] + [""]
            hi = ax.pcolormesh(xp / 1e3, np.arange(len(wl)), f(xp)[:, i])
            ax.set_yticks(np.arange(len(wl)) + 0.5)
            ax.set_yticklabels(wl)
            ax.set_ylim(0, len(wl) - 1)
            # end hack
            ax.set_ylabel(r"wavelength $\AA$")
            ax.set_xlabel("eastward dist. (km)")
        elif parm.ndim == 2:
            f = interp.interp2d(grid["x2"][inds2], grid["x1"][inds1], parm, bounds_error=False)
            plot12(xp[i], zp, f(xp, zp)[:, i], name, cmap, vmin, vmax, fg, ax)
        elif parm.ndim == 1:  # phitop
            f = interp.interp1d(grid["x2"][inds2], parm, bounds_error=False)
            plot1d2(xp, f(xp), name, fg, ax)
        else:
            raise ValueError(f"{name}: only 2D and 1D data are expected--squeeze data")
    elif grid["lx"][1] == 1:  # alt./lat. slice
        ax = fg.gca()
        ax.set_title(f"{name}: {time.isoformat()}  {gitrev()}")
        # so north dist, east dist., alt.
        # slice expects the first dim. to be "y"
        # %% CONVERT ANGULAR COORDINATES TO MLAT,MLON
        i = np.argsort(yp)  # FIXME: this was in Matlab code--what is its purpose?

        if name == "rayleighs":
            # FIXME: this needs to be tested
            f = interp.interp1d(grid["x3"][inds3], parm, axis=1, bounds_error=False)
            # hack for pcolormesh to put labels in center of pixel
            wl = kwargs["wavelength"] + [""]
            hi = ax.pcolormesh(np.arange(len(wl)), yp / 1e3, f(yp)[:, i].T)
            ax.set_xticks(np.arange(len(wl)) + 0.5)
            ax.set_xticklabels(wl)
            ax.set_xlim(0, len(wl) - 1)
            # end hack
            ax.set_xlabel(r"wavelength $\AA$")
            ax.set_ylabel("northward dist. (km)")
        elif parm.ndim == 2:
            f = interp.interp2d(grid["x3"][inds3], grid["x1"][inds1], parm, bounds_error=False)
            parmp = f(yp, zp).reshape((lzp, lyp))
            plot13(yp[i], zp, parmp[:, i], name, cmap, vmin, vmax, fg, ax)
        elif parm.ndim == 1:  # phitop
            f = interp.interp1d(grid["x3"][inds3], parm, bounds_error=False)
            plot1d3(yp, f(yp), name, fg, ax)
        else:
            raise ValueError(f"{name}: only 2D and 1D data are expected--squeeze data")

    else:  # 3-panel plot, vs. single-panel plots of 2-D cases
        if name == "rayleighs":
            axs = fg.subplots(2, 2, sharey=True, sharex=True).ravel()
            fg.suptitle(f"{name}: {time.isoformat()}  {gitrev()}", y=0.99)
            # arbitrary pick of which emission lines to plot lat/lon slices
            for j, i in enumerate([1, 3, 4, 8]):
                f = interp.interp2d(grid["x3"][inds3], grid["x2"][inds2], parm[i, :, :], bounds_error=False)
                hi = axs[j].pcolormesh(xp / 1e3, yp / 1e3, f(yp, xp))
                axs[j].set_title(kwargs["wavelength"][i] + r"$\AA$")
                fg.colorbar(hi, ax=axs[j], label="Rayleighs")
            axs[2].set_xlabel("eastward dist. (km)")
            axs[2].set_ylabel("northward dist. (km)")
            return
        elif parm.ndim == 3:
            fg.set_size_inches((18, 5))
            axs = fg.subplots(1, 3, sharey=False, sharex=False)
            fg.suptitle(f"{name}: {time.isoformat()}  {gitrev()}", y=0.99)
        else:
            # like phitop, SINGLE plot
            ax = fg.gca()
            f = interp.interp2d(grid["x3"][inds3], grid["x2"][inds2], parm, bounds_error=False)
            hi = ax.pcolormesh(xp / 1e3, yp / 1e3, f(yp, xp), cmap=cmap, vmin=vmin, vmax=vmax)
            ax.set_xlabel("eastward dist. (km)")
            ax.set_ylabel("northward dist. (km)")
            fg.colorbar(hi, ax=ax, label=CB_LBL[name])
            return

        # %% CONVERT TO DISTANCE UP, EAST, NORTH (left panel)
        # JUST PICK AN X3 LOCATION FOR THE MERIDIONAL SLICE PLOT,
        # AND AN ALTITUDE FOR THE LAT./LON. SLICE
        ix3 = lx3 // 2 - 1  # arbitrary slice, to match Matlab
        f = interp.interp2d(grid["x2"][inds2], grid["x1"][inds1], parm[:, :, ix3], bounds_error=False)
        # CONVERT ANGULAR COORDINATES TO MLAT,MLON
        ix = np.argsort(xp)
        iy = np.argsort(yp)
        plot12(xp[ix], zp, f(xp, zp)[:, ix], name, cmap, vmin, vmax, fg, axs[0])
        # %% LAT./LONG. SLICE COORDINATES (center panel)
        zp2 = REF_ALT
        X3, Y3, Z3 = np.meshgrid(xp, yp, zp2 * 1e3)
        # transpose: so north dist, east dist., alt.
        parmp = interp.interpn(
            points=(grid["x1"][inds1], grid["x2"][inds2], grid["x3"][inds3]),
            values=parm,
            xi=np.column_stack((Z3.ravel(), Y3.ravel(), X3.ravel())),
            bounds_error=False,
        ).reshape((1, lyp, lxp))

        parmp = parmp[:, iy, :]  # must be indexed in two steps
        plot23(xp[ix], yp[iy], parmp[0, :, ix], name, cmap, vmin, vmax, fg, axs[1])
        # %% ALT/LAT SLICE (right panel)
        ix2 = lx2 // 2 - 1  # arbitrary slice, to match Matlab
        f = interp.interp2d(grid["x3"][inds3], grid["x1"][inds1], parm[:, ix2, :], bounds_error=False)
        hi = plot13(yp[iy], zp, f(yp, zp)[:, iy], name, cmap, vmin, vmax, fg, axs[2])

        fg.colorbar(hi, ax=axs, aspect=60, pad=0.01)
コード例 #23
0
def Workflow_cardio_fbl_100x(
    struct_img: np.ndarray,
    rescale_ratio: float = -1,
    output_type: str = "default",
    output_path: Union[str, Path] = None,
    fn: Union[str, Path] = None,
    output_func=None,
):
    """
    classic segmentation workflow wrapper for structure Cardio FBL 100x

    Parameter:
    -----------
    struct_img: np.ndarray
        the 3D image to be segmented
    rescale_ratio: float
        an optional parameter to allow rescale the image before running the
        segmentation functions, default is no rescaling
    output_type: str
        select how to handle output. Currently, four types are supported:
        1. default: the result will be saved at output_path whose filename is
            original name without extention + "_struct_segmentaiton.tiff"
        2. array: the segmentation result will be simply returned as a numpy array
        3. array_with_contour: segmentation result will be returned together with
            the contour of the segmentation
        4. customize: pass in an extra output_func to do a special save. All the
            intermediate results, names of these results, the output_path, and the
            original filename (without extension) will be passed in to output_func.
    """
    ##########################################################################
    # PARAMETERS:
    #   note that these parameters are supposed to be fixed for the structure
    #   and work well accross different datasets

    intensity_norm_param = [0.5, 3]
    gaussian_smoothing_sigma = 1
    gaussian_smoothing_truncate_range = 3.0
    dot_2d_sigma = 1
    dot_2d_cutoff = 0.015
    minArea = 1
    low_level_min_size = 600
    ##########################################################################

    out_img_list = []
    out_name_list = []

    ###################
    # PRE_PROCESSING
    ###################
    # intenisty normalization (min/max)
    struct_img = intensity_normalization(struct_img,
                                         scaling_param=intensity_norm_param)

    out_img_list.append(struct_img.copy())
    out_name_list.append("im_norm")

    # rescale if needed
    if rescale_ratio > 0:
        struct_img = zoom(struct_img, (1, rescale_ratio, rescale_ratio),
                          order=2)

        struct_img = (struct_img - struct_img.min() +
                      1e-8) / (struct_img.max() - struct_img.min() + 1e-8)
        gaussian_smoothing_truncate_range = (
            gaussian_smoothing_truncate_range * rescale_ratio)

    # smoothing with gaussian filter
    structure_img_smooth = image_smoothing_gaussian_3d(
        struct_img,
        sigma=gaussian_smoothing_sigma,
        truncate_range=gaussian_smoothing_truncate_range,
    )

    out_img_list.append(structure_img_smooth.copy())
    out_name_list.append("im_smooth")

    ###################
    # core algorithm
    ###################

    # step 1: low level thresholding
    # global_otsu = threshold_otsu(structure_img_smooth)
    global_tri = threshold_triangle(structure_img_smooth)
    global_median = np.percentile(structure_img_smooth, 50)

    th_low_level = (global_tri + global_median) / 2
    bw_low_level = structure_img_smooth > th_low_level
    bw_low_level = remove_small_objects(bw_low_level,
                                        min_size=low_level_min_size,
                                        connectivity=1,
                                        in_place=True)

    # step 2: high level thresholding
    local_cutoff = 0.333 * threshold_otsu(structure_img_smooth)
    bw_high_level = np.zeros_like(bw_low_level)
    lab_low, num_obj = label(bw_low_level, return_num=True, connectivity=1)
    for idx in range(num_obj):
        single_obj = lab_low == (idx + 1)
        local_otsu = threshold_otsu(structure_img_smooth[single_obj])
        if local_otsu > local_cutoff:
            bw_high_level[np.logical_and(
                structure_img_smooth > 1.1 * local_otsu, single_obj)] = 1

    out_img_list.append(bw_high_level.copy())
    out_name_list.append("interm_high")

    # step 3: finer segmentation
    response2d = dot_slice_by_slice(structure_img_smooth,
                                    log_sigma=dot_2d_sigma)

    bw_finer = remove_small_objects(response2d > dot_2d_cutoff,
                                    min_size=minArea,
                                    connectivity=1,
                                    in_place=True)

    out_img_list.append(bw_finer.copy())
    out_name_list.append("bw_fine")

    # merge finer level detection into high level coarse segmentation
    # to include outside dim parts
    bw_high_level[bw_finer > 0] = 1

    ###################
    # POST-PROCESSING
    ###################
    seg = remove_small_objects(bw_high_level > 0,
                               min_size=minArea,
                               connectivity=1,
                               in_place=False)

    # output
    seg = seg > 0
    seg = seg.astype(np.uint8)
    seg[seg > 0] = 255

    out_img_list.append(seg.copy())
    out_name_list.append("bw_coarse")

    if output_type == "default":
        # the default final output, simply save it to the output path
        save_segmentation(seg, False, Path(output_path), fn)
    elif output_type == "customize":
        # the hook for passing in a customized output function
        # use "out_img_list" and "out_name_list" in your hook to
        # customize your output functions
        output_func(out_img_list, out_name_list, Path(output_path), fn)
    elif output_type == "array":
        return seg
    elif output_type == "array_with_contour":
        return (seg, generate_segmentation_contour(seg))
    else:
        raise NotImplementedError("invalid output type: {output_type}")
コード例 #24
0
 def __normalize_column(values: np.ndarray):
     max_val = values.max(initial=None)
     min_val = values.min(initial=None)
     return np.asarray([(val - min_val) / (max_val - min_val)
                        for val in values])
コード例 #25
0
 def act(X: np.ndarray) -> np.ndarray:
     expX = np.exp(X - X.max())
     return expX / np.sum(expX, axis=1)[:, None]
コード例 #26
0
ファイル: helpers.py プロジェクト: dschaffner/PlasmaPy
def check_sweep(
    voltage: np.ndarray,
    current: np.ndarray,
    strip_units: bool = True,
) -> (np.ndarray, np.ndarray):
    """
    Function for checking that the voltage and current arrays are properly
    formatted for analysis by `plasmapy.analysis.swept_langmuir`.

    Parameters
    ----------
    voltage: `numpy.ndarray`
        1D `numpy.ndarray` representing the voltage of the swept Langmuir trace.
        Voltage should be monotonically increasing.  *No units are assumed or
        checked, but values should be in volts.*

    current: `numpy.ndarray`
        1D `numpy.ndarray` representing the current of the swept Langmuir trace.
        Values should start from a negative ion-saturation current and increase
        to a positive electron-saturation current.  *No units are assumed or
        checked, but values should be in amperes.*

    strip_units: `bool`
        (Default: `True`) If `True`, then the units on ``voltage`` and/or
        ``current`` will be stripped if either are passed in as an Astropy
        `~astropy.units.Quantity`.

    Returns
    -------
    voltage : `numpy.ndarray`
        Input argument ``voltage`` after it goes through all of its checks
        and conditioning.

    current : `numpy.ndarray`
        Input argument ``current`` after it goes through all of its checks
        and conditioning.

    Raises
    ------
    `TypeError`
        If either the ``voltage`` or ``current`` arrays are not instances of a
        `numpy.ndarray`.

    `ValueError`:
        If either the ``voltage`` or ``current`` arrays are not 1D.

    `ValueError`
        If the ``voltage`` array is not monotonically increasing.

    `ValueError`
        If the ``current`` array never crosses zero (i.e. has no floating
        potential).

    `ValueError`
        If the ``current`` array does not start form a negative ion-saturation
        current and increases to a positive electron-saturation current.

    `ValueError`
        If either the ``voltage`` or ``current`` array does not have a
        `numpy.dtype` of either `numpy.integer` or `numpy.floating`.

    """
    # -- examine voltage array --
    # check type
    if isinstance(voltage, np.ndarray):
        pass
    elif isinstance(voltage, (list, tuple)):
        voltage = np.array(voltage)
    else:
        raise TypeError(
            f"Expected 1D numpy array for voltage, but got {type(voltage)}.", )

    # check array structure
    if not (np.issubdtype(voltage.dtype, np.floating)
            or np.issubdtype(voltage.dtype, np.integer)):
        raise ValueError(
            f"Expected 1D numpy array of floats or integers for voltage, but"
            f" got an array with dytpe '{voltage.dtype}'.")
    elif voltage.ndim != 1:
        raise ValueError(
            f"Expected 1D numpy array for voltage, but got array with "
            f"{voltage.ndim} dimensions.", )
    elif not np.all(np.diff(voltage) >= 0):
        raise ValueError("The voltage array is not monotonically increasing.")

    # strip units
    if isinstance(voltage, u.Quantity) and strip_units:
        voltage = voltage.value

    # -- examine current array --
    # check type
    if isinstance(current, np.ndarray):
        pass
    elif isinstance(current, (list, tuple)):
        current = np.array(current)
    else:
        raise TypeError(
            f"Expected 1D numpy array for current, but got {type(current)}.", )

    # check array structure
    if not (np.issubdtype(current.dtype, np.floating)
            or np.issubdtype(current.dtype, np.integer)):
        raise ValueError(
            f"Expected 1D numpy array of floats or integers for current, but"
            f" got an array with dytpe '{current.dtype}'.")
    elif current.ndim != 1:
        raise ValueError(
            f"Expected 1D numpy array for current, but got array with "
            f"{current.ndim} dimensions.", )
    elif current.min() > 0.0 or current.max() < 0:
        raise ValueError(
            "Invalid swept Langmuir trace, the current never crosses zero "
            "'current = 0'.")
    elif current[0] > 0.0 or current[-1] < 0.0:
        raise ValueError(
            "The current array needs to start from a negative ion-saturation current"
            " to a positive electron-saturation current.")

    if voltage.size != current.size:
        raise ValueError(
            f"Incompatible arrays, 'voltage' size {voltage.size} must be the same"
            f" as the 'current' size {current.size}.")

    # strip units
    if isinstance(current, u.Quantity) and strip_units:
        current = current.value

    return voltage, current
コード例 #27
0
def sparsery(mov: np.ndarray, high_pass: int, neuropil_high_pass: int, batch_size: int, spatial_scale: int, threshold_scaling,
             max_iterations: int, yrange, xrange) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
    """Returns stats and ops from 'mov' using correlations in time."""

    mov = temporal_high_pass_filter(mov=mov, width=int(high_pass))
    max_proj = mov.max(axis=0)

    sdmov = utils.standard_deviation_over_time(mov, batch_size=batch_size)
    mov = neuropil_subtraction(mov=mov / sdmov, filter_size=neuropil_high_pass)  # subtract low-pass filtered movie

    _, Lyc, Lxc = mov.shape
    LL = np.meshgrid(np.arange(Lxc), np.arange(Lyc))
    gxy = [np.array(LL).astype('float32')]
    dmov = mov
    movu = []

    # downsample movie at various spatial scales
    Lyp, Lxp = np.zeros(5, 'int32'), np.zeros(5, 'int32')  # downsampled sizes
    for j in range(5):
        movu0 = square_convolution_2d(dmov, 3)
        dmov = 2 * utils.downsample(dmov)
        gxy0 = utils.downsample(gxy[j], False)
        gxy.append(gxy0)
        _, Lyp[j], Lxp[j] = movu0.shape
        movu.append(movu0)

    # spline over scales
    I = np.zeros((len(gxy), gxy[0].shape[1], gxy[0].shape[2]))
    for movu0, gxy0, I0 in zip(movu, gxy, I):
        gmodel = RectBivariateSpline(gxy0[1, :, 0], gxy0[0, 0, :], movu0.max(axis=0),
                                     kx=min(3, gxy0.shape[1] - 1), ky=min(3, gxy0.shape[2] - 1))
        I0[:] = gmodel(gxy[0][1, :, 0], gxy[0][0, 0, :])
    v_corr = I.max(axis=0)

    # to set threshold, find best scale based on scale of top peaks
    im, estimate_mode = find_best_scale(I=I, spatial_scale=spatial_scale)
    spatscale_pix = 3 * 2 ** im
    Th2 = threshold_scaling * 5 * max(1, im)  # threshold for accepted peaks (scale it by spatial scale)
    vmultiplier = max(1, mov.shape[0] / 1200)
    print('NOTE: %s spatial scale ~%d pixels, time epochs %2.2f, threshold %2.2f ' % (estimate_mode.value, spatscale_pix, vmultiplier, vmultiplier * Th2))

    # get standard deviation for pixels for all values > Th2
    v_map = [utils.threshold_reduce(movu0, Th2) for movu0 in movu]
    movu = [movu0.reshape(movu0.shape[0], -1) for movu0 in movu]

    mov = np.reshape(mov, (-1, Lyc * Lxc))
    lxs = 3 * 2**np.arange(5)
    nscales = len(lxs)

    v_max = np.zeros(max_iterations)
    ihop = np.zeros(max_iterations)
    v_split = np.zeros(max_iterations)
    V1 = deepcopy(v_map)
    stats = []
    for tj in range(max_iterations):
        # find peaks in stddev's
        v0max = np.array([V1[j].max() for j in range(5)])
        imap = np.argmax(v0max)
        imax = np.argmax(V1[imap])
        yi, xi = np.unravel_index(imax, (Lyp[imap], Lxp[imap]))
        # position of peak
        yi, xi = gxy[imap][1,yi,xi], gxy[imap][0,yi,xi]

        # check if peak is larger than threshold * max(1,nbinned/1200)
        v_max[tj] = v0max.max()
        if v_max[tj] < vmultiplier*Th2:
            break
        ls = lxs[imap]

        ihop[tj] = imap

        # make square of initial pixels based on spatial scale of peak
        ypix0, xpix0, lam0 = add_square(int(yi), int(xi), ls, Lyc, Lxc)
        
        # project movie into square to get time series
        tproj = mov[:, ypix0*Lxc + xpix0] @ lam0
        active_frames = np.nonzero(tproj>Th2)[0] # frames with activity > Th2
        
        # extend mask based on activity similarity
        for j in range(3):
            ypix0, xpix0, lam0 = iter_extend(ypix0, xpix0, mov, Lyc, Lxc, active_frames)
            tproj = mov[:, ypix0*Lxc+ xpix0] @ lam0
            active_frames = np.nonzero(tproj>Th2)[0]
            if len(active_frames)<1:
                break
        if len(active_frames)<1:
            break

        # check if ROI should be split
        v_split[tj], ipack = two_comps(mov[:, ypix0 * Lxc + xpix0], lam0, Th2)
        if v_split[tj] > 1.25:
            lam0, xp, active_frames = ipack
            tproj[active_frames] = xp
            ix = lam0 > lam0.max() / 5
            xpix0 = xpix0[ix]
            ypix0 = ypix0[ix]
            lam0 = lam0[ix]

        # update residual on raw movie
        mov[np.ix_(active_frames, ypix0*Lxc+ xpix0)] -= tproj[active_frames][:,np.newaxis] * lam0
        # update filtered movie
        ys, xs, lms = multiscale_mask(ypix0,xpix0,lam0, Lyp, Lxp)
        for j in range(nscales):
            movu[j][np.ix_(active_frames, xs[j]+Lxp[j]*ys[j])] -= np.outer(tproj[active_frames], lms[j])
            Mx = movu[j][:,xs[j]+Lxp[j]*ys[j]]
            V1[j][ys[j], xs[j]] = (Mx**2 * np.float32(Mx>Th2)).sum(axis=0)**.5

        stats.append({
            'ypix': ypix0 + yrange[0],
            'xpix': xpix0 + xrange[0],
            'lam': lam0 * sdmov[ypix0, xpix0],
            'footprint': ihop[tj]
        })

        if tj % 1000 == 0:
            print('%d ROIs, score=%2.2f' % (tj, v_max[tj]))
    
    new_ops = {
        'max_proj': max_proj,
        'Vmax': v_max,
        'ihop': ihop,
        'Vsplit': v_split,
        'Vcorr': v_corr,
        'Vmap': v_map,
        'spatscale_pix': spatscale_pix,
    }

    return new_ops, stats
コード例 #28
0
    def _calculate_phase_change_level(
        self,
        wet_bulb_temp: ndarray,
        wb_integral: ndarray,
        orography: ndarray,
        max_nbhood_orog: ndarray,
        land_sea_data: ndarray,
        heights: ndarray,
        height_points: ndarray,
        highest_height: float,
    ) -> ndarray:
        """
        Calculate phase change level and fill in missing points

        .. See the documentation for a more detailed discussion of the steps.
        .. include:: extended_documentation/psychrometric_calculations/
           psychrometric_calculations/_calculate_phase_change_level.rst

        Args:
            wet_bulb_temp:
                Wet bulb temperature data
            wb_integral:
                Wet bulb temperature integral
            orography:
                Orography heights
            max_nbhood_orog:
                Maximum orography height in neighbourhood (used to determine points that
                can be used for interpolation)
            land_sea_data:
                Mask of binary land / sea data
            heights:
                All heights of wet bulb temperature input
            height_points:
                Heights on wet bulb temperature integral slice
            highest_height:
                Height of the highest level to which the wet bulb
                temperature has been integrated

        Returns:
            Level at which phase changes
        """
        phase_change_data = self.find_falling_level(wb_integral, orography,
                                                    height_points)

        # Fill in missing data
        self.fill_in_high_phase_change_falling_levels(phase_change_data,
                                                      orography,
                                                      wb_integral.max(axis=0),
                                                      highest_height)
        self.fill_in_sea_points(
            phase_change_data,
            land_sea_data,
            wb_integral.max(axis=0),
            wet_bulb_temp,
            heights,
        )

        # Any unset points at this stage are set to np.nan; these will be
        # lands points where the phase-change-level is below the orography.
        # These can be filled by optional horizontal interpolation.
        if self.horizontal_interpolation:
            phase_change_data = self._horizontally_interpolate_phase(
                phase_change_data, orography, max_nbhood_orog)

        # Mask any points that are still set to np.nan; this should be no
        # points if horizontal interpolation has been used.
        phase_change_data = np.ma.masked_invalid(phase_change_data)

        return phase_change_data
コード例 #29
0
ファイル: fonts.py プロジェクト: oneengineer/cogai
def normalize_0_1(img: np.ndarray):
    assert img.ndim == 2
    m = img.max()
    if m < IMG_EPS:
        return np.ones_like(img, dtype=np.float) * IMG_EPS
    return img / m
コード例 #30
0
 def norm_by_pic(pic: np.ndarray) -> np.ndarray:
     """
     Normalize the image based on it's own values
     """
     pic = (pic - pic.min()) / (pic.max() - pic.min())
     return pic
コード例 #31
0
ファイル: mlnb_model.py プロジェクト: AlexFridman/code_temp
 def scale(x: np.ndarray) -> np.ndarray:
     max_x = x.max()
     min_x = x.min()
     return (x - min_x) / (max_x - min_x)
コード例 #32
0
ファイル: utils.py プロジェクト: justindujardin/fragile
def statistics_from_array(x: numpy.ndarray):
    """Return the (mean, std, max, min) of an array."""
    try:
        return x.mean(), x.std(), x.max(), x.min()
    except AttributeError:
        return numpy.nan, numpy.nan, numpy.nan, numpy.nan
コード例 #33
0
ファイル: shaders.py プロジェクト: vanyakosmos/rep-counter
def norm_img(img: np.ndarray):
    img = (img - img.min()) / (img.max() - img.min())
    img *= 255
    img = img.astype(np.uint8)
    return img
コード例 #34
0
ファイル: find_edge_model.py プロジェクト: matosjr/realadv
def cw_loss_vec(score: np.ndarray, label):
    score = score.copy()
    t = score[label]
    score[label] = float('-inf')
    return t - score.max()
コード例 #35
0
ファイル: graph.py プロジェクト: tfgraph/tfgraph
    def __init__(self,
                 sess: tf.Session,
                 name: str,
                 writer: tf.summary.FileWriter = None,
                 edges_np: np.ndarray = None,
                 n: int = None,
                 is_sparse: bool = False) -> None:
        """ Class Constructor of the Graph

    This method is called to construct a Graph object. This block of code
    initializes all the variables necessaries for this class to properly works.

    This class can be initialized using an edge list, that fill the graph at
    this moment, or can be construct it from the cardinality of vertices set
    given by `n` parameter.

    Args:
      sess (:obj:`tf.Session`): This attribute represents the session that runs
        the TensorFlow operations.
      name (str): This attribute represents the name of the object in
        TensorFlow's op Graph.
      writer (:obj:`tf.summary.FileWriter`, optional): This attribute represents
        a TensorFlow's Writer, that is used to obtain stats. The default value
        is `None`.
      edges_np (:obj:`np.ndarray`, optional): The edge set of the graph codifies
        as `edges_np[:,0]` represents the sources and `edges_np[:,1]` the
        destinations of the edges. The default value is `None`.
      n (int, optional): Represents the cardinality of the vertex set. The
        default value is `None`.
      is_sparse (bool, optional): Use sparse Tensors if it's set to `True`. The
        default value is False` Not implemented yet. Show the Todo for more
        information.

    Todo:
      * Implement variables as sparse when it's possible. Waiting to
        TensorFlow for it.

    """
        TensorFlowObject.__init__(self, sess, name, writer, is_sparse)
        UpdateEdgeNotifier.__init__(self)

        if edges_np is not None:
            if n is not None:
                self.n = max(n, int(edges_np.max(axis=0).max() + 1))
            else:
                self.n = int(edges_np.max(axis=0).max() + 1)
            self.m = int(edges_np.shape[0])
            A_init = tf.scatter_nd(edges_np.tolist(), self.m * [1.0],
                                   [self.n, self.n])
        elif n is not None:
            self.n = n
            self.m = 0
            A_init = tf.zeros([self.n, self.n])
        else:
            raise ValueError('Graph constructor must be have edges or n')

        self.n_tf = tf.Variable(float(self.n),
                                tf.float32,
                                name=self.name + "_n")
        self.A_tf = tf.Variable(A_init, tf.float64, name=self.name + "_A")
        self.out_degrees_tf = tf.Variable(tf.reduce_sum(self.A_tf,
                                                        1,
                                                        keep_dims=True),
                                          name=self.name + "_d_out")
        self.in_degrees_tf = tf.Variable(tf.reduce_sum(self.A_tf,
                                                       0,
                                                       keep_dims=True),
                                         name=self.name + "_d_in")
        self.run_tf(tf.variables_initializer([self.A_tf, self.n_tf]))
        self.run_tf(
            tf.variables_initializer([self.out_degrees_tf,
                                      self.in_degrees_tf]))
コード例 #36
0
def check_bandit_feedback_inputs(
    context: np.ndarray,
    action: np.ndarray,
    reward: np.ndarray,
    position: Optional[np.ndarray] = None,
    pscore: Optional[np.ndarray] = None,
    action_context: Optional[np.ndarray] = None,
) -> Optional[AssertionError]:
    """Check inputs for bandit learning or simulation.

    Parameters
    -----------
    context: array-like, shape (n_rounds, dim_context)
        Context vectors in each round, i.e., :math:`x_t`.

    action: array-like, shape (n_rounds,)
        Sampled (realized) actions by behavior policy in each round, i.e., :math:`a_t`.

    reward: array-like, shape (n_rounds,)
        Observed rewards (or outcome) in each round, i.e., :math:`r_t`.

    position: array-like, shape (n_rounds,), default=None
        Positions of each round in the given logged bandit feedback.

    pscore: array-like, shape (n_rounds,), default=None
        Propensity scores, the probability of selecting each action by behavior policy,
        in the given logged bandit feedback.

    action_context: array-like, shape (n_actions, dim_action_context)
        Context vectors characterizing each action.

    """
    assert isinstance(context, np.ndarray), "context must be ndarray"
    assert context.ndim == 2, "context must be 2-dimensional"
    assert isinstance(action, np.ndarray), "action must be ndarray"
    assert action.ndim == 1, "action must be 1-dimensional"
    assert isinstance(reward, np.ndarray), "reward must be ndarray"
    assert reward.ndim == 1, "reward must be 1-dimensional"

    if pscore is not None:
        assert isinstance(pscore, np.ndarray), "pscore must be ndarray"
        assert pscore.ndim == 1, "pscore must be 1-dimensional"
        assert (context.shape[0] == action.shape[0] == reward.shape[0] ==
                pscore.shape[0]
                ), "context, action, reward, and pscore must be the same size."
    if position is not None:
        assert isinstance(position, np.ndarray), "position must be ndarray"
        assert position.ndim == 1, "position must be 1-dimensional"
        assert (
            context.shape[0] == action.shape[0] == reward.shape[0] ==
            position.shape[0]
        ), "context, action, reward, and position must be the same size."
    else:
        assert (context.shape[0] == action.shape[0] == reward.shape[0]
                ), "context, action, and reward must be the same size."
    if action_context is not None:
        assert isinstance(action_context,
                          np.ndarray), "action_context must be ndarray"
        assert action_context.ndim == 2, "action_context must be 2-dimensional"
        assert (action.max() + 1) == action_context.shape[
            0], "the number of action and the size of the first dimension of action_context must be same."
コード例 #37
0
    def fit(
        self,
        context: np.ndarray,
        action: np.ndarray,
        reward: np.ndarray,
        pscore_cascade: np.ndarray,
        evaluation_policy_pscore_cascade: np.ndarray,
        evaluation_policy_action_dist: np.ndarray,
    ):
        """Fit the regression model on given logged bandit data.

        Parameters
        ----------
        context: array-like, shape (n_rounds, dim_context)
            Context vectors observed for each data, i.e., :math:`x_i`.

        action: array-like, (n_rounds * len_list,)
            Actions observed at each slot in a ranking/slate in logged bandit data, i.e., :math:`a_{i}(l)`,
            which is chosen by the behavior policy :math:`\\pi_b`.

        reward: array-like, shape (n_rounds * len_list,)
            Slot-level rewards observed for each data in logged bandit data, i.e., :math:`r_{i}(l)`.

        pscore_cascade: array-like, shape (n_rounds * len_list,)
            Joint probabilities of behavior policy choosing a particular sequence of actions from the top position to the :math:`l`-th position (:math:`a_{1:l}`).

        evaluation_policy_pscore_cascade: array-like, shape (n_rounds * len_list,)
            Joint probabilities of evaluation policy choosing a particular sequence of actions from the top position to the :math:`l`-th position (:math:`a_{1:l}`). This type of action choice probabilities corresponds to the cascade model.

        evaluation_policy_action_dist: array-like (n_rounds * len_list * n_unique_actions, )
            Plackett-luce style action distribution induced by evaluation policy
            (action choice probabilities at each slot given previous action choices)
            , i.e., :math:`\\pi_e({a'}_t(k) | x_i, a_i(1), \\ldots, a_i(l-1)) \\forall {a'}_t(k) \\in \\mathcal{A}`.

        """
        check_array(array=context, name="context", expected_dim=2)
        check_array(array=action, name="action", expected_dim=1)
        check_array(array=reward, name="reward", expected_dim=1)
        check_array(array=pscore_cascade,
                    name="pscore_cascade",
                    expected_dim=1)
        check_array(
            array=evaluation_policy_pscore_cascade,
            name="evaluation_policy_pscore_cascade",
            expected_dim=1,
        )
        check_array(
            array=evaluation_policy_action_dist,
            name="evaluation_policy_action_dist",
            expected_dim=1,
        )
        if not (action.shape == reward.shape == pscore_cascade.shape ==
                evaluation_policy_pscore_cascade.shape ==
                (context.shape[0] * self.len_list, )):
            raise ValueError(
                "Expected `action.shape == reward.shape == pscore_cascade.shape == evaluation_policy_pscore_cascade.shape"
                " == (context.shape[0] * len_list, )`"
                ", but found it False")
        if evaluation_policy_action_dist.shape != (
                context.shape[0] * self.len_list * self.n_unique_action, ):
            raise ValueError(
                "Expected `evaluation_policy_action_dist.shape == (context.shape[0] * len_list * n_unique_action, )`"
                ", but found it False")
        if not (np.issubdtype(action.dtype, np.integer) and action.min() >= 0
                and action.max() < self.n_unique_action):
            raise ValueError(
                "`action` elements must be integers in the range of [0, n_unique_action)"
            )
        if np.any(pscore_cascade <= 0) or np.any(pscore_cascade > 1):
            raise ValueError("`pscore_cascade` must be in the range of (0, 1]")
        if np.any(evaluation_policy_pscore_cascade <= 0) or np.any(
                evaluation_policy_pscore_cascade > 1):
            raise ValueError(
                "`evaluation_policy_pscore_cascade` must be in the range of (0, 1]"
            )
        if not np.allclose(
                np.ones(
                    evaluation_policy_action_dist.reshape(
                        (-1, self.n_unique_action)).shape[0]),
                evaluation_policy_action_dist.reshape(
                    (-1, self.n_unique_action)).sum(axis=1),
        ):
            raise ValueError(
                "`evaluation_policy_action_dist[i * n_unique_action : (i+1) * n_unique_action]` "
                "must sum up to one for all i.")
        # (n_rounds_ * len_list, ) -> (n_rounds_, len_list)
        action = action.reshape((-1, self.len_list))
        reward = reward.reshape((-1, self.len_list))
        iw = (evaluation_policy_pscore_cascade / pscore_cascade).reshape(
            (-1, self.len_list))

        # (n_rounds_, )
        n_rounds_ = len(action)
        sample_weight = np.ones(n_rounds_)

        for pos_ in range(self.len_list)[::-1]:
            X, y = self._preprocess_for_reg_model(
                context=context,
                action=action,
                reward=reward,
                evaluation_policy_action_dist=evaluation_policy_action_dist,
                position_=pos_,
            )

            if self.fitting_method == "iw":
                sample_weight = iw[:, pos_]

            self.base_model_list[pos_].fit(X, y, sample_weight=sample_weight)
コード例 #38
0
 def _update_lims(self, X: numpy.ndarray):
     """Update the x axis boundaries to match the values of the plotted distribution."""
     self.xlim = (X.min(), X.max())
コード例 #39
0
ファイル: Program.py プロジェクト: nomadedge/Isat
def nrmse(actual: numpy.ndarray, predicted: numpy.ndarray):
    return rmse(actual, predicted) / (actual.max() - actual.min())
コード例 #40
0
def decode_mst(energy: numpy.ndarray,
               length: int,
               has_labels: bool = True) -> Tuple[numpy.ndarray, numpy.ndarray]:
    """
    Decode the optimal MST tree with the Chu-Liu-Edmonds algorithm for
    minimum spanning arboresences on graphs.

    Parameters
    ----------
    energy : ``numpy.ndarray``, required.
        A tensor with shape (num_labels, timesteps, timesteps)
        containing the energy of each edge. If has_labels is ``False``,
        the tensor should have shape (timesteps, timesteps) instead.
    length : ``int``, required.
        The length of this sequence, as the energy may have come
        from a padded batch.
    has_labels : ``bool``, optional, (default = True)
        Whether the graph has labels or not.
    """
    if has_labels and energy.ndim != 3:
        raise ConfigurationError(
            "The dimension of the energy array is not equal to 3.")
    elif not has_labels and energy.ndim != 2:
        raise ConfigurationError(
            "The dimension of the energy array is not equal to 2.")
    input_shape = energy.shape
    max_length = input_shape[-1]

    # Our energy matrix might have been batched -
    # here we clip it to contain only non padded tokens.
    if has_labels:
        energy = energy[:, :length, :length]
        # get best label for each edge.
        label_id_matrix = energy.argmax(axis=0)
        energy = energy.max(axis=0)
    else:
        energy = energy[:length, :length]
        label_id_matrix = None
    # get original score matrix
    original_score_matrix = energy
    # initialize score matrix to original score matrix
    score_matrix = numpy.array(original_score_matrix, copy=True)

    old_input = numpy.zeros([length, length], dtype=numpy.int32)
    old_output = numpy.zeros([length, length], dtype=numpy.int32)
    current_nodes = [True for _ in range(length)]
    representatives: List[Set[int]] = []

    for node1 in range(length):
        original_score_matrix[node1, node1] = 0.0
        score_matrix[node1, node1] = 0.0
        representatives.append({node1})

        for node2 in range(node1 + 1, length):
            old_input[node1, node2] = node1
            old_output[node1, node2] = node2

            old_input[node2, node1] = node2
            old_output[node2, node1] = node1

    final_edges: Dict[int, int] = {}

    # The main algorithm operates inplace.
    chu_liu_edmonds(length, score_matrix, current_nodes, final_edges,
                    old_input, old_output, representatives)

    heads = numpy.zeros([max_length], numpy.int32)
    if has_labels:
        head_type = numpy.ones([max_length], numpy.int32)
        # Set the head type of the symbolic head to be zero, arbitrarily.
        head_type[0] = 0
    else:
        head_type = None

    for child, parent in final_edges.items():
        heads[child] = parent
        if has_labels and child != 0:
            head_type[child] = label_id_matrix[parent, child]

    # Set the head of the symbolic head to be zero, arbitrarily.
    heads[0] = 0

    return heads, head_type
コード例 #41
0
def determine_topology_parameters(
    trace_length_array: np.ndarray,
    area: float,
    branches_defined: bool,
    correct_mauldon: bool,
    node_counts: Optional[Dict[str, Number]],
    branch_length_array: Optional[np.ndarray],
) -> Dict[str, float]:
    """
    Determine geometric (and topological) parameters.

    Number of traces (and branches) are determined by node counting.

    The passed ``trace_length_array`` should be non-weighted.
    """
    radius = np.sqrt(area / np.pi)
    if len(trace_length_array) == 0:
        min_length_traces, max_length_traces, characteristic_length_traces = (
            0.0,
            0.0,
            0.0,
        )
    else:
        min_length_traces, max_length_traces, characteristic_length_traces = (
            float(trace_length_array.min()),
            float(trace_length_array.max()),
            float(trace_length_array.mean()),
        )
    assert isinstance(min_length_traces, float), type(min_length_traces)
    assert isinstance(max_length_traces, float)
    assert isinstance(characteristic_length_traces, float)
    # characteristic_length_traces = (
    #     trace_length_array.mean() if len(trace_length_array) > 0 else 0.0
    # )
    fracture_intensity = trace_length_array.sum() / area
    dimensionless_intensity_traces = fracture_intensity * characteristic_length_traces

    # Collect parameters that do not require topology determination into a dict
    params_without_topology = {
        Param.FRACTURE_INTENSITY_B21.value.name: fracture_intensity,
        Param.FRACTURE_INTENSITY_P21.value.name: fracture_intensity,
        Param.TRACE_MIN_LENGTH.value.name: min_length_traces,
        Param.TRACE_MAX_LENGTH.value.name: max_length_traces,
        Param.TRACE_MEAN_LENGTH.value.name: characteristic_length_traces,
        Param.DIMENSIONLESS_INTENSITY_P22.value.name: dimensionless_intensity_traces,
        Param.AREA.value.name: area,
        Param.NUMBER_OF_TRACES_TRUE.value.name: len(trace_length_array),
    }

    if not branches_defined:
        # Return dict with nans in place of topological parameters
        nan_dict = {
            param.value.name: np.nan
            for param in Param
            if param.value.name not in params_without_topology
        }
        all_params_without_topo = {**params_without_topology, **nan_dict}
        assert all(param.value.name in all_params_without_topo for param in Param)

        return all_params_without_topo

    assert isinstance(trace_length_array, np.ndarray)
    assert isinstance(node_counts, dict)
    assert isinstance(area, float)

    if branch_length_array is None:
        raise ValueError("Expected branch_length_array to not be None.")

    number_of_traces = (node_counts[Y_node] + node_counts[I_node]) / 2

    # Handle empty branch length array
    if len(branch_length_array) == 0:
        min_length_branches, max_length_branches = 0.0, 0.0
    else:
        min_length_branches, max_length_branches = (
            float(branch_length_array.min()),
            float(branch_length_array.max()),
        )
    aerial_frequency_traces = number_of_traces / area
    number_of_branches = (
        (node_counts[X_node] * 4) + (node_counts[Y_node] * 3) + node_counts[I_node]
    ) / 2
    aerial_frequency_branches = number_of_branches / area
    characteristic_length_branches = (
        (trace_length_array.sum() / number_of_branches)
        if number_of_branches > 0
        else 0.0
    )
    dimensionless_intensity_branches = (
        fracture_intensity * characteristic_length_branches
    )
    connections_per_trace = (
        (2 * (node_counts[Y_node] + node_counts[X_node]) / number_of_traces)
        if number_of_traces > 0
        else 0.0
    )
    connections_per_branch = (
        (3 * node_counts[Y_node] + 4 * node_counts[X_node]) / number_of_branches
        if number_of_branches > 0
        else 0.0
    )
    if correct_mauldon:
        trace_mean_length_mauldon = (
            (
                ((np.pi * radius) / 2)
                * (node_counts[E_node] / (node_counts[I_node] + node_counts[Y_node]))
            )
            if node_counts[I_node] + node_counts[Y_node] > 0
            else 0.0
        )
        fracture_density_mauldon = (node_counts[I_node] + node_counts[Y_node]) / (
            area * 2
        )
        fracture_intensity_mauldon = (
            trace_mean_length_mauldon * fracture_density_mauldon
        )
    else:
        # If Network target area is not circular mauldon parameters cannot be
        # determined.
        (
            trace_mean_length_mauldon,
            fracture_density_mauldon,
            fracture_intensity_mauldon,
        ) = (np.nan, np.nan, np.nan)
    connection_frequency = (node_counts[Y_node] + node_counts[X_node]) / area

    params_with_topology = {
        Param.NUMBER_OF_TRACES.value.name: number_of_traces,
        Param.BRANCH_MIN_LENGTH.value.name: min_length_branches,
        Param.BRANCH_MAX_LENGTH.value.name: max_length_branches,
        Param.BRANCH_MEAN_LENGTH.value.name: characteristic_length_branches,
        Param.AREAL_FREQUENCY_B20.value.name: aerial_frequency_branches,
        Param.AREAL_FREQUENCY_P20.value.name: aerial_frequency_traces,
        Param.DIMENSIONLESS_INTENSITY_B22.value.name: dimensionless_intensity_branches,
        Param.CONNECTIONS_PER_TRACE.value.name: connections_per_trace,
        Param.CONNECTIONS_PER_BRANCH.value.name: connections_per_branch,
        Param.FRACTURE_INTENSITY_MAULDON.value.name: fracture_intensity_mauldon,
        Param.FRACTURE_DENSITY_MAULDON.value.name: fracture_density_mauldon,
        Param.TRACE_MEAN_LENGTH_MAULDON.value.name: trace_mean_length_mauldon,
        Param.CONNECTION_FREQUENCY.value.name: connection_frequency,
        Param.NUMBER_OF_BRANCHES.value.name: number_of_branches,
        Param.NUMBER_OF_BRANCHES_TRUE.value.name: len(branch_length_array),
    }

    all_parameters = {**params_without_topology, **params_with_topology}
    assert len(all_parameters) == sum(
        [len(params_without_topology), len(params_with_topology)]
    )
    assert all(
        param in {param.value.name for param in Param} for param in all_parameters
    )

    return all_parameters
コード例 #42
0
def print_it(a: np.ndarray, name: str = ''):
    print(name, a.shape, a.dtype, a.min(), a.mean(), a.max())
コード例 #43
0
def scale(x: np.ndarray):
    mean_x = x.mean()
    max_x = x.max()
    min_x = x.min()
    return (x - min_x) / (max_x - min_x)