def get_rotations(num_rotations, max_absolute_rotation_angle_deg):
    """Creates an array of rotation angles.

    These angles are meant for use in `rotate_radar_images`.

    N = number of rotations

    :param num_rotations: Number of rotations.  Image will be rotated only in
        the xy-plane (about the z-axis).
    :param max_absolute_rotation_angle_deg: Max absolute rotation angle
        (degrees).  In general, the image will be rotated both clockwise and
        counterclockwise, up to this angle.
    :return: ccw_rotation_angles_deg: length-N numpy array of counterclockwise
        rotation angles (degrees).
    """

    error_checking.assert_is_integer(num_rotations)
    if num_rotations == 0:
        return numpy.array([], dtype=float)

    error_checking.assert_is_greater(num_rotations, 0)
    error_checking.assert_is_geq(max_absolute_rotation_angle_deg,
                                 MIN_ABSOLUTE_ROTATION_ANGLE_DEG)
    error_checking.assert_is_leq(max_absolute_rotation_angle_deg,
                                 MAX_ABSOLUTE_ROTATION_ANGLE_DEG)

    absolute_rotation_angles_deg = numpy.random.uniform(
        low=1., high=max_absolute_rotation_angle_deg, size=num_rotations)

    possible_signs = numpy.array([-1, 1], dtype=int)
    return absolute_rotation_angles_deg * numpy.random.choice(
        possible_signs, size=num_rotations, replace=True)
def sia_for_closed_polygon(
        polygon_object,
        num_vertices_in_half_window=NUM_VERTICES_IN_HALF_WINDOW_DEFAULT,
        num_iterations=NUM_ITERATIONS_DEFAULT, check_input_args=True):
    """Implements the SIA algorithm for a closed polygon.

    This method smooths only the exterior of the polygon, ignoring the interior
    (holes).

    V = number of exterior vertices

    :param polygon_object: Instance of `shapely.geometry.Polygon`.
    :param num_vertices_in_half_window: Number of vertices in smoothing half-
        window.  Number of vertices in full window =
        2 * num_vertices_in_half_window + 1.
    :param num_iterations: Number of iterations.
    :param check_input_args: Boolean flag.  If True, will error-check input
        arguments.  If False, will not.
    :return: vertex_x_coords_smoothed: length-V numpy array with smoothed
        x-coordinates of vertices.
    :return: vertex_y_coords_smoothed: length-V numpy array with smoothed
        y-coordinates of vertices.
    """

    num_vertices = len(polygon_object.exterior.xy[0]) - 1

    if check_input_args:
        error_checking.assert_is_geq(
            num_vertices, MIN_VERTICES_IN_POLYGON_OR_LINE)
        error_checking.assert_is_integer(num_vertices_in_half_window)
        error_checking.assert_is_geq(num_vertices_in_half_window, 1)
        error_checking.assert_is_integer(num_iterations)
        error_checking.assert_is_geq(num_iterations, 1)

    num_vertices_in_half_window = numpy.min(
        numpy.array([num_vertices_in_half_window, num_vertices - 1]))

    for i in range(num_iterations):
        if i == 0:
            this_polygon_object = copy.deepcopy(polygon_object)
        else:
            this_polygon_object = polygons.vertex_arrays_to_polygon_object(
                vertex_x_coords_smoothed, vertex_y_coords_smoothed)

        vertex_x_coords_padded, vertex_y_coords_padded = (
            shape_utils.pad_closed_polygon(
                this_polygon_object,
                num_padding_vertices=num_vertices_in_half_window,
                check_input_args=False))

        vertex_x_coords_smoothed, vertex_y_coords_smoothed = _sia_one_iteration(
            vertex_x_coords_padded, vertex_y_coords_padded,
            num_vertices_in_half_window)

    vertex_x_coords_smoothed = numpy.concatenate((
        vertex_x_coords_smoothed, numpy.array([vertex_x_coords_smoothed[0]])))
    vertex_y_coords_smoothed = numpy.concatenate((
        vertex_y_coords_smoothed, numpy.array([vertex_y_coords_smoothed[0]])))

    return vertex_x_coords_smoothed, vertex_y_coords_smoothed
def get_noisings(num_noisings, max_standard_deviation):
    """Creates an array of standard deviations for Gaussian noising.

    These standard deviations are meant for use in `noise_radar_images`.

    N = number of noisings

    :param num_noisings: Number of times to noise the image.
    :param max_standard_deviation: Max standard deviation of Gaussian noise.
    :return: standard_deviations: length-N numpy array of standard deviations.
    """

    error_checking.assert_is_integer(num_noisings)
    if num_noisings == 0:
        return numpy.array([], dtype=float)

    error_checking.assert_is_greater(num_noisings, 0)
    error_checking.assert_is_geq(max_standard_deviation,
                                 MIN_NOISE_STANDARD_DEVIATION)
    error_checking.assert_is_leq(max_standard_deviation,
                                 MAX_NOISE_STANDARD_DEVIATION)

    return numpy.random.uniform(low=0.,
                                high=max_standard_deviation,
                                size=num_noisings)
示例#4
0
def do_2d_pooling(feature_matrix,
                  stride_length_px=2,
                  pooling_type_string=MAX_POOLING_TYPE_STRING):
    """Pools 2-D feature maps.

    m = number of rows after pooling
    n = number of columns after pooling

    :param feature_matrix: Input feature maps (numpy array).  Dimensions must be
        M x N x C or 1 x M x N x C.
    :param stride_length_px: Stride length (pixels).  The pooling window will
        move by this many rows or columns at a time as it slides over each input
        feature map.
    :param pooling_type_string: Pooling type (must be accepted by
        `_check_pooling_type`).
    :return: feature_matrix: Output feature maps (numpy array).  Dimensions will
        be 1 x m x n x C.
    """

    error_checking.assert_is_numpy_array_without_nan(feature_matrix)
    error_checking.assert_is_integer(stride_length_px)
    error_checking.assert_is_geq(stride_length_px, 2)
    _check_pooling_type(pooling_type_string)

    if len(feature_matrix.shape) == 3:
        feature_matrix = numpy.expand_dims(feature_matrix, axis=0)
    error_checking.assert_is_numpy_array(feature_matrix, num_dimensions=4)

    feature_tensor = K.pool2d(x=K.variable(feature_matrix),
                              pool_mode=pooling_type_string,
                              pool_size=(stride_length_px, stride_length_px),
                              strides=(stride_length_px, stride_length_px),
                              padding='valid',
                              data_format='channels_last')
    return feature_tensor.eval(session=K.get_session())
示例#5
0
def subset_by_time(example_dict, first_time_unix_sec, last_time_unix_sec):
    """Subsets examples by time.

    :param example_dict: Dictionary of examples (in the format returned by
        `example_io.read_file`).
    :param first_time_unix_sec: Earliest time to keep.
    :param last_time_unix_sec: Latest time to keep.
    :return: example_dict: Same as input but with fewer examples.
    :return: example_indices: 1-D numpy array with indices of examples kept.
    """

    error_checking.assert_is_integer(first_time_unix_sec)
    error_checking.assert_is_integer(last_time_unix_sec)
    error_checking.assert_is_geq(last_time_unix_sec, first_time_unix_sec)

    good_indices = numpy.where(
        numpy.logical_and(
            example_dict[VALID_TIMES_KEY] >= first_time_unix_sec,
            example_dict[VALID_TIMES_KEY] <= last_time_unix_sec))[0]

    for this_key in ONE_PER_EXAMPLE_KEYS:
        if isinstance(example_dict[this_key], list):
            example_dict[this_key] = [
                example_dict[this_key][k] for k in good_indices
            ]
        else:
            example_dict[this_key] = (example_dict[this_key][good_indices,
                                                             ...])

    return example_dict, good_indices
示例#6
0
def find_file(directory_name, year, raise_error_if_missing=True):
    """Finds NetCDF file with RRTM data.

    :param directory_name: Name of directory where file is expected.
    :param year: Year (integer).
    :param raise_error_if_missing: Boolean flag.  If file is missing and
        `raise_error_if_missing == True`, will throw error.  If file is missing
        and `raise_error_if_missing == False`, will return *expected* file path.
    :return: rrtm_file_name: File path.
    :raises: ValueError: if file is missing
        and `raise_error_if_missing == True`.
    """

    error_checking.assert_is_string(directory_name)
    error_checking.assert_is_integer(year)
    error_checking.assert_is_boolean(raise_error_if_missing)

    rrtm_file_name = '{0:s}/rrtm_output_{1:04d}.nc'.format(
        directory_name, year)

    if raise_error_if_missing and not os.path.isfile(rrtm_file_name):
        error_string = 'Cannot find file.  Expected at: "{0:s}"'.format(
            rrtm_file_name)
        raise ValueError(error_string)

    return rrtm_file_name
示例#7
0
def create_histogram(input_values, num_bins, min_value, max_value):
    """Creates a histogram with uniform bin-spacing.

    N = number of input values
    K = number of bins

    :param input_values: length-N numpy array of input values (to be binned).
    :param num_bins: Number of bins.
    :param min_value: Minimum value to include in histogram.  Any input value <
        `min_value` will be assigned to the first bin.
    :param max_value: Maximum value to include in histogram.  Any input value >
        `max_value` will be assigned to the last bin.
    :return: input_to_bin_indices: length-N numpy array of bin indices.  If
        input_values[i] = j, the [i]th input value belongs in the [j]th bin.
    :return: num_examples_by_bin: length-K numpy array, where the [j]th value is
        the number of inputs assigned to the [j]th bin.
    """

    error_checking.assert_is_numpy_array_without_nan(input_values)
    error_checking.assert_is_numpy_array(input_values, num_dimensions=1)
    error_checking.assert_is_integer(num_bins)
    error_checking.assert_is_geq(num_bins, 2)
    error_checking.assert_is_greater(max_value, min_value)

    bin_cutoffs = numpy.linspace(min_value, max_value, num=num_bins + 1)
    input_to_bin_indices = numpy.digitize(
        input_values, bin_cutoffs, right=False) - 1
    input_to_bin_indices[input_to_bin_indices < 0] = 0
    input_to_bin_indices[input_to_bin_indices > num_bins - 1] = num_bins - 1

    num_examples_by_bin = numpy.full(num_bins, -1, dtype=int)
    for j in range(num_bins):
        num_examples_by_bin[j] = numpy.sum(input_to_bin_indices == j)

    return input_to_bin_indices, num_examples_by_bin
示例#8
0
def _check_input_args(num_iterations,
                      learning_rate,
                      l2_weight=None,
                      radar_constraint_weight=None,
                      minmax_constraint_weight=None,
                      ideal_activation=None):
    """Error-checks input args for backwards optimization.

    :param num_iterations: See doc for `_do_gradient_descent`.
    :param learning_rate: Same.
    :param l2_weight: Same.
    :param radar_constraint_weight: Weight used to multiply part of loss
        function with radar constraints (see doc for
        `_radar_constraints_to_loss_fn`).
    :param minmax_constraint_weight: Weight used to multiply part of loss
        function with min-max constraints (see doc for
        `_minmax_constraints_to_loss_fn`).
    :param ideal_activation: See doc for `optimize_input_for_neuron_activation`
        or `optimize_input_for_channel_activation`.
    """

    error_checking.assert_is_integer(num_iterations)
    error_checking.assert_is_greater(num_iterations, 0)
    error_checking.assert_is_greater(learning_rate, 0.)
    error_checking.assert_is_less_than(learning_rate, 1.)

    if l2_weight is not None:
        error_checking.assert_is_greater(l2_weight, 0.)
    if radar_constraint_weight is not None:
        error_checking.assert_is_greater(radar_constraint_weight, 0.)
    if minmax_constraint_weight is not None:
        error_checking.assert_is_greater(minmax_constraint_weight, 0.)

    if ideal_activation is not None:
        error_checking.assert_is_greater(ideal_activation, 0.)
示例#9
0
def find_processed_file(directory_name, year, raise_error_if_missing=True):
    """Finds processed file with tornado reports.

    See `write_processed_file` for the definition of a "processed file".

    :param directory_name: Name of directory.
    :param year: Year (integer).
    :param raise_error_if_missing: Boolean flag.  If file is missing and
        raise_error_if_missing = True, this method will error out.
    :return: processed_file_name: Path to file.  If file is missing and
        raise_error_if_missing = True, this will be the *expected* path.
    :raises: ValueError: if file is missing and raise_error_if_missing = True.
    """

    error_checking.assert_is_string(directory_name)
    error_checking.assert_is_integer(year)
    error_checking.assert_is_boolean(raise_error_if_missing)

    processed_file_name = '{0:s}/tornado_reports_{1:04d}.csv'.format(
        directory_name, year)

    if raise_error_if_missing and not os.path.isfile(processed_file_name):
        error_string = (
            'Cannot find processed file with tornado reports.  Expected at: '
            '{0:s}').format(processed_file_name)
        raise ValueError(error_string)

    return processed_file_name
示例#10
0
def _check_args_one_step(predictor_matrix, permuted_flag_matrix,
                         scalar_channel_flags, shuffle_profiles_together,
                         num_bootstrap_reps):
    """Checks input args for `run_*_test_one_step`.

    :param predictor_matrix: See doc for `run_forward_test_one_step` or
        `run_backwards_test_one_step`.
    :param permuted_flag_matrix: Same.
    :param scalar_channel_flags: Same.
    :param shuffle_profiles_together: Same.
    :param num_bootstrap_reps: Same.
    :return: num_bootstrap_reps: Same as input but maxxed with 1.
    """

    error_checking.assert_is_numpy_array_without_nan(predictor_matrix)
    num_predictor_dim = len(predictor_matrix.shape)
    error_checking.assert_is_geq(num_predictor_dim, 3)
    error_checking.assert_is_leq(num_predictor_dim, 3)

    error_checking.assert_is_boolean_numpy_array(permuted_flag_matrix)
    these_expected_dim = numpy.array(predictor_matrix.shape[1:], dtype=int)
    error_checking.assert_is_numpy_array(permuted_flag_matrix,
                                         exact_dimensions=these_expected_dim)

    error_checking.assert_is_boolean_numpy_array(scalar_channel_flags)
    these_expected_dim = numpy.array([predictor_matrix.shape[-1]], dtype=int)
    error_checking.assert_is_numpy_array(scalar_channel_flags,
                                         exact_dimensions=these_expected_dim)

    error_checking.assert_is_boolean(shuffle_profiles_together)
    error_checking.assert_is_integer(num_bootstrap_reps)

    return numpy.maximum(num_bootstrap_reps, 1)
示例#11
0
def plot_parallels(basemap_object,
                   axes_object,
                   min_latitude_deg=None,
                   max_latitude_deg=None,
                   num_parallels=DEFAULT_NUM_PARALLELS,
                   line_width=DEFAULT_GRID_LINE_WIDTH,
                   line_colour=DEFAULT_GRID_LINE_COLOUR,
                   z_order=DEFAULT_GRID_LINE_Z_ORDER):
    """Plots parallels (grid lines for latitude).

    If `min_latitude_deg` and `max_latitude_deg` are both None, this method will
    take plotting limits from `basemap_object`.

    :param basemap_object: See doc for `plot_countries`.
    :param axes_object: Same.
    :param min_latitude_deg: Minimum latitude for grid lines.
    :param max_latitude_deg: Max latitude for grid lines.
    :param num_parallels: Number of parallels.
    :param line_width: See doc for `plot_countries`.
    :param line_colour: Same.
    :param z_order: Same.
    """

    if min_latitude_deg is None or max_latitude_deg is None:
        min_latitude_deg = basemap_object.llcrnrlat
        max_latitude_deg = basemap_object.urcrnrlat

    error_checking.assert_is_valid_latitude(min_latitude_deg)
    error_checking.assert_is_valid_latitude(max_latitude_deg)
    error_checking.assert_is_greater(max_latitude_deg, min_latitude_deg)

    error_checking.assert_is_integer(num_parallels)
    error_checking.assert_is_geq(num_parallels, 2)

    parallel_spacing_deg = ((max_latitude_deg - min_latitude_deg) /
                            (num_parallels - 1))

    if parallel_spacing_deg < 1.:
        parallel_spacing_deg = number_rounding.round_to_nearest(
            parallel_spacing_deg, 0.1)
    else:
        parallel_spacing_deg = numpy.round(parallel_spacing_deg)

    min_latitude_deg = number_rounding.ceiling_to_nearest(
        min_latitude_deg, parallel_spacing_deg)
    max_latitude_deg = number_rounding.floor_to_nearest(
        max_latitude_deg, parallel_spacing_deg)
    num_parallels = 1 + int(
        numpy.round(
            (max_latitude_deg - min_latitude_deg) / parallel_spacing_deg))
    latitudes_deg = numpy.linspace(min_latitude_deg,
                                   max_latitude_deg,
                                   num=num_parallels)

    basemap_object.drawparallels(latitudes_deg,
                                 color=colour_from_numpy_to_tuple(line_colour),
                                 linewidth=line_width,
                                 labels=[True, False, False, False],
                                 ax=axes_object,
                                 zorder=z_order)
def check_component_metadata(
        component_type_string, target_class=None, layer_name=None,
        neuron_indices=None, channel_index=None):
    """Checks metadata for model component.

    :param component_type_string: Component type (must be accepted by
        `check_component_type`).
    :param target_class: [used only if component_type_string = "class"]
        Target class.  Integer from 0...(K - 1), where K = number of classes.
    :param layer_name:
        [used only if component_type_string = "neuron" or "channel"]
        Name of layer containing neuron or channel.
    :param neuron_indices: [used only if component_type_string = "neuron"]
        1-D numpy array with indices of neuron.
    :param channel_index: [used only if component_type_string = "channel"]
        Index of channel.
    """

    check_component_type(component_type_string)
    if component_type_string == CLASS_COMPONENT_TYPE_STRING:
        error_checking.assert_is_integer(target_class)
        error_checking.assert_is_geq(target_class, 0)

    if component_type_string in [NEURON_COMPONENT_TYPE_STRING,
                                 CHANNEL_COMPONENT_TYPE_STRING]:
        error_checking.assert_is_string(layer_name)

    if component_type_string == NEURON_COMPONENT_TYPE_STRING:
        error_checking.assert_is_integer_numpy_array(neuron_indices)
        error_checking.assert_is_geq_numpy_array(neuron_indices, 0)
        error_checking.assert_is_numpy_array(neuron_indices, num_dimensions=1)

    if component_type_string == CHANNEL_COMPONENT_TYPE_STRING:
        error_checking.assert_is_integer(channel_index)
        error_checking.assert_is_geq(channel_index, 0)
def dimensions_to_grid(num_rows, num_columns):
    """Determines grid from dimensions.

    :param num_rows: Number of rows (unique y-coordinates of grid points).
    :param num_columns: Number of columns (unique x-coordinates of grid points).
    :return: grid_name: Grid name.
    :raises: ValueError: if dimensions do not match a known grid.
    """

    error_checking.assert_is_integer(num_rows)
    error_checking.assert_is_integer(num_columns)
    grid_dimensions = numpy.array([num_rows, num_columns], dtype=int)

    for this_grid_name in NARR_GRID_NAMES:
        these_dimensions = numpy.array(get_grid_dimensions(
            model_name=NARR_MODEL_NAME, grid_name=this_grid_name),
                                       dtype=int)

        if numpy.array_equal(these_dimensions, grid_dimensions):
            return this_grid_name

    for this_grid_name in RUC_GRID_NAMES:
        these_dimensions = numpy.array(get_grid_dimensions(
            model_name=RUC_MODEL_NAME, grid_name=this_grid_name),
                                       dtype=int)

        if numpy.array_equal(these_dimensions, grid_dimensions):
            return this_grid_name

    error_string = 'Cannot find grid with {0:d} rows and {1:d} columns.'.format(
        num_rows, num_columns)
    raise ValueError(error_string)
示例#14
0
def close_frontal_image(ternary_image_matrix, num_iterations=1):
    """Applies binary closing to both warm and cold fronts in image.

    :param ternary_image_matrix: See doc for `_check_frontal_image`.
    :param num_iterations: Number of iterations of binary closing.  The more
        iterations, the more frontal pixels will be created.
    :return: ternary_image_matrix: Same as input, but after closing.
    """

    _check_frontal_image(image_matrix=ternary_image_matrix,
                         assert_binary=False)
    error_checking.assert_is_integer(num_iterations)
    error_checking.assert_is_greater(num_iterations, 0)

    binary_warm_front_matrix = binary_closing(
        (ternary_image_matrix == WARM_FRONT_INTEGER_ID).astype(int),
        structure=STRUCTURE_MATRIX_FOR_BINARY_CLOSING,
        origin=0,
        iterations=num_iterations)
    binary_cold_front_matrix = binary_closing(
        (ternary_image_matrix == COLD_FRONT_INTEGER_ID).astype(int),
        structure=STRUCTURE_MATRIX_FOR_BINARY_CLOSING,
        origin=0,
        iterations=num_iterations)

    ternary_image_matrix[numpy.where(
        binary_warm_front_matrix)] = WARM_FRONT_INTEGER_ID
    ternary_image_matrix[numpy.where(
        binary_cold_front_matrix)] = COLD_FRONT_INTEGER_ID

    return ternary_image_matrix
示例#15
0
def do_3d_pooling(feature_matrix,
                  stride_length_px=2,
                  pooling_type_string=MAX_POOLING_TYPE_STRING):
    """Pools 3-D feature maps.

    :param feature_matrix: Input feature maps (numpy array).  Dimensions must be
        M x N x H x C or 1 x M x N x H x C.
    :param stride_length_px: See doc for `do_2d_pooling`.import tensorflow.python.keras.backend as K
    :param pooling_type_string: Pooling type (must be accepted by
        `_check_pooling_type`).
    :return: feature_matrix: Output feature maps (numpy array).  Dimensions will
        be 1 x m x n x h x C.
    """

    error_checking.assert_is_numpy_array_without_nan(feature_matrix)
    error_checking.assert_is_integer(stride_length_px)
    error_checking.assert_is_geq(stride_length_px, 2)
    _check_pooling_type(pooling_type_string)

    if len(feature_matrix.shape) == 4:
        feature_matrix = numpy.expand_dims(feature_matrix, axis=0)

    error_checking.assert_is_numpy_array(feature_matrix, num_dimensions=5)

    feature_tensor = K.pool3d(x=K.variable(feature_matrix),
                              pool_mode=pooling_type_string,
                              pool_size=(stride_length_px, stride_length_px,
                                         stride_length_px),
                              strides=(stride_length_px, stride_length_px,
                                       stride_length_px),
                              padding='valid',
                              data_format='channels_last')

    return feature_tensor.numpy()
def _check_model_fields(field_matrix, field_name, pressure_level_pascals,
                        valid_times_unix_sec):
    """Checks model fields for errors.

    M = number of rows (unique grid-point y-coordinates)
    N = number of columns (unique grid-point x-coordinates)
    T = number of time steps

    :param field_matrix: T-by-M-by-N numpy array with values of a single field
        (atmospheric variable).
    :param field_name: Field name in GewitterGefahr format.
    :param pressure_level_pascals: Pressure level (integer Pascals).
    :param valid_times_unix_sec: length-T numpy array of valid times.
    """

    check_field_name(field_name, require_standard=False)
    error_checking.assert_is_integer(pressure_level_pascals)

    error_checking.assert_is_integer_numpy_array(valid_times_unix_sec)
    error_checking.assert_is_numpy_array(valid_times_unix_sec,
                                         num_dimensions=1)
    num_times = len(valid_times_unix_sec)

    num_grid_rows, num_grid_columns = nwp_model_utils.get_grid_dimensions(
        model_name=nwp_model_utils.NARR_MODEL_NAME)

    error_checking.assert_is_real_numpy_array(field_matrix)
    error_checking.assert_is_numpy_array(field_matrix, num_dimensions=3)
    error_checking.assert_is_numpy_array(field_matrix,
                                         exact_dimensions=numpy.array([
                                             num_times, num_grid_rows,
                                             num_grid_columns
                                         ]))
def check_metadata(component_type_string,
                   target_class=None,
                   layer_name=None,
                   ideal_activation=None,
                   neuron_indices=None,
                   channel_index=None):
    """Error-checks metadata for saliency calculations.

    :param component_type_string: Component type (must be accepted by
        `model_interpretation.check_component_type`).
    :param target_class: See doc for `get_saliency_maps_for_class_activation`.
    :param layer_name: See doc for `get_saliency_maps_for_neuron_activation` or
        `get_saliency_maps_for_channel_activation`.
    :param ideal_activation: Same.
    :param neuron_indices: See doc for
        `get_saliency_maps_for_neuron_activation`.
    :param channel_index: See doc for `get_saliency_maps_for_class_activation`.

    :return: metadata_dict: Dictionary with the following keys.
    metadata_dict['component_type_string']: See input doc.
    metadata_dict['target_class']: Same.
    metadata_dict['layer_name']: Same.
    metadata_dict['ideal_activation']: Same.
    metadata_dict['neuron_indices']: Same.
    metadata_dict['channel_index']: Same.
    """

    model_interpretation.check_component_type(component_type_string)
    if (component_type_string ==
            model_interpretation.CLASS_COMPONENT_TYPE_STRING):
        error_checking.assert_is_integer(target_class)
        error_checking.assert_is_geq(target_class, 0)

    if component_type_string in [
            model_interpretation.NEURON_COMPONENT_TYPE_STRING,
            model_interpretation.CHANNEL_COMPONENT_TYPE_STRING
    ]:
        error_checking.assert_is_string(layer_name)
        if ideal_activation is not None:
            error_checking.assert_is_greater(ideal_activation, 0.)

    if (component_type_string ==
            model_interpretation.NEURON_COMPONENT_TYPE_STRING):
        error_checking.assert_is_integer_numpy_array(neuron_indices)
        error_checking.assert_is_geq_numpy_array(neuron_indices, 0)
        error_checking.assert_is_numpy_array(neuron_indices, num_dimensions=1)

    if (component_type_string ==
            model_interpretation.CHANNEL_COMPONENT_TYPE_STRING):
        error_checking.assert_is_integer(channel_index)
        error_checking.assert_is_geq(channel_index, 0)

    return {
        COMPONENT_TYPE_KEY: component_type_string,
        TARGET_CLASS_KEY: target_class,
        LAYER_NAME_KEY: layer_name,
        IDEAL_ACTIVATION_KEY: ideal_activation,
        NEURON_INDICES_KEY: neuron_indices,
        CHANNEL_INDEX_KEY: channel_index
    }
示例#18
0
def do_2d_upsampling(feature_matrix, upsampling_factor=2,
                     use_linear_interp=True):
    """Upsamples 2-D feature maps.

    m = number of rows after upsampling
    n = number of columns after upsampling

    :param feature_matrix: Input feature maps (numpy array).  Dimensions must be
        M x N x C or 1 x M x N x C.
    :param upsampling_factor: Upsampling factor (integer > 1).
    :param use_linear_interp: Boolean flag.  If True (False), will use linear
        (nearest-neighbour) interpolation.
    :return: feature_matrix: Output feature maps (numpy array).  Dimensions will
        be 1 x m x n x C.
    """

    error_checking.assert_is_numpy_array_without_nan(feature_matrix)
    error_checking.assert_is_integer(upsampling_factor)
    error_checking.assert_is_geq(upsampling_factor, 2)
    error_checking.assert_is_boolean(use_linear_interp)

    if len(feature_matrix.shape) == 3:
        feature_matrix = numpy.expand_dims(feature_matrix, axis=0)

    error_checking.assert_is_numpy_array(feature_matrix, num_dimensions=4)
示例#19
0
def trim_whitespace(input_file_name,
                    output_file_name,
                    border_width_pixels=10,
                    convert_exe_name=DEFAULT_CONVERT_EXE_NAME):
    """Trims whitespace around edge of image.

    :param input_file_name: Path to input file (may be in any format handled by
        ImageMagick).
    :param output_file_name: Path to output file.
    :param border_width_pixels: Desired border width (whitespace).
    :param convert_exe_name: Path to executable file for ImageMagick's "convert"
        function.  If you installed ImageMagick with root access, this should be
        the default.  Regardless, the pathless file name should be just
        "convert".
    :raises: ValueError: if ImageMagick command (which is ultimately a Unix
        command) fails.
    """

    error_checking.assert_file_exists(input_file_name)
    file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name)
    error_checking.assert_is_integer(border_width_pixels)
    error_checking.assert_is_geq(border_width_pixels, 0)
    error_checking.assert_file_exists(convert_exe_name)

    command_string = (
        '"{0:s}" "{1:s}" -trim -bordercolor White -border {2:d} "{3:s}"'
    ).format(convert_exe_name, input_file_name, border_width_pixels,
             output_file_name)

    exit_code = os.system(command_string)
    if exit_code == 0:
        return
    raise ValueError(ERROR_STRING)
示例#20
0
def _check_input_args(list_of_baseline_matrices, list_of_trial_matrices,
                      num_iterations, confidence_level):
    """Error-checks input args for Monte Carlo test.

    :param list_of_baseline_matrices: See doc for `run_monte_carlo_test`.
    :param list_of_trial_matrices: Same.
    :param num_iterations: Same.
    :param confidence_level: Same.
    :raises: ValueError: if number of baseline matrices (input tensors to model)
        != number of trial matrices.
    :raises: TypeError: if all "input matrices" are None.
    :return: num_examples_per_set: Number of examples in each set.
    """

    error_checking.assert_is_integer(num_iterations)
    error_checking.assert_is_geq(num_iterations, 100)
    error_checking.assert_is_geq(confidence_level, 0.)
    error_checking.assert_is_leq(confidence_level, 1.)

    num_baseline_matrices = len(list_of_baseline_matrices)
    num_trial_matrices = len(list_of_trial_matrices)

    if num_baseline_matrices != num_trial_matrices:
        error_string = (
            'Number of baseline matrices ({0:d}) should = number of trial '
            'matrices ({1:d}).').format(num_baseline_matrices,
                                        num_trial_matrices)

        raise ValueError(error_string)

    num_matrices = num_trial_matrices
    num_examples_per_set = None

    for i in range(num_matrices):
        if (list_of_baseline_matrices[i] is None
                and list_of_trial_matrices[i] is None):
            continue

        error_checking.assert_is_numpy_array(list_of_baseline_matrices[i])

        if num_examples_per_set is None:
            num_examples_per_set = list_of_baseline_matrices[i].shape[0]

        these_expected_dim = numpy.array(
            (num_examples_per_set, ) + list_of_baseline_matrices[i].shape[1:],
            dtype=int)

        error_checking.assert_is_numpy_array(
            list_of_baseline_matrices[i], exact_dimensions=these_expected_dim)

        these_expected_dim = numpy.array(list_of_baseline_matrices[i].shape,
                                         dtype=int)

        error_checking.assert_is_numpy_array(
            list_of_trial_matrices[i], exact_dimensions=these_expected_dim)

    if num_examples_per_set is None:
        raise TypeError('All "input matrices" are None.')

    return num_examples_per_set
def check_metadata(layer_name, neuron_indices, ideal_activation,
                   num_iterations, learning_rate, l2_weight):
    """Checks metadata for errors.

    :param layer_name: Name of layer with relevant neuron.
    :param neuron_indices: 1-D numpy array with indices of relevant neuron.
        Must have length D - 1, where D = number of dimensions in layer output.
        The first dimension is the batch dimension, which always has length
        `None` in Keras.
    :param ideal_activation: Ideal neuron activation, used to define loss
        function.  The loss function will be
        (neuron_activation - ideal_activation)**2.
    :param num_iterations: Number of iterations for gradient descent.
    :param learning_rate: Learning rate for gradient descent.
    :param l2_weight: L2 weight (penalty for difference between initial and
        final predictor matrix) in loss function.
    """

    error_checking.assert_is_string(layer_name)

    error_checking.assert_is_integer_numpy_array(neuron_indices)
    error_checking.assert_is_geq_numpy_array(neuron_indices, 0)
    error_checking.assert_is_numpy_array(neuron_indices, num_dimensions=1)

    error_checking.assert_is_not_nan(ideal_activation)

    error_checking.assert_is_integer(num_iterations)
    error_checking.assert_is_greater(num_iterations, 0)

    error_checking.assert_is_greater(learning_rate, 0.)
    error_checking.assert_is_less_than(learning_rate, 1.)

    error_checking.assert_is_geq(l2_weight, 0.)
示例#22
0
def create_fake_heights(real_heights_m_agl, num_padding_heights):
    """Creates fake heights for padding at top of profile.

    :param real_heights_m_agl: 1-D numpy array of real heights (metres above
        ground level).
    :param num_padding_heights: Number of heights to pad at top.
    :return: heights_m_agl: 1-D numpy array with all heights (real followed by
        fake).
    """

    error_checking.assert_is_numpy_array(real_heights_m_agl, num_dimensions=1)
    error_checking.assert_is_geq_numpy_array(real_heights_m_agl, 0.)
    assert numpy.allclose(real_heights_m_agl,
                          numpy.sort(real_heights_m_agl),
                          atol=TOLERANCE)

    error_checking.assert_is_integer(num_padding_heights)
    error_checking.assert_is_geq(num_padding_heights, 0)

    if num_padding_heights == 0:
        return real_heights_m_agl

    fake_heights_m_agl = numpy.linspace(1,
                                        num_padding_heights,
                                        num=num_padding_heights,
                                        dtype=float)
    fake_heights_m_agl = real_heights_m_agl[-1] + 1e6 * fake_heights_m_agl

    return numpy.concatenate((real_heights_m_agl, fake_heights_m_agl), axis=0)
def find_local_raw_file(year,
                        directory_name=None,
                        raise_error_if_missing=True):
    """Finds raw file on local machine.

    This file should contain all storm reports for one year.

    :param year: [integer] Will look for file from this year.
    :param directory_name: Name of directory with Storm Events files.
    :param raise_error_if_missing: Boolean flag.  If True and file is missing,
        this method will raise an error.
    :return: raw_file_name: File path.  If raise_error_if_missing = False and
        file is missing, this will be the *expected* path.
    :raises: ValueError: if raise_error_if_missing = True and file is missing.
    """

    error_checking.assert_is_integer(year)
    error_checking.assert_is_string(directory_name)
    error_checking.assert_is_boolean(raise_error_if_missing)

    raw_file_name = '{0:s}/{1:s}{2:s}{3:s}'.format(
        directory_name, PATHLESS_RAW_FILE_PREFIX, _year_number_to_string(year),
        RAW_FILE_EXTENSION)

    if raise_error_if_missing and not os.path.isfile(raw_file_name):
        raise ValueError('Cannot find raw file.  Expected at location: ' +
                         raw_file_name)

    return raw_file_name
示例#24
0
def resize_image(input_file_name,
                 output_file_name,
                 output_size_pixels,
                 convert_exe_name=DEFAULT_CONVERT_EXE_NAME):
    """Resizes image.

    :param input_file_name: Path to input file (may be in any format handled by
        ImageMagick).
    :param output_file_name: Path to output file.
    :param output_size_pixels: Output size.
    :param convert_exe_name: See doc for `trim_whitespace`.
    :raises: ValueError: if ImageMagick command (which is ultimately a Unix
        command) fails.
    """

    error_checking.assert_file_exists(input_file_name)
    file_system_utils.mkdir_recursive_if_necessary(file_name=output_file_name)
    error_checking.assert_is_integer(output_size_pixels)
    error_checking.assert_is_greater(output_size_pixels, 0)
    error_checking.assert_file_exists(convert_exe_name)

    command_string = '"{0:s}" "{1:s}" -resize {2:d}@ "{3:s}"'.format(
        convert_exe_name, input_file_name, output_size_pixels,
        output_file_name)

    exit_code = os.system(command_string)
    if exit_code == 0:
        return

    raise ValueError(ERROR_STRING)
def find_file(year, directory_name, raise_error_if_missing=True):
    """Finds Storm Events file.

    This file should contain all storm reports for one year.

    :param year: Year (integer).
    :param directory_name: Name of directory with Storm Events files.
    :param raise_error_if_missing: Boolean flag.  If file is missing and
        raise_error_if_missing = True, this method will error out.
    :return: storm_event_file_name: Path to Storm Events file.  If file is
        missing and raise_error_if_missing = False, this will be the *expected*
        path.
    :raises: ValueError: if file is missing and raise_error_if_missing = True.
    """

    error_checking.assert_is_integer(year)
    error_checking.assert_is_string(directory_name)
    error_checking.assert_is_boolean(raise_error_if_missing)

    storm_event_file_name = '{0:s}/{1:s}{2:s}{3:s}'.format(
        directory_name, PATHLESS_FILE_PREFIX, _year_number_to_string(year),
        FILE_EXTENSION)

    if raise_error_if_missing and not os.path.isfile(storm_event_file_name):
        error_string = ('Cannot find Storm Events file.  Expected at: {0:s}'.
                        format(storm_event_file_name))
        raise ValueError(error_string)

    return storm_event_file_name
示例#26
0
def plot_multipass_test(permutation_dict,
                        axes_object=None,
                        num_predictors_to_plot=None,
                        plot_percent_increase=False,
                        confidence_level=DEFAULT_CONFIDENCE_LEVEL,
                        bar_face_colour=None):
    """Plots results of multi-pass (Lakshmanan) permutation test.

    :param permutation_dict: See doc for `plot_single_pass_test`.
    :param axes_object: Same.
    :param num_predictors_to_plot: Same.
    :param plot_percent_increase: Same.
    :param confidence_level: Same.
    :param bar_face_colour: Same.
    """

    # Check input args.
    predictor_names = permutation_dict[permutation_utils.BEST_PREDICTORS_KEY]
    if num_predictors_to_plot is None:
        num_predictors_to_plot = len(predictor_names)

    error_checking.assert_is_integer(num_predictors_to_plot)
    error_checking.assert_is_greater(num_predictors_to_plot, 0)
    num_predictors_to_plot = min(
        [num_predictors_to_plot, len(predictor_names)])

    error_checking.assert_is_boolean(plot_percent_increase)

    # Set up plotting args.
    backwards_flag = permutation_dict[permutation_utils.BACKWARDS_FLAG]
    perturbed_cost_matrix = permutation_dict[
        permutation_utils.BEST_COST_MATRIX_KEY]

    perturbed_cost_matrix = perturbed_cost_matrix[:num_predictors_to_plot, :]
    predictor_names = predictor_names[:num_predictors_to_plot]

    original_cost_array = permutation_dict[
        permutation_utils.ORIGINAL_COST_ARRAY_KEY]
    original_cost_matrix = numpy.reshape(original_cost_array,
                                         (1, original_cost_array.size))
    cost_matrix = numpy.concatenate(
        (original_cost_matrix, perturbed_cost_matrix), axis=0)

    # Do plotting.
    if backwards_flag:
        clean_cost_array = permutation_dict[
            permutation_utils.BEST_COST_MATRIX_KEY][-1, :]
    else:
        clean_cost_array = original_cost_array

    _plot_bars(cost_matrix=cost_matrix,
               clean_cost_array=clean_cost_array,
               predictor_names=predictor_names,
               plot_percent_increase=plot_percent_increase,
               backwards_flag=backwards_flag,
               multipass_flag=True,
               confidence_level=confidence_level,
               axes_object=axes_object,
               bar_face_colour=bar_face_colour)
示例#27
0
def find_single_field_file(init_time_unix_sec,
                           lead_time_hours=None,
                           model_name=None,
                           grid_id=None,
                           grib1_field_name=None,
                           top_directory_name=None,
                           raise_error_if_missing=True):
    """Finds with single field on local machine.

    "Single field" = one variable at one time step and all grid cells.

    :param init_time_unix_sec: Model-initialization time (Unix format).
    :param lead_time_hours: Lead time (valid time minus init time).  If model is
        a reanalysis, you can leave this as None (always zero).
    :param model_name: Name of model.
    :param grid_id: String ID for model grid.
    :param grib1_field_name: Field name in grib1 format.
    :param top_directory_name: Name of top-level directory with single-field
        files for the given model/grib combo.
    :param raise_error_if_missing:
    :param raise_error_if_missing: Boolean flag.  If True and file is missing,
        will raise an error.
    :return: single_field_file_name: Path to single-field file.  If file is
        missing but raise_error_if_missing = False, this will be the *expected*
        path.
    :raises: ValueError: if raise_error_if_missing = True and file is missing.
    """

    error_checking.assert_is_string(grib1_field_name)
    error_checking.assert_is_string(top_directory_name)
    error_checking.assert_is_boolean(raise_error_if_missing)

    nwp_model_utils.check_model_name(model_name)
    if model_name == nwp_model_utils.NARR_MODEL_NAME:
        lead_time_hours = 0

    error_checking.assert_is_integer(lead_time_hours)
    error_checking.assert_is_geq(lead_time_hours, 0)

    pathless_file_name = _get_pathless_single_field_file_name(
        init_time_unix_sec,
        lead_time_hours=lead_time_hours,
        model_name=model_name,
        grid_id=grid_id,
        grib1_field_name=grib1_field_name)

    single_field_file_name = '{0:s}/{1:s}/{2:s}'.format(
        top_directory_name,
        time_conversion.unix_sec_to_string(init_time_unix_sec,
                                           TIME_FORMAT_MONTH),
        pathless_file_name)

    if raise_error_if_missing and not os.path.isfile(single_field_file_name):
        raise ValueError('Cannot find single-field file.  Expected at: ' +
                         single_field_file_name)

    return single_field_file_name
示例#28
0
def time_to_spc_date_string(unix_time_sec):
    """Converts time in Unix format to SPC date in string format.

    :param unix_time_sec: Time in Unix format.
    :return: spc_date_string: SPC date in format "yyyymmdd".
    """

    error_checking.assert_is_integer(unix_time_sec)
    return unix_sec_to_string(unix_time_sec - DAYS_TO_SECONDS // 2,
                              SPC_DATE_FORMAT)
def _get_grid_points_in_storms(storm_object_table,
                               num_grid_rows=None,
                               num_grid_columns=None):
    """Finds grid points in all storm objects.

    N = number of storm objects
    P = number of grid points in a storm object

    :param storm_object_table: N-row pandas DataFrame in format specified by
        `storm_tracking_io.write_processed_file`.
    :param num_grid_rows: Number of rows (unique grid-point latitudes).
    :param num_grid_columns: Number of columns (unique grid-point longitudes).
    :return: grid_points_in_storms_table: P-row pandas DataFrame with the
        following columns.
    grid_points_in_storms_table.flattened_index: Flattened index (integer) of
        grid point.
    grid_points_in_storms_table.storm_id: String ID for storm cell.
    """

    error_checking.assert_is_integer(num_grid_rows)
    error_checking.assert_is_greater(num_grid_rows, 0)
    error_checking.assert_is_integer(num_grid_columns)
    error_checking.assert_is_greater(num_grid_columns, 0)

    grid_point_row_indices = numpy.array([])
    grid_point_column_indices = numpy.array([])
    grid_point_storm_ids = []

    num_storms = len(storm_object_table.index)
    for i in range(num_storms):
        grid_point_row_indices = numpy.concatenate(
            (grid_point_row_indices,
             storm_object_table[tracking_io.GRID_POINT_ROW_COLUMN].values[i]))
        grid_point_column_indices = numpy.concatenate(
            (grid_point_column_indices,
             storm_object_table[tracking_io.GRID_POINT_COLUMN_COLUMN].values[i]
             ))

        this_num_grid_points = len(
            storm_object_table[tracking_io.GRID_POINT_ROW_COLUMN].values[i])
        this_storm_id_list = (
            [storm_object_table[tracking_io.STORM_ID_COLUMN].values[i]] *
            this_num_grid_points)
        grid_point_storm_ids += this_storm_id_list

    grid_point_flattened_indices = numpy.ravel_multi_index(
        (grid_point_row_indices.astype(int),
         grid_point_column_indices.astype(int)),
        (num_grid_rows, num_grid_columns))

    grid_points_in_storms_dict = {
        FLATTENED_INDEX_COLUMN: grid_point_flattened_indices,
        STORM_ID_COLUMN: grid_point_storm_ids
    }
    return pandas.DataFrame.from_dict(grid_points_in_storms_dict)
示例#30
0
def zero_top_heating_rate_function(heating_rate_channel_index, height_index):
    """Returns function that zeroes predicted heating rate at top of profile.

    :param heating_rate_channel_index: Channel index for heating rate.
    :param height_index: Will zero out heating rate at this height.
    :return: zeroing_function: Function handle (see below).
    """

    error_checking.assert_is_integer(heating_rate_channel_index)
    error_checking.assert_is_geq(heating_rate_channel_index, 0)
    error_checking.assert_is_integer(height_index)
    error_checking.assert_is_geq(height_index, 0)

    def zeroing_function(orig_prediction_tensor):
        """Zeroes out predicted heating rate at top of profile.

        :param orig_prediction_tensor: Keras tensor with model predictions.
        :return: new_prediction_tensor: Same as input but with top heating rate
            zeroed out.
        """

        num_heights = orig_prediction_tensor.get_shape().as_list()[-2]
        num_channels = orig_prediction_tensor.get_shape().as_list()[-1]

        zero_tensor = K.greater_equal(
            orig_prediction_tensor[..., height_index,
                                   heating_rate_channel_index], 1e12)
        zero_tensor = K.cast(zero_tensor, dtype=K.floatx())

        heating_rate_tensor = K.concatenate(
            (orig_prediction_tensor[..., heating_rate_channel_index][
                ..., :height_index], K.expand_dims(zero_tensor, axis=-1)),
            axis=-1)

        if height_index != num_heights - 1:
            heating_rate_tensor = K.concatenate(
                (heating_rate_tensor,
                 orig_prediction_tensor[..., heating_rate_channel_index][..., (
                     height_index + 1):]),
                axis=-1)

        new_prediction_tensor = K.concatenate(
            (orig_prediction_tensor[..., :heating_rate_channel_index],
             K.expand_dims(heating_rate_tensor, axis=-1)),
            axis=-1)

        if heating_rate_channel_index == num_channels - 1:
            return new_prediction_tensor

        return K.concatenate(
            (new_prediction_tensor,
             orig_prediction_tensor[..., (heating_rate_channel_index + 1):]),
            axis=-1)

    return zeroing_function