Exemple #1
0
    def evaluate_expression(self, expression, context={}):
        try:
            # Operations against masked arrays are really slow, so take a regular array view, then back to a masked
            # array afterwards. Todo: find a better solution long-term
            expr_context = {
                k: v.view(ndarray) if is_masked(v) else v
                for k, v in context.items()
            }

            result = Parser().evaluate(expression, context=expr_context)

            if is_ndarray(result):
                for value in context.values():
                    if is_masked(value):
                        if is_masked(result) and is_masked(value):
                            result.mask = result.mask | value.mask
                        elif is_masked(value):
                            result = masked_array(result, mask=value.mask)

                        result = Raster(result, value.extent, value.x_dim,
                                        value.y_dim, value.y_increasing)
                        break

            return result

        except (SyntaxError, NameError) as e:
            raise ExecutionError(
                'The expression is invalid ({0}): {1}\nContext: {2}'.format(
                    str(e), expression, str(context)), self)
Exemple #2
0
    def evaluate_expression(self, expression, context={}):
        try:
            # Operations against masked arrays are really slow, so take a regular array view, then back to a masked
            # array afterwards. Todo: find a better solution long-term
            expr_context = {k: v.view(numpy.ndarray) if is_masked(v) else v for k, v in six.iteritems(context)}

            result = Parser().evaluate(expression, context=expr_context)

            if is_ndarray(result):
                for value in six.itervalues(context):
                    if is_masked(value):
                        if is_masked(result) and is_masked(value):
                            result.mask = result.mask | value.mask
                        elif is_masked(value):
                            result = numpy.ma.masked_array(result, mask=value.mask)

                        result = Raster(result, value.extent, value.x_dim, value.y_dim, value.y_increasing)
                        break

            return result

        except (SyntaxError, NameError) as e:
            raise ExecutionError(
                'The expression is invalid ({0}): {1}\nContext: {2}'.format(str(e), expression, str(context)),
                self
            )
Exemple #3
0
def to_paletted_png(arr, palette, nodata=None):
    """
    Render an array as a paletted PNG.
    
    Parameters
    ----------
    arr : input array or masked array, must have dtype of uint8
    palette : numpy array of 8 bit tuples [(r, g, b), ...], where the index corresponds to the value in the image
    nodata : nodata value, will be set as transparent in the image (optional, default: None)

    Returns
    -------
    PNG bytes
    """

    if arr.dtype.kind not in ("u", "i"):
        raise ValueError("Input array must be integer type")

    # TODO: validate palette: must be of a small enough size and have rgb tuples

    nodata_index = None
    if is_masked(arr) or nodata is not None:
        # If it is masked, fill with index at end of palette
        nodata_index = len(palette)

        # add nodata color to palette (set as transparent below)
        palette = np.append(palette, (0, 0, 0))

        if is_masked(arr):
            arr = arr.filled(nodata_index)
        else:
            arr[arr == nodata] = nodata_index

    else:
        # return the underlying ndarray
        arr = arr.data

    img = Image.frombuffer("P", (arr.shape[1], arr.shape[0]), arr, "raw", "P",
                           0, 1)
    # palette must be a list of [r, g, b, r, g, b, ...]  values
    img.putpalette(palette.flatten().tolist(), "RGB")

    if nodata_index is not None:
        img.info["transparency"] = nodata_index

    buf = BytesIO()
    img.save(buf, "PNG")
    buf.seek(0)  # rewind to beginning of buffer
    return buf.read()
Exemple #4
0
    def handle_request(self, request, **kwargs):
        try:
            configurations = self.get_identify_configurations(request, **kwargs)
            if not configurations:
                return HttpResponse()

            data = {}

            for config in configurations:
                variable = config.variable
                service = variable.service

                if service.supports_time and variable.supports_time:
                    time_index = config.time_index or 0
                else:
                    time_index = None

                geometry = project_geometry(
                    config.geometry, config.projection, pyproj.Proj(str(self.service.projection))
                )
                assert isinstance(geometry, Point)  # Only point-based identify is supported

                dimensions = self.get_grid_spatial_dimensions(config.variable)
                cell_size = (
                    float(variable.full_extent.width) / dimensions[0],
                    float(variable.full_extent.height) / dimensions[1]
                )
                cell_index = [
                    int(float(geometry.x-variable.full_extent.xmin) / cell_size[0]),
                    int(float(geometry.y-variable.full_extent.ymin) / cell_size[1])
                ]
                if not self.is_y_increasing(variable):
                    cell_index[1] = dimensions[1] - cell_index[1] - 1

                variable_data = self.get_grid_for_variable(
                    config.variable, time_index=time_index, x_slice=(cell_index[0], cell_index[0] + 1),
                    y_slice=(cell_index[1], cell_index[1] + 1)
                )

                if len(variable_data):
                    value = variable_data[0][0]
                    data[variable] = None if is_masked(value) else float(value)

            data, content_type = self.serialize_data(data)
            return self.create_response(request, data, content_type=content_type)

        except ConfigurationError:
            return HttpResponseBadRequest()
        finally:
            self.close_dataset()
Exemple #5
0
    def get_queryset(self):
        if not self.request.query_params.get('point'):
            return self.queryset
        else:
            try:
                x, y = [float(x) for x in self.request.query_params['point'].split(',')]
            except ValueError:
                raise ParseError()

            elevation = get_elevation_at_point(Point(x, y))

            if elevation is None or is_masked(elevation):
                return self.queryset.none()

            # Elevation bands are stored in feet
            return self.queryset.filter(
                Q(low__lt=elevation/0.3048, high__gte=elevation/0.3048) | Q(low__isnull=True, high__isnull=True)
            )
Exemple #6
0
    def __new__(cls, arr, extent, x_dim, y_dim, y_increasing=False):
        """Create a new Raster from a numpy array and a `BBox` object."""

        assert len(arr.shape) > 1
        assert isinstance(extent, BBox)

        obj = numpy.asarray(arr).view(cls)
        obj.x_dim = x_dim
        obj.y_dim = y_dim
        obj.extent = extent
        obj.y_increasing = y_increasing
        obj.__array_priority__ = 100

        if is_masked(arr):
            obj._mask = arr._mask
            obj._fill_value = arr._fill_value

        return obj
Exemple #7
0
    def get_queryset(self):
        if not self.request.query_params.get('point'):
            return self.queryset
        else:
            try:
                x, y = [
                    float(x)
                    for x in self.request.query_params['point'].split(',')
                ]
            except ValueError:
                raise ParseError()

            elevation = get_elevation_at_point(Point(x, y))

            if elevation is None or is_masked(elevation):
                return self.queryset.none()

            # Elevation bands are stored in feet
            return self.queryset.filter(
                Q(low__lt=elevation / 0.3048, high__gte=elevation / 0.3048)
                | Q(low__isnull=True, high__isnull=True))
Exemple #8
0
def process_web_outputs(results, job, publish_raster_results=False, renderer_or_fn=None):
    outputs = results.format_args()

    for k, v in six.iteritems(outputs):
        if is_raster(v) and publish_raster_results:
            service_name = '{0}/{1}'.format(job.uuid, k)
            rel_path = '{}.nc'.format(service_name)
            abs_path = os.path.join(SERVICE_DATA_ROOT, rel_path)
            os.makedirs(os.path.dirname(abs_path))

            with Dataset(abs_path, 'w', format='NETCDF4') as ds:
                if v.extent.projection.is_latlong():
                    x_var = 'longitude'
                    y_var = 'latitude'
                else:
                    x_var = 'x'
                    y_var = 'y'

                coord_vars = SpatialCoordinateVariables.from_bbox(v.extent, *reversed(v.shape))
                coord_vars.add_to_dataset(ds, x_var, y_var)

                fill_value = v.fill_value if is_masked(v) else None
                data_var = ds.createVariable('data', v.dtype, dimensions=(y_var, x_var), fill_value=fill_value)
                data_var[:] = v
                set_crs(ds, 'data', v.extent.projection)

            if callable(renderer_or_fn):
                renderer = renderer_or_fn(v)
            elif renderer_or_fn is None:
                renderer = StretchedRenderer(
                    [(numpy.min(v).item(), Color(0, 0, 0)), (numpy.max(v).item(), Color(255, 255, 255))]
                )
            else:
                renderer = renderer_or_fn

            with transaction.atomic():
                service = Service.objects.create(
                    name=service_name,
                    description='This service has been automatically generated from the result of a geoprocessing job.',
                    data_path=rel_path,
                    projection=v.extent.projection.srs,
                    full_extent=v.extent,
                    initial_extent=v.extent,
                )
                Variable.objects.create(
                    service=service,
                    index=0,
                    variable='data',
                    projection=v.extent.projection.srs,
                    x_dimension=x_var,
                    y_dimension=y_var,
                    name='data',
                    renderer=renderer,
                    full_extent=v.extent
                )
                ProcessingResultService.objects.create(job=job, service=service)

            outputs[k] = service_name

        elif is_ndarray(v):
            if v.size < numpy.get_printoptions()['threshold']:
                outputs[k] = v.tolist()
            else:
                outputs[k] = str(v)

    return outputs
Exemple #9
0
def to_smallest_png(arr, image_type=None):
    """
    Convert an array to PNG, using the smallest PNG bit depth that
    will contain the data range: 8, 24, or 32.

    If the input is a masked array, the maximum value of the data type
    will be used to fill nodata.

    You can pre-fill nodata with a different value, but if you use a value well
    outside your value range, this may force use of a larger output PNG bit depth
    than is ideal.
    
    Parameters
    ----------
    arr: input array or masked array, must have dtype of uint8, uint16, or uint32

    Returns
    -------
    PNG bytes
    """

    if arr.dtype.kind not in ("u", "i"):
        raise ValueError("Input array must be integer type")

    if image_type is None:
        image_type = get_smallest_image_type(arr)

    else:
        if not image_type in ("L", "RGB", "RGBA"):
            raise ValueError("Image type must be one of: L, RGB, RGBA")

    # Temporary: either remove RGBA support or make it work in browsers
    if image_type == "RGBA":
        raise ValueError(
            "RGBA is not currently supported due to variable decoding in browsers"
        )

    if is_masked(arr):
        # If it is masked, fill it with appropriate nodata value
        image_type_max = MAX_VALUE[image_type]
        if arr.max() < image_type_max:
            arr = arr.filled(image_type_max)
        else:
            raise ValueError("Max of value range must be max of data type - 1"
                             "to allow max of data type to be nodata value")
    else:
        # return the underlying ndarray
        arr = arr.data

    if image_type == "L":
        image_data = arr

    elif image_type == "RGB":
        image_data = to_rgb_array(np.asarray(arr))

    elif image_type == "RGBA":
        image_data = to_rgba_array(np.asarray(arr))

    else:
        raise NotImplementedError(
            "values require an image type that is not supported")

    img = Image.frombuffer(image_type, (arr.shape[1], arr.shape[0]),
                           image_data, "raw", image_type, 0, 1)

    buf = BytesIO()
    img.save(buf, "PNG")
    buf.seek(0)  # rewind to beginning of buffer
    return buf.read()
Exemple #10
0
def main(in_pattern, out_pattern, boundary, single, varname):
    """
    Clips and masks large NetCDF datasets to regional datasets based on the boundary. The in_pattern and out_pattern
    arguments should be filename patterns (can include path) with the pattern: /path/to/in_netcdf_{variable}.nc.

    Example usage: python cut_to_region.py NorthAmerica/NA_{variable}.nc USWest/west_{variable}.nc west.shp
    """

    if single and not varname:
        print('--varname is required when --single is used')
        sys.exit(-1)

    if single:
        if not os.path.exists(in_pattern):
            print('Input file {} does not exist.'.format(in_pattern))
            sys.exit(-1)

        input_paths = [(in_pattern, varname)]
    else:
        input_paths = [(in_pattern.format(variable=x), x) for x in VARIABLES]

        for path, _ in input_paths:
            if not os.path.exists(path):
                print('Input file {} does not exist.'.format(path))
                sys.exit(-1)

    with fiona.open(boundary, 'r') as shp:
        features = []
        wgs84 = Proj('+init=EPSG:4326')
        shp_projection = Proj(shp.crs)
        bounds = shp.bounds

        ll = transform(shp_projection, wgs84, bounds[0], bounds[1])
        ur = transform(shp_projection, wgs84, bounds[2], bounds[3])

        bbox = BBox([*ll, *ur], projection=wgs84)

        for feature in shp.items():
            geometry = transform_geom(shp.crs, {'init': 'EPSG: 4326'},
                                      feature[1]['geometry'])
            features.append(geometry)

    for in_path, variable in input_paths:
        if single:
            out_path = out_pattern
        else:
            out_path = out_pattern.format(variable=variable)

        if os.path.exists(out_path):
            confirm = input(
                "The output file '{}' already exists? Do you with to replace it? [y/n] "
                .format(out_path))
            if confirm.lower().strip() not in ['y', 'yes']:
                print('Exiting...')
                sys.exit()

        with Dataset(in_path, 'r') as ds:
            coords = SpatialCoordinateVariables.from_dataset(
                ds, x_name='longitude', y_name='latitude')

            x_start, x_stop = coords.x.indices_for_range(bbox.xmin, bbox.xmax)
            y_start, y_stop = coords.y.indices_for_range(bbox.ymin, bbox.ymax)

            x_slice = slice(x_start, x_stop)
            y_slice = slice(y_start, y_stop)

            clipped_coords = coords.slice_by_bbox(bbox)

            grid = ds.variables[variable][y_slice, x_slice]

        if is_masked(grid):
            mask = grid.mask.astype('uint8')
        else:
            mask = numpy.zeros(grid.shape, dtype='uint8')

        mask |= rasterize(((x, 0) for x in features),
                          out_shape=mask.shape,
                          transform=clipped_coords.affine,
                          fill=1,
                          default_value=0)
        grid = numpy.ma.masked_where(mask == 1, grid.data)

        print('Writing {}...'.format(out_path))
        with Dataset(out_path, 'w', format='NETCDF4') as ds:
            clipped_coords.add_to_dataset(ds, 'longitude', 'latitude')
            data_var = ds.createVariable(variable,
                                         grid.dtype,
                                         dimensions=('latitude', 'longitude'),
                                         fill_value=grid.fill_value)

            if data_var.shape != grid.shape:
                grid = grid[:data_var.shape[0], :data_var.shape[1]]

            data_var[:] = grid
            set_crs(ds, variable, Proj('+init=EPSG:4326'))