Exemplo n.º 1
0
 def __init__(self, backend_configs=None, rescale_configs=None, **kwargs):
     self.rescale_configs = rescale_configs or [DEFAULT_NINJO_RCONFIG]
     self.backend_configs = backend_configs or [DEFAULT_NINJO_CONFIG]
     self.rescaler = Rescaler(*self.rescale_configs)
     self.band_config_reader = NinjoBandConfigReader(*self.backend_configs)
     self.sat_config_reader = NinJoSatConfigReader(*self.backend_configs)
     super(Backend, self).__init__(**kwargs)
Exemplo n.º 2
0
 def __init__(self, backend_configs=None, rescale_configs=None, **kwargs):
     self.rescale_configs = rescale_configs or [DEFAULT_NINJO_RCONFIG]
     self.backend_configs = backend_configs or [DEFAULT_NINJO_CONFIG]
     self.rescaler = Rescaler(*self.rescale_configs)
     self.band_config_reader = NinjoBandConfigReader(*self.backend_configs)
     self.grid_config_reader = NinjoGridConfigReader(*self.backend_configs)
     self.sat_config_reader = NinJoSatConfigReader(*self.backend_configs)
     super(Backend, self).__init__(**kwargs)
Exemplo n.º 3
0
class Backend(roles.BackendRole):
    def __init__(self, rescale_configs=None, **kwargs):
        self.rescale_configs = rescale_configs or [DEFAULT_RCONFIG]
        self.rescaler = Rescaler(*self.rescale_configs)
        super(Backend, self).__init__(**kwargs)

    @property
    def known_grids(self):
        # should work regardless of grid
        return None

    def create_output_from_product(self, gridded_product, output_pattern=None,
                                   data_type=None, inc_by_one=None, fill_value=0, **kwargs):
        data_type = data_type or numpy.uint8
        etype = np2etype[data_type]
        inc_by_one = inc_by_one or False
        grid_def = gridded_product["grid_definition"]
        if not output_pattern:
            output_pattern = DEFAULT_OUTPUT_PATTERN
        if "{" in output_pattern:
            # format the filename
            of_kwargs = gridded_product.copy(as_dict=True)
            of_kwargs["data_type"] = data_type
            output_filename = self.create_output_filename(output_pattern,
                                                          grid_name=grid_def["grid_name"],
                                                          rows=grid_def["height"],
                                                          columns=grid_def["width"],
                                                          **of_kwargs)
        else:
            output_filename = output_pattern

        if os.path.isfile(output_filename):
            if not self.overwrite_existing:
                LOG.error("Geotiff file already exists: %s", output_filename)
                raise RuntimeError("Geotiff file already exists: %s" % (output_filename,))
            else:
                LOG.warning("Geotiff file already exists, will overwrite: %s", output_filename)

        try:
            LOG.debug("Scaling %s data to fit in geotiff...", gridded_product["product_name"])
            data = self.rescaler.rescale_product(gridded_product, data_type,
                                                 inc_by_one=inc_by_one, fill_value=fill_value)

            # Create the geotiff
            # X and Y rotation are 0 in most cases so we just hard-code it
            geotransform = gridded_product["grid_definition"].gdal_geotransform
            create_geotiff(data, output_filename, grid_def["proj4_definition"], geotransform,
                           etype=etype, **kwargs)
        except StandardError:
            if not self.keep_intermediate and os.path.isfile(output_filename):
                os.remove(output_filename)
            raise

        return output_filename
Exemplo n.º 4
0
 def __init__(self, rescale_configs=None, **kwargs):
     self.rescale_configs = rescale_configs or [DEFAULT_RCONFIG]
     self.rescaler = Rescaler(*self.rescale_configs)
     super(Backend, self).__init__(**kwargs)
Exemplo n.º 5
0
class Backend(roles.BackendRole):
    """Simple backend for renaming or rescaling binary files created from remapping.

    Rescale configuration files are only used for non-float data types.
    """
    def __init__(self, rescale_configs=None, **kwargs):
        self.rescale_configs = rescale_configs or [DEFAULT_RCONFIG]
        self.rescaler = Rescaler(*self.rescale_configs)
        super(Backend, self).__init__(**kwargs)

    @property
    def known_grids(self):
        # should work regardless of grid
        return None

    def create_output_from_product(self,
                                   gridded_product,
                                   output_pattern=None,
                                   data_type=None,
                                   inc_by_one=None,
                                   fill_value=None,
                                   **kwargs):
        inc_by_one = inc_by_one or False
        data_type = data_type or gridded_product["data_type"]
        fill_value = fill_value or gridded_product["fill_value"]
        same_fill = numpy.isnan(fill_value) and numpy.isnan(
            gridded_product["fill_value"]
        ) or fill_value == gridded_product["fill_value"]
        grid_def = gridded_product["grid_definition"]
        if not output_pattern:
            output_pattern = DEFAULT_OUTPUT_PATTERN
        if "{" in output_pattern:
            # format the filename
            of_kwargs = gridded_product.copy(as_dict=True)
            of_kwargs["data_type"] = data_type
            output_filename = self.create_output_filename(
                output_pattern,
                grid_name=grid_def["grid_name"],
                rows=grid_def["height"],
                columns=grid_def["width"],
                **of_kwargs)
        else:
            output_filename = output_pattern

        if os.path.isfile(output_filename):
            if not self.overwrite_existing:
                LOG.error("Geotiff file already exists: %s", output_filename)
                raise RuntimeError("Geotiff file already exists: %s" %
                                   (output_filename, ))
            else:
                LOG.warning("Geotiff file already exists, will overwrite: %s",
                            output_filename)

        # if we have a floating point data type, then scaling doesn't make much sense
        if data_type == gridded_product["data_type"] and same_fill:
            LOG.info("Saving product %s to binary file %s",
                     gridded_product["product_name"], output_filename)
            shutil.copyfile(gridded_product["grid_data"], output_filename)
            return output_filename
        elif numpy.issubclass_(data_type, numpy.floating):
            # we didn't rescale any data, but we need to convert it
            data = gridded_product.get_data_array()
        else:
            try:
                LOG.debug("Scaling %s data to fit data type",
                          gridded_product["product_name"])
                data = self.rescaler.rescale_product(gridded_product,
                                                     data_type,
                                                     inc_by_one=inc_by_one,
                                                     fill_value=fill_value)
                data = clip_to_data_type(data, data_type)
            except ValueError:
                if not self.keep_intermediate and os.path.isfile(
                        output_filename):
                    os.remove(output_filename)
                raise

        LOG.info("Saving product %s to binary file %s",
                 gridded_product["product_name"], output_filename)
        data = data.astype(data_type)
        fill_mask = gridded_product.get_data_mask()
        data[fill_mask] = fill_value
        data.tofile(output_filename)

        return output_filename
Exemplo n.º 6
0
 def __init__(self, backend_configs=None, rescale_configs=None, **kwargs):
     backend_configs = backend_configs or [DEFAULT_AWIPS_CONFIG]
     rescale_configs = rescale_configs or [DEFAULT_RCONFIG]
     self.awips_config_reader = AWIPS2ConfigReader(*backend_configs)
     self.rescaler = Rescaler(*rescale_configs, **kwargs)
     super(Backend, self).__init__(**kwargs)
Exemplo n.º 7
0
class Backend(roles.BackendRole):
    def __init__(self, backend_configs=None, rescale_configs=None, **kwargs):
        backend_configs = backend_configs or [DEFAULT_AWIPS_CONFIG]
        rescale_configs = rescale_configs or [DEFAULT_RCONFIG]
        self.awips_config_reader = AWIPS2ConfigReader(*backend_configs)
        self.rescaler = Rescaler(*rescale_configs, **kwargs)
        super(Backend, self).__init__(**kwargs)

    @property
    def known_grids(self):
        return self.awips_config_reader.known_grids

    def create_output_from_product(self, gridded_product, **kwargs):
        data_type = DTYPE_UINT8
        inc_by_one = False
        fill_value = 0
        grid_def = gridded_product["grid_definition"]

        # awips_info = self.awips_config_reader.get_product_options(gridded_product)
        try:
            awips_info = self.awips_config_reader.get_product_info(
                gridded_product)
        except NoSectionError as e:
            LOG.error(
                "Could not get information on product from backend configuration file"
            )
            # NoSectionError is not a "StandardError" so it won't be caught normally
            raise RuntimeError(e.message)

        try:
            awips_info.update(self.awips_config_reader.get_grid_info(grid_def))
        except NoSectionError as e:
            LOG.error(
                "Could not get information on grid from backend configuration file"
            )
            # NoSectionError is not a "StandardError" so it won't be caught normally
            raise RuntimeError(e.msg)

        if "filename_scheme" in awips_info:
            # Let individual products have special names if needed (mostly for weird product naming)
            fn_format = awips_info.pop("filename_scheme")
        else:
            fn_format = self.awips_config_reader.get_filename_format()

        output_filename = self.create_output_filename(
            fn_format,
            grid_name=grid_def["grid_name"],
            rows=grid_def["height"],
            columns=grid_def["width"],
            **gridded_product)

        if os.path.isfile(output_filename):
            if not self.overwrite_existing:
                LOG.error("AWIPS file already exists: %s", output_filename)
                raise RuntimeError("AWIPS file already exists: %s" %
                                   (output_filename, ))
            else:
                LOG.warning("AWIPS file already exists, will overwrite: %s",
                            output_filename)

        # Create the netcdf file
        try:
            LOG.debug("Scaling %s data to fit in netcdf file...",
                      gridded_product["product_name"])
            data = self.rescaler.rescale_product(gridded_product,
                                                 data_type,
                                                 inc_by_one=inc_by_one,
                                                 fill_value=fill_value)

            LOG.info("Writing product %s to AWIPS NetCDF file",
                     gridded_product["product_name"])
            create_awips2_netcdf3(output_filename, data,
                                  gridded_product["begin_time"], **awips_info)
        except StandardError:
            LOG.error("Error while filling in NC file with data: %s",
                      output_filename)
            if not self.keep_intermediate and os.path.isfile(output_filename):
                os.remove(output_filename)
            raise

        return output_filename
Exemplo n.º 8
0
class Backend(roles.BackendRole):
    def __init__(self, backend_configs=None, rescale_configs=None, **kwargs):
        self.rescale_configs = rescale_configs or [DEFAULT_NINJO_RCONFIG]
        self.backend_configs = backend_configs or [DEFAULT_NINJO_CONFIG]
        self.rescaler = Rescaler(*self.rescale_configs)
        self.band_config_reader = NinjoBandConfigReader(*self.backend_configs)
        self.sat_config_reader = NinJoSatConfigReader(*self.backend_configs)
        super(Backend, self).__init__(**kwargs)

    @property
    def known_grids(self):
        # should work regardless of grid
        return None

    def create_output_from_product(self,
                                   gridded_product,
                                   output_pattern=None,
                                   data_type=None,
                                   inc_by_one=None,
                                   fill_value=0,
                                   **kwargs):
        # FIXME: Previous version had -999.0 as the fill value...really?
        grid_def = gridded_product["grid_definition"]
        data_type = data_type or numpy.uint8
        inc_by_one = inc_by_one or False
        band_config_info = self.band_config_reader.get_config_options(
            product_name=gridded_product["product_name"],
            satellite=gridded_product["satellite"],
            instrument=gridded_product["instrument"],
            data_type=gridded_product["data_type"],
            data_kind=gridded_product["data_kind"],
            allow_default=False,
        )
        band_config_info[
            "satellite_id"] = self.sat_config_reader.get_satellite_id(
                gridded_product)

        if not output_pattern:
            output_pattern = DEFAULT_OUTPUT_PATTERN
        if "{" in output_pattern:
            # format the filename
            of_kwargs = gridded_product.copy(as_dict=True)
            of_kwargs["data_type"] = dtype_to_str(data_type)
            output_filename = self.create_output_filename(
                output_pattern,
                grid_name=grid_def["grid_name"],
                rows=grid_def["height"],
                columns=grid_def["width"],
                **of_kwargs)
        else:
            output_filename = output_pattern

        if os.path.isfile(output_filename):
            if not self.overwrite_existing:
                LOG.error("NinJo TIFF file already exists: %s",
                          output_filename)
                raise RuntimeError("NinJo TIFF file already exists: %s" %
                                   (output_filename, ))
            else:
                LOG.warning(
                    "NinJo TIFF file already exists, will overwrite: %s",
                    output_filename)

        try:
            LOG.debug("Scaling %s data to fit in ninjotiff...",
                      gridded_product["product_name"])
            data = self.rescaler.rescale_product(gridded_product,
                                                 data_type,
                                                 inc_by_one=inc_by_one,
                                                 fill_value=fill_value)

            # Create the geotiff
            save(
                data,
                grid_def,
                output_filename,
                sat_id=band_config_info["satellite_id"],
                chan_id=band_config_info["band_id"],
                data_source=band_config_info["data_source"],
                data_cat=band_config_info["data_category"],
                fill_value=fill_value,
                data_kind=gridded_product["data_kind"],
                begin_time=gridded_product["begin_time"],
                product_name=gridded_product["product_name"],
            )
        except (OSError, RuntimeError, ValueError, KeyError):
            if not self.keep_intermediate and os.path.isfile(output_filename):
                os.remove(output_filename)
            raise

        return output_filename
Exemplo n.º 9
0
class Backend(roles.BackendRole):
    def __init__(self, backend_configs=None, rescale_configs=None, **kwargs):
        self.rescale_configs = rescale_configs or [DEFAULT_NINJO_RCONFIG]
        self.backend_configs = backend_configs or [DEFAULT_NINJO_CONFIG]
        self.rescaler = Rescaler(*self.rescale_configs)
        self.band_config_reader = NinjoBandConfigReader(*self.backend_configs)
        self.grid_config_reader = NinjoGridConfigReader(*self.backend_configs)
        self.sat_config_reader = NinJoSatConfigReader(*self.backend_configs)
        super(Backend, self).__init__(**kwargs)

    @property
    def known_grids(self):
        return self.grid_config_reader.known_grids

    def create_output_from_product(self,
                                   gridded_product,
                                   output_pattern=None,
                                   data_type=None,
                                   inc_by_one=None,
                                   fill_value=-999.0,
                                   **kwargs):
        # FIXME: Previous version had -999.0 as the fill value...really?
        grid_def = gridded_product["grid_definition"]
        grid_name = grid_def["grid_name"]
        data_type = data_type or numpy.uint8
        inc_by_one = inc_by_one or False
        grid_config_info = self.grid_config_reader.get_config_options(
            grid_name=grid_name, allow_default=False)
        band_config_info = self.band_config_reader.get_config_options(
            product_name=gridded_product["product_name"],
            satellite=gridded_product["satellite"],
            instrument=gridded_product["instrument"],
            data_type=gridded_product["data_type"],
            data_kind=gridded_product["data_kind"],
            allow_default=False,
        )
        band_config_info[
            "satellite_id"] = self.sat_config_reader.get_satellite_id(
                gridded_product)

        if not output_pattern:
            output_pattern = DEFAULT_OUTPUT_PATTERN
        if "{" in output_pattern:
            # format the filename
            of_kwargs = gridded_product.copy(as_dict=True)
            of_kwargs["data_type"] = dtype_to_str(data_type)
            output_filename = self.create_output_filename(
                output_pattern,
                grid_name=grid_def["grid_name"],
                rows=grid_def["height"],
                columns=grid_def["width"],
                **of_kwargs)
        else:
            output_filename = output_pattern

        if os.path.isfile(output_filename):
            if not self.overwrite_existing:
                LOG.error("NinJo TIFF file already exists: %s",
                          output_filename)
                raise RuntimeError("NinJo TIFF file already exists: %s" %
                                   (output_filename, ))
            else:
                LOG.warning(
                    "NinJo TIFF file already exists, will overwrite: %s",
                    output_filename)

        try:
            LOG.debug("Extracting additional information from grid projection")
            map_origin_lon, map_origin_lat = grid_def.lonlat_upperleft
            proj_dict = grid_def.proj4_dict
            equ_radius = proj_dict["a"]
            pol_radius = proj_dict["b"]
            central_meridian = proj_dict.get("lon_0", None)
            ref_lat1 = proj_dict.get("lat_ts", None)

            LOG.debug("Scaling %s data to fit in ninjotiff...",
                      gridded_product["product_name"])
            data = self.rescaler.rescale_product(gridded_product,
                                                 data_type,
                                                 inc_by_one=inc_by_one,
                                                 fill_value=fill_value)

            # Create the geotiff
            create_ninjo_tiff(data,
                              output_filename,
                              pixel_xres=grid_config_info["xres"],
                              pixel_yres=grid_config_info["yres"],
                              projection=grid_config_info["projection"],
                              origin_lat=map_origin_lat,
                              origin_lon=map_origin_lon,
                              radius_a=equ_radius,
                              radius_b=pol_radius,
                              central_meridian=central_meridian,
                              ref_lat1=ref_lat1,
                              is_calibrated=1,
                              sat_id=band_config_info["satellite_id"],
                              chan_id=band_config_info["band_id"],
                              data_source=band_config_info["data_source"],
                              data_cat=band_config_info["data_category"],
                              image_dt=gridded_product["begin_time"],
                              data_kind=gridded_product["data_kind"])
        except StandardError:
            if not self.keep_intermediate and os.path.isfile(output_filename):
                os.remove(output_filename)
            raise

        return output_filename
Exemplo n.º 10
0
 def __init__(self, rescale_configs=None, **kwargs):
     self.rescale_configs = rescale_configs or [DEFAULT_RCONFIG]
     self.rescaler = Rescaler(*self.rescale_configs)
     super(Backend, self).__init__(**kwargs)
Exemplo n.º 11
0
class Backend(roles.BackendRole):
    """Simple backend for renaming or rescaling binary files created from remapping.

    Rescale configuration files are only used for non-float data types.
    """
    def __init__(self, rescale_configs=None, **kwargs):
        self.rescale_configs = rescale_configs or [DEFAULT_RCONFIG]
        self.rescaler = Rescaler(*self.rescale_configs)
        super(Backend, self).__init__(**kwargs)

    @property
    def known_grids(self):
        # should work regardless of grid
        return None

    def create_output_from_product(self, gridded_product, output_pattern=None,
                                   data_type=None, inc_by_one=None, fill_value=None, **kwargs):
        inc_by_one = inc_by_one or False
        data_type = data_type or gridded_product["data_type"]
        fill_value = fill_value or gridded_product["fill_value"]
        same_fill = numpy.isnan(fill_value) and numpy.isnan(gridded_product["fill_value"]) or fill_value == gridded_product["fill_value"]
        grid_def = gridded_product["grid_definition"]
        if not output_pattern:
            output_pattern = DEFAULT_OUTPUT_PATTERN
        if "{" in output_pattern:
            # format the filename
            of_kwargs = gridded_product.copy(as_dict=True)
            of_kwargs["data_type"] = data_type
            output_filename = self.create_output_filename(output_pattern,
                                                          grid_name=grid_def["grid_name"],
                                                          rows=grid_def["height"],
                                                          columns=grid_def["width"],
                                                          **of_kwargs)
        else:
            output_filename = output_pattern

        if os.path.isfile(output_filename):
            if not self.overwrite_existing:
                LOG.error("Geotiff file already exists: %s", output_filename)
                raise RuntimeError("Geotiff file already exists: %s" % (output_filename,))
            else:
                LOG.warning("Geotiff file already exists, will overwrite: %s", output_filename)

        # if we have a floating point data type, then scaling doesn't make much sense
        if data_type == gridded_product["data_type"] and same_fill:
            LOG.info("Saving product %s to binary file %s", gridded_product["product_name"], output_filename)
            shutil.copyfile(gridded_product["grid_data"], output_filename)
            return output_filename
        elif numpy.issubclass_(data_type, numpy.floating):
            # we didn't rescale any data, but we need to convert it
            data = gridded_product.get_data_array()
        else:
            try:
                LOG.debug("Scaling %s data to fit data type", gridded_product["product_name"])
                data = self.rescaler.rescale_product(gridded_product, data_type,
                                                     inc_by_one=inc_by_one, fill_value=fill_value)
                data = clip_to_data_type(data, data_type)
            except ValueError:
                if not self.keep_intermediate and os.path.isfile(output_filename):
                    os.remove(output_filename)
                raise

        LOG.info("Saving product %s to binary file %s", gridded_product["product_name"], output_filename)
        data = data.astype(data_type)
        fill_mask = gridded_product.get_data_mask()
        data[fill_mask] = fill_value
        data.tofile(output_filename)

        return output_filename
Exemplo n.º 12
0
class Backend(roles.BackendRole):
    def __init__(self, rescale_configs=None, **kwargs):
        self.rescale_configs = rescale_configs or [DEFAULT_RCONFIG]
        self.rescaler = Rescaler(*self.rescale_configs)
        super(Backend, self).__init__(**kwargs)

    @property
    def known_grids(self):
        # should work regardless of grid
        return None

    def create_output_from_product(self, gridded_product, output_pattern=None,
                                   data_type=None, inc_by_one=None, fill_value=0,
                                   tiled=False, blockxsize=None, blockysize=None, **kwargs):
        data_type = data_type or np.uint8
        etype = np2etype[data_type]
        inc_by_one = inc_by_one or False
        grid_def = gridded_product["grid_definition"]
        if not output_pattern:
            output_pattern = DEFAULT_OUTPUT_PATTERN
        if "{" in output_pattern:
            # format the filename
            of_kwargs = gridded_product.copy(as_dict=True)
            of_kwargs["data_type"] = data_type
            output_filename = self.create_output_filename(output_pattern,
                                                          grid_name=grid_def["grid_name"],
                                                          rows=grid_def["height"],
                                                          columns=grid_def["width"],
                                                          **of_kwargs)
        else:
            output_filename = output_pattern

        if os.path.isfile(output_filename):
            if not self.overwrite_existing:
                LOG.error("Geotiff file already exists: %s", output_filename)
                raise RuntimeError("Geotiff file already exists: %s" % (output_filename,))
            else:
                LOG.warning("Geotiff file already exists, will overwrite: %s", output_filename)

        try:
            if np.issubdtype(data_type, np.floating):
                # assume they don't want to scale floating point
                data = gridded_product.get_data_array()
                rescale_options = {}
            else:
                LOG.debug("Scaling %s data to fit in geotiff...", gridded_product["product_name"])
                rescale_options = self.rescaler.get_rescale_options(gridded_product,
                                                                    data_type,
                                                                    inc_by_one=inc_by_one,
                                                                    fill_value=fill_value)
                data = self.rescaler.rescale_product(gridded_product, data_type, rescale_options=rescale_options.copy())

            # Create the geotiff
            # X and Y rotation are 0 in most cases so we just hard-code it
            geotransform = gridded_product["grid_definition"].gdal_geotransform
            gtiff = create_geotiff(data, output_filename, grid_def["proj4_definition"], geotransform,
                                   etype=etype, tiled=tiled, blockxsize=blockxsize, blockysize=blockysize,
                                   **kwargs)

            if rescale_options.get("method") == "linear" and "min_in" in rescale_options and "max_in" in rescale_options:
                LOG.debug("Setting geotiff metadata for linear min/max values")
                gtiff.SetMetadataItem("min_in", str(rescale_options["min_in"]))
                gtiff.SetMetadataItem("max_in", str(rescale_options["max_in"]))
        except StandardError:
            if not self.keep_intermediate and os.path.isfile(output_filename):
                os.remove(output_filename)
            raise

        return output_filename
Exemplo n.º 13
0
 def __init__(self, backend_configs=None, rescale_configs=None, **kwargs):
     backend_configs = backend_configs or [DEFAULT_AWIPS_CONFIG]
     rescale_configs = rescale_configs or [DEFAULT_RCONFIG]
     self.awips_config_reader = AWIPS2ConfigReader(*backend_configs)
     self.rescaler = Rescaler(*rescale_configs, **kwargs)
     super(Backend, self).__init__(**kwargs)
Exemplo n.º 14
0
class Backend(roles.BackendRole):
    def __init__(self, backend_configs=None, rescale_configs=None, **kwargs):
        backend_configs = backend_configs or [DEFAULT_AWIPS_CONFIG]
        rescale_configs = rescale_configs or [DEFAULT_RCONFIG]
        self.awips_config_reader = AWIPS2ConfigReader(*backend_configs)
        self.rescaler = Rescaler(*rescale_configs, **kwargs)
        super(Backend, self).__init__(**kwargs)

    @property
    def known_grids(self):
        return self.awips_config_reader.known_grids

    def create_output_from_product(self, gridded_product, **kwargs):
        data_type = DTYPE_UINT8
        inc_by_one = False
        fill_value = 0
        grid_def = gridded_product["grid_definition"]

        # awips_info = self.awips_config_reader.get_product_options(gridded_product)
        try:
            awips_info = self.awips_config_reader.get_product_info(gridded_product)
        except NoSectionError as e:
            LOG.error("Could not get information on product from backend configuration file")
            # NoSectionError is not a "StandardError" so it won't be caught normally
            raise RuntimeError(e.message)

        try:
            awips_info.update(self.awips_config_reader.get_grid_info(grid_def))
        except NoSectionError as e:
            LOG.error("Could not get information on grid from backend configuration file")
            # NoSectionError is not a "StandardError" so it won't be caught normally
            raise RuntimeError(e.msg)

        if "filename_scheme" in awips_info:
            # Let individual products have special names if needed (mostly for weird product naming)
            fn_format = awips_info.pop("filename_scheme")
        else:
            fn_format = self.awips_config_reader.get_filename_format()

        output_filename = self.create_output_filename(fn_format,
                                                       grid_name=grid_def["grid_name"],
                                                       rows=grid_def["height"],
                                                       columns=grid_def["width"],
                                                       **gridded_product)

        if os.path.isfile(output_filename):
            if not self.overwrite_existing:
                LOG.error("AWIPS file already exists: %s", output_filename)
                raise RuntimeError("AWIPS file already exists: %s" % (output_filename,))
            else:
                LOG.warning("AWIPS file already exists, will overwrite: %s", output_filename)

        # Create the netcdf file
        try:
            LOG.debug("Scaling %s data to fit in netcdf file...", gridded_product["product_name"])
            data = self.rescaler.rescale_product(gridded_product, data_type,
                                                 inc_by_one=inc_by_one, fill_value=fill_value)

            LOG.info("Writing product %s to AWIPS NetCDF file", gridded_product["product_name"])
            create_awips2_netcdf3(output_filename, data, gridded_product["begin_time"], **awips_info)
        except StandardError:
            LOG.error("Error while filling in NC file with data: %s", output_filename)
            if not self.keep_intermediate and os.path.isfile(output_filename):
                os.remove(output_filename)
            raise

        return output_filename
Exemplo n.º 15
0
class Backend(roles.BackendRole):
    def __init__(self, backend_configs=None, rescale_configs=None, **kwargs):
        self.rescale_configs = rescale_configs or [DEFAULT_NINJO_RCONFIG]
        self.backend_configs = backend_configs or [DEFAULT_NINJO_CONFIG]
        self.rescaler = Rescaler(*self.rescale_configs)
        self.band_config_reader = NinjoBandConfigReader(*self.backend_configs)
        self.grid_config_reader = NinjoGridConfigReader(*self.backend_configs)
        self.sat_config_reader = NinJoSatConfigReader(*self.backend_configs)
        super(Backend, self).__init__(**kwargs)

    @property
    def known_grids(self):
        return self.grid_config_reader.known_grids

    def create_output_from_product(self, gridded_product, output_pattern=None,
                                   data_type=None, inc_by_one=None, fill_value=-999.0, **kwargs):
        # FIXME: Previous version had -999.0 as the fill value...really?
        grid_def = gridded_product["grid_definition"]
        grid_name = grid_def["grid_name"]
        data_type = data_type or numpy.uint8
        inc_by_one = inc_by_one or False
        grid_config_info = self.grid_config_reader.get_config_options(grid_name=grid_name, allow_default=False)
        band_config_info = self.band_config_reader.get_config_options(
            product_name=gridded_product["product_name"],
            satellite=gridded_product["satellite"],
            instrument=gridded_product["instrument"],
            data_type=gridded_product["data_type"],
            data_kind=gridded_product["data_kind"],
            allow_default=False,
        )
        band_config_info["satellite_id"] = self.sat_config_reader.get_satellite_id(gridded_product)

        if not output_pattern:
            output_pattern = DEFAULT_OUTPUT_PATTERN
        if "{" in output_pattern:
            # format the filename
            of_kwargs = gridded_product.copy(as_dict=True)
            of_kwargs["data_type"] = dtype_to_str(data_type)
            output_filename = self.create_output_filename(output_pattern,
                                                          grid_name=grid_def["grid_name"],
                                                          rows=grid_def["height"],
                                                          columns=grid_def["width"],
                                                          **of_kwargs)
        else:
            output_filename = output_pattern

        if os.path.isfile(output_filename):
            if not self.overwrite_existing:
                LOG.error("NinJo TIFF file already exists: %s", output_filename)
                raise RuntimeError("NinJo TIFF file already exists: %s" % (output_filename,))
            else:
                LOG.warning("NinJo TIFF file already exists, will overwrite: %s", output_filename)

        try:
            LOG.debug("Extracting additional information from grid projection")
            map_origin_lon, map_origin_lat = grid_def.lonlat_upperleft
            proj_dict = grid_def.proj4_dict
            equ_radius = proj_dict["a"]
            pol_radius = proj_dict["b"]
            central_meridian = proj_dict.get("lon_0", None)
            ref_lat1 = proj_dict.get("lat_ts", None)

            LOG.debug("Scaling %s data to fit in ninjotiff...", gridded_product["product_name"])
            data = self.rescaler.rescale_product(gridded_product, data_type,
                                                 inc_by_one=inc_by_one, fill_value=fill_value)

            # Create the geotiff
            create_ninjo_tiff(data, output_filename,
                              pixel_xres=grid_config_info["xres"],
                              pixel_yres=grid_config_info["yres"],
                              projection=grid_config_info["projection"],
                              origin_lat=map_origin_lat,
                              origin_lon=map_origin_lon,
                              radius_a=equ_radius,
                              radius_b=pol_radius,
                              central_meridian=central_meridian,
                              ref_lat1=ref_lat1,
                              is_calibrated=1,
                              sat_id=band_config_info["satellite_id"],
                              chan_id=band_config_info["band_id"],
                              data_source=band_config_info["data_source"],
                              data_cat=band_config_info["data_category"],
                              image_dt=gridded_product["begin_time"],
                              data_kind=gridded_product["data_kind"]
                              )
        except StandardError:
            if not self.keep_intermediate and os.path.isfile(output_filename):
                os.remove(output_filename)
            raise

        return output_filename
Exemplo n.º 16
0
class Backend(roles.BackendRole):
    def __init__(self, rescale_configs=None, **kwargs):
        self.rescale_configs = rescale_configs or [DEFAULT_RCONFIG]
        self.rescaler = Rescaler(*self.rescale_configs)
        super(Backend, self).__init__(**kwargs)

    @property
    def known_grids(self):
        # should work regardless of grid
        return None

    def create_output_from_product(self,
                                   gridded_product,
                                   output_pattern=None,
                                   data_type=None,
                                   inc_by_one=None,
                                   fill_value=0,
                                   **kwargs):
        data_type = data_type or numpy.uint8
        etype = np2etype[data_type]
        inc_by_one = inc_by_one or False
        grid_def = gridded_product["grid_definition"]
        if not output_pattern:
            output_pattern = DEFAULT_OUTPUT_PATTERN
        if "{" in output_pattern:
            # format the filename
            of_kwargs = gridded_product.copy(as_dict=True)
            of_kwargs["data_type"] = data_type
            output_filename = self.create_output_filename(
                output_pattern,
                grid_name=grid_def["grid_name"],
                rows=grid_def["height"],
                columns=grid_def["width"],
                **of_kwargs)
        else:
            output_filename = output_pattern

        if os.path.isfile(output_filename):
            if not self.overwrite_existing:
                LOG.error("Geotiff file already exists: %s", output_filename)
                raise RuntimeError("Geotiff file already exists: %s" %
                                   (output_filename, ))
            else:
                LOG.warning("Geotiff file already exists, will overwrite: %s",
                            output_filename)

        try:
            LOG.debug("Scaling %s data to fit in geotiff...",
                      gridded_product["product_name"])
            data = self.rescaler.rescale_product(gridded_product,
                                                 data_type,
                                                 inc_by_one=inc_by_one,
                                                 fill_value=fill_value)

            # Create the geotiff
            # X and Y rotation are 0 in most cases so we just hard-code it
            geotransform = gridded_product["grid_definition"].gdal_geotransform
            create_geotiff(data,
                           output_filename,
                           grid_def["proj4_definition"],
                           geotransform,
                           etype=etype,
                           **kwargs)
        except StandardError:
            if not self.keep_intermediate and os.path.isfile(output_filename):
                os.remove(output_filename)
            raise

        return output_filename