Ejemplo n.º 1
0
    def __validate_data_bound_axes(self, user_axes, number_of_dimensions):
        """
        Check if number of user axes defined in the ingredients file + dataBound:false axes
        should match with number of dimensions in the first input file
        """
        number_of_data_bounded_axes = 0
        for user_axis in user_axes:
            if user_axis.dataBound == True:
                number_of_data_bounded_axes += 1

        if number_of_data_bounded_axes != number_of_dimensions:
            raise RuntimeException("Number of specified axes in the ingredient without dataBound=false != number of axes of the input file,"
                                   " given: {} and {} axes. \n"
                                   "Hint: set irregular axis with \"dataBound\": false in the ingredients file.".format(number_of_data_bounded_axes, number_of_dimensions))
Ejemplo n.º 2
0
 def get_milli_by_uom(self, time_uom):
     """
     Based on time_uom to return the correct milliseconds (e.g: 1d = 24 * 60 * 60 * 1000 milliseconds)
     :param time_uom: uom of time crs (e.g: ansidate: d, unixtime: s)
     :return: long
     """
     # return the uom value in milliseconds
     milli_seconds = self.__UOM_TIME_MAP_CACHE__.get(time_uom)
     if milli_seconds is None:
         raise RuntimeException(
             "Time uom: " + time_uom +
             " does not support yet, cannot calculate time value in milliseconds from time uom."
         )
     return milli_seconds
Ejemplo n.º 3
0
 def _parse_compound_crs(self):
     # http://kahlua.eecs.jacobs-university.de:8080/def/crs-compound?1=http://www.opengis.net/def/crs/EPSG/0/28992&2=http://www.opengis.net/def/crs/EPSG/0/5709
     try:
         url_parts = urlparse.urlparse(self.crs_url)
         get_params = urlparse.parse_qs(url_parts.query)
         index = 1
         while str(index) in get_params:
             crs = get_params[str(index)][0]
             self._parse_single_crs(crs)
             index += 1
     except Exception as ex:
         raise RuntimeException("Failed parsing the compound crs at: {}. "
                                "Detailed error: {}".format(
                                    self.crs_url, str(ex)))
Ejemplo n.º 4
0
 def __parse_axes_elements_from_single_crs(crs):
     """
     Parse the axes XML elements out of the CRS definition
     :param str crs: a complete CRS request  (e.g: http://localhost:8080/def/crs/EPSG/0/4326)
     """
     try:
         gml = validate_and_read_url(crs)
         root = etree.fromstring(gml)
         cselem = root.xpath("./*[contains(local-name(), 'CS')]")[0]
         xml_axes = cselem.xpath(
             ".//*[contains(local-name(), 'SystemAxis')]")
         return xml_axes
     except Exception as ex:
         raise RuntimeException("Failed parsing the crs at: {}. "
                                "Detail error: {}".format(crs, str(ex)))
Ejemplo n.º 5
0
 def get_crs(self):
     """
     Returns the CRS associated with this dataset. If none is found the default for the session is returned
     :rtype: str
     """
     wkt = self.gdal_dataset.GetProjection()
     spatial_ref = self._get_spatial_ref(wkt)
     crs = ConfigManager.default_crs
     if spatial_ref.GetAuthorityName(None) is not None:
         crs = CRSUtil.get_crs_url(spatial_ref.GetAuthorityName(None),
                                   spatial_ref.GetAuthorityCode(None))
     if crs is None:
         raise RuntimeException("Cannot implicitly detect EPSG code from WKT of input file. "
                                "Please explicitly specify the CRS in the ingredient (option \"default_crs\").")
     return crs
Ejemplo n.º 6
0
    def read_file_to_string(file_path):
        """
        Read a file path to a string
        :param str file_path: absolute path
        :return: file content
        """

        if not os.access(file_path, os.R_OK):
            raise RuntimeException("Cannot read content from file '" +
                                   file_path + "' as string.")

        with open(file_path, "r") as myfile:
            data = myfile.read()

            return data
Ejemplo n.º 7
0
 def evaluate(self, expression, evaluator_slice):
     """
     Evaluates a wcst import ingredient expression
     :param str expression: the expression to evaluate
     :param NetcdfEvaluatorSlice evaluator_slice: the slice on which to evaluate the expression
     :rtype: str
     """
     if not isinstance(evaluator_slice, NetcdfEvaluatorSlice):
         raise RuntimeException(
             "Cannot evaluate a netcdf expression on something that is not a netcdf valid file. Given expression: {}".format(
                 expression))
     # extract the axis and operation in the expression only
     # ${netcdf:variable:E:min} -> variable:E:min
     expression = expression.replace("netcdf:", "").replace(self.PREFIX, "").replace(self.POSTFIX, "")
     return self._resolve(expression, evaluator_slice.get_dataset())
Ejemplo n.º 8
0
def import_pygrib():
    """
    Import pygrib which is used for importing GRIB file.
    """
    global imported_pygrib
    if imported_pygrib is None:
        try:
            import pygrib
            imported_pygrib = pygrib
        except ImportError as e:
            raise RuntimeException(
                "Cannot import GRIB data, please install pygrib first (sudo pip3 install pygrib). "
                "Reason: {}.".format(e))

    return imported_pygrib
Ejemplo n.º 9
0
    def get_time_crs_origin(self, crs_uri):
        """
        Based on crs_uri (e.g: http://localhost:8080/def/crs/OGC/0/AnsiDate) to get the datetime origin
        :param crs_uri: string
        :return:
        """
        date_origin = None
        for key, value in self.__TIME_CRS_MAP_CACHE__.iteritems():
            # e.g: AnsiDate in crs_uri
            if key in crs_uri:
                date_origin = value
                break

        if date_origin is None:
            raise RuntimeException("Time crs: " + crs_uri + " does not support yet, cannot get the date origin from the time crs.")
        return date_origin
Ejemplo n.º 10
0
    def evaluate(self, expression, evaluator_slice):
        """
        Evaluates a wcst import ingredient expression
        :param str expression: the expression to evaluate
        :param FileEvaluatorSlice evaluator_slice: the slice on which to evaluate the expression
        :rtype: str
        """
        if not isinstance(evaluator_slice, FileEvaluatorSlice):
            raise RuntimeException(
                "Cannot evaluate a file expression on something that is not a valid file. Given expression: {}".format(
                    expression))

        # e.g: file:path -> path
        expression = expression.split(":")[1]

        return self._resolve(expression, evaluator_slice.get_file())
Ejemplo n.º 11
0
def import_netcdf4():
    """
    Import netCDF4 which is used for importing netCDF file.
    """
    global imported_netcdf
    if imported_netcdf is None:
        try:
            import netCDF4
            imported_netcdf = netCDF4
        except ImportError as e:
            raise RuntimeException(
                "Cannot import netCDF data, please install netCDF4 first \
                                    (sudo pip3 install netCDF4)."
                "Reason: {}.".format(e))

    return imported_netcdf
Ejemplo n.º 12
0
 def _get_coverage(self):
     """
     Returns the coverage to be used for the importer
     :rtype: master.importer.coverage.Coverage
     """
     recipe_type = self.options['coverage']['slicer']['type']
     if recipe_type == GdalToCoverageConverter.RECIPE_TYPE:
         coverage = self._get_gdal_coverage(recipe_type)
     elif recipe_type == NetcdfToCoverageConverter.RECIPE_TYPE:
         coverage = self._get_netcdf_coverage(recipe_type)
     elif recipe_type == GRIBToCoverageConverter.RECIPE_TYPE:
         coverage = self._get_grib_coverage(recipe_type)
     else:
         raise RuntimeException(
             "No valid slicer could be found, given: " + recipe_type)
     return coverage
Ejemplo n.º 13
0
    def _init_coverage_options(self):
        covopts = self.options["coverage"]
        self.epsg_xy_crs = None

        self._init_epsg_xy_crs()
        if self.epsg_xy_crs is None:
            raise RuntimeException(
                "Cannot detect geo CRS from input files, make sure the files exist and are readable by gdal."
            )

        compound_crs = self.CRS_TEMPLATE.replace(self.EPSG_XY_CRS,
                                                 self.epsg_xy_crs)
        self.crs = self._set_option(covopts, "crs",
                                    self._resolve_crs(compound_crs))
        self._set_option(covopts, "slicer", {})
        self._init_slicer_options(covopts)
Ejemplo n.º 14
0
    def get_axis_labels(self):
        """
        Returns the axis labels as a list
        :rtype list[str]
        """
        try:
            service_call = self.wcs_service + "?service=WCS&request=DescribeCoverage&version=" + \
                           Session.get_WCS_VERSION_SUPPORTED() + "&coverageId=" + self.coverage_id
            response = validate_and_read_url(service_call)

            return response.split("axisLabels=\"")[1].split('"')[0].split(" ")
        except Exception as ex:
            raise RuntimeException(
                "Could not retrieve the axis labels. "
                "Check that the WCS service is up and running on url: {}. "
                "Detail error: {}".format(self.wcs_service, str(ex)))
Ejemplo n.º 15
0
    def _parse_single_crs(self, crs):
        """
        Parses the axes out of the CRS definition
        str crs: a complete CRS request (e.g: http://localhost:8080/def/crs/EPSG/0/4326)
        """
        try:
            xml_axes = self.__parse_axes_elements_from_single_crs(crs)
            axis_labels = []

            for xml_axis in xml_axes:
                axis_label = xml_axis.xpath(
                    ".//*[contains(local-name(), 'axisAbbrev')]")[0].text
                axis_type = CRSAxis.get_axis_type_by_name(axis_label)

                # e.g: http://localhost:8080/def/uom/EPSG/0/9122
                uom_url = xml_axis.attrib['uom']
                try:
                    # NOTE: as opengis.net will redirect to another web page for UoM description, so don't follow it
                    if CRSAxis.UOM_UCUM in uom_url:
                        axis_uom = self.__get_axis_uom_from_uom_crs(uom_url)
                    else:
                        uom_gml = validate_and_read_url(uom_url)
                        uom_root_element = etree.fromstring(uom_gml)

                        # e.g: degree (supplier to define representation)
                        uom_name = uom_root_element.xpath(
                            ".//*[contains(local-name(), 'name')]")[0].text
                        axis_uom = uom_name.split(" ")[0]
                except Exception:
                    # e.g: def/uom/OGC/1.0/GridSpacing does not exist -> GridSpacing
                    axis_uom = self.__get_axis_uom_from_uom_crs(uom_url)

                # With OGC Time axes, check axis type in different way
                if re.search(DateTimeUtil.CRS_CODE_ANSI_DATE, crs) is not None \
                   or re.search(DateTimeUtil.CRS_CODE_UNIX_TIME, crs) is not None:
                    axis_type = CRSAxis.get_axis_type_by_uom(axis_uom)

                crs_axis = CRSAxis(crs, axis_label, axis_type, axis_uom)
                axis_labels.append(axis_label)

                self.axes.append(crs_axis)

            # add to the list of individual crs to axes
            self.individual_crs_axes[crs] = axis_labels
        except Exception as ex:
            raise RuntimeException("Failed parsing the crs at: {}. "
                                   "Reason: {}".format(crs, str(ex)))
Ejemplo n.º 16
0
    def _global_metadata_fields(self):
        """
        Returns the global metadata fields
        :rtype: dict
        """

        if "coverage" in self.options and "metadata" in self.options['coverage']:

            for file_path in self.session.get_files():
                try:
                    if "global" in self.options['coverage']['metadata']:
                        global_metadata = self.options['coverage']['metadata']['global']

                        # global_metadata is manually defined with { ... some values }
                        if type(global_metadata) is dict:
                            metadata_dict = self.options['coverage']['metadata']['global']
                        else:
                            # global metadata is defined with "a string"
                            if global_metadata != "auto":
                                raise RuntimeException("No valid global metadata attribute for gdal slicer could be found, "
                                                       "given: " + global_metadata)
                            else:
                                # global metadata is defined with auto, then parse the metadata from the first file to a dict
                                if self.options['coverage']['slicer']['type'] == "gdal":
                                    metadata_dict = GdalToCoverageConverter.parse_gdal_global_metadata(file_path)
                                elif self.options['coverage']['slicer']['type'] == "netcdf":
                                    metadata_dict = NetcdfToCoverageConverter.parse_netcdf_global_metadata(file_path)
                    else:
                        # global is not specified in ingredient file, it is considered as "global": "auto"
                        if self.options['coverage']['slicer']['type'] == "gdal":
                            metadata_dict = GdalToCoverageConverter.parse_gdal_global_metadata(file_path)
                        elif self.options['coverage']['slicer']['type'] == "netcdf":
                            metadata_dict = NetcdfToCoverageConverter.parse_netcdf_global_metadata(file_path)

                    self.__add_color_palette_table_to_global_metadata(metadata_dict, file_path)

                    result = escape_metadata_dict(metadata_dict)
                    return result
                except Exception as e:
                    if ConfigManager.skip == True:
                        # Error with opening the first file, then try with another file as skip is true
                        pass
                    else:
                        raise e

        return {}
Ejemplo n.º 17
0
    def _parse_single_crs(self, crs):
        """
        Parses the axes out of the CRS definition
        """
        try:
            contents = validate_and_read_url(crs)
            root = etree.fromstring(contents)
            cselem = root.xpath("./*[contains(local-name(), 'CS')]")[0]
            xml_axes = cselem.xpath(
                ".//*[contains(local-name(), 'SystemAxis')]")
            axesLabels = []
            for xml_axis in xml_axes:
                label = xml_axis.xpath(
                    ".//*[contains(local-name(), 'axisAbbrev')]")[0].text
                direction = xml_axis.xpath(
                    ".//*[contains(local-name(), 'axisDirection')]")[0].text

                # IndexND crses do not define the direction properly so try to detect them here based
                # on their labels and direction
                if direction.find(
                        "indexedAxisPositive") != -1 and label == "i":
                    direction = "east"
                if direction.find(
                        "indexedAxisPositive") != -1 and label == "j":
                    direction = "north"
                if "future" in direction:
                    uom = root.xpath(
                        ".//*[contains(local-name(), 'CoordinateSystemAxis')]"
                    )[0].attrib['uom']
                else:
                    uom = ""

                # in some crs definitions the axis direction is not properly set, override
                if label in self.X_AXES:
                    direction = "east"
                elif label in self.Y_AXES:
                    direction = "north"

                crsAxis = CRSAxis(crs, label, direction, uom)
                axesLabels.append(label)
                self.axes.append(crsAxis)
            # add to the list of individual crs to axes
            self.individual_crs_axes[crs] = axesLabels
        except Exception as ex:
            raise RuntimeException("Failed parsing the crs at: {}. "
                                   "Detail error: {}".format(crs, str(ex)))
Ejemplo n.º 18
0
def url_read_exception(url, exception_message):
    """
    Open url which could return an exception (e.g: DescribeCoverage when coverage does not exist in Petascope).
    If it returns an exception, check the exception_message is in exception, if not -> not valid.

    :param str url: the url to open
    :param str exception_message: the error message could be in the exception from the requested URL
    :return: boolean (false: if status is 200 or true if message does exist)
    """
    ret = urllib.urlopen(url)
    response = ret.read()
    if ret.getcode() == 200:
        return False
    elif ret.getcode() != 200 and exception_message in response:
        return True

    # Exception message is not in response and status is not 200, so it is an error
    raise RuntimeException("Failed opening connection to '{}'. Check that the service is up and running.".format(url))
Ejemplo n.º 19
0
 def _resolve(self, expression, file):
     """
     Resolves the expression in the context of a file container
     :param str expression: the expression to resolve
     :param File file: the file for which to resolve the expression
     :return:
     """
     file_dictionary = {
         "path": file.get_filepath(),
         "name": os.path.basename(file.get_filepath())
     }
     if expression in file_dictionary:
         value = file_dictionary[expression]
     else:
         raise RuntimeException(
             "Cannot evaluate the given file expression {} on the given container."
             .format(str(expression)))
     return value
Ejemplo n.º 20
0
    def _user_axis(self, user_axis, evaluator_slice):
        """
        Returns an evaluated user axis from a user supplied axis
        The user supplied axis contains for each attribute an expression that can be evaluated.
         We need to return a user axis that contains the actual values derived from the expression evaluation
        :param UserAxis | IrregularUserAxis user_axis: the user axis to evaluate
        :param evaluator_slice: the sentence evaluator for the slice
        :rtype: UserAxis | IrregularUserAxis
        """
        if isinstance(user_axis, IrregularUserAxis):
            if not user_axis.dataBound and user_axis.directPositions != self.DIRECT_POSITIONS_SLICING:
                raise RuntimeException(
                    "The dataBound option is set to false for irregular axis '{}', "
                    "so directPositions option can not be set in the ingredients file for this axis."
                    .format(user_axis.name))

        min = self.sentence_evaluator.evaluate(user_axis.interval.low,
                                               evaluator_slice,
                                               user_axis.statements)
        max = None
        if user_axis.interval.high:
            max = self.sentence_evaluator.evaluate(user_axis.interval.high,
                                                   evaluator_slice,
                                                   user_axis.statements)
        resolution = self.sentence_evaluator.evaluate(user_axis.resolution,
                                                      evaluator_slice,
                                                      user_axis.statements)
        if isinstance(user_axis, RegularUserAxis):
            return RegularUserAxis(user_axis.name, resolution, user_axis.order,
                                   min, max, user_axis.type,
                                   user_axis.dataBound)
        else:
            if GribExpressionEvaluator.FORMAT_TYPE in user_axis.directPositions:
                # grib irregular axis will be calculated later when all the messages is evaluated
                direct_positions = user_axis.directPositions
            else:
                direct_positions = self.sentence_evaluator.evaluate(
                    user_axis.directPositions, evaluator_slice,
                    user_axis.statements)

            return IrregularUserAxis(user_axis.name, resolution,
                                     user_axis.order, min, direct_positions,
                                     max, user_axis.type, user_axis.dataBound,
                                     [], user_axis.slice_group_size)
Ejemplo n.º 21
0
 def __init__(self, datetime, dt_format=None, time_crs=None):
     """
     :param str datetime: the datetime value
     :param str dt_format: the datetime format, if none is given we'll try to guess
     """
     self.init_cache()
     try:
         if dt_format is None:
             self.datetime = arrow.get(datetime)
         else:
             self.datetime = arrow.get(datetime, dt_format)
         if time_crs is None:
             self.time_crs_code = self.CRS_CODE_ANSI_DATE
         else:
             tmp_crs = CRSUtil(time_crs)
             self.time_crs_code = tmp_crs.get_crs_code()
     except ParserError as pe:
         dt_format_err = "auto" if dt_format is None else dt_format
         raise RuntimeException("Failed to parse the date " + datetime +
                                " using format " + dt_format_err)
Ejemplo n.º 22
0
    def _file_band_nil_values(self, index):
        """
        This is used to get the null values (Only 1) from the given band index if one exists when nilValue was not defined
        in ingredient file
        :param integer index: the current band index to get the nilValues (GRIB seems only support 1 band)
        :rtype: List[RangeTypeNilValue] with only 1 element
        """
        if len(self.files) < 1:
            raise RuntimeException("No files to import were specified.")

        # NOTE: all files should have same bands's metadata
        try:
            nil_value = self.dataset.message(1)["missingValue"]
        except KeyError:
            # missingValue is not defined in grib file
            nil_value = None

        if nil_value is None:
            return None
        else:
            return [nil_value]
Ejemplo n.º 23
0
    def _file_band_nil_values(self, index):
        """
        This is used to get the null values (Only 1) from the given band index if one exists when nilValue was not defined
        in ingredient file
        :param integer index: the current band index to get the nilValues
        :rtype: List[RangeTypeNilValue] with only 1 element
        """
        if len(self.files) < 1:
            raise RuntimeException("No gdal files given for import!")

        import osgeo.gdal as gdal
        # NOTE: all files should have same bands's metadata, so first file is ok
        gdal_dataset = gdal.Open(self.files[0].filepath)
        # band in gdal starts with 1
        gdal_band = gdal_dataset.GetRasterBand(index + 1)
        nil_value = gdal_band.GetNoDataValue()

        if nil_value is None:
            return None
        else:
            return [nil_value]
Ejemplo n.º 24
0
 def exists(self):
     """
     Returns true if the coverage exist, false otherwise
     :rtype bool
     """
     try:
         service_call = self.wcs_service + "?service=WCS&request=DescribeCoverage&version=" + \
                        Session.get_WCS_VERSION_SUPPORTED() + "&coverageId=" + self.coverage_id
         # Check if exception is thrown in the response
         ret = url_read_exception(service_call,
                                  'exceptionCode="NoSuchCoverage"')
         if ret:
             # exception is in the response, coverage does not exist
             return False
         else:
             # exception is not the in the response, coverage does exist
             return True
     except Exception as ex:
         raise RuntimeException(
             "Could not check if the coverage exists. "
             "Check that the WCS service is up and running on url: {}. "
             "Detail error: {}".format(self.wcs_service, str(ex)))
Ejemplo n.º 25
0
    def exists(self):
        """
        Returns true if the coverage exist, false otherwise
        :rtype bool
        """
        try:
            # Check if coverage exists in WCS GetCapabilities result
            service_call = self.wcs_service + "?service=WCS&request=GetCapabilities&acceptVersions=" + \
                           Session.get_WCS_VERSION_SUPPORTED()
            response = validate_and_read_url(service_call)

            root = etree.fromstring(response)
            coverage_id_elements = root.xpath(
                "//*[local-name() = 'CoverageId']")
            # Iterate all <CoverageId> elements to check coverageId exists already
            for element in coverage_id_elements:
                if self.coverage_id == element.text:
                    return True
            return False
        except Exception as ex:
            raise RuntimeException("Could not check if the coverage exists. "
                                   "Detail error: {}".format(str(ex)))
Ejemplo n.º 26
0
 def __load_imported_data_from_resume_file(self, coverage_id):
     """
     Try to load a resume file coverage_id.resume.json from input data folder.
     :param str coverage_id: coverage id of current importer to find the resume file.
     """
     if coverage_id not in Resumer.__IMPORTED_DATA_DICT:
         resume_file_path = ConfigManager.resumer_dir_path + coverage_id + Resumer.__RESUMER_FILE_SUFFIX
         Resumer.__RESUMER_FILE_NAME_DICT[coverage_id] = resume_file_path
         try:
             if os.path.isfile(resume_file_path) \
                     and os.access(resume_file_path, os.R_OK):
                 log.info(
                     "We found a resumer file in the ingredients folder. The slices listed in '" + resume_file_path
                     + "' will not be imported.")
                 file = open(Resumer.__RESUMER_FILE_NAME_DICT[coverage_id])
                 data = json.loads(file.read())
                 Resumer.__IMPORTED_DATA_DICT[coverage_id] = data
                 file.close()
         except IOError as e:
             raise RuntimeException("Could not read the resume file, full error message: " + str(e))
         except ValueError as e:
             log.warn("The resumer JSON file could not be parsed. A new one will be created.")
Ejemplo n.º 27
0
    def _file_band_nil_values(self, index):
        """
        This is used to get the null values (Only 1) from the given band index if one exists when nilValue was not defined
        in ingredient file
        :param integer index: the current band index to get the nilValues
        :rtype: List[RangeTypeNilValue] with only 1 element
        """
        if len(self.files) < 1:
            raise RuntimeException("No gdal files given for import!")

        if len(self.default_null_values) > 0:
            return self.default_null_values

        # NOTE: all files should have same bands's metadata, so 1 file is ok
        gdal_dataset = GDALGmlUtil.open_gdal_dataset_from_any_file(self.files)
        # band in gdal starts with 1
        gdal_band = gdal_dataset.get_raster_band(index + 1)
        nil_value = gdal_band.GetNoDataValue()

        if nil_value is None:
            return None
        else:
            return [nil_value]
Ejemplo n.º 28
0
    def exists(self):
        """
        Returns true if the coverage exist, false otherwise
        :rtype bool
        """
        try:
            # Check if coverage exists in WCS DescribeCoverage result
            service_call = self.wcs_service + "?service=WCS&request=DescribeCoverage&version=" + \
                           Session.get_WCS_VERSION_SUPPORTED() \
                           + "&coverageId=" + self.coverage_id

            response = validate_and_read_url(service_call)
            if decode_res(response).strip() != "":
                return True
            return False
        except Exception as ex:
            if not "NoSuchCoverage" in str(ex):
                raise RuntimeException(
                    "Could not check if the coverage exists. "
                    "Detail error: {}".format(str(ex)))
            else:
                # coverage doesn't exist
                return False
Ejemplo n.º 29
0
    def _apply_operation(self, nc_dataset, nc_obj_name, operation):
        """
        Applies operation on a given variable which contains a list of values (e.g: lat = [0, 1, 2, 3,...]),
        (e.g: find the min of time variable ${netcdf:variable:time:min})
        :param netCDF4 nc_dataset: the netcdf dataset
        :param str nc_obj_name: name of netCDF variable or netCDF dimension
        :param str operation: the operation to apply:
        :return: str value: The value from the applied operation with precession
        """
        """ NOTE: min or max of list(variable) with values like [148654.08425925925,...]
        will return 148654.084259 in float which will cause problem with calculate coefficient as the first coeffcient should be 0
        but due to this change of min/max value, the coefficient is like 0.00000000001 (case: PML)
        "min": "${netcdf:variable:ansi:min} * 24 * 3600 - 11644560000.0", -> return: 1199152879.98
        "directPositions": "[float(x) * 24 * 3600 - 11644560000.0 for x in ${netcdf:variable:ansi}]", -> return 1199152880.0

        So we must use the values in the list by string and split it to a list to get the same values
        """
        MAX = "max"
        MIN = "min"
        LAST = "last"
        FIRST = "first"
        RESOLUTION = "resolution"
        METADATA = "metadata"

        # List of support operation on a netCDF variable
        supported_operations = [MAX, MIN, LAST, FIRST, RESOLUTION, METADATA]

        import_util.import_numpy()
        import numpy as np

        if nc_obj_name in nc_dataset.variables:
            nc_obj = nc_dataset.variables[nc_obj_name]

            if operation not in supported_operations:
                # it must be an attribute of variable
                return nc_obj.__getattribute__(operation)

            # It must be an operation that could be applied on a netCDF variable
            # convert list of string values to list of decimal values
            values = nc_obj[:].flatten()
        elif nc_obj_name in nc_dataset.dimensions:
            nc_obj = nc_dataset.dimensions[nc_obj_name]
            # Cannot determine list of values from variable but only dimension (e.g: station = 758)
            values = np.arange(0, nc_obj.size)
        else:
            raise Exception("Cannot find '" + nc_obj_name + "' from list of netCDF variables and dimensions.")

        if operation == MAX:
            return to_decimal(np.amax(values))
        elif operation == MIN:
            return to_decimal(np.amin(values))
        elif operation == LAST:
            last_index = len(nc_obj) - 1
            return to_decimal(values[last_index])
        elif operation == FIRST:
            return to_decimal(values[0])
        elif operation == RESOLUTION:
            # NOTE: only netCDF needs this expression to calculate resolution automatically
            # for GDAL: it uses: ${gdal:resolutionX} and GRIB: ${grib:jDirectionIncrementInDegrees} respectively
            return self.__calculate_netcdf_resolution(values)
        elif operation == METADATA:
            # return a dict of variable (axis) metadata with keys, values as string
            tmp_dict = {}
            for attr in nc_obj.ncattrs():
                try:
                    tmp_dict[attr] = escape(getattr(nc_obj, attr))
                except:
                    log.warn("Attribute '" + attr + "' of variable '" + nc_obj._getname() + "' cannot be parsed as string, ignored.")
            return tmp_dict

        # Not supported operation and not valid attribute of netCDF variable
        raise RuntimeException(
            "Invalid operation on netcdf variable: " + operation
            + ". Currently supported: " + ', '.join(supported_operations) + " or any metadata entry of the variable.")
Ejemplo n.º 30
0
 def to_unknown(self):
     """
     Handle unknown time CRS
     :return: throw RuntimeException
     """
     raise RuntimeException("Unsupported time CRS " + self.time_crs_code)