def __validate_data_bound_axes(self, user_axes, number_of_dimensions): """ Check if number of user axes defined in the ingredients file + dataBound:false axes should match with number of dimensions in the first input file """ number_of_data_bounded_axes = 0 for user_axis in user_axes: if user_axis.dataBound == True: number_of_data_bounded_axes += 1 if number_of_data_bounded_axes != number_of_dimensions: raise RuntimeException("Number of specified axes in the ingredient without dataBound=false != number of axes of the input file," " given: {} and {} axes. \n" "Hint: set irregular axis with \"dataBound\": false in the ingredients file.".format(number_of_data_bounded_axes, number_of_dimensions))
def get_milli_by_uom(self, time_uom): """ Based on time_uom to return the correct milliseconds (e.g: 1d = 24 * 60 * 60 * 1000 milliseconds) :param time_uom: uom of time crs (e.g: ansidate: d, unixtime: s) :return: long """ # return the uom value in milliseconds milli_seconds = self.__UOM_TIME_MAP_CACHE__.get(time_uom) if milli_seconds is None: raise RuntimeException( "Time uom: " + time_uom + " does not support yet, cannot calculate time value in milliseconds from time uom." ) return milli_seconds
def _parse_compound_crs(self): # http://kahlua.eecs.jacobs-university.de:8080/def/crs-compound?1=http://www.opengis.net/def/crs/EPSG/0/28992&2=http://www.opengis.net/def/crs/EPSG/0/5709 try: url_parts = urlparse.urlparse(self.crs_url) get_params = urlparse.parse_qs(url_parts.query) index = 1 while str(index) in get_params: crs = get_params[str(index)][0] self._parse_single_crs(crs) index += 1 except Exception as ex: raise RuntimeException("Failed parsing the compound crs at: {}. " "Detailed error: {}".format( self.crs_url, str(ex)))
def __parse_axes_elements_from_single_crs(crs): """ Parse the axes XML elements out of the CRS definition :param str crs: a complete CRS request (e.g: http://localhost:8080/def/crs/EPSG/0/4326) """ try: gml = validate_and_read_url(crs) root = etree.fromstring(gml) cselem = root.xpath("./*[contains(local-name(), 'CS')]")[0] xml_axes = cselem.xpath( ".//*[contains(local-name(), 'SystemAxis')]") return xml_axes except Exception as ex: raise RuntimeException("Failed parsing the crs at: {}. " "Detail error: {}".format(crs, str(ex)))
def get_crs(self): """ Returns the CRS associated with this dataset. If none is found the default for the session is returned :rtype: str """ wkt = self.gdal_dataset.GetProjection() spatial_ref = self._get_spatial_ref(wkt) crs = ConfigManager.default_crs if spatial_ref.GetAuthorityName(None) is not None: crs = CRSUtil.get_crs_url(spatial_ref.GetAuthorityName(None), spatial_ref.GetAuthorityCode(None)) if crs is None: raise RuntimeException("Cannot implicitly detect EPSG code from WKT of input file. " "Please explicitly specify the CRS in the ingredient (option \"default_crs\").") return crs
def read_file_to_string(file_path): """ Read a file path to a string :param str file_path: absolute path :return: file content """ if not os.access(file_path, os.R_OK): raise RuntimeException("Cannot read content from file '" + file_path + "' as string.") with open(file_path, "r") as myfile: data = myfile.read() return data
def evaluate(self, expression, evaluator_slice): """ Evaluates a wcst import ingredient expression :param str expression: the expression to evaluate :param NetcdfEvaluatorSlice evaluator_slice: the slice on which to evaluate the expression :rtype: str """ if not isinstance(evaluator_slice, NetcdfEvaluatorSlice): raise RuntimeException( "Cannot evaluate a netcdf expression on something that is not a netcdf valid file. Given expression: {}".format( expression)) # extract the axis and operation in the expression only # ${netcdf:variable:E:min} -> variable:E:min expression = expression.replace("netcdf:", "").replace(self.PREFIX, "").replace(self.POSTFIX, "") return self._resolve(expression, evaluator_slice.get_dataset())
def import_pygrib(): """ Import pygrib which is used for importing GRIB file. """ global imported_pygrib if imported_pygrib is None: try: import pygrib imported_pygrib = pygrib except ImportError as e: raise RuntimeException( "Cannot import GRIB data, please install pygrib first (sudo pip3 install pygrib). " "Reason: {}.".format(e)) return imported_pygrib
def get_time_crs_origin(self, crs_uri): """ Based on crs_uri (e.g: http://localhost:8080/def/crs/OGC/0/AnsiDate) to get the datetime origin :param crs_uri: string :return: """ date_origin = None for key, value in self.__TIME_CRS_MAP_CACHE__.iteritems(): # e.g: AnsiDate in crs_uri if key in crs_uri: date_origin = value break if date_origin is None: raise RuntimeException("Time crs: " + crs_uri + " does not support yet, cannot get the date origin from the time crs.") return date_origin
def evaluate(self, expression, evaluator_slice): """ Evaluates a wcst import ingredient expression :param str expression: the expression to evaluate :param FileEvaluatorSlice evaluator_slice: the slice on which to evaluate the expression :rtype: str """ if not isinstance(evaluator_slice, FileEvaluatorSlice): raise RuntimeException( "Cannot evaluate a file expression on something that is not a valid file. Given expression: {}".format( expression)) # e.g: file:path -> path expression = expression.split(":")[1] return self._resolve(expression, evaluator_slice.get_file())
def import_netcdf4(): """ Import netCDF4 which is used for importing netCDF file. """ global imported_netcdf if imported_netcdf is None: try: import netCDF4 imported_netcdf = netCDF4 except ImportError as e: raise RuntimeException( "Cannot import netCDF data, please install netCDF4 first \ (sudo pip3 install netCDF4)." "Reason: {}.".format(e)) return imported_netcdf
def _get_coverage(self): """ Returns the coverage to be used for the importer :rtype: master.importer.coverage.Coverage """ recipe_type = self.options['coverage']['slicer']['type'] if recipe_type == GdalToCoverageConverter.RECIPE_TYPE: coverage = self._get_gdal_coverage(recipe_type) elif recipe_type == NetcdfToCoverageConverter.RECIPE_TYPE: coverage = self._get_netcdf_coverage(recipe_type) elif recipe_type == GRIBToCoverageConverter.RECIPE_TYPE: coverage = self._get_grib_coverage(recipe_type) else: raise RuntimeException( "No valid slicer could be found, given: " + recipe_type) return coverage
def _init_coverage_options(self): covopts = self.options["coverage"] self.epsg_xy_crs = None self._init_epsg_xy_crs() if self.epsg_xy_crs is None: raise RuntimeException( "Cannot detect geo CRS from input files, make sure the files exist and are readable by gdal." ) compound_crs = self.CRS_TEMPLATE.replace(self.EPSG_XY_CRS, self.epsg_xy_crs) self.crs = self._set_option(covopts, "crs", self._resolve_crs(compound_crs)) self._set_option(covopts, "slicer", {}) self._init_slicer_options(covopts)
def get_axis_labels(self): """ Returns the axis labels as a list :rtype list[str] """ try: service_call = self.wcs_service + "?service=WCS&request=DescribeCoverage&version=" + \ Session.get_WCS_VERSION_SUPPORTED() + "&coverageId=" + self.coverage_id response = validate_and_read_url(service_call) return response.split("axisLabels=\"")[1].split('"')[0].split(" ") except Exception as ex: raise RuntimeException( "Could not retrieve the axis labels. " "Check that the WCS service is up and running on url: {}. " "Detail error: {}".format(self.wcs_service, str(ex)))
def _parse_single_crs(self, crs): """ Parses the axes out of the CRS definition str crs: a complete CRS request (e.g: http://localhost:8080/def/crs/EPSG/0/4326) """ try: xml_axes = self.__parse_axes_elements_from_single_crs(crs) axis_labels = [] for xml_axis in xml_axes: axis_label = xml_axis.xpath( ".//*[contains(local-name(), 'axisAbbrev')]")[0].text axis_type = CRSAxis.get_axis_type_by_name(axis_label) # e.g: http://localhost:8080/def/uom/EPSG/0/9122 uom_url = xml_axis.attrib['uom'] try: # NOTE: as opengis.net will redirect to another web page for UoM description, so don't follow it if CRSAxis.UOM_UCUM in uom_url: axis_uom = self.__get_axis_uom_from_uom_crs(uom_url) else: uom_gml = validate_and_read_url(uom_url) uom_root_element = etree.fromstring(uom_gml) # e.g: degree (supplier to define representation) uom_name = uom_root_element.xpath( ".//*[contains(local-name(), 'name')]")[0].text axis_uom = uom_name.split(" ")[0] except Exception: # e.g: def/uom/OGC/1.0/GridSpacing does not exist -> GridSpacing axis_uom = self.__get_axis_uom_from_uom_crs(uom_url) # With OGC Time axes, check axis type in different way if re.search(DateTimeUtil.CRS_CODE_ANSI_DATE, crs) is not None \ or re.search(DateTimeUtil.CRS_CODE_UNIX_TIME, crs) is not None: axis_type = CRSAxis.get_axis_type_by_uom(axis_uom) crs_axis = CRSAxis(crs, axis_label, axis_type, axis_uom) axis_labels.append(axis_label) self.axes.append(crs_axis) # add to the list of individual crs to axes self.individual_crs_axes[crs] = axis_labels except Exception as ex: raise RuntimeException("Failed parsing the crs at: {}. " "Reason: {}".format(crs, str(ex)))
def _global_metadata_fields(self): """ Returns the global metadata fields :rtype: dict """ if "coverage" in self.options and "metadata" in self.options['coverage']: for file_path in self.session.get_files(): try: if "global" in self.options['coverage']['metadata']: global_metadata = self.options['coverage']['metadata']['global'] # global_metadata is manually defined with { ... some values } if type(global_metadata) is dict: metadata_dict = self.options['coverage']['metadata']['global'] else: # global metadata is defined with "a string" if global_metadata != "auto": raise RuntimeException("No valid global metadata attribute for gdal slicer could be found, " "given: " + global_metadata) else: # global metadata is defined with auto, then parse the metadata from the first file to a dict if self.options['coverage']['slicer']['type'] == "gdal": metadata_dict = GdalToCoverageConverter.parse_gdal_global_metadata(file_path) elif self.options['coverage']['slicer']['type'] == "netcdf": metadata_dict = NetcdfToCoverageConverter.parse_netcdf_global_metadata(file_path) else: # global is not specified in ingredient file, it is considered as "global": "auto" if self.options['coverage']['slicer']['type'] == "gdal": metadata_dict = GdalToCoverageConverter.parse_gdal_global_metadata(file_path) elif self.options['coverage']['slicer']['type'] == "netcdf": metadata_dict = NetcdfToCoverageConverter.parse_netcdf_global_metadata(file_path) self.__add_color_palette_table_to_global_metadata(metadata_dict, file_path) result = escape_metadata_dict(metadata_dict) return result except Exception as e: if ConfigManager.skip == True: # Error with opening the first file, then try with another file as skip is true pass else: raise e return {}
def _parse_single_crs(self, crs): """ Parses the axes out of the CRS definition """ try: contents = validate_and_read_url(crs) root = etree.fromstring(contents) cselem = root.xpath("./*[contains(local-name(), 'CS')]")[0] xml_axes = cselem.xpath( ".//*[contains(local-name(), 'SystemAxis')]") axesLabels = [] for xml_axis in xml_axes: label = xml_axis.xpath( ".//*[contains(local-name(), 'axisAbbrev')]")[0].text direction = xml_axis.xpath( ".//*[contains(local-name(), 'axisDirection')]")[0].text # IndexND crses do not define the direction properly so try to detect them here based # on their labels and direction if direction.find( "indexedAxisPositive") != -1 and label == "i": direction = "east" if direction.find( "indexedAxisPositive") != -1 and label == "j": direction = "north" if "future" in direction: uom = root.xpath( ".//*[contains(local-name(), 'CoordinateSystemAxis')]" )[0].attrib['uom'] else: uom = "" # in some crs definitions the axis direction is not properly set, override if label in self.X_AXES: direction = "east" elif label in self.Y_AXES: direction = "north" crsAxis = CRSAxis(crs, label, direction, uom) axesLabels.append(label) self.axes.append(crsAxis) # add to the list of individual crs to axes self.individual_crs_axes[crs] = axesLabels except Exception as ex: raise RuntimeException("Failed parsing the crs at: {}. " "Detail error: {}".format(crs, str(ex)))
def url_read_exception(url, exception_message): """ Open url which could return an exception (e.g: DescribeCoverage when coverage does not exist in Petascope). If it returns an exception, check the exception_message is in exception, if not -> not valid. :param str url: the url to open :param str exception_message: the error message could be in the exception from the requested URL :return: boolean (false: if status is 200 or true if message does exist) """ ret = urllib.urlopen(url) response = ret.read() if ret.getcode() == 200: return False elif ret.getcode() != 200 and exception_message in response: return True # Exception message is not in response and status is not 200, so it is an error raise RuntimeException("Failed opening connection to '{}'. Check that the service is up and running.".format(url))
def _resolve(self, expression, file): """ Resolves the expression in the context of a file container :param str expression: the expression to resolve :param File file: the file for which to resolve the expression :return: """ file_dictionary = { "path": file.get_filepath(), "name": os.path.basename(file.get_filepath()) } if expression in file_dictionary: value = file_dictionary[expression] else: raise RuntimeException( "Cannot evaluate the given file expression {} on the given container." .format(str(expression))) return value
def _user_axis(self, user_axis, evaluator_slice): """ Returns an evaluated user axis from a user supplied axis The user supplied axis contains for each attribute an expression that can be evaluated. We need to return a user axis that contains the actual values derived from the expression evaluation :param UserAxis | IrregularUserAxis user_axis: the user axis to evaluate :param evaluator_slice: the sentence evaluator for the slice :rtype: UserAxis | IrregularUserAxis """ if isinstance(user_axis, IrregularUserAxis): if not user_axis.dataBound and user_axis.directPositions != self.DIRECT_POSITIONS_SLICING: raise RuntimeException( "The dataBound option is set to false for irregular axis '{}', " "so directPositions option can not be set in the ingredients file for this axis." .format(user_axis.name)) min = self.sentence_evaluator.evaluate(user_axis.interval.low, evaluator_slice, user_axis.statements) max = None if user_axis.interval.high: max = self.sentence_evaluator.evaluate(user_axis.interval.high, evaluator_slice, user_axis.statements) resolution = self.sentence_evaluator.evaluate(user_axis.resolution, evaluator_slice, user_axis.statements) if isinstance(user_axis, RegularUserAxis): return RegularUserAxis(user_axis.name, resolution, user_axis.order, min, max, user_axis.type, user_axis.dataBound) else: if GribExpressionEvaluator.FORMAT_TYPE in user_axis.directPositions: # grib irregular axis will be calculated later when all the messages is evaluated direct_positions = user_axis.directPositions else: direct_positions = self.sentence_evaluator.evaluate( user_axis.directPositions, evaluator_slice, user_axis.statements) return IrregularUserAxis(user_axis.name, resolution, user_axis.order, min, direct_positions, max, user_axis.type, user_axis.dataBound, [], user_axis.slice_group_size)
def __init__(self, datetime, dt_format=None, time_crs=None): """ :param str datetime: the datetime value :param str dt_format: the datetime format, if none is given we'll try to guess """ self.init_cache() try: if dt_format is None: self.datetime = arrow.get(datetime) else: self.datetime = arrow.get(datetime, dt_format) if time_crs is None: self.time_crs_code = self.CRS_CODE_ANSI_DATE else: tmp_crs = CRSUtil(time_crs) self.time_crs_code = tmp_crs.get_crs_code() except ParserError as pe: dt_format_err = "auto" if dt_format is None else dt_format raise RuntimeException("Failed to parse the date " + datetime + " using format " + dt_format_err)
def _file_band_nil_values(self, index): """ This is used to get the null values (Only 1) from the given band index if one exists when nilValue was not defined in ingredient file :param integer index: the current band index to get the nilValues (GRIB seems only support 1 band) :rtype: List[RangeTypeNilValue] with only 1 element """ if len(self.files) < 1: raise RuntimeException("No files to import were specified.") # NOTE: all files should have same bands's metadata try: nil_value = self.dataset.message(1)["missingValue"] except KeyError: # missingValue is not defined in grib file nil_value = None if nil_value is None: return None else: return [nil_value]
def _file_band_nil_values(self, index): """ This is used to get the null values (Only 1) from the given band index if one exists when nilValue was not defined in ingredient file :param integer index: the current band index to get the nilValues :rtype: List[RangeTypeNilValue] with only 1 element """ if len(self.files) < 1: raise RuntimeException("No gdal files given for import!") import osgeo.gdal as gdal # NOTE: all files should have same bands's metadata, so first file is ok gdal_dataset = gdal.Open(self.files[0].filepath) # band in gdal starts with 1 gdal_band = gdal_dataset.GetRasterBand(index + 1) nil_value = gdal_band.GetNoDataValue() if nil_value is None: return None else: return [nil_value]
def exists(self): """ Returns true if the coverage exist, false otherwise :rtype bool """ try: service_call = self.wcs_service + "?service=WCS&request=DescribeCoverage&version=" + \ Session.get_WCS_VERSION_SUPPORTED() + "&coverageId=" + self.coverage_id # Check if exception is thrown in the response ret = url_read_exception(service_call, 'exceptionCode="NoSuchCoverage"') if ret: # exception is in the response, coverage does not exist return False else: # exception is not the in the response, coverage does exist return True except Exception as ex: raise RuntimeException( "Could not check if the coverage exists. " "Check that the WCS service is up and running on url: {}. " "Detail error: {}".format(self.wcs_service, str(ex)))
def exists(self): """ Returns true if the coverage exist, false otherwise :rtype bool """ try: # Check if coverage exists in WCS GetCapabilities result service_call = self.wcs_service + "?service=WCS&request=GetCapabilities&acceptVersions=" + \ Session.get_WCS_VERSION_SUPPORTED() response = validate_and_read_url(service_call) root = etree.fromstring(response) coverage_id_elements = root.xpath( "//*[local-name() = 'CoverageId']") # Iterate all <CoverageId> elements to check coverageId exists already for element in coverage_id_elements: if self.coverage_id == element.text: return True return False except Exception as ex: raise RuntimeException("Could not check if the coverage exists. " "Detail error: {}".format(str(ex)))
def __load_imported_data_from_resume_file(self, coverage_id): """ Try to load a resume file coverage_id.resume.json from input data folder. :param str coverage_id: coverage id of current importer to find the resume file. """ if coverage_id not in Resumer.__IMPORTED_DATA_DICT: resume_file_path = ConfigManager.resumer_dir_path + coverage_id + Resumer.__RESUMER_FILE_SUFFIX Resumer.__RESUMER_FILE_NAME_DICT[coverage_id] = resume_file_path try: if os.path.isfile(resume_file_path) \ and os.access(resume_file_path, os.R_OK): log.info( "We found a resumer file in the ingredients folder. The slices listed in '" + resume_file_path + "' will not be imported.") file = open(Resumer.__RESUMER_FILE_NAME_DICT[coverage_id]) data = json.loads(file.read()) Resumer.__IMPORTED_DATA_DICT[coverage_id] = data file.close() except IOError as e: raise RuntimeException("Could not read the resume file, full error message: " + str(e)) except ValueError as e: log.warn("The resumer JSON file could not be parsed. A new one will be created.")
def _file_band_nil_values(self, index): """ This is used to get the null values (Only 1) from the given band index if one exists when nilValue was not defined in ingredient file :param integer index: the current band index to get the nilValues :rtype: List[RangeTypeNilValue] with only 1 element """ if len(self.files) < 1: raise RuntimeException("No gdal files given for import!") if len(self.default_null_values) > 0: return self.default_null_values # NOTE: all files should have same bands's metadata, so 1 file is ok gdal_dataset = GDALGmlUtil.open_gdal_dataset_from_any_file(self.files) # band in gdal starts with 1 gdal_band = gdal_dataset.get_raster_band(index + 1) nil_value = gdal_band.GetNoDataValue() if nil_value is None: return None else: return [nil_value]
def exists(self): """ Returns true if the coverage exist, false otherwise :rtype bool """ try: # Check if coverage exists in WCS DescribeCoverage result service_call = self.wcs_service + "?service=WCS&request=DescribeCoverage&version=" + \ Session.get_WCS_VERSION_SUPPORTED() \ + "&coverageId=" + self.coverage_id response = validate_and_read_url(service_call) if decode_res(response).strip() != "": return True return False except Exception as ex: if not "NoSuchCoverage" in str(ex): raise RuntimeException( "Could not check if the coverage exists. " "Detail error: {}".format(str(ex))) else: # coverage doesn't exist return False
def _apply_operation(self, nc_dataset, nc_obj_name, operation): """ Applies operation on a given variable which contains a list of values (e.g: lat = [0, 1, 2, 3,...]), (e.g: find the min of time variable ${netcdf:variable:time:min}) :param netCDF4 nc_dataset: the netcdf dataset :param str nc_obj_name: name of netCDF variable or netCDF dimension :param str operation: the operation to apply: :return: str value: The value from the applied operation with precession """ """ NOTE: min or max of list(variable) with values like [148654.08425925925,...] will return 148654.084259 in float which will cause problem with calculate coefficient as the first coeffcient should be 0 but due to this change of min/max value, the coefficient is like 0.00000000001 (case: PML) "min": "${netcdf:variable:ansi:min} * 24 * 3600 - 11644560000.0", -> return: 1199152879.98 "directPositions": "[float(x) * 24 * 3600 - 11644560000.0 for x in ${netcdf:variable:ansi}]", -> return 1199152880.0 So we must use the values in the list by string and split it to a list to get the same values """ MAX = "max" MIN = "min" LAST = "last" FIRST = "first" RESOLUTION = "resolution" METADATA = "metadata" # List of support operation on a netCDF variable supported_operations = [MAX, MIN, LAST, FIRST, RESOLUTION, METADATA] import_util.import_numpy() import numpy as np if nc_obj_name in nc_dataset.variables: nc_obj = nc_dataset.variables[nc_obj_name] if operation not in supported_operations: # it must be an attribute of variable return nc_obj.__getattribute__(operation) # It must be an operation that could be applied on a netCDF variable # convert list of string values to list of decimal values values = nc_obj[:].flatten() elif nc_obj_name in nc_dataset.dimensions: nc_obj = nc_dataset.dimensions[nc_obj_name] # Cannot determine list of values from variable but only dimension (e.g: station = 758) values = np.arange(0, nc_obj.size) else: raise Exception("Cannot find '" + nc_obj_name + "' from list of netCDF variables and dimensions.") if operation == MAX: return to_decimal(np.amax(values)) elif operation == MIN: return to_decimal(np.amin(values)) elif operation == LAST: last_index = len(nc_obj) - 1 return to_decimal(values[last_index]) elif operation == FIRST: return to_decimal(values[0]) elif operation == RESOLUTION: # NOTE: only netCDF needs this expression to calculate resolution automatically # for GDAL: it uses: ${gdal:resolutionX} and GRIB: ${grib:jDirectionIncrementInDegrees} respectively return self.__calculate_netcdf_resolution(values) elif operation == METADATA: # return a dict of variable (axis) metadata with keys, values as string tmp_dict = {} for attr in nc_obj.ncattrs(): try: tmp_dict[attr] = escape(getattr(nc_obj, attr)) except: log.warn("Attribute '" + attr + "' of variable '" + nc_obj._getname() + "' cannot be parsed as string, ignored.") return tmp_dict # Not supported operation and not valid attribute of netCDF variable raise RuntimeException( "Invalid operation on netcdf variable: " + operation + ". Currently supported: " + ', '.join(supported_operations) + " or any metadata entry of the variable.")
def to_unknown(self): """ Handle unknown time CRS :return: throw RuntimeException """ raise RuntimeException("Unsupported time CRS " + self.time_crs_code)