def test_get_keyword_from_file(self): """Get keyword from a filesystem file's .keyword file.""" raster_shake_path = test_data_path('hazard', 'jakarta_flood_design.tif') vector_path = test_data_path('exposure', 'buildings_osm_4326.shp') raster_tsunami_path = test_data_path('hazard', 'tsunami_wgs84.tif') keyword = read_file_keywords(raster_shake_path, 'layer_purpose') expected_keyword = 'hazard' message = ( 'The keyword "layer_purpose" for %s is %s. Expected keyword is: ' '%s') % (raster_shake_path, keyword, expected_keyword) self.assertEqual(keyword, expected_keyword, message) # Test we get an exception if keyword is not found self.assertRaises(KeywordNotFoundError, read_file_keywords, raster_shake_path, 'boguskeyword') # Test if all the keywords are all ready correctly keywords = read_file_keywords(raster_shake_path) expected_keywords = { 'hazard_category': 'single_event', 'hazard': 'flood', 'continuous_hazard_unit': 'metres', 'layer_purpose': 'hazard', 'layer_mode': 'continuous', 'title': 'Jakarta flood like 2007 with structural improvements', 'keyword_version': inasafe_keyword_version } message = 'Expected:\n%s\nGot:\n%s\n' % (expected_keywords, keywords) self.assertDictEqual(keywords, expected_keywords, message) # Test reading keywords from vector layer keywords = read_file_keywords(vector_path) expected_keywords = { 'keyword_version': inasafe_keyword_version, 'structure_class_field': 'FLOODED', 'title': 'buildings_osm_4326', 'layer_geometry': 'polygon', 'layer_purpose': 'exposure', 'layer_mode': 'classified', 'exposure': 'structure' } message = 'Expected:\n%s\nGot:\n%s\n' % (expected_keywords, keywords) self.assertDictEqual(keywords, expected_keywords, message) # tsunami example keywords = read_file_keywords(raster_tsunami_path) expected_keywords = { 'hazard_category': 'single_event', 'title': 'Tsunami', 'hazard': 'tsunami', 'continuous_hazard_unit': 'metres', 'layer_geometry': 'raster', 'layer_purpose': 'hazard', 'layer_mode': 'continuous', 'keyword_version': inasafe_keyword_version } message = 'Expected:\n%s\nGot:\n%s\n' % (expected_keywords, keywords) self.assertEqual(keywords, expected_keywords, message)
def test_get_keyword_from_file(self): """Get keyword from a filesystem file's .keyword file.""" raster_layer = clone_raster_layer( 'jakarta_flood_design', '.tif', False, test_data_path('hazard')) raster_layer_path = raster_layer.source() keyword_file = test_data_path('other', 'jakarta_flood_design.keywords') raster_keyword_path = ( os.path.splitext(raster_layer_path)[0] + '.keywords') shutil.copy2(keyword_file, raster_keyword_path) keyword = read_file_keywords(raster_layer_path, 'layer_purpose') expected_keyword = 'hazard' self.assertEqual(keyword, expected_keyword) # Test we get an exception if keyword is not found self.assertRaises( KeywordNotFoundError, read_file_keywords, raster_layer_path, 'boguskeyword') # Test if all the keywords are all ready correctly keywords = read_file_keywords(raster_layer_path) expected_keywords = { 'hazard_category': 'single_event', 'hazard': 'flood', 'continuous_hazard_unit': 'metres', 'layer_purpose': 'hazard', 'layer_mode': 'continuous', 'title': 'Jakarta flood like 2007 with structural improvements', 'keyword_version': '3.2' } self.assertDictEqual(keywords, expected_keywords)
def test_get_keyword_from_file(self): """Get keyword from a filesystem file's .keyword file.""" raster_shake_path = test_data_path("hazard", "jakarta_flood_design.tif") vector_path = test_data_path("exposure", "buildings_osm_4326.shp") raster_tsunami_path = test_data_path("hazard", "tsunami_wgs84.tif") keyword = read_file_keywords(raster_shake_path, "layer_purpose") expected_keyword = "hazard" message = ('The keyword "layer_purpose" for %s is %s. Expected keyword is: ' "%s") % ( raster_shake_path, keyword, expected_keyword, ) self.assertEqual(keyword, expected_keyword, message) # Test we get an exception if keyword is not found self.assertRaises(KeywordNotFoundError, read_file_keywords, raster_shake_path, "boguskeyword") # Test if all the keywords are all ready correctly keywords = read_file_keywords(raster_shake_path) expected_keywords = { "hazard_category": "single_event", "hazard": "flood", "continuous_hazard_unit": "metres", "layer_purpose": "hazard", "layer_mode": "continuous", "title": "Jakarta flood like 2007 with structural improvements", "keyword_version": inasafe_keyword_version, } message = "Expected:\n%s\nGot:\n%s\n" % (expected_keywords, keywords) self.assertDictEqual(keywords, expected_keywords, message) # Test reading keywords from vector layer keywords = read_file_keywords(vector_path) expected_keywords = { "keyword_version": inasafe_keyword_version, "structure_class_field": "FLOODED", "title": "buildings_osm_4326", "layer_geometry": "polygon", "layer_purpose": "exposure", "layer_mode": "classified", "exposure": "structure", } message = "Expected:\n%s\nGot:\n%s\n" % (expected_keywords, keywords) self.assertDictEqual(keywords, expected_keywords, message) # tsunami example keywords = read_file_keywords(raster_tsunami_path) expected_keywords = { "hazard_category": "single_event", "title": "Tsunami", "hazard": "tsunami", "continuous_hazard_unit": "metres", "layer_geometry": "raster", "layer_purpose": "hazard", "layer_mode": "continuous", "keyword_version": inasafe_keyword_version, } message = "Expected:\n%s\nGot:\n%s\n" % (expected_keywords, keywords) self.assertEqual(keywords, expected_keywords, message)
def test_get_keyword_from_file(self): """Get keyword from a filesystem file's .keyword file.""" raster_shake_path = test_data_path( 'hazard', 'jakarta_flood_design.tif') vector_path = test_data_path( 'exposure', 'buildings_osm_4326.shp') raster_tsunami_path = test_data_path( 'hazard', 'tsunami_wgs84.tif') keyword = read_file_keywords(raster_shake_path, 'category') expected_keyword = 'hazard' message = ('The keyword "category" for %s is %s. Expected keyword is: ' '%s') % (raster_shake_path, keyword, expected_keyword) self.assertEqual(keyword, expected_keyword, message) # Test we get an exception if keyword is not found self.assertRaises( KeywordNotFoundError, read_file_keywords, raster_shake_path, 'boguskeyword') # Test if all the keywords are all ready correctly keywords = read_file_keywords(raster_shake_path) expected_keywords = { 'category': 'hazard', 'subcategory': 'flood', 'data_type': 'continuous', 'unit': 'metres_depth', 'title': 'Jakarta flood like 2007 with structural improvements'} message = 'Expected:\n%s\nGot:\n%s\n' % (expected_keywords, keywords) self.assertEqual(keywords, expected_keywords, message) # Test reading keywords from vector layer keywords = read_file_keywords(vector_path) expected_keywords = { 'category': 'exposure', 'datatype': 'osm', 'subcategory': 'structure', 'title': 'buildings_osm_4326', 'purpose': 'dki'} message = 'Expected:\n%s\nGot:\n%s\n' % (expected_keywords, keywords) self.assertEqual(keywords, expected_keywords, message) # tsunami example keywords = read_file_keywords(raster_tsunami_path) expected_keywords = { 'category': 'hazard', 'unit': 'metres_depth', 'subcategory': 'tsunami', 'data_type': 'continuous', 'title': 'Tsunami' } message = 'Expected:\n%s\nGot:\n%s\n' % (expected_keywords, keywords) self.assertEqual(keywords, expected_keywords, message)
def test_get_keyword_from_file(self): """Get keyword from a filesystem file's .keyword file.""" raster_shake_path = test_data_path('hazard', 'jakarta_flood_design.tif') vector_path = test_data_path('exposure', 'buildings_osm_4326.shp') raster_tsunami_path = test_data_path('hazard', 'tsunami_wgs84.tif') keyword = read_file_keywords(raster_shake_path, 'category') expected_keyword = 'hazard' message = ('The keyword "category" for %s is %s. Expected keyword is: ' '%s') % (raster_shake_path, keyword, expected_keyword) self.assertEqual(keyword, expected_keyword, message) # Test we get an exception if keyword is not found self.assertRaises(KeywordNotFoundError, read_file_keywords, raster_shake_path, 'boguskeyword') # Test if all the keywords are all ready correctly keywords = read_file_keywords(raster_shake_path) expected_keywords = { 'category': 'hazard', 'subcategory': 'flood', 'data_type': 'continuous', 'unit': 'metres_depth', 'title': 'Jakarta flood like 2007 with structural improvements' } message = 'Expected:\n%s\nGot:\n%s\n' % (expected_keywords, keywords) self.assertEqual(keywords, expected_keywords, message) # Test reading keywords from vector layer keywords = read_file_keywords(vector_path) expected_keywords = { 'category': 'exposure', 'datatype': 'osm', 'subcategory': 'structure', 'title': 'buildings_osm_4326', 'purpose': 'dki' } message = 'Expected:\n%s\nGot:\n%s\n' % (expected_keywords, keywords) self.assertEqual(keywords, expected_keywords, message) # tsunami example keywords = read_file_keywords(raster_tsunami_path) expected_keywords = { 'category': 'hazard', 'unit': 'metres_depth', 'subcategory': 'tsunami', 'data_type': 'continuous', 'title': 'Tsunami' } message = 'Expected:\n%s\nGot:\n%s\n' % (expected_keywords, keywords) self.assertEqual(keywords, expected_keywords, message)
def read_keywords(self, layer, keyword=None): """Read keywords for a datasource and return them as a dictionary. This is a wrapper method that will 'do the right thing' to fetch keywords for the given datasource. In particular, if the datasource is remote (e.g. a database connection) it will fetch the keywords from the keywords store. :param layer: A QGIS QgsMapLayer instance that you want to obtain the keywords for. :type layer: QgsMapLayer, QgsRasterLayer, QgsVectorLayer, QgsPluginLayer :param keyword: If set, will extract only the specified keyword from the keywords dict. :type keyword: str :returns: A dict if keyword is omitted, otherwise the value for the given key if it is present. :rtype: dict, str TODO: Don't raise generic exceptions. :raises: HashNotFoundError, Exception, OperationalError, NoKeywordsFoundError, KeywordNotFoundError, InvalidParameterError, UnsupportedProviderError """ source = layer.source() # Try to read from ISO metadata first. try: return read_iso19115_metadata(source, keyword) except (MetadataReadError, NoKeywordsFoundError): pass try: flag = self.are_keywords_file_based(layer) except UnsupportedProviderError: raise try: if flag: keywords = read_file_keywords(source) else: uri = self.normalize_uri(layer) keywords = self.read_keyword_from_uri(uri) return write_read_iso_19115_metadata(source, keywords, keyword) except (HashNotFoundError, Exception, OperationalError, NoKeywordsFoundError, KeywordNotFoundError, InvalidParameterError, UnsupportedProviderError): raise
def read_keywords_file(cls, filename, keyword=None): """Read keywords from a keywords file and return as dictionary This serves as a wrapper function that should be provided by Keyword IO. Use this if you are sure that the filename is a keyword file. :param filename: The filename of the keyword, typically with .xml or .keywords extension. If not, will raise exceptions :type filename: str :param keyword: If set, will extract only the specified keyword from the keywords dict. :type keyword: str :returns: A dict if keyword is omitted, otherwise the value for the given key if it is present. :rtype: dict, str :raises: KeywordNotFoundError, InvalidParameterError """ # Try to read from ISO metadata first. _, ext = os.path.splitext(filename) dictionary = {} if ext == '.xml': try: dictionary = read_iso19115_metadata(filename) except (MetadataReadError, NoKeywordsFoundError): pass elif ext == '.keywords': try: dictionary = read_file_keywords(filename) # update to xml based metadata write_read_iso_19115_metadata(filename, dictionary) except (HashNotFoundError, Exception, OperationalError, NoKeywordsFoundError, KeywordNotFoundError, InvalidParameterError, UnsupportedProviderError): raise else: raise InvalidParameterError( 'Keywords file have .xml or .keywords extension') # if no keyword was supplied, just return the dict if keyword is None: return dictionary if keyword not in dictionary: message = tr('No value was found in file %s for keyword %s' % ( filename, keyword)) raise KeywordNotFoundError(message) return dictionary[keyword]
def test_get_admissible_plugins(self): """Test for get_admissible_plugins function.""" functions = get_admissible_plugins() message = 'No functions available (len=%ss)' % len(functions) self.assertTrue(len(functions) > 0, message) # Also test if it works when we give it two layers # to see if we can determine which functions will # work for them. keywords1 = read_file_keywords(self.raster_shake_path) keywords2 = read_file_keywords(self.vector_path) # We need to explicitly add the layer type to each keyword list keywords1['layertype'] = 'raster' keywords2['layertype'] = 'vector' functions = [keywords1, keywords2] functions = get_admissible_plugins(functions) message = 'No functions available (len=%ss)' % len(functions) self.assertTrue(len(functions) > 0, message)
def test_copy_keywords(self): """Test we can copy the keywords.""" out_path = unique_filename( prefix='test_copy_keywords', suffix='.keywords') self.keyword_io.copy_keywords(self.raster_layer, out_path) copied_keywords = read_file_keywords(out_path) expected_keywords = self.expected_raster_keywords message = 'Got:\n%s\nExpected:\n%s\nSource:\n%s' % ( copied_keywords, expected_keywords, out_path) self.assertEquals(copied_keywords, expected_keywords, message)
def test_copy_keywords(self): """Test we can copy the keywords.""" out_path = unique_filename( prefix='test_copy_keywords', suffix='.keywords') self.keyword_io.copy_keywords(self.raster_layer, out_path) copied_keywords = read_file_keywords(out_path) expected_keywords = self.expected_raster_keywords message = 'Got:\n%s\nExpected:\n%s\nSource:\n%s' % ( copied_keywords, expected_keywords, out_path) self.assertDictEqual(copied_keywords, expected_keywords, message)
def load_layer(layer_path): """Helper to load and return a single QGIS layer :param layer_path: Path name to raster or vector file. :type layer_path: str :returns: tuple containing layer and its layer_purpose. :rtype: (QgsMapLayer, str) """ # Extract basename and absolute path file_name = os.path.split(layer_path)[-1] # In case path was absolute base_name, extension = os.path.splitext(file_name) # Determine if layer is hazard or exposure layer_purpose = 'undefined' try: try: keywords = read_iso19115_metadata(layer_path) except: try: keywords = read_file_keywords(layer_path) keywords = write_read_iso_19115_metadata(layer_path, keywords) except NoKeywordsFoundError: keywords = {} if 'layer_purpose' in keywords: layer_purpose = keywords['layer_purpose'] except NoKeywordsFoundError: pass # Create QGis Layer Instance if extension in ['.asc', '.tif']: layer = QgsRasterLayer(layer_path, base_name) elif extension in ['.shp']: layer = QgsVectorLayer(layer_path, base_name, 'ogr') else: message = 'File %s had illegal extension' % layer_path raise Exception(message) # noinspection PyUnresolvedReferences message = 'Layer "%s" is not valid' % layer.source() # noinspection PyUnresolvedReferences if not layer.isValid(): print message # noinspection PyUnresolvedReferences if not layer.isValid(): raise Exception(message) return layer, layer_purpose
def _clip_raster_layer( layer, extent, cell_size=None, extra_keywords=None): """Clip a Hazard or Exposure raster layer to the extents provided. The layer must be a raster layer or an exception will be thrown. .. note:: The extent *must* be in EPSG:4326. The output layer will always be in WGS84/Geographic. :param layer: A valid QGIS raster layer in EPSG:4326 :type layer: QgsRasterLayer :param extent: An array representing the exposure layer extents in the form [xmin, ymin, xmax, ymax]. It is assumed that the coordinates are in EPSG:4326 although currently no checks are made to enforce this. or: A QgsGeometry of type polygon. **Polygon clipping currently only supported for vector datasets.** :type extent: list(float), QgsGeometry :param cell_size: Cell size (in GeoCRS) which the layer should be resampled to. If not provided for a raster layer (i.e. theCellSize=None), the native raster cell size will be used. :type cell_size: float :returns: Output clipped layer (placed in the system temp dir). :rtype: QgsRasterLayer :raises: InvalidProjectionError - if input layer is a density layer in projected coordinates. See issue #123. """ if not layer or not extent: message = tr('Layer or Extent passed to clip is None.') raise InvalidParameterError(message) if layer.type() != QgsMapLayer.RasterLayer: message = tr( 'Expected a raster layer but received a %s.' % str(layer.type())) raise InvalidParameterError(message) working_layer = layer.source() # Check for existence of keywords file base, _ = os.path.splitext(working_layer) keywords_path = base + '.keywords' message = tr( 'Input file to be clipped "%s" does not have the ' 'expected keywords file %s' % ( working_layer, keywords_path )) verify(os.path.isfile(keywords_path), message) # Raise exception if layer is projected and refers to density (issue #123) # FIXME (Ole): Need to deal with it - e.g. by automatically reprojecting # the layer at this point and setting the native resolution accordingly # in its keywords. keywords = read_file_keywords(keywords_path) if 'datatype' in keywords and keywords['datatype'] == 'count': if str(layer.crs().authid()) != 'EPSG:4326': # This layer is not WGS84 geographic message = ( 'Layer %s represents count but has spatial reference "%s". ' 'Count layers must be given in WGS84 geographic coordinates, ' 'so please reproject and try again. For more information, see ' 'issue https://github.com/AIFDR/inasafe/issues/123' % ( working_layer, layer.crs().toProj4() )) raise InvalidProjectionError(message) # We need to provide gdalwarp with a dataset for the clip # because unlike gdal_translate, it does not take projwin. clip_kml = extent_to_kml(extent) # Create a filename for the clipped, resampled and reprojected layer handle, filename = tempfile.mkstemp('.tif', 'clip_', temp_dir()) os.close(handle) os.remove(filename) # If no cell size is specified, we need to run gdalwarp without # specifying the output pixel size to ensure the raster dims # remain consistent. binary_list = which('gdalwarp') LOGGER.debug('Path for gdalwarp: %s' % binary_list) if len(binary_list) < 1: raise CallGDALError( tr('gdalwarp could not be found on your computer')) # Use the first matching gdalwarp found binary = binary_list[0] if cell_size is None: command = ( '"%s" -q -t_srs EPSG:4326 -r near -cutline %s -crop_to_cutline ' '-ot Float64 -of GTiff "%s" "%s"' % ( binary, clip_kml, working_layer, filename)) else: command = ( '"%s" -q -t_srs EPSG:4326 -r near -tr %s %s -cutline %s ' '-crop_to_cutline -ot Float64 -of GTiff "%s" "%s"' % ( binary, repr(cell_size), repr(cell_size), clip_kml, working_layer, filename)) LOGGER.debug(command) result = QProcess().execute(command) # For QProcess exit codes see # http://qt-project.org/doc/qt-4.8/qprocess.html#execute if result == -2: # cannot be started message_detail = tr('Process could not be started.') message = tr( '<p>Error while executing the following shell command:' '</p><pre>%s</pre><p>Error message: %s' % (command, message_detail)) raise CallGDALError(message) elif result == -1: # process crashed message_detail = tr('Process crashed.') message = tr( '<p>Error while executing the following shell command:</p>' '<pre>%s</pre><p>Error message: %s' % (command, message_detail)) raise CallGDALError(message) # .. todo:: Check the result of the shell call is ok keyword_io = KeywordIO() keyword_io.copy_keywords(layer, filename, extra_keywords=extra_keywords) base_name = '%s clipped' % layer.name() layer = QgsRasterLayer(filename, base_name) return layer
def test_get_keyword_from_file(self): """Get keyword from a filesystem file's .keyword file.""" raster_shake_path = test_data_path( 'hazard', 'jakarta_flood_design.tif') vector_path = test_data_path( 'exposure', 'buildings_osm_4326.shp') raster_tsunami_path = test_data_path( 'hazard', 'tsunami_wgs84.tif') keyword = read_file_keywords(raster_shake_path, 'layer_purpose') expected_keyword = 'hazard' message = ( 'The keyword "layer_purpose" for %s is %s. Expected keyword is: ' '%s') % (raster_shake_path, keyword, expected_keyword) self.assertEqual(keyword, expected_keyword, message) # Test we get an exception if keyword is not found self.assertRaises( KeywordNotFoundError, read_file_keywords, raster_shake_path, 'boguskeyword') # Test if all the keywords are all ready correctly keywords = read_file_keywords(raster_shake_path) expected_keywords = { 'hazard_category': 'single_event', 'hazard': 'flood', 'continuous_hazard_unit': 'metres', 'layer_purpose': 'hazard', 'layer_mode': 'continuous', 'title': 'Jakarta flood like 2007 with structural improvements', 'keyword_version': inasafe_keyword_version } message = 'Expected:\n%s\nGot:\n%s\n' % (expected_keywords, keywords) self.assertDictEqual(keywords, expected_keywords, message) # Test reading keywords from vector layer keywords = read_file_keywords(vector_path) expected_keywords = { 'keyword_version': inasafe_keyword_version, 'structure_class_field': 'FLOODED', 'title': 'buildings_osm_4326', 'layer_geometry': 'polygon', 'layer_purpose': 'exposure', 'layer_mode': 'classified', 'exposure': 'structure' } message = 'Expected:\n%s\nGot:\n%s\n' % (expected_keywords, keywords) self.assertDictEqual(keywords, expected_keywords, message) # tsunami example keywords = read_file_keywords(raster_tsunami_path) expected_keywords = { 'hazard_category': 'single_event', 'title': 'Tsunami', 'hazard': 'tsunami', 'continuous_hazard_unit': 'metres', 'layer_geometry': 'raster', 'layer_purpose': 'hazard', 'layer_mode': 'continuous', 'keyword_version': inasafe_keyword_version } message = 'Expected:\n%s\nGot:\n%s\n' % (expected_keywords, keywords) self.assertEqual(keywords, expected_keywords, message)