class AhnWebCoverageDatasource(object): def __init__(self): try: self.wcs_ahn3 = WebCoverageService( 'https://geodata.nationaalgeoregister.nl/ahn3/wcs', version='1.0.0') self.wcs_ahn2 = WebCoverageService( 'https://geodata.nationaalgeoregister.nl/ahn2/wcs', version='1.0.0') except: print("AHN WCS host unavailable") def retrieve_tile_ahn3(self, srid, x_min, y_min, x_max, y_max): output_ahn3 = self.wcs_ahn3.getCoverage( identifier='ahn3_05m_dtm', bbox=[x_min, y_min, x_max, y_max], format='GEOTIFF_FLOAT32', crs=srid, resx=0.5, resy=0.5) return output_ahn3 def retrieve_tile_ahn2(self, srid, x_min, y_min, x_max, y_max): output_ahn2 = self.wcs_ahn2.getCoverage( identifier='ahn2_05m_int', bbox=[x_min, y_min, x_max, y_max], format='GEOTIFF_FLOAT32', crs=srid, resx=0.5, resy=0.5) return output_ahn2
class WCSHelper(object): """docstring for WCSHelper""" def __init__(self, url, dates, variable, bbox, single=False): super(WCSHelper, self).__init__() self.url = url self.single = single self.dates = dates self.variable = variable self.bbox = bbox self.owslib_log = logging.getLogger('owslib') self.owslib_log.setLevel(logging.DEBUG) self.wcs = WebCoverageService(url, version="1.0.0") def __repr__(self): return str(self.wcs) def getData(self): #print '-'*20 #print self.bbox #print self.dates if self.single : output = self.wcs.getCoverage(identifier=self.variable, time=[self.dates], bbox=self.bbox, format="NetCDF3") else: output = self.wcs.getCoverage(identifier=self.variable, time=self.dates, bbox=self.bbox, format="NetCDF3") return output
def wcsRequest(self, layer='AGRICULTURE'): self.layer = layer ID = 'your ID' wcs_url = 'https://services.sentinel-hub.com/ogc/wcs/'+ID wcs = WebCoverageService(wcs_url, version='1.0.0') self.x, self.y = self.deg2num(self.lat_center, self.lon_center, self.zoom) self.lat_max, self.lon_min = self.num2deg(self.x, self.y, self.zoom) self.lat_min, self.lon_max = self.num2deg(self.x+1, self.y+1, self.zoom) inProj = Proj(init='epsg:4326') outProj = Proj(init='epsg:3857') x1,y1 = transform(inProj,outProj,self.lon_min,self.lat_min) x2,y2 = transform(inProj,outProj,self.lon_max,self.lat_max) bb=(x1, y1, x2, y2) self.wcsOut = wcs.getCoverage(identifier=self.layer, time=None, width=800, height=800, bbox = bb, format = 'GeoTIFF') self.imgTiff = Image.open(self.wcsOut) self.imgArr = np.array(self.imgTiff) imgurl = image_to_url(image=self.imgArr) self.map.add_layer(ImageOverlay(url=imgurl, bounds=[[self.lat_min, self.lon_min], [self.lat_max, self.lon_max]]))
def test_ows_interfaces_wcs(): service = WebCoverageService(WCS_SERVICE_URL, version='1.0.0') # Check each service instance conforms to OWSLib interface service.alias = 'WCS' isinstance(service, owslib.coverage.wcs100.WebCoverageService_1_0_0) # URL attribute assert service.url == WCS_SERVICE_URL # version attribute assert service.version == '1.0.0' # Identification object assert hasattr(service, 'identification') # Check all ServiceIdentification attributes assert service.identification.type == 'OGC:WCS' for attribute in ['type', 'version', 'title', 'abstract', 'keywords', 'fees']: assert hasattr(service.identification, attribute) # Check all ServiceProvider attributes for attribute in ['name', 'url', 'contact']: assert hasattr(service.provider, attribute) # Check all operations implement IOperationMetadata for op in service.operations: for attribute in ['name', 'methods']: assert hasattr(op, attribute) # Check all contents implement IContentMetadata as a dictionary isinstance(service.contents, dict) # Check any item (WCS coverage, WMS layer etc) from the contents of each service # Check it conforms to IContentMetadata interface # get random item from contents dictionary -has to be a nicer way to do this! content = service.contents[list(service.contents.keys())[0]] for attribute in ['id', 'title', 'boundingBox', 'boundingBoxWGS84', 'crsOptions', 'styles', 'timepositions']: assert hasattr(content, attribute)
def test_ows_interfaces_wcs(): service = WebCoverageService(WCS_SERVICE_URL, version='1.0.0') # Check each service instance conforms to OWSLib interface service.alias = 'WCS' isinstance(service, owslib.coverage.wcs100.WebCoverageService_1_0_0) # URL attribute assert service.url == WCS_SERVICE_URL # version attribute assert service.version == '1.0.0' # Identification object assert hasattr(service, 'identification') # Check all ServiceIdentification attributes assert service.identification.type == 'OGC:WCS' for attribute in [ 'type', 'version', 'title', 'abstract', 'keywords', 'fees' ]: assert hasattr(service.identification, attribute) # Check all ServiceProvider attributes for attribute in ['name', 'url', 'contact']: assert hasattr(service.provider, attribute) # Check all operations implement IOperationMetadata for op in service.operations: for attribute in ['name', 'methods']: assert hasattr(op, attribute) # Check all contents implement IContentMetadata as a dictionary isinstance(service.contents, dict) # Check any item (WCS coverage, WMS layer etc) from the contents of each service # Check it conforms to IContentMetadata interface # get random item from contents dictionary -has to be a nicer way to do this! content = service.contents[list(service.contents.keys())[0]] for attribute in [ 'id', 'title', 'boundingBox', 'boundingBoxWGS84', 'crsOptions', 'styles', 'timepositions' ]: assert hasattr(content, attribute)
def test_metadata_available_after_upload(self): """Test metadata is available after upload """ # Upload hazard data for this test name = 'jakarta_flood_design.tif' exposure_filename = os.path.join(UNITDATA, 'hazard', name) exposure_layer = save_to_geonode(exposure_filename, user=self.user, overwrite=True) layer_name = exposure_layer.typename server_url = settings.GEOSERVER_BASE_URL + '/ows' wcs = WebCoverageService(server_url, version='1.0.0') layer_appears_immediately = layer_name in wcs.contents wait_time = 0.5 time.sleep(wait_time) wcs2 = WebCoverageService(server_url, version='1.0.0') layer_appears_afterwards = layer_name in wcs2.contents msg = ('Layer %s was not found after %s seconds in WxS contents ' 'on server %s.\n' 'WCS contents: %s\n' % (layer_name, wait_time, server_url, wcs.contents)) assert layer_appears_afterwards, msg msg = ('Layer %s was not found in WxS contents on server %s.\n' 'WCS contents: %s\n' % (layer_name, server_url, wcs.contents)) assert layer_appears_immediately, msg
def get_products(products, dt, bbox=None, fcast='000', outpath='.'): ''' Get a list of products for a given date and time Example: >> get_products(['Temperature'], datetime(2010,10,10,12,0)) ''' saved_files = [] url_params = (dt.strftime('%Y%m'), dt.strftime('%Y%m%d'), dt.strftime('%Y%m%d'), dt.strftime('%H%M'), fcast) server_url = 'http://nomads.ncdc.noaa.gov/thredds/wcs/ruc13/%s/%s/ruc2_130_%s_%s_%s.grb2' % url_params try: wcs = WebCoverageService(server_url,version='1.0.0') except: print >> sys.stderr, "Bad URL!" else: if set(products).issubset(wcs.contents.keys()): print >> sys.stderr, 'Requested products are a subset of available products' else: print >> sys.stderr, 'Some product(s) selected do not exist' if bbox is None: bbox = (-139, -57.995, 16, 55) # CONUS date_time = dt.isoformat() + 'Z' for product in products: filename = os.path.join(outpath, model + '_' + date_time.replace(':','') + '_' + product + '.tif') if os.path.exists(filename): # Check if file exists so we don't download duplicates pass else: try: output = wcs.getCoverage(identifier = product, time = [date_time], bbox= bbox, format='GeoTiff_float') except urllib2.HTTPError: print "Bad URL" f = open(filename, 'wb') f.write(output.read()) f.close() saved_files.append(filename) print >> sys.stderr, 'Saved: ' + filename return saved_files
def getGeotiffFromWebCoverageService(url, version, bbox, identifier, frmt, directory, filename): """ Obtain data from WebCoverageService. :url: url from which to obtain the data :version: version of the data to obtain :bbox: bounding box :identifier: user-specified filename :format: data format to obtain :directory: folder in which file is to be written :filename: filename of file to write """ # create path direc = "./" + directory + "/" dirFile = direc + filename + ".tif" # check if directory and file exist if not os.path.exists(direc): os.makedirs(direc) if not os.path.isfile(dirFile): wcs = WebCoverageService(url, version=version) response = wcs.getCoverage(identifier=identifier, bbox=bbox, format=frmt, crs='urn:ogc:def:crs:EPSG::28992', resx=0.5, resy=0.5) with open(dirFile, 'wb') as file: file.write(response.read())
def test_wcs1_getcoverage_netcdf(ows_server): # Use owslib to confirm that we have a somewhat compliant WCS service wcs = WebCoverageService(url=ows_server.url + "/wcs", version="1.0.0") # Ensure that we have at least some layers available contents = list(wcs.contents) test_layer_name = contents[0] test_layer = wcs.contents[test_layer_name] bbox = test_layer.boundingBoxWGS84 output = wcs.getCoverage(identifier=contents[0], format='netCDF', bbox=pytest.helpers.enclosed_bbox(bbox), crs='EPSG:4326', width=400, height=300) assert output assert output.info()['Content-Type'] == 'application/x-netcdf' output = wcs.getCoverage(identifier=contents[0], format='netCDF', bbox=pytest.helpers.enclosed_bbox(bbox), crs='I-CANT-BELIEVE-ITS-NOT-EPSG:4326', width=400, height=300) assert output assert output.info()['Content-Type'] == 'application/x-netcdf'
def test_metadata_available_after_upload(self): """Test metadata is available after upload """ # Upload exposure data for this test name = 'Population_2010' exposure_filename = '%s/%s.asc' % (TESTDATA, name) exposure_layer = save_to_geonode(exposure_filename, user=self.user, overwrite=True) layer_name = exposure_layer.typename server_url = settings.GEOSERVER_BASE_URL + '/ows' wcs = WebCoverageService(server_url, version='1.0.0') layer_appears_immediately = layer_name in wcs.contents wait_time = 0.5 import time time.sleep(wait_time) wcs2 = WebCoverageService(server_url, version='1.0.0') layer_appears_afterwards = layer_name in wcs2.contents msg = ('Layer %s was not found after %s seconds in WxS contents ' 'on server %s.\n' 'WCS contents: %s\n' % (layer_name, wait_time, server_url, wcs.contents)) assert layer_appears_afterwards, msg msg = ('Layer %s was not found in WxS contents on server %s.\n' 'WCS contents: %s\n' % (layer_name, server_url, wcs.contents)) assert layer_appears_immediately, msg
def loadWCS(folder, out_name, WCS_URL, Version, layerName, bbox, srid): folder = folder urls = WCS_URL version = Version bbox = bbox input_value_raster = layerName crs_num = 'urn:ogc:def:crs:EPSG::' + str(srid) if urls == 'None': return ("Please provide WCS parameters") wcs = WebCoverageService(urls, version) #print(list(wcs.contents)) # # print([op.name for op in wcs.operations]) cvg = wcs.contents[input_value_raster] if bbox == 'None': bbox = cvg.boundingBoxWGS84 response = wcs.getCoverage(identifier=input_value_raster, bbox=bbox, format='GEOTIFF_FLOAT32', crs='urn:ogc:def:crs:EPSG::28992', resx=0.5, resy=0.5) temp_raster = folder + '//' + out_name + '.tif' with open(temp_raster, 'wb') as file: file.write(response.read())
def getCube(self, maptype, depths, datatypes, bounds, overwrite=False): #check if all files are already downloaded fpaths = [ '../tmp/Hadocha_' + maptype + '_' + depth + 'cm_' + datatype + '.tif' for depth in depths for datatype in datatypes ] if (min([os.path.isfile(fpath) for fpath in fpaths]) and not overwrite): print('all files already downloaded for: ' + maptype) return fpaths #set up the Web Coverage Service. This sometimes raises errors, so retry print('setting up WCS for maptype: ' + maptype) for attempt in range(5): try: wcs = WebCoverageService( 'http://maps.isric.org/mapserv?map=/map/' + maptype + '.map', version='1.0.0') except: print('failed to connect to WCS ' + str(attempt + 1) + ' times to: ' + maptype) else: break else: print('failed to connect to WCS, try again later') return [] #start retrieving map data for depth in depths: for datatype in datatypes: fpath = '../tmp/Hadocha_' + maptype + '_' + depth + 'cm_' + datatype + '.tif' #also handle exceptions for retrieving files if (not os.path.isfile(fpath) or overwrite): for attempt in range(5): try: response = wcs.getCoverage( identifier=maptype + '_' + depth + 'cm_' + datatype, crs='urn:ogc:def:crs:EPSG::152160', bbox=(bounds.minx, bounds.miny, bounds.maxx, bounds.maxy), resx=20, resy=20, format='GEOTIFF_INT16') except: print('coverage failed ' + str(attempt + 1) + ' times for: ' + fpath) else: with open(fpath, 'wb') as file: file.write(response.read()) # print('file: '+fpath+'downloaded') break else: print('file not retrieved: ' + fpath) fpaths.remove(fpath) # else: # print('file already downloaded: '+fpath) # print('map at location: '+fpath) return fpaths
def get_raster_wcs( coordinates: Union[Iterable, Sequence[Union[float, str]]], geographic: bool = True, layer: str = None, geoserver: str = GEO_URL, ) -> bytes: """Return a subset of a raster image from the local GeoServer via WCS 2.0.1 protocol. For geographic rasters, subsetting is based on WGS84 (Long, Lat) boundaries. If not geographic, subsetting based on projected coordinate system (Easting, Northing) boundaries. Parameters ---------- coordinates : Sequence[Union[int, float, str]] Geographic coordinates of the bounding box (left, down, right, up) geographic : bool If True, uses "Long" and "Lat" in WCS call. Otherwise uses "E" and "N". layer : str Layer name of raster exposed on GeoServer instance, e.g. 'public:CEC_NALCMS_LandUse_2010' geoserver: str The address of the geoserver housing the layer to be queried. Default: http://pavics.ouranos.ca/geoserver/. Returns ------- bytes A GeoTIFF array. """ (left, down, right, up) = coordinates if geographic: x, y = "Long", "Lat" else: x, y = "E", "N" wcs = WebCoverageService(url=urljoin(geoserver, "ows"), version="2.0.1") try: resp = wcs.getCoverage( identifier=[layer], format="image/tiff", subsets=[(x, left, right), (y, down, up)], timeout=120, ) except Exception as e: raise Exception(e) data = resp.read() try: etree.fromstring(data) # The response is an XML file describing the server error. raise ChildProcessError(data) except etree.XMLSyntaxError: # The response is the DEM array. return data
def GetWCSLayer(u, p): start = time.time() # Manage the WFS URL & the layer name split_url = u.split('?') server_url = split_url[0] ows = server_url[-3:] print 'The OGC standard is: '+ ows if ows == 'ows' or ows == 'wcs': server_url = server_url[:-3]+ 'wcs' spacename_wcs = split_url[1] chemin = p + spacename_wcs +'.tif' if not os.path.exists(chemin): # Get the raster layer using OGC WCS standard wcs = WebCoverageService(server_url ,version='1.0.0') image = wcs[spacename_wcs] # Download the GeoTIFF image file info = (image.boundingboxes)[0] epsg = info['nativeSrs'] bboxx = info['bbox'] offset = image.grid.offsetvectors cellsize_x= offset[0] x = cellsize_x[0] X = str(abs(float(x))) cellsize_y= offset[1] y = cellsize_y[1] Y = str(abs(float(y))) # img_formats = image.supportedFormats # img_format = img_formats[0] img_format = 'GeoTIFF' print "Downloading the GeoTIFF file... : "+spacename_wcs print "From: "+server_url output = wcs.getCoverage(identifier = spacename_wcs, bbox = bboxx, crs = epsg, format = img_format, resx = X, resy = Y) data = output.read() f = open(chemin,'wb') f.write(data) f.close() # Calculat time temps =time.time() - start tps = round(temps,2) temps_ms = str(tps) print "GetWCSLayer download time : " + temps_ms +" ms"
def __init__(self, url, dates, variable, bbox, single=False): super(WCSHelper, self).__init__() self.url = url self.single = single self.dates = dates self.variable = variable self.bbox = bbox self.owslib_log = logging.getLogger('owslib') self.owslib_log.setLevel(logging.DEBUG) self.wcs = WebCoverageService(url, version="1.0.0")
def __init__(self): try: self.wcs_ahn3 = WebCoverageService( 'https://geodata.nationaalgeoregister.nl/ahn3/wcs', version='1.0.0') self.wcs_ahn2 = WebCoverageService( 'https://geodata.nationaalgeoregister.nl/ahn2/wcs', version='1.0.0') except: print("AHN WCS host unavailable")
def wcs_links(wcs_url, identifier, bbox=None, crs=None, height=None, width=None, exclude_formats=True, quiet=True, version='1.0.0'): # FIXME(Ariel): This would only work for layers marked for public view, # what about the ones with permissions enabled? try: wcs = WebCoverageService(wcs_url, version=version) except ServiceException as err: err_msg = 'WCS server returned exception: %s' % err if not quiet: logger.warn(err_msg) raise GeoNodeException(err_msg) msg = ('Could not create WCS links for layer "%s",' ' it was not in the WCS catalog,' ' the available layers were: "%s"' % (identifier, wcs.contents.keys())) output = [] formats = [] if identifier not in wcs.contents: if not quiet: raise RuntimeError(msg) else: logger.warn(msg) else: coverage = wcs.contents[identifier] formats = coverage.supportedFormats for f in formats: if exclude_formats and f in DEFAULT_EXCLUDE_FORMATS: continue # roundabout, hacky way to accomplish getting a getCoverage url. # nonetheless, it's better than having to load an entire large # coverage just to generate a URL fakeUrl = wcs.getCoverage(identifier=coverage.id, format=f, bbox=bbox, crs=crs, height=20, width=20).geturl() url = sub(r'(height=)20(\&width=)20', r'\g<1>{0}\g<2>{1}', fakeUrl).format(height, width) # The outputs are: (ext, name, mime, url) # FIXME(Ariel): Find a way to get proper ext, name and mime # using format as a default for all is not good enough output.append((f, f, f, url)) return output
def get_raster_wcs(coordinates, geographic=True, layer=None): """Return a subset of a raster image from the local GeoServer via WCS 2.0.1 protocol. For geoggraphic rasters, subsetting is based on WGS84 (Long, Lat) boundaries. If not geographic, subsetting based on projected coordinate system (Easting, Northing) boundries. Parameters ---------- coordinates : sequence Geographic coordinates of the bounding box (left, down, right, up) geographic : bool If True, uses "Long" and "Lat" in WCS call. Otherwise uses "E" and "N". layer : str Layer name of raster exposed on GeoServer instance. E.g. 'public:CEC_NALCMS_LandUse_2010' Returns ------- bytes A GeoTIFF array. """ from owslib.wcs import WebCoverageService from lxml import etree (left, down, right, up) = coordinates if geographic: x, y = 'Long', 'Lat' else: x, y = 'E', 'N' wcs = WebCoverageService('http://boreas.ouranos.ca/geoserver/ows', version='2.0.1') try: resp = wcs.getCoverage(identifier=[ layer, ], format='image/tiff', subsets=[(x, left, right), (y, down, up)]) except Exception as e: raise Exception(e) data = resp.read() try: etree.fromstring(data) # The response is an XML file describing the server error. raise ChildProcessError(data) except etree.XMLSyntaxError: # The response is the DEM array. return data
def wcs_links( wcs_url, identifier, bbox=None, crs=None, height=None, width=None, exclude_formats=True, quiet=True, version='1.0.0'): # FIXME(Ariel): This would only work for layers marked for public view, # what about the ones with permissions enabled? try: wcs = WebCoverageService(wcs_url, version=version) except ServiceException as err: err_msg = 'WCS server returned exception: %s' % err if not quiet: logger.warn(err_msg) raise GeoNodeException(err_msg) msg = ('Could not create WCS links for layer "%s",' ' it was not in the WCS catalog,' ' the available layers were: "%s"' % ( identifier, wcs.contents.keys())) output = [] formats = [] if identifier not in wcs.contents: if not quiet: raise RuntimeError(msg) else: logger.warn(msg) else: coverage = wcs.contents[identifier] formats = coverage.supportedFormats for f in formats: if exclude_formats and f in DEFAULT_EXCLUDE_FORMATS: continue # roundabout, hacky way to accomplish getting a getCoverage url. # nonetheless, it's better than having to load an entire large # coverage just to generate a URL fakeUrl = wcs.getCoverage(identifier=coverage.id, format=f, bbox=bbox, crs=crs, height=20, width=20).geturl() url = sub(r'(height=)20(\&width=)20', r'\g<1>{0}\g<2>{1}', fakeUrl).format(height, width) # The outputs are: (ext, name, mime, url) # FIXME(Ariel): Find a way to get proper ext, name and mime # using format as a default for all is not good enough output.append((f, f, f, url)) return output
def test_wcs1_describecoverage(ows_server): # Use owslib to confirm that we have a somewhat compliant WCS service wcs = WebCoverageService(url=ows_server.url + "/wcs", version="1.0.0") # Ensure that we have at least some layers available contents = list(wcs.contents) test_layer_name = contents[0] resp = wcs.getDescribeCoverage(test_layer_name) gc_xds = get_xsd("1.0.0/describeCoverage.xsd") assert gc_xds.validate(resp)
def get_dtm(path_out, minlong,maxlong,minlat,maxlat): bbox=(minlong,minlat,maxlong,maxlat) url="http://services.ga.gov.au/gis/services/DEM_SRTM_1Second_over_Bathymetry_Topography/MapServer/WCSServer?" wcs = WebCoverageService(url,version='1.0.0') cvg=wcs.getCoverage(identifier='1', bbox=bbox, format='GeoTIFF', crs=4326, width=200, height=200) f = open(path_out, 'wb') bytes_written = f.write(cvg.read()) f.close() print("dtm geotif saved as",path_out)
def test_wcs_200(): """ Web Coverage Service WCS Version 2.0.x rewritten doctest/wcs_200.txt """ wcs = WebCoverageService(SERVICE_URL, version="2.0.1") assert wcs.version == '2.0.1' assert wcs.url == SERVICE_URL assert wcs.identification.title == 'rasdaman' assert wcs.identification.service == 'OGC WCS' assert wcs.provider.name == 'Jacobs University Bremen' assert 'AvgLandTemp' in wcs.contents.keys() assert len(wcs.contents.keys()) >= 20 cvg = wcs.contents['AvgLandTemp'] assert cvg.boundingboxes[0]['bbox'] == (-90, -180, 90, 180) assert cvg.timelimits == [datetime.datetime(2000, 2, 1, 0, 0), datetime.datetime(2015, 6, 1, 0, 0)] assert cvg.timepositions[0:5] == [datetime.datetime(2000, 2, 1, 0, 0), datetime.datetime(2000, 3, 1, 0, 0), datetime.datetime(2000, 4, 1, 0, 0), datetime.datetime(2000, 5, 1, 0, 0), datetime.datetime(2000, 6, 1, 0, 0)] assert cvg.supportedFormats == ['application/gml+xml', 'image/jpeg', 'image/png', 'image/tiff', 'image/bmp', 'image/jp2', 'application/netcdf', 'text/csv', 'application/json', 'application/dem', 'application/x-ogc-dted', 'application/x-ogc-ehdr', 'application/x-ogc-elas', 'application/x-ogc-envi', 'application/x-ogc-ers', 'application/x-ogc-fit', 'application/x-ogc-fits', 'image/gif', 'application/x-netcdf-gmt', 'application/x-ogc-gs7bg', 'application/x-ogc-gsag', 'application/x-ogc-gsbg', 'application/x-ogc-gta', 'application/x-ogc-hf2', 'application/x-erdas-hfa', 'application/x-ogc-ida', 'application/x-ogc-ingr', 'application/x-ogc-isis2', 'application/x-erdas-lan', 'application/x-ogc-mff2', 'application/x-ogc-nitf', 'application/x-ogc-paux', 'application/x-ogc-pcidsk', 'application/x-ogc-pcraster', 'application/x-ogc-pdf', 'application/x-ogc-pnm', 'text/x-r', 'application/x-ogc-rmf', 'image/x-sgi', 'application/x-ogc-vrt', 'image/xpm', 'application/x-ogc-zmap'] assert cvg.grid.axislabels == ['Lat', 'Long', 'ansi'] assert cvg.grid.dimension == 3 assert cvg.grid.lowlimits == ['0', '0', '0'] assert cvg.grid.highlimits == ['1799', '3599', '184'] covID = 'AvgLandTemp' time_subset = ("ansi", "2000-02-01T00:00:00Z") lat_subset = ('Lat', 40, 50) long_subset = ('Long', -10, 0) formatType = 'application/netcdf' output = wcs.getCoverage(identifier=[covID], format=formatType, subsets=[long_subset, lat_subset, time_subset]) f = open(scratch_file('test_wcs_200.nc'), 'wb') bytes_written = f.write(output.read()) f.close()
def test_wcs20_getcoverage_netcdf(ows_server): # Use owslib to confirm that we have a somewhat compliant WCS service wcs = WebCoverageService(url=ows_server.url + "/wcs", version="2.0.0") # Ensure that we have at least some layers available contents = list(wcs.contents) output = wcs.getCoverage( identifier=[contents[0]], format='netCDF', subsets=[('x', 144, 144.3), ('y', -42.4, -42), ('time', '2019-11-05')], subsettingcrs="EPSG:4326", scalesize="x(400),y(300)", ) assert output assert output.info()['Content-Type'] == 'application/x-netcdf'
def __init__(self, config, grid): """ Current Issues: - only 'now' works as starttime in the model. any other model gives an error at line 132 (req_url). Somethingsomething dictionary not fed the correct keys. """ # # Initialize the provider. This sets up a temporary directory and some # general stuff that all providers need. # provider.Provider.__init__(self, config, grid) # # First figure out which GFS forecast has been most recently uploaded. # gfs_layers=[] self.gfs_wcs_access_urls={} gfs_dt=[] start = round_datetime(dt=datetime.datetime.utcnow(), seconds=-21600) logger.debug("Looking for most recent GFS dataset") for offset in xrange(-21600*1, -21600*12, -21600): logger.debug("trying date") gfs_latest = start + datetime.timedelta(seconds=offset) gfs_dt.append(gfs_latest) logger.debug("create date") wcs_url=gfs_latest.strftime("http://nomads.ncdc.noaa.gov/thredds/wcs/gfs-004/%Y%m/%Y%m%d/gfs_4_%Y%m%d_%H%M_000.grb2") try: logger.debug("Trying WCS at %s"%(wcs_url)) wcs = WebCoverageService(wcs_url, version='1.0.0') logger.debug("Fine...") except: logger.debug("Exception! Continue to next loop!!") continue logger.debug("moving on...") logger.debug("Connected to GFS WCS at %s"%(wcs_url)) contents = list(wcs.contents) if len(contents) > 0: for layer in contents: gfs_layers.append(layer) self.available_layers.append(layer) for hours in xrange(0,300,3): timestamp = gfs_latest + datetime.timedelta(hours=hours) timestamp = timestamp.replace(tzinfo=pytz.utc) hr = "%.3d"%(hours) wcs_access_url = gfs_latest.strftime("http://nomads.ncdc.noaa.gov/thredds/wcs/gfs-004/%Y%m/%Y%m%d/gfs_4_%Y%m%d_%H%M_"+hr+".grb2") cache_key = gfs_latest.strftime("gfsrun-%Y%m%d%H%M-"+hr+"-") self.gfs_wcs_access_urls.update({timestamp.isoformat():{'url':wcs_access_url,'cache_key':cache_key}}) self.gfs_run_timestamp = gfs_latest break else: continue logger.debug("The following GFS WCS urls will be used for timesteps:") for k in sorted(self.gfs_wcs_access_urls): logger.debug("Timestamp: %s WCS URL: %s"%(k,self.gfs_wcs_access_urls[k]))
def __init__(self, config, grid): """ Initialization code for this specific provider. First initialize the base provider and then to some custom setup stuff specific to the wcs provider. The provider needs to update the self.available_layers attribute and append all the layers that it can provide to this attribute. Since a provider can literally provide any sort of layer and any number of them, this cannot be done automatically, and therefore must be done explicitly in the provider's __init__. """ provider.Provider.__init__(self, config, grid) try: self._layers = {} for wcs_url in config: wcs = WebCoverageService(wcs_url, version='1.0.0') contents = list(wcs.contents) for layer in contents: self.available_layers.append(layer) #make a mapping in the _layers variable about which wcs url #we need to query to fetch a particular layer self._layers.update({layer: wcs_url}) except: logger.debug( " - %s provider couldn't find any layers to make available.")
def getGeoTransformImStack(mSet, verbose_query=True, remove=True): from osgeo import gdal import os wcs = WebCoverageService( 'http://saocompute.eurac.edu/sincohmap/rasdaman/ows?', version='2.0.1') mContents = wcs.contents[mSet] indxDate = getIndexLabelFromWCScontents('date', mContents) subset = 'date(' + mContents.grid.origin[indxDate] + ')' query = 'for c in ( ' + mSet + ' ) return encode (c[' + subset + '], "tiff", "nodata=-999")' if verbose_query: print(query) raster_path = wcps_rasdaman(query, ip='saocompute.eurac.edu/sincohmap', verbose=verbose_query) raster = gdal.Open(raster_path) if remove: os.remove(raster_path) if verbose_query: print('Temporary file has been deleted') return (raster.GetGeoTransform())
def __init__(self, config, grid): self.name = "example" self._config = config self._layers = {} print "Initializing wxs provider:" print grid print config ## # # todo: move all this grid/caching stuff to the base class? providers # shouldnt have to worry about this, and really not about reprojecting # to a equidistant projction either. all that a provider needs to do # is provide data for a layer name # self._grid = grid self._cache = os.path.join( os.environ.get('GEM_WORKING_DIR', '/tmp/gem'), "wcs_cache", self._grid['uuid']) if not os.path.isdir(self._cache): os.makedirs(self._cache) #create the cache directory for wcs_url in config: print "* init url=" print wcs_url wcs = WebCoverageService(wcs_url, version='1.0.0') contents = list(wcs.contents) for layer in contents: #make a mapping in the _layers variable about which wcs url #we need to query to fetch a particular layer self._layers.update({layer: wcs_url})
def test_wcs1_server(ows_server): # Use owslib to confirm that we have a somewhat compliant WCS service wcs = WebCoverageService(url=ows_server.url + "/wcs", version="1.0.0") # Ensure that we have at least some layers available contents = list(wcs.contents) assert contents
def init_config(self, config_file): _wcs = WebCoverageService(self.url, version="2.0.1") config = {} _covs = _wcs.contents.keys() config['coverages'] = {} for _cov in _covs: #print "writting coverage info for "+_cov #if _cov not in self.banned: try: t_cov = {} try: t_cov['time_axis_name'] = [ item for item in _wcs.contents[_cov].grid.axislabels if item not in self.axis_removes ][0] except Exception, e: t_cov['time_axis_name'] = '' spatial_axis = [ item for item in _wcs.contents[_cov].grid.axislabels if item in self.axis_removes ] del _wcs.contents[_cov] for p in spatial_axis: t_cov[get_lat_long(p) + '_axis_name'] = p # gather X and Y coords definition t_cov['name'] = _cov config['coverages'][_cov] = t_cov self.coverages[_cov] = t_cov except Exception, e: print e print "coverage {} failed you should check it".format(_cov)
def getReferenceImage(mSet, verbose_query=True, rotate=False, x0=0, xN=0, y0=0, yN=0): if x0 != 0 and xN != 0 and y0 != 0 and yN != 0: DO_CROP = 1 else: DO_CROP = 0 print(DO_CROP) # Select WCS server and service version wcs = WebCoverageService( 'http://saocompute.eurac.edu/sincohmap/rasdaman/ows?', version='2.0.1') mContents = wcs.contents[mSet] indxMaster = getIndexLabelFromWCScontents('master_date', mContents) indxSlave = getIndexLabelFromWCScontents('slave_date', mContents) indxN = getIndexLabelFromWCScontents('N', mContents) indxE = getIndexLabelFromWCScontents('E', mContents) masterDates = getMasterTimeAxis(mSet, mContents, indxSlave, indxN, indxE) slaveDates = getSlaveTimeAxis(mSet, mContents, indxMaster, indxN, indxE) date0M = masterDates[0].strftime('%Y-%m-%d') date0S = slaveDates[0].strftime('%Y-%m-%d') print(date0M) print(date0S) #subset = 'master_date(\"' + date0M +'\"), slave_date(\"' + date0S +'\")' if DO_CROP: subset = 'E(' + str(x0) + ':' + str(xN) + '), N(' + str(y0) + ':' + str(yN) + \ '), master_date(\"' + date0M +'\"), slave_date(\"' + date0S +'\")' else: subset = 'master_date(\"' + date0M + '\"), slave_date(\"' + date0S + '\")' query = 'for c in ( ' + mSet + ' ) return encode (c[' + subset + '], "tiff")' subset_coherence = wcps_rasdaman(query, ip='saocompute.eurac.edu/sincohmap', verbose=verbose_query) if rotate: refPath = mSet + '_reference_image_rotated.tiff' print(subset_coherence) print(refPath) rotate_tiff_file(subset_coherence, refPath) os.remove(subset_coherence) else: refPath = mSet + '_reference_image.tiff' shutil.move(subset_coherence, refPath) return (refPath)
def test_wcs_idee(): """ COWS Web Coverage Service WCS Version 1.0.0 rewritten doctest/wcs_idee.txt """ wcs = WebCoverageService(SERVICE_URL) assert wcs.version == '1.0.0' assert wcs.url == SERVICE_URL assert wcs.identification.title == 'WCS UTM30N - MDT Peninsula y Baleares' assert wcs.identification.service == 'IDEE-WCS-UTM30N' assert wcs.provider.name == u'Instituto Geogr\xe1fico Nacional' assert sorted(wcs.contents.keys()) == [ 'MDT1000_peninsula_baleares', 'MDT1000_peninsula_baleares_aspecto', 'MDT1000_peninsula_baleares_pendientes', 'MDT25_peninsula_ZIP', 'MDT25_peninsula_aspecto', 'MDT25_peninsula_pendientes', 'MDT500_peninsula_baleares', 'MDT500_peninsula_baleares_aspecto', 'MDT500_peninsula_baleares_pendientes', 'MDT_peninsula_baleares', 'MDT_peninsula_baleares_aspecto', 'MDT_peninsula_baleares_pendientes'] cvg = wcs['MDT25_peninsula_pendientes'] assert cvg.title == 'MDT25 Pendientes Peninsula' assert cast_tuple_int_list(cvg.boundingBoxWGS84) == [-8, 35, 3, 43] assert cvg.timelimits == [] assert sorted(cvg.supportedFormats) == ['AsciiGrid', 'FloatGrid_Zip', 'GeoTIFF'] assert sorted(map(lambda x: x.getcode(), cvg.supportedCRS)) == [ 'EPSG:23028', 'EPSG:23029', 'EPSG:23030', 'EPSG:23030', 'EPSG:23031', 'EPSG:4230', 'EPSG:4326'] output = wcs.getCoverage( identifier='MDT25_peninsula_pendientes', bbox=(600000, 4200000, 601000, 4201000), crs='EPSG:23030', format='AsciiGrid', resX=25, resY=25) f = open(scratch_file('test_idee.grd'), 'wb') bytes_written = f.write(output.read()) f.close()
def main(input_zone, input_value_raster, earsource, **kwargs): wcs = kwargs.get("WCS", 'no') if wcs == 'yes': folder = kwargs.get("WCS_temp", '/home/') urls = kwargs.get("WCS_url", 'None') version = kwargs.get("WCS_version", '1.0.0') bbox = kwargs.get("bbox", 'None') if urls == 'None': return ("Please provide WCS parameters") wcs = WebCoverageService(url, version) ##print(list(wcs.contents)) #print([op.name for op in wcs.operations]) cvg = wcs.contents[input_value_raster] if bbox == 'None': bbox = cvg.boundingBoxWGS84 response = wcs.getCoverage(identifier=input_value_raster, bbox=bbox, format='GEOTIFF_FLOAT32', crs='urn:ogc:def:crs:EPSG::4326', resx=0.5, resy=0.5) temp_raster = folder + '//' + input_value_raster + '.tif' with open(temp_raster, 'wb') as file: file.write(response.read()) input_value_raster = temp_raster if earsource == "shp": return loop_zonal_stats(input_zone, input_value_raster) elif earsource == "pgtable": connString = kwargs.get('connString', "None") if (connString == "None"): print("Please Supply valid connection string") #print("please supply valid data and their source") return loop_zonal_statsPG(input_zone, input_value_raster, connString) elif earsource == "wfs": wfsURL = kwargs.get('wfsURL', "None") if (wfsURL == "None"): return ("Please provide valid WFS URL") return loop_zonal_statsWFS(input_zone, input_value_raster, wfsURL) else: print("please supply valid data and their source")
def test_wcs21_getcoverage(ows_server): # Use owslib to confirm that we have a somewhat compliant WCS service wcs = WebCoverageService(url=ows_server.url + "/wcs", version="2.0.1") # Ensure that we have at least some layers available contents = list(wcs.contents) output = wcs.getCoverage( identifier=[contents[0]], format='image/geotiff', subsets=[('x', 144, 144.3), ('y', -42.4, -42)], # timeSequence=['2019-11-05'], subsettingcrs="EPSG:4326", subset='time("2019-11-05")', scalesize="x(400),y(300)") assert output assert output.info()['Content-Type'] == 'image/geotiff'
def test_wcs20_getcoverage_multidate(ows_server): # Use owslib to confirm that we have a somewhat compliant WCS service wcs = WebCoverageService(url=ows_server.url + "/wcs", version="2.0.0") # Ensure that we have at least some layers available contents = list(wcs.contents) try: resp = wcs.getCoverage( identifier=[contents[0]], format='image/geotiff', subsets=[('x', 144, 144.3), ('y', -42.4, -42), ('time', '2019-11-05', "2019-12-05")], subsettingcrs="EPSG:4326", scalesize="x(400),y(300)", ) except ServiceException as e: assert 'Format does not support multi-time datasets' in str(e)
def get_wcs(server_url, spacename_wcs): chemin = '/home/tmp/'+spacename_wcs+'.tif' if not os.path.exists(chemin): wcs = WebCoverageService(server_url +"/wcs/",version='1.0.0') image = wcs[spacename_wcs] info = (image.boundingboxes)[0] epsg = info['nativeSrs'] bboxx = info['bbox'] offset = image.grid.offsetvectors cellsize_x= offset[0] x = cellsize_x[0] X = str(abs(float(x))) cellsize_y= offset[1] y = cellsize_y[1] Y = str(abs(float(y))) # img_formats = image.supportedFormats # img_format = img_formats[0] img_format = 'GeoTIFF' print "Downloading the WCS: "+spacename_wcs print "From: "+server_url output = wcs.getCoverage(identifier = spacename_wcs, bbox = bboxx, crs = epsg, format = img_format, resx = X, resy = Y) data = output.read() f = open(chemin,'wb') f.write(data) f.close() print "Done" else: print "Done" return chemin
def ws_init(access_type, endpoint): '''Initialise a Web Service object''' if access_type == 'wfs': ws = WebFeatureService(endpoint) elif access_type == 'wms: ws = WebMapService(endpoint) else: ws = WebCoverageService(endpoint) return ws
def _getWCSObj(self, endpoint): oldProxy = proxyFix(endpoint) try: log.debug("wcs endpoint = %s" % (endpoint,)) getCapabilitiesEndpoint = parseEndpointString(endpoint, {'Service':'WCS', 'Request':'GetCapabilities'}) log.debug("wcs endpoint = %s" % (getCapabilitiesEndpoint,)) #requires OWSLib with cookie support wcs=WebCoverageService(getCapabilitiesEndpoint, version='1.0.0',cookies= request.headers.get('Cookie', '')) layers = [x[0] for x in wcs.items()] finally: resetProxy(oldProxy) return wcs, layers
def run_test_resource(resource_type, url): """tests a CSW service and provides run metrics""" if resource_type not in RESOURCE_TYPES.keys(): msg = gettext('Invalid resource type') msg2 = '%s: %s' % (msg, resource_type) LOGGER.error(msg2) raise RuntimeError(msg2) title = None start_time = datetime.datetime.utcnow() message = None try: if resource_type == 'OGC:WMS': ows = WebMapService(url) elif resource_type == 'OGC:WFS': ows = WebFeatureService(url) elif resource_type == 'OGC:WCS': ows = WebCoverageService(url) elif resource_type == 'OGC:WPS': ows = WebProcessingService(url) elif resource_type == 'OGC:CSW': ows = CatalogueServiceWeb(url) elif resource_type == 'OGC:SOS': ows = SensorObservationService(url) elif resource_type in ['WWW:LINK', 'urn:geoss:waf']: ows = urlopen(url) if resource_type == 'WWW:LINK': import re try: title_re = re.compile("<title>(.+?)</title>") title = title_re.search(ows.read()).group(1) except: title = url elif resource_type == 'urn:geoss:waf': title = 'WAF %s %s' % (gettext('for'), urlparse(url).hostname) elif resource_type == 'FTP': ows = urlopen(url) title = urlparse(url).hostname success = True if resource_type.startswith('OGC:'): title = ows.identification.title if title is None: title = '%s %s %s' % (resource_type, gettext('for'), url) except Exception as err: msg = str(err) LOGGER.exception(msg) message = msg success = False end_time = datetime.datetime.utcnow() delta = end_time - start_time response_time = '%s.%s' % (delta.seconds, delta.microseconds) return [title, success, response_time, message, start_time]
def _getWCSObj(self, endpoint): oldProxy = proxyFix(endpoint) try: log.debug("wcs endpoint = %s" % (endpoint,)) getCapabilitiesEndpoint = parseEndpointString(endpoint, {"Service": "WCS", "Request": "GetCapabilities"}) log.debug("wcs endpoint = %s" % (getCapabilitiesEndpoint,)) # requires OWSLib with cookie support wcs = WebCoverageService( getCapabilitiesEndpoint, version="1.0.0", cookies=request.headers.get("Cookie", "") ) layers = [x[0] for x in wcs.items()] finally: resetProxy(oldProxy) return wcs, layers
def provide(self,name,options={}): """ Return a numpy array of the requested data todo: - split into a download() or get_from_cache() """ logger.debug("Looking for layer '%s' at timestamp '%s'"%(name,options['timestamp'])) if name not in self.available_layers: logger.error("Layer '%s' is not available in this provider."%(name)) if options['timestamp'].isoformat() not in self.gfs_wcs_access_urls: logger.error("Timestamp %s was not found in the list of available GFS timesteps."%(options['timestamp'].isoformat())) logger.debug("Downloading coverage from url:") logger.debug(self.gfs_wcs_access_urls.get(options['timestamp'].isoformat())) extent=box(*self._grid["bounds"]).buffer(1.0) logger.debug("request bounds: %s"%(str(extent.bounds))) req_url=self.gfs_wcs_access_urls[options['timestamp'].isoformat()]['url'] cache_key=self.gfs_wcs_access_urls[options['timestamp'].isoformat()]['cache_key']+name logger.debug("req_url=%s"%(req_url)) wcs=WebCoverageService(req_url, version='1.0.0') meta=wcs.contents[name] #cov = wcs.getCoverage(identifier=name,bbox=extent.bounds, format="GeoTIFF_Float") try: cov = wcs.getCoverage(identifier=name, bbox=extent.bounds, format="GeoTIFF_Float") filename=os.path.join(self._cache,"%s.tif"%(cache_key)) logger.debug("WCS: saving file to: %s"%(filename)) with open(filename,'w') as f: f.write(cov.read()) dataset = gdal.Open(filename, gdalconst.GA_ReadOnly) except Exception as e: logger.error("WCS: failure: %s"%(e)) return self.warp_to_grid(dataset)
def get_coverage(wcs_url, layer, verbose=False): """Get coverage from Web Coverage Service (WCS) in GeoTIFF format Input: wcs_url: URL for web ceature service. E.g. http://www.aifdr.org:8080/geoserver/ows? layer: Coverage layer name as <workspace>:<layer> verbose [optional]: Flag controlling the verbosity level. Default is False. Output: GeoTIFF data or None. """ if verbose: print('Retrieving %s from %s' % (layer, wcs_url)) wcs = WebCoverageService(wcs_url, version='1.1.1') if layer not in wcs.contents.keys(): return None response = wcs.getcoverage(typename=[layer], format='GeoTIFF') return response
from owslib.wcs import WebCoverageService from scipy.io import netcdf from pyproj import Proj import io import datetime import json def to_celsius(kelvin): return kelvin - 273.15 wcs = WebCoverageService('http://thredds.met.no/thredds/wcs/arome25/arome_metcoop_test2_5km_latest.nc?service=WCS&version=1.0.0&request=GetCapabilities') def geojson_feature(lon, lat): return { 'type': 'Feature', 'properties': {}, 'geometry': {'type': 'Point', 'coordinates': [lon, lat]} } def lambert_to_latlon(x, y): proj_str = '+proj=lcc +lat_0=63 +lon_0=15 +lat_1=63 +lat_2=63 +no_defs +R=6.371e+06 +units=m' p1 = Proj(proj_str) lon, lat = p1(x, y, inverse=True) return lat, lon def get_coverage(identifier, bbox):
def provide(self, name, options={}): """ The providers' provide() method returns a numpy with the correct data type and propotions given the layer name and possibly some extra options. The readmap() in the model will convert the numpy array to a pcraster map when it is requested. """ logging.debug("WCS: provide request for layer '%s'" % (name)) target_file = os.path.join(self._cache, "%s" % (name)) crs = None srid = None dataset = None logging.debug("WCS: geometry: %s" % (self._geom.wkt)) url = urlparse(self._layers[name]) qs = parse_qs(url.query) query_params = {} for p in list(qs): query_params.update({p: qs[p][0]}) wcs = WebCoverageService(url.geturl(), version="1.0.0") meta = wcs.contents[name] mapformat = meta.supportedFormats[0] supported_crses = [crs_.code for crs_ in meta.supportedCRS] logging.debug("WCS: Service supports the following crs: %s" % (", ".join(map(str, supported_crses)))) if self._grid["srid"] in supported_crses: crs = meta.supportedCRS[supported_crses.index(self._grid["srid"])] srid = self._grid["srid"] elif 4326 in supported_crses: crs = meta.supportedCRS[supported_crses.index(4326)] srid = 4326 else: crs = meta.supportedCRS[supported_crses.index(supported_crses[0])] srid = supported_crses[0] if crs is not None and srid is not None: logging.debug("WCS: using %s (epsg:%d) to fetch the file from the wcs server" % (crs, srid)) else: logging.debug("WCS: could not agree upon a projection format to fetch data with") raise Exception("WCS: no valid projections found") logging.debug("WCS: reprojecting the chunk mask to the required projection") project = partial( pyproj.transform, pyproj.Proj(init="epsg:%d" % (self._grid["srid"])), pyproj.Proj(init="epsg:%d" % (srid)) ) # Add a small buffer to the request so we fetch an area slightly larger # than what we really need. This will prevent some edge effects due # to the reprojection. projected_geom = transform(project, self._geom.buffer(200)) logging.debug("WCS: original geom: %s" % (self._geom.wkt)) logging.debug("WCS: reprojected geom: %s" % (projected_geom.wkt)) try: logging.debug("WCS: fetching wcs data in %s" % (crs)) logging.debug("WCS: saving to: %s" % (target_file + ".tif")) cov = wcs.getCoverage( identifier=name, crs=crs, bbox=projected_geom.bounds, format=mapformat, width=self._grid["cols"], height=self._grid["rows"], **query_params ) with open(target_file + ".tif", "w") as f: f.write(cov.read()) dataset = gdal.Open(target_file + ".tif", gdalconst.GA_ReadOnly) except Exception as e: logger.error("WCS: failure: %s" % (e)) utm_data = self.warp_to_grid(dataset) dataset = None return utm_data
# <nbformat>3.0</nbformat> # <headingcell level=1> # Extract data from USGS CMG WCS Service # <codecell> from owslib.wcs import WebCoverageService import numpy as np import numpy.ma as ma endpoint='http://coastalmap.marine.usgs.gov/cmgp/services/EastCoast/Mass_Seafloor/MapServer/WCSServer?request=GetCapabilities&service=WCS' # <codecell> wcs = WebCoverageService(endpoint,version='1.0.0',timeout=60) # <codecell> for k,v in wcs.contents.iteritems(): print v.title # <codecell> lidar = wcs['3'] print lidar.title print lidar.boundingBoxWGS84 print lidar.timelimits print lidar.supportedFormats # <codecell>
def get_records(post_code_for_bounding_box): pc_info = utils.post_codes[int(post_code_for_bounding_box)] print "Getting images for: %s - %s, %s" % (post_code_for_bounding_box, pc_info['suburb'], pc_info['state']) lower_corner = '%s %s' % (float(pc_info['lon']) - 1.0, float(pc_info['lat']) - 0.5) lower_corner_sml = '%s %s' % (float(pc_info['lon']) - 0.2, float(pc_info['lat']) - 0.1) upper_corner = '%f %f' % (float(pc_info['lon']) + 1.0, float(pc_info['lat']) + 0.5) upper_corner_sml = '%f %f' % (float(pc_info['lon']) + 0.2, float(pc_info['lat']) + 0.1) body = """<?xml version="1.0" encoding="UTF-8"?> <csw:GetRecords xmlns:gml="http://www.opengis.net/gml" xmlns:ogc="http://www.opengis.net/ogc" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:csw="http://www.opengis.net/cat/csw/2.0.2" outputSchema="http://www.opengis.net/cat/csw/2.0.2" outputFormat="application/xml" version="2.0.2" service="CSW" resultType="results" maxRecords="10" nextRecord="0" xsi:schemaLocation="http://www.opengis.net/cat/csw/2.0.2 http://schemas.opengis.net/csw/2.0.2/CSW-discovery.xsd"> <csw:Query typeNames="csw:Record"> <csw:ElementSetName>full</csw:ElementSetName> <csw:Constraint version="1.1.0"> <ogc:Filter> <ogc:And> <ogc:PropertyIsLike escape="\" singleChar="_" wildCard="%"> <ogc:PropertyName>Title</ogc:PropertyName> <ogc:Literal>%Landsat%</ogc:Literal> </ogc:PropertyIsLike> <ogc:BBOX> <ogc:PropertyName>ows:BoundingBox</ogc:PropertyName> <gml:Envelope> <gml:lowerCorner>""" + lower_corner_sml + """</gml:lowerCorner> <gml:upperCorner>""" + upper_corner_sml + """</gml:upperCorner> </gml:Envelope> </ogc:BBOX> </ogc:And> </ogc:Filter> </csw:Constraint> <ogc:SortBy> <ogc:SortProperty> <ogc:PropertyName>apiso:TempExtent_begin</ogc:PropertyName> <ogc:SortOrder>ASC</ogc:SortOrder> </ogc:SortProperty> </ogc:SortBy> </csw:Query> </csw:GetRecords>""" headers = { 'Accept-Encoding': 'gzip,deflate', 'Content-Type': 'text/xml;charset=UTF-8', 'Content-Length': len(body), 'Host': HOST_NAME, 'Connection': 'Keep-Alive', 'User-Agent': 'GovHack - Team A Kicking Wheel' } #request_path = '%s?%s&format=json' % (HOST_PATH, query_params) request_path = '%s?request=GetRecords' % (HOST_PATH) conn = httplib.HTTPConnection(HOST_NAME, HOST_PORT) conn.request('POST', request_path, body, headers) response = conn.getresponse() print 'GetRecords: %d %s' % (response.status, response.reason) if response.status == 200: data = response.read() conn.close() result_tree = ET.ElementTree(ET.fromstring(data)) namespaces = { 'csw': 'http://www.opengis.net/cat/csw/2.0.2', 'dc': 'http://purl.org/dc/elements/1.1/', 'ows': 'http://www.opengis.net/ows', } results = result_tree.getroot().findall( 'csw:SearchResults/csw:Record', namespaces=namespaces ) for result in results: get_capabilities_response = result.find( "dc:URI[@protocol='OGC:WCS']", namespaces=namespaces ) wcs_url = get_capabilities_response.text print 'Record URL: %s' % wcs_url service = WebCoverageService(wcs_url, version='1.0.0') for content in service.contents: bounding_box_strs = lower_corner_sml.split(' ') + upper_corner_sml.split(' ') bounding_box = tuple([float(i) for i in bounding_box_strs]) print 'Bounding Box: ' + str(bounding_box) img = service.getCoverage(identifier=content, bbox=bounding_box, format='GeoTIFF') identifier = result.find('dc:identifier', namespaces=namespaces).text directory = 'landsat_images/%s' % pc_info['suburb'] if not os.path.exists(directory): os.makedirs(directory) file_name = '%s/%s_%s.png' % (directory, identifier, content) print 'Writing file: %s' % file_name out = open(file_name, 'wb') out.write(img.read()) out.close()
def fetch(self,name): #print " * Fetching layer '%s'"%(name) if name in self._layers: target_file=os.path.join(self._cache,"%s"%(name)) url=urlparse(self._layers[name]) qs=parse_qs(url.query) query_params={} for p in list(qs): query_params.update({p:qs[p][0]}) print "Creating a coverage url:" print url.geturl() wcs=WebCoverageService(url.geturl(), version='1.0.0') meta=wcs.contents[name] mapformat=meta.supportedFormats[0] supported_crses=[crs.code for crs in meta.supportedCRS] try: crs=meta.supportedCRS[supported_crses.index(self._grid['srid'])] except ValueError: raise Exception("WCS provider could not retrieve layer '%s' because it is not available in the UTM grid that the model needs to run at. We need a raster in epsg %d, available are only epsg %s."%(name,self._grid['srid']," ".join(map(str,supported_crses)))) resx=self._grid['cellsize'] resy=self._grid['cellsize'] bbox=self._grid['bbox'] cov=wcs.getCoverage(identifier=name,crs=crs,bbox=bbox,format=mapformat,resx=resx,resy=resy,**query_params) #Dict used for converting geotiffs to pcraster format. Any data #types not set explicitly will become Float32/VS_SCALARS pcraster_valuescale = defaultdict(lambda: ('Float32','VS_SCALAR')) pcraster_valuescale.update({ 'Float32': ('Float32','VS_SCALAR'), 'Int32': ('Int32','VS_NOMINAL'), 'Int16': ('Int32','VS_NOMINAL'), 'Byte': ('Int32','VS_NOMINAL') }) print " * Downloading %s"%(cov.url) with open(target_file+".tif",'w') as f: f.write(cov.read()) print " * Converting to PCRaster format..." dataset=gdal.Open(target_file+".tif",GA_ReadOnly) print 'Driver: ', dataset.GetDriver().ShortName,'/', dataset.GetDriver().LongName print 'Size is ',dataset.RasterXSize,'x',dataset.RasterYSize, 'x',dataset.RasterCount print 'Projection is ',dataset.GetProjection() print 'Geotransform is ',dataset.GetGeoTransform() band = dataset.GetRasterBand(1) gdal_type=gdal.GetDataTypeName(band.DataType) print 'Band Type=',gdal_type pcraster_type=pcraster_valuescale.get(gdal_type) print "Pcraster type=",pcraster_type try: #-of PCRaster -co "PCRASTER_VALUESCALE=VS_SCALAR" -ot Float32 -a_nodata 500.1 c=[ '/usr/bin/gdal_translate','-q', '-ot',pcraster_type[0], '-of','PCRaster', '-co','PCRASTER_VALUESCALE=%s'%(pcraster_type[1]), target_file+".tif",target_file+".map" ] print "Conversion command: " print " ".join(c) rc=subprocess.call(c) except Exception as e: print " * Conversion to pcraster format failed!!! Hint: %s"%(e) print " * Completed!" else: print " * Layer not found" return None
# Very simple script demonstrating how to interact with a THREDDS based WCS. # --- # # The GetCapabilities and DescribeCoverage requests for this dataset are: # http://cida.usgs.gov/thredds/wcs/prism?service=WCS&version=1.0.0&request=GetCapabilities # http://cida.usgs.gov/thredds/wcs/prism?service=WCS&version=1.0.0&request=DescribeCoverage # # The equivalent GetCoverage request that is equivalent ot hte example is: # http://cida.usgs.gov/thredds/wcs/prism?request=GetCoverage&version=1.0.0&service=WCS&format=GeoTIFF&coverage=tmx&time=1895-01-01T00:00:00Z&bbox=-90,40,-89,41 # --- # # Example to find the equivalent information using OWSLib: # from owslib.wcs import WebCoverageService wcs=WebCoverageService('http://cida.usgs.gov/thredds/wcs/prism',version='1.0.0') # Take a look at the contents (coverages) of the wcs. print wcs.contents tmax=wcs['tmx'] # Take a look at the attributes of the coverage dir(tmax) print tmax.boundingBoxWGS84 print tmax.timepositions print tmax.supportedFormats # mock up a simple GetCoverage request. output=wcs.getCoverage(identifier='tmx',time=['1895-01-01T00:00:00Z'],bbox=(-90,40,-89,41),format='GeoTIFF') # Write the file out to disk. f=open('foo.tif','wb') f.write(output.read()) f.close()
# Extract data from USGS ScienceBase # <codecell> %matplotlib inline # <codecell> from owslib.wcs import WebCoverageService import numpy as np import numpy.ma as ma endpoint='https://www.sciencebase.gov/catalogMaps/mapping/ows/5638cf1fe4b0d6133fe73040?service=wcs&request=getcapabilities&version=1.1.1' # <codecell> wcs = WebCoverageService(endpoint,version='1.1',timeout=60) # <codecell> wcs.contents # <codecell> for k,v in wcs.contents.iteritems(): print v.title # <codecell> g = wcs['bh_30mbathy'] print g.title print g.boundingBoxWGS84
# -*- coding: utf-8 -*- # <nbformat>3.0</nbformat> # <headingcell level=1> # Test ESRI WCS with OWSLib # <codecell> from owslib.wcs import WebCoverageService endpoint='http://olga.er.usgs.gov/stpgis/services/lidar/Bare_Earth_Lidar/MapServer/WCSServer?request=GetCapabilities&service=WCS' # <codecell> wcs = WebCoverageService(endpoint,version='1.0.0',timeout=60) # <codecell> for k,v in wcs.contents.iteritems(): print v.title # <codecell> wcs['1'].title # <codecell> cvg = wcs['1'] print cvg.title print cvg.boundingBoxWGS84
def test_raster_wcs_reprojection(self): """UTM Raster can be reprojected by Geoserver and downloaded correctly """ # FIXME (Ole): Still need to do this with assertions filename = 'tsunami_max_inundation_depth_BB_utm.asc' projected_tif_file = os.path.join(TESTDATA, filename) #projected_tif = file_upload(projected_tif_file, overwrite=True) projected_tif = save_to_geonode(projected_tif_file, user=self.user, overwrite=True) check_layer(projected_tif) wcs_url = settings.GEOSERVER_BASE_URL + 'wcs' wcs = WebCoverageService(wcs_url, version='1.0.0') #logger.info(wcs.contents) metadata = wcs.contents[projected_tif.typename] #logger.info(metadata.grid) bboxWGS84 = metadata.boundingBoxWGS84 #logger.info(bboxWGS84) resx = metadata.grid.offsetvectors[0][0] resy = abs(float(metadata.grid.offsetvectors[1][1])) #logger.info("resx=%s resy=%s" % (str(resx), str(resy))) formats = metadata.supportedFormats #logger.info(formats) supportedCRS = metadata.supportedCRS #logger.info(supportedCRS) width = metadata.grid.highlimits[0] height = metadata.grid.highlimits[1] #logger.info("width=%s height=%s" % (width, height)) gs_cat = Layer.objects.gs_catalog cvg_store = gs_cat.get_store(projected_tif.name) cvg_layer = gs_cat.get_resource(projected_tif.name, store=cvg_store) #logger.info(cvg_layer.request_srs_list) #logger.info(cvg_layer.response_srs_list) # FIXME: A patch was submitted OWSlib 20110808 # Can delete the following once patch appears # In the future get bboxNative and nativeSRS from get_metadata descCov = metadata._service.getDescribeCoverage(projected_tif.typename) envelope = (descCov.find(ns('CoverageOffering/') + ns('domainSet/') + ns('spatialDomain/') + '{http://www.opengis.net/gml}Envelope')) nativeSrs = envelope.attrib['srsName'] #logger.info(nativeSrs) gmlpositions = envelope.findall('{http://www.opengis.net/gml}pos') lc = gmlpositions[0].text uc = gmlpositions[1].text bboxNative = (float(lc.split()[0]), float(lc.split()[1]), float(uc.split()[0]), float(uc.split()[1])) #logger.info(bboxNative) # ---- END PATCH # Make a temp dir to store the saved files tempdir = '/tmp/%s' % str(time.time()) os.mkdir(tempdir) # Check that the layer can be downloaded in its native projection cvg = wcs.getCoverage(identifier=projected_tif.typename, format='GeoTIFF', crs=nativeSrs, bbox=bboxNative, resx=resx, resy=resy) t = tempfile.NamedTemporaryFile(delete=False, dir=tempdir) out = open(t.name, 'wb') out.write(cvg.read()) out.close() #logger.info("GeoTIFF in %s = %s" % (nativeSrs, t.name)) # TODO: Verify that the file is a valid GeoTiff and that it is # _exactly_ the same size and bbox of the original # Test that the layer can be downloaded in ARCGRID format cvg_layer.supported_formats = cvg_layer.supported_formats + ['ARCGRID'] gs_cat.save(cvg_layer) cvg = wcs.getCoverage(identifier=projected_tif.typename, format='ARCGRID', crs=nativeSrs, bbox=bboxNative, resx=resx, resy=resy) t = tempfile.NamedTemporaryFile(delete=False, dir=tempdir) out = open(t.name, 'wb') out.write(cvg.read()) out.close() #logger.info("ARCGRID in %s = %s" % (nativeSrs, t.name)) # Check that the downloaded file is a valid ARCGRID file and that it # the required projection information # (FIXME: There is no prj file here. GS bug) # Check that the layer can downloaded in WGS84 cvg_layer.request_srs_list += ['EPSG:4326'] cvg_layer.response_srs_list += ['EPSG:4326'] gs_cat.save(cvg_layer) #logger.info(cvg_layer.request_srs_list) #logger.info(cvg_layer.response_srs_list) cvg = wcs.getCoverage(identifier=projected_tif.typename, format='GeoTIFF', crs='EPSG:4326', bbox=bboxWGS84, #resx=0.000202220898116, # Should NOT be hard-coded! # How do we convert #resy=0.000202220898116) # See comments in riab issue #103 width=width, height=height) t = tempfile.NamedTemporaryFile(delete=False, dir=tempdir) out = open(t.name, 'wb') out.write(cvg.read()) out.close() #logger.info("GeoTIFF in %s = %s" % ("EPSG:4326", t.name)) # TODO: Verify that the file is a valid GeoTiff and that it is # the correct size and bbox based on the resx and resy or width # and height specified # Check that we can download the layer in another projection cvg_layer.request_srs_list += ['EPSG:32356'] cvg_layer.response_srs_list += ['EPSG:32356'] cvg_layer.request_srs_list += ['EPSG:900913'] cvg_layer.response_srs_list += ['EPSG:900913'] gs_cat.save(cvg_layer) #logger.info(cvg_layer.request_srs_list) #logger.info(cvg_layer.response_srs_list) # How do we get the bboxes for the newly assigned # request/response SRS?? cvg = wcs.getCoverage(identifier=projected_tif.typename, format='GeoTIFF', crs='EPSG:32356', # Should not be hardcoded for a test, # or should use 900913 (need bbox) bbox=bboxNative, #resx=0.000202220898116, # Should NOT be hard-coded! # How do we convert #resy=0.000202220898116) # See comments in riab issue #103 width=width, height=height) t = tempfile.NamedTemporaryFile(delete=False, dir=tempdir) out = open(t.name, 'wb') out.write(cvg.read()) out.close()
def download_scenario(user, scenario_id): """ Generate a working directory for running ANUGA Create/Copy all of the files to that directory that are necessary for running the simulation. Generate a json run file and call run_tsudat. Notes here: https://github.com/AIFDR/tsudat2/wiki/Create-anuga-run-script """ create_dir() # Get the scenario object from the Database scenario = Scenario.objects.get(id=scenario_id) # the base of the TsuDAT user directory structures from settings.py TsuDATBase = settings.TSUDAT_BASE_DIR TsuDATMux = settings.TSUDAT_MUX_DIR # change setup value to one of expected strings print('original scenario.model_setup=%s' % scenario.model_setup) trial_edit = {'t': 'trial', 'T': 'trial', 'trial': 'trial', 'TRIAL': 'trial', 'f': 'final', 'F': 'final', 'final': 'final', 'FINAL': 'final'} actual_setup = trial_edit.get(scenario.model_setup, 'trial') print('actual_setup=%s' % actual_setup) # fake a prject name ##? if not scenario.project.name: ##? scenario.project.name = _slugify(scenario.name) ##? # create the user working directory (work_dir, raw_elevations, boundaries, meshes, polygons, gauges, topographies, user_dir) = run_tsudat.make_tsudat_dir(TsuDATBase, user.username, _slugify(scenario.project.name), _slugify(scenario.name), ##? scenario.model_setup, actual_setup, scenario.event.tsudat_id) project_geom = scenario.project.geom project_extent = scenario.project.geom.extent centroid = project_geom.centroid # This somewhat naively assumes that the whole bounding polygon is in the same zone (UTMZone, UTMEasting, UTMNorthing) = LLtoUTM(23, centroid.coords[1], centroid.coords[0]) if(len(UTMZone) == 3): utm_zone = int(UTMZone[0:2]) else: utm_zone = int(UTMZone[0:1]) if(centroid.coords[1] > 0): srid_base = 32600 else: srid_base = 32700 srid = srid_base + utm_zone scenario.project.srid = srid scenario.project.save() project_geom.transform(srid) # Polygons print polygons bounding_polygon_file = open(os.path.join(polygons, 'bounding_polygon.csv'), 'w') for coord in project_geom.coords[0][:-1]: bounding_polygon_file.write('%f,%f\n' % (coord[0], coord[1])) bounding_polygon_file.close() # Internal Polygons internal_polygons = InternalPolygon.objects.filter(project=scenario.project).order_by('value') count = 0 InteriorRegions = [] for ip in internal_polygons: ipfile = open(os.path.join(polygons, 'ip%s.csv' % count), 'w') geom = ip.geom geom.transform(srid) for coord in geom.coords[0][:-1]: ipfile.write('%f,%f\n' % (coord[0], coord[1])) if(ip.type == 1): type = "resolution" elif(ip.type == 2): type = "friction" elif(ip.type == 3): type = "aoi" InteriorRegions.append([type, ipfile.name, ip.value]) ipfile.close() geom = ipfile = None count += 1 # Raw Elevation Files RawElevationFiles = [] elevation_files = [] wcs_url = settings.GEOSERVER_BASE_URL + 'wcs' wcs = WebCoverageService(wcs_url, version='1.0.0') pds = ProjectDataSet.objects.filter(project=scenario.project).order_by('ranking') srs = osr.SpatialReference() srs.ImportFromEPSG(srid) dst_wkt = srs.ExportToPrettyWkt() eResampleAlg = None create_options = None output_format = "AAIGrid" driver = gdal.GetDriverByName(output_format) for ds in pds: layer = Layer.objects.get(typename=ds.dataset.typename) elevation_files.append(layer.typename) logger.info(wcs.contents) metadata = wcs.contents[layer.typename] print metadata.grid resx = metadata.grid.offsetvectors[0][0] resy = abs(float(metadata.grid.offsetvectors[1][1])) formats = metadata.supportedFormats print formats cvg = wcs.getCoverage(identifier=layer.typename, format='GeoTIFF', crs="EPSG:4326", bbox=(project_extent[0], project_extent[1], project_extent[2], project_extent[3]), resx=resx, resy=resy) # Need to make sure the ranking numbers are unique for each project (enforced with DB constraint?) tif_file_name = '%s.tif' % ds.ranking tif_file_path = os.path.join(raw_elevations, tif_file_name) asc_file_name = '%s.asc' % ds.ranking asc_file_path = os.path.join(raw_elevations, asc_file_name) out = open(tif_file_path, 'wb') out.write(cvg.read()) out.close() # Warp to UTM cmd = "/usr/bin/gdalwarp -srcnodata -9999 -dstnodata -9999 -t_srs EPSG:%d %s %s.tmp" % (srid, tif_file_path, tif_file_path) os.system(cmd) # Convert to AAIGrid cmd = "/usr/bin/gdal_translate -a_nodata -9999 -of %s %s.tmp %s" % (output_format, tif_file_path, asc_file_path) os.system(cmd) # Remove Intermediate files #os.remove(tif_file_path) #os.remove(tif_file_path + ".tmp") # Rename the .prj file to .prj.wkt shutil.move(asc_file_path.replace('.asc', '.prj'), asc_file_path.replace('.asc', '.prj.wkt')) # Generate a prj.adf style prj file # NOTE: Not sure if this will work in all cases? prj_file_name = '%s.prj' % ds.ranking prj_file = open(os.path.join(raw_elevations, prj_file_name), 'w') prj_file.write('Projection UTM\n') prj_file.write('Zone %d\n' % utm_zone) prj_file.write('Datum WGS1984\n') prj_file.write('Zunits NO\n') prj_file.write('Units METERS\n') prj_file.write('Spheroid WGS_1984\n') prj_file.write('Xshift 500000\n') prj_file.write('Yshift 10000000\n') prj_file.write('Parameters\n') prj_file.write('NODATA_value -9999') prj_file.close() RawElevationFiles.append(asc_file_path) ''' src_ds = gdal.Open( str(tif_file_path), GA_ReadOnly ) dst_ds_tmp = driver.CreateCopy( str(asc_file_name + '.tmp'), src_ds, 0) dst_ds = driver.Create( str(asc_file_path), dst_ds_tmp.RasterXSize, dst_ds_tmp.RasterYSize) gdal.ReprojectImage(src_ds, dst_ds, None, dst_wkt) dst_ds = None dst_ds_tmp = None src_ds = None ''' # Landward Boundary # Iterate over the in the project geometry and add a l or s flag and call landward.landward with them points_list = [] for coord in project_geom.coords[0][:-1]: pnt_wkt = 'SRID=%s;POINT(%f %f)' % (srid, coord[0], coord[1]) land = Land.objects.filter(the_geom__intersects=pnt_wkt) if(land.count() > 0): points_list.append((coord[0], coord[1], "l")) else: points_list.append((coord[0], coord[1], "s")) print('points_list=%s' % str(points_list)) landward_points = landward.landward(points_list) print('landward_points=%s' % str(landward_points)) # Write out the landward points to a file landward_boundary_file = open(os.path.join(boundaries, 'landward_boundary.csv'), 'w') for pt in landward_points: landward_boundary_file.write('%f,%f\n' % (pt[0], pt[1])) landward_boundary_file.close() # Interior Hazard Points File interior_hazard_points_file = open(os.path.join(boundaries, 'interior_hazard_points.csv'), 'w') hps = HazardPoint.objects.filter(geom__intersects=project_geom).order_by('tsudat_id') for hp in hps: the_geom = hp.geom latitude=the_geom.coords[1] longitude=the_geom.coords[0] the_geom.transform(srid) interior_hazard_points_file.write('%d,%f,%f,%f,%f\n' % (hp.tsudat_id,longitude,latitude,the_geom.coords[0], the_geom.coords[1])) interior_hazard_points_file.close() # Gauges gauge_file = open(os.path.join(gauges, 'gauges.csv'), 'w') gauge_file.write('easting,northing,name,elevation\n') gauge_points = GaugePoint.objects.filter(project=scenario.project) for gauge in gauge_points: gauge_geom = gauge.geom gauge_geom.transform(srid) gauge_file.write('%f,%f,%s,%f\n' % (gauge_geom.coords[0], gauge_geom.coords[1], gauge.name, 0.0)) gauge_file.close() # Layers scenario_layers = scenario.output_layers.all() layers = [] for layer in scenario_layers: layers.append(layer.name) # build the scenario json data file date_time = strftime("%Y%m%d%H%M%S", gmtime()) json_file = os.path.join(work_dir, '%s.%s.json' % (_slugify(scenario.name), date_time)) json_dict = { 'user': user.username, 'user_directory': user_dir, 'project': _slugify(scenario.project.name), 'project_id': scenario.project.id, 'scenario': _slugify(scenario.name), 'scenario_id': scenario.id, ##? 'setup': scenario.model_setup, 'setup': actual_setup, 'event_number': scenario.event.tsudat_id, 'working_directory': TsuDATBase, 'mux_directory': TsuDATMux, 'initial_tide': scenario.initial_tidal_stage, 'start_time': scenario.start_time, 'end_time': scenario.end_time, 'smoothing': scenario.smoothing_param, 'bounding_polygon_file': bounding_polygon_file.name, 'raw_elevation_directory': raw_elevations, 'elevation_data_list': RawElevationFiles, 'mesh_friction': scenario.default_friction_value, 'raster_resolution': scenario.raster_resolution, 'export_area': "AOI" if scenario.use_aoi == True else "ALL", 'gauge_file': gauge_file.name, 'bounding_polygon_maxarea': scenario.project.max_area, 'interior_regions_list': InteriorRegions, 'interior_hazard_points_file': interior_hazard_points_file.name, 'landward_boundary_file': landward_boundary_file.name, 'zone_number': utm_zone, 'layers_list': layers, 'get_results_max': True, 'get_timeseries': True } with open(json_file, 'w') as fd: json.dump(json_dict, fd, indent=2, separators=(',', ':')) scenario.tsudat_payload = json.dumps(json_dict) scenario.save() # now run the simulation run_tsudat.run_tsudat(json_file) scenario.anuga_status = "QUEUE" scenario.save() return True
def getNLCDRasterDataForBoundingBox(config, outputDir, bbox, coverage=DEFAULT_COVERAGE, filename='NLCD', srs='EPSG:4326', resx=0.000277777777778, resy=0.000277777777778, interpolation='near', fmt=FORMAT_GEOTIFF, overwrite=False, verbose=False, outfp=sys.stdout): """ Download NLCD rasters from http://raster.nationalmap.gov/arcgis/rest/services/LandCover/USGS_EROS_LandCover_NLCD/MapServer @param config A Python ConfigParser (not currently used) @param outputDir String representing the absolute/relative path of the directory into which output raster should be written @param bbox Dict representing the lat/long coordinates and spatial reference of the bounding box area for which the raster is to be extracted. The following keys must be specified: minX, minY, maxX, maxY, srs. @param srs String representing the spatial reference of the raster to be returned. @param resx Float representing the X resolution of the raster(s) to be returned @param resy Float representing the Y resolution of the raster(s) to be returned @param interpolation String representing resampling method to use. Must be one of spatialdatalib.utils.RASTER_RESAMPLE_METHOD. @param fmt String representing format of raster file. Must be one of FORMATS. @param overwrite Boolean True if existing data should be overwritten @param verbose Boolean True if detailed output information should be printed to outfp @param outfp File-like object to which verbose output should be printed @return A dictionary mapping soil property names to soil property file path and WCS URL, i.e. dict[soilPropertyName] = (soilPropertyFilePath, WCS URL) @exception Exception if coverage is not known @exception Exception if interpolation method is not known @exception Exception if fmt is not a known format @exception Exception if output already exists by overwrite is False """ if coverage not in COVERAGES: raise Exception("Coverage {0} is not known".format(coverage)) if interpolation not in INTERPOLATION_METHODS: raise Exception("Interpolation method {0} is not of a known method {1}".format(interpolation, INTERPOLATION_METHODS.keys())) if fmt not in FORMATS: raise Exception("Format {0} is not of a known format {1}".format(fmt, str(FORMATS))) if verbose: outfp.write("Acquiring NLCD coverage {lctype} from {pub}\n".format(lctype=coverage, pub=DC_PUBLISHER)) outFilename = os.path.extsep.join([filename, FORMAT_EXT[fmt]]) outFilepath = os.path.join(outputDir, outFilename) delete = False if os.path.exists(outFilepath): if not overwrite: raise Exception("File {0} already exists, and overwrite is false".format(outFilepath)) else: delete = True try: if delete: os.unlink(outFilepath) wcs = WebCoverageService(URL_BASE, version='1.0.0') bbox = [bbox['minX'], bbox['minY'], bbox['maxX'], bbox['maxY']] wcsfp = wcs.getCoverage(identifier=COVERAGES[coverage], bbox=bbox, crs=srs, response_crs=srs, resx=resx, # their WCS seems to accept resx, resy in meters resy=resy, format=fmt, interpolation=INTERPOLATION_METHODS[interpolation], **{'band': '1'}) url = urllib.unquote(wcsfp.geturl()) f = open(outFilepath, 'wb') f.write(wcsfp.read()) f.close() return (True, url, outFilename) except Exception as e: traceback.print_exc(file=outfp) raise(e) finally: # Clean-up pass
def getSoilsRasterDataForBoundingBox(config, outputDir, bbox, srs='EPSG:4326', resx=0.000277777777778, resy=0.000277777777778, interpolation='bilinear', fmt=FORMAT_GEOTIFF, overwrite=False, verbose=False, outfp=sys.stdout): """ Download soil property rasters from http://www.clw.csiro.au/aclep/soilandlandscapegrid/ For each property, rasters for the first 1-m of the soil profile will be downloaded from which the depth-weighted mean of the property will be calculated and stored in outpufDir @param config A Python ConfigParser (not currently used) @param outputDir String representing the absolute/relative path of the directory into which output raster should be written @param bbox Dict representing the lat/long coordinates and spatial reference of the bounding box area for which the raster is to be extracted. The following keys must be specified: minX, minY, maxX, maxY, srs. @param srs String representing the spatial reference of the raster to be returned. @param resx Float representing the X resolution of the raster(s) to be returned @param resy Float representing the Y resolution of the raster(s) to be returned @param interpolation String representing resampling method to use. Must be one of spatialdatalib.utils.RASTER_RESAMPLE_METHOD. @param fmt String representing format of raster file. Must be one of FORMATS. @param overwrite Boolean True if existing data should be overwritten @param verbose Boolean True if detailed output information should be printed to outfp @param outfp File-like object to which verbose output should be printed @return A dictionary mapping soil property names to soil property file path and WCS URL, i.e. dict[soilPropertyName] = (soilPropertyFilePath, WCS URL) @exception Exception if interpolation method is not known @exception Exception if fmt is not a known format @exception Exception if output already exists by overwrite is False @exception Exception if a gdal_calc.py command fails """ if interpolation not in RASTER_RESAMPLE_METHOD: raise Exception("Interpolation method {0} is not of a known method {1}".format(interpolation, RASTER_RESAMPLE_METHOD)) if fmt not in FORMATS: raise Exception("Format {0} is not of a known format {1}".format(fmt, str(FORMATS))) if verbose: outfp.write("Acquiring soils data from {0}\n".format(DC_PUBLISHER)) soilPropertyRasters = {} #import logging #logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) #owslib_log = logging.getLogger('owslib') # Add formatting and handlers as needed #owslib_log.setLevel(logging.DEBUG) # Set-up gdal_calc.py command gdalBase = None try: gdalBase = config.get('GDAL/OGR', 'GDAL_BASE') except ConfigParser.NoOptionError: gdalBase = os.path.dirname(config.get('GDAL/OGR', 'PATH_OF_GDAL_WARP')) gdalCmdPath = os.path.join(gdalBase, 'gdal_calc.py') if not os.access(gdalCmdPath, os.X_OK): raise IOError(errno.EACCES, "The gdal_calc.py binary at %s is not executable" % gdalCmdPath) gdalCmdPath = os.path.abspath(gdalCmdPath) tmpdir = tempfile.mkdtemp() #print(tmpdir) bbox = [bbox['minX'], bbox['minY'], bbox['maxX'], bbox['maxY']] # For each soil variable, download desired depth layers for v in VARIABLE.keys(): variable = VARIABLE[v] soilPropertyName = "soil_raster_pct{var}".format(var=v) soilPropertyFilename = "{name}.tif".format(name=soilPropertyName) soilPropertyFilepathTmp = os.path.join(tmpdir, soilPropertyFilename) soilPropertyFilepath = os.path.join(outputDir, soilPropertyFilename) if verbose: outfp.write("Getting attribute {0} ...\n".format(soilPropertyName)) delete = False if os.path.exists(soilPropertyFilepath): if not overwrite: raise Exception("File {0} already exists, and overwrite is false".format(soilPropertyFilepath)) else: delete = True url = URL_BASE.format(variable=variable) wcs = WebCoverageService(url, version='1.0.0') (coverages, weights_abs) = _getCoverageIDsAndWeightsForCoverageTitle(wcs, variable) outfiles = [] weights = [] for c in coverages.keys(): coverage = coverages[c] weights.append(weights_abs[c]) #coverage = c.format(variable=variable) wcsfp = wcs.getCoverage(identifier=coverage, bbox=bbox, crs='EPSG:4326', resx=resx, # their WCS seems to accept resx, resy in meters resy=resy, format=fmt) filename = os.path.join(tmpdir, "{coverage}.tif".format(coverage=c)) outfiles.append(filename) f = open(filename, 'wb') f.write(wcsfp.read()) f.close() # Compute depth-length weighted-average for each coverage using gdal_calc.py assert(len(outfiles) == len(COVERAGES)) gdalCommand = gdalCmdPath calcStr = '0' # Identity element for addition for (i, outfile) in enumerate(outfiles): ord = i + 1 var_label = ordinalToAlpha(ord) gdalCommand += " -{var} {outfile}".format(var=var_label, outfile=outfile) calcStr += "+({weight}*{var})".format(weight=weights[i], var=var_label) gdalCommand += " --calc='{calc}' --outfile={outfile} --type='Float32' --format=GTiff --co='COMPRESS=LZW'".format(calc=calcStr, outfile=soilPropertyFilepathTmp) #print("GDAL command:\n{0}".format(gdalCommand)) process = Popen(gdalCommand, cwd=outputDir, shell=True, stdout=PIPE, stderr=PIPE) (process_stdout, process_stderr) = process.communicate() if process.returncode != 0: raise Exception("GDAL command {0} failed, returning {1}\nstdout:\n{2}\nstderr:\n{3}\n.".format(gdalCommand, process.returncode, process_stdout, process_stderr)) if verbose: outfp.write(process_stdout) outfp.write(process_stderr) # Resample raster if delete: os.unlink(soilPropertyFilepath) resampleRaster(config, outputDir, soilPropertyFilepathTmp, soilPropertyFilename, 'EPSG:4326', srs, resx, resy, resampleMethod=interpolation) soilPropertyRasters[soilPropertyName] = (soilPropertyFilepath, wcs.url) # Clean-up shutil.rmtree(tmpdir) return soilPropertyRasters