def main(argv): i = 1 output_format = None in_filename = None out_filename = None ovr_level = None while i < len(argv): if argv[i] == "-f": output_format = argv[i + 1] i = i + 1 elif argv[i] == "-ovr": ovr_level = int(argv[i + 1]) i = i + 1 elif argv[i][0] == '-': return Usage() elif in_filename is None: in_filename = argv[i] elif out_filename is None: out_filename = argv[i] else: return Usage() i = i + 1 if out_filename is None: return Usage() if output_format is None: output_format = GetOutputDriverFor(out_filename, is_raster=False) src_ds = gdal.Open(in_filename) out_ds = gdal.GetDriverByName(output_format).Create( out_filename, 0, 0, 0, gdal.GDT_Unknown) first_band = src_ds.GetRasterBand(1) main_gt = src_ds.GetGeoTransform() for i in ([ovr_level] if ovr_level is not None else range(1 + first_band.GetOverviewCount())): src_band = first_band if i == 0 else first_band.GetOverview(i - 1) out_lyr = out_ds.CreateLayer('main_image' if i == 0 else ('overview_%d' % i), geom_type=ogr.wkbPolygon, srs=src_ds.GetSpatialRef()) blockxsize, blockysize = src_band.GetBlockSize() nxblocks = (src_band.XSize + blockxsize - 1) // blockxsize nyblocks = (src_band.YSize + blockysize - 1) // blockysize gt = [ main_gt[0], main_gt[1] * first_band.XSize / src_band.XSize, 0, main_gt[3], 0, main_gt[5] * first_band.YSize / src_band.YSize ] for y in range(nyblocks): ymax = gt[3] + y * blockysize * gt[5] ymin = ymax + blockysize * gt[5] for x in range(nxblocks): xmin = gt[0] + x * blockxsize * gt[1] xmax = xmin + blockxsize * gt[1] f = ogr.Feature(out_lyr.GetLayerDefn()) wkt = 'POLYGON((%.18g %.18g,%.18g %.18g,%.18g %.18g,%.18g %.18g,%.18g %.18g))' % ( xmin, ymin, xmin, ymax, xmax, ymax, xmax, ymin, xmin, ymin) f.SetGeometryDirectly(ogr.CreateGeometryFromWkt(wkt)) out_lyr.CreateFeature(f) out_ds = None return 0
def get_src_ds(self): 'get src dataset, convert to RGB(A) if required' #---------------------------- self.src_path = self.src if os.path.exists(self.src): self.src_path = os.path.abspath(self.src) #~ pf('') ld('self.src_path', self.src_path, self.src) # check for source raster type src_ds = gdal.Open(self.src_path, GA_ReadOnly) self.src_ds = src_ds self.description = self.src_ds.GetMetadataItem('DESCRIPTION') # source is successfully opened, then create destination dir os.makedirs(self.dest) src_geotr = src_ds.GetGeoTransform() src_proj = txt2proj4(src_ds.GetProjection()) gcps = src_ds.GetGCPs() if gcps: ld('src GCPsToGeoTransform', gdal.GCPsToGeoTransform(gcps)) if not src_proj and gcps: src_proj = txt2proj4(src_ds.GetGCPProjection()) override_srs = self.options.srs if override_srs is not None: src_proj = txt2proj4(override_srs) ld('src_proj', src_proj, 'src geotr', src_geotr) assert src_proj, 'The source does not have a spatial reference system assigned' src_bands = src_ds.RasterCount band1 = src_ds.GetRasterBand(1) if src_bands == 1 and band1.GetColorInterpretation( ) == GCI_PaletteIndex: # source is a paletted raster transparency = None if self.base_resampling == 'NearestNeighbour' and self.resampling == Image.NEAREST: # check if src can be rendered in paletted mode color_table = band1.GetColorTable() ncolors = color_table.GetCount() palette = [ color_table.GetColorEntry(i) for i in range(ncolors) ] r, g, b, a = zip(*palette) pil_palette = flatten(zip( r, g, b)) # PIL doesn't support RGBA palettes if self.options.dst_nodata is not None: transparency = int(self.options.dst_nodata.split(',')[0]) elif min(a) == 0: transparency = a.index(0) elif ncolors < 256: pil_palette += [0, 0, 0 ] # the last color index is a transparency transparency = len(pil_palette) / 3 - 1 ld('transparency', transparency) if transparency is not None: # render in paletted mode self.transparency = transparency self.palette = pil_palette ld('self.palette', self.palette) else: # convert src to rgb VRT if not src_geotr or src_geotr == (0.0, 1.0, 0.0, 0.0, 0.0, 1.0): geotr_txt = '' else: geotr_txt = geotr_templ % src_geotr gcplst_txt = '' if gcps: gcp_lst = '\n'.join( (gcp_templ % (g.Id, g.GCPPixel, g.GCPLine, g.GCPX, g.GCPY, g.GCPZ) for g in gcps)) gcp_proj = txt2proj4(src_ds.GetGCPProjection( )) if override_srs is None else src_proj gcplst_txt = gcplst_templ % (gcp_proj, gcp_lst) metadata = src_ds.GetMetadata() ld('metadata', metadata) if metadata: mtd_lst = [ xml_txt('MDI', metadata[mdkey], 4, key=mdkey) for mdkey in metadata ] meta_txt = meta_templ % '\n'.join(mtd_lst) else: meta_txt = '' xsize, ysize = (src_ds.RasterXSize, src_ds.RasterYSize) blxsize, blysize = band1.GetBlockSize() band_lst = ''.join( (band_templ % { 'band': band, 'color': color, 'src': cgi.escape(self.src_path, quote=True), 'srcband': 1, 'xsize': xsize, 'ysize': ysize, 'blxsize': blxsize, 'blysize': blysize, } for band, color in ((1, 'Red'), (2, 'Green'), (3, 'Blue')))) vrt_txt = vrt_templ % { 'xsize': xsize, 'ysize': ysize, 'metadata': meta_txt, 'srs': (srs_templ % src_proj) if src_proj else '', 'geotr': geotr_txt, 'gcp_list': gcplst_txt, 'band_list': band_lst, } src_vrt = os.path.abspath( os.path.join(self.dest, self.base + '.src.vrt')) # auxilary VRT file self.temp_files.append(src_vrt) self.src_path = src_vrt with open(src_vrt, 'w') as f: f.write(vrt_txt.encode('utf-8')) self.src_ds = gdal.Open(src_vrt, GA_ReadOnly) return # rgb VRT created # finished with a paletted raster if override_srs is not None: # src SRS needs to be relpaced src_vrt = os.path.join(self.dest, self.base + '.src.vrt') # auxilary VRT file self.temp_files.append(src_vrt) self.src_path = src_vrt vrt_drv = gdal.GetDriverByName('VRT') self.src_ds = vrt_drv.CreateCopy(src_vrt, src_ds) # replace src dataset ld('override_srs', override_srs, 'txt2wkt(override_srs)', txt2wkt(override_srs)) self.src_ds.SetProjection( txt2wkt(override_srs)) # replace source SRS gcps = self.src_ds.GetGCPs() if gcps: self.src_ds.SetGCPs(gcps, txt2wkt(override_srs))
def dn2ref(out_dir, zip_file): # zip所在父目录 zip_dir = os.path.dirname(zip_file) zip_name = os.path.splitext(os.path.basename(zip_file))[0] zip_file_name = zip_name temp_dir = os.path.normpath(os.path.join(out_dir, zip_name + '_un_zip')) if not os.path.isdir(temp_dir): os.mkdir(temp_dir) zip_value = un_zip(zip_file, temp_dir) if zip_value == 0: shutil.rmtree(temp_dir) return # 增加用于兼容不同网站下载的Sen2数据 tag = zip_name[0:3] if tag == 'L1C': zip_name = list(os.walk(temp_dir))[0][1][0][0:-5] # 使用Sen2Cor计算地表反射率 safe_dir = os.path.join(temp_dir, '%s.SAFE' % zip_name) # 定义返回内容的字典 ext_info = {} # 返回影像时间 date_time_str = zip_name.split('_')[2][0:8] if not os.path.isdir(safe_dir): sys.exit('No %s.SAFE dir' % zip_name) subprocess.call('L2A_Process.bat --resolution 10 --refresh %s' % safe_dir) # os.system('/home/zhaoshaoshuai/S2/Sen2Cor/bin/L2A_Process --refresh %s' % safe_dir) L2_dir_list = list(zip_name.split('_')) L2_dir_list[1] = 'MSIL2A' L2_dir_name = '_'.join(L2_dir_list) L2_dir = os.path.join(temp_dir, '%s.SAFE' % L2_dir_name) L2_data_dir = os.path.join(L2_dir, 'GRANULE') xml_files = search_file(L2_data_dir, '.xml') for xml_file in xml_files: xml_dir = os.path.dirname(xml_file) jp2_files = search_file(xml_dir, '.jp2') if jp2_files == []: continue xml_name = os.path.basename(xml_dir) jp2_10_files = get_10_jp2(jp2_files) if jp2_10_files == []: continue jp2_20_files = get_20_jp2(jp2_files) if jp2_20_files == []: continue # 近红外波段重采样 # resam_nir_name_list = list(os.path.splitext(os.path.basename(jp2_20_files[0]))[0].split('_')) # resam_nir_name_list[2] = 'B08' # resam_nir_file = os.path.join(os.path.dirname(jp2_20_files[0]), '%s.jp2' % '_'.join(resam_nir_name_list)) # reproj_resample(jp2_10_files[-1], jp2_20_files[0], resam_nir_file) # jp2_20_files.insert(-3, jp2_10_files[-1]) vrt_10_file = os.path.join(safe_dir, '%s_10m.vrt' % xml_name) vrt_20_file = os.path.join(safe_dir, '%s_20m.vrt' % xml_name) vrt_10_dataset = gdal.BuildVRT( vrt_10_file, jp2_10_files, separate=True) vrt_20_dataset = gdal.BuildVRT(vrt_20_file, jp2_20_files[:-1], resolution='user', xRes=20, yRes=20, separate=True, options=['-r', 'bilinear']) vrt_10_dataset.FlushCache() vrt_20_dataset.FlushCache() vrt_10_dataset = None vrt_20_dataset = None isub_ref_dir = os.path.join(out_dir, zip_file_name) if not os.path.isdir(isub_ref_dir): os.mkdir(isub_ref_dir) out_driver = gdal.GetDriverByName('GTiff') out_10_file = os.path.join(isub_ref_dir, '%s_ref_10m.tif' % xml_name) out_20_file = os.path.join(isub_ref_dir, '%s_ref_20m.tif' % xml_name) class_file = os.path.join(isub_ref_dir, '%s_SCL_20m.tif' % xml_name) # 输出处理后的结果 print("Start exporting images at 10 meters resolution") out_10_sds = out_driver.CreateCopy( out_10_file, gdal.Open(vrt_10_file), callback=progress) print("Start exporting images at 20 meters resolution") out_20_sds = out_driver.CreateCopy( out_20_file, gdal.Open(vrt_20_file), callback=progress) out_class_sds = out_driver.CreateCopy( class_file, gdal.Open(jp2_20_files[-1]), callback=progress) # 输出投影信息 PRJ_str = out_10_sds.GetProjection() oSRC = osr.SpatialReference(PRJ_str) PRJ = oSRC.GetAttrValue('PROJCS') out_10_sds = None out_20_sds = None out_class_sds = None shutil.rmtree(temp_dir) # 返回辅助信息 ext_info["name"] = zip_name ext_info["date_time"] = date_time_str ext_info["PRJ"] = PRJ ext_info["resolution_m"] = ['10', '20', '20'] ext_info["resolution_n"] = ['10', '20', '20'] return ext_info
def zonal_stats(vector_path, raster_path, nodata_value=None, global_src_extent=False): rds = gdal.Open(raster_path, GA_ReadOnly) assert(rds) rb = rds.GetRasterBand(1) rgt = rds.GetGeoTransform() if nodata_value: nodata_value = float(nodata_value) rb.SetNoDataValue(nodata_value) vds = ogr.Open(vector_path, GA_ReadOnly) # TODO maybe open update if we want to write stats assert(vds) vlyr = vds.GetLayer(0) # create an in-memory numpy array of the source raster data # covering the whole extent of the vector layer if global_src_extent: # use global source extent # useful only when disk IO or raster scanning inefficiencies are your limiting factor # advantage: reads raster data in one pass # disadvantage: large vector extents may have big memory requirements src_offset = bbox_to_pixel_offsets(rgt, vlyr.GetExtent()) src_array = rb.ReadAsArray(*src_offset) # calculate new geotransform of the layer subset new_gt = ( (rgt[0] + (src_offset[0] * rgt[1])), rgt[1], 0.0, (rgt[3] + (src_offset[1] * rgt[5])), 0.0, rgt[5] ) mem_drv = ogr.GetDriverByName('Memory') driver = gdal.GetDriverByName('MEM') # Loop through vectors stats = [] feat = vlyr.GetNextFeature() while feat is not None: if not global_src_extent: # use local source extent # fastest option when you have fast disks and well indexed raster (ie tiled Geotiff) # advantage: each feature uses the smallest raster chunk # disadvantage: lots of reads on the source raster src_offset = bbox_to_pixel_offsets(rgt, feat.geometry().GetEnvelope()) src_array = rb.ReadAsArray(*src_offset) # calculate new geotransform of the feature subset new_gt = ( (rgt[0] + (src_offset[0] * rgt[1])), rgt[1], 0.0, (rgt[3] + (src_offset[1] * rgt[5])), 0.0, rgt[5] ) # Create a temporary vector layer in memory mem_ds = mem_drv.CreateDataSource('out') mem_layer = mem_ds.CreateLayer('poly', None, ogr.wkbPolygon) mem_layer.CreateFeature(feat.Clone()) # Rasterize it rvds = driver.Create('', src_offset[2], src_offset[3], 1, gdal.GDT_Byte) rvds.SetGeoTransform(new_gt) gdal.RasterizeLayer(rvds, [1], mem_layer, burn_values=[1]) rv_array = rvds.ReadAsArray() # Mask the source data array with our current feature # we take the logical_not to flip 0<->1 to get the correct mask effect # we also mask out nodata values explictly masked = np.ma.MaskedArray( src_array, mask=np.logical_or( src_array == nodata_value, np.logical_not(rv_array) ) ) feature_stats = { 'min': float(masked.min()), 'mean': float(masked.mean()), 'max': float(masked.max()), 'std': float(masked.std()), 'sum': float(masked.sum()), 'count': int(masked.count()), 'fid': int(feat.GetFID())} stats.append(feature_stats) rvds = None mem_ds = None feat = vlyr.GetNextFeature() vds = Noney rds = None return stats
# Create dummy shapefile to story features geometry in (necessary for rasterizing) drv_mem = ogr.GetDriverByName('Memory') ds = drv_mem.CreateDataSource("") ds_lyr = ds.CreateLayer("", SpatialReferenceFromRaster(dem), ogr.wkbPolygon) featureDefn = ds_lyr.GetLayerDefn() out_feat = ogr.Feature(featureDefn) out_feat.SetGeometry(p_geom_trans) ds_lyr.CreateFeature(out_feat) out_feat = None # CopySHPDisk(ds_lyr, "tryout.shp") #If you wish to check the shp # Create the destination data source x_res = math.ceil((x_max - x_min) / gt[1]) y_res = math.ceil((y_max - y_min) / gt[1]) target_ds = gdal.GetDriverByName('MEM').Create('', x_res, y_res, gdal.GDT_Byte) target_ds.GetRasterBand(1).SetNoDataValue(-9999) target_ds.SetProjection(pr) target_ds.SetGeoTransform((x_min, gt[1], 0, y_max, 0, gt[5])) # Rasterization gdal.RasterizeLayer( target_ds, [1], ds_lyr, burn_values=[1], options=[ 'ALL_TOUCHED=TRUE' ]) # ['ALL_TOUCHED=TRUE'] command necessary, however does not work yet target_array = target_ds.ReadAsArray() # target_ds = None # Convert data from the DEM to the extent of the envelope of the polygon (to array) inv_gt = gdal.InvGeoTransform(gt) offsets_ul = gdal.ApplyGeoTransform(inv_gt, x_min, y_max)
def main(params, out_dir=None, xy_txt=None, kernel=False, resolution=30, tile_id_field=None): t0 = time.time() # Read params. Make variables from each line of the 1-line variables inputs, df_vars = read_params(params) for var in inputs: exec("{0} = str({1})").format(var, inputs[var]) columns = df_vars.columns expected_cols = [ 'var_name', 'data_type', 'data_band', 'search_str', 'basepath', 'by_tile', 'nodata', 'path_filter' ] #missing_cols = print '\nExtracting variable values for samples:\n%s\n' % xy_txt if 'out_dir' not in inputs: out_dir = os.path.dirname(xy_txt) if not os.path.isdir(out_dir): #raise IOError('Output directory does not exist: %s' % out_dir) print 'Output directory does not exist. Creating new directory at', out_dir os.mkdir(out_dir) shutil.copy2(params, out_dir) if not os.path.exists(mosaic_path): raise IOError('mosaic_path does not exist: %s' % mosaic_path) if 'years' in locals(): years = [int(yr) for yr in years.split(',')] else: try: year_start = int(year_start) year_end = int(year_end) years = range(year_start, year_end + 1) except NameError: raise NameError('No list of years or year_start/year_end specified in' +\ ' param file:\n%s\n. Re-run script with either of these' +\ ' parameters given.' % params) if 'kernel' in inputs: if inputs['kernel'].lower() == 'true': kernel = True # Get the TSA mosaic as an array #if (df_vars.by_tsa == 1).any(): print 'Reading mosaic dataset...%s\n' % time.ctime(time.time()) try: mosaic_ds = ogr.Open(mosaic_path) if 'resolution' not in inputs: warnings.warn( 'Resolution not specified. Assuming default of 30...\n') mosaic = mosaic_ds.GetLayer() min_x, max_x, min_y, max_y = mosaic.GetExtent() xsize = int((max_x - min_x) / resolution) ysize = int((max_y - min_y) / resolution) prj = mosaic.GetSpatialRef().ExportToWkt() driver = gdal.GetDriverByName('envi') x_res = resolution y_res = -resolution mosaic_tx = min_x, x_res, 0, max_y, 0, y_res except: exc_type, exc_msg, _ = sys.exc_info() try: mosaic_ds = gdal.Open(mosaic_path) exc_type_str = str(exc_type).split('.')[-1].replace("'", '').replace( '>', '') msg = ('Could not open mosaic_path as vector : dataset %s.\n%s: %s' +\ '\n\nAttempting to open as raster...\n') % (mosaic_path, exc_type_str, exc_msg) except: exc_type, exc_msg, _ = sys.exc_info() exc_type_str = str(exc_type).split('.')[-1].replace("'", '').replace( '>', '') msg = 'Could not open mosaic_path as vector or raster: dataset %s.\n%s: %s' % ( mosaic_path, exc_type_str, exc_msg) raise IOError(msg) mosaic_tx = mosaic_ds.GetGeoTransform() mosaic = mosaic_ds.ReadAsArray() #tile_ids = np.unique(df_tile.tile_id) # Assumes there are coords in all TSAs''' # Only need to do this once per xy sample set # Get the TSA at each xy location and the row and col print 'Extracting tile IDs... %s\n' % time.ctime(time.time()) df_xy = pd.read_csv(xy_txt, sep='\t', index_col='obs_id') if np.any(df_vars.by_tile): extract_at_xy(df_xy, mosaic, mosaic_tx, tile_id_field) # Gets val inplace df_mosaic = attributes_to_df(mosaic_path) df_xy = df_xy[ ~df_xy.tile_fid.isnull()] #drop samples that weren't inside a tile df_xy['tile_id'] = df_mosaic.ix[df_xy['tile_fid'], tile_id_field].tolist() #tile_id_field = 'tile_id' ul_x, x_res, x_rot, ul_y, y_rot, y_res = mosaic_tx df_xy['row'] = [int((y - ul_y) / y_res) for y in df_xy.y] df_xy['col'] = [int((x - ul_x) / x_res) for x in df_xy.x] #df_xy = df_xy[df_xy.tile_id > 0] #df_xy[tile_id_field] = 0 tile_ids = df_xy['tile_id'].unique() if 'tile_txt' in inputs: df_tile = pd.read_csv(tile_txt, sep='\t', dtype={'tsa_str': object}) df_tile['tile_str'] = df_tile['tsa_str'] else: df_tile = pd.DataFrame({ 'tile_id': tile_ids, 'tile_str': tile_ids }) # id string is to ensure compatibility with TSAs # For each year, do extractions c = 1 #For keeping count of files processed n_years = len(years) #n_vars = len(df_vars) #import pdb; pdb.set_trace() n_files = (len(df_tile) * len(df_vars[df_vars.by_tile == 1])) * n_years +\ n_years * (len(df_vars[df_vars.data_band < 0]) + len(df_vars[df_vars.data_band > 0])) xy_txt_bn = os.path.basename(xy_txt) #out_dir = os.path.join(out_dir, xy_txt_bn.split('.')[0]) #if not os.path.exists(out_dir): os.mkdir(out_dir) last_file = 1 for year in years: t1 = time.time() df_yr = pd.DataFrame() for var_name, var_row in df_vars.iterrows(): #var_name is index col # Get variables from row search_str = var_row.search_str basepath = var_row.basepath path_filter = var_row.path_filter data_type = var_row.data_type by_tile = var_row.by_tile path_filter = '' #data_band = var_row.data_band if np.isnan(var_row.nodata): nodata = None else: nodata = int(var_row.nodata) if int(var_row.data_band) < 0: data_band = year - 1984 + 1 else: data_band = int(var_row.data_band) df_var, last_file = extract_var(year, var_name, by_tile, data_band, data_type, df_tile, df_xy, basepath, search_str, path_filter, mosaic_tx, last_file, n_files, nodata, kernel) df_yr[df_var.columns] = df_var #last_file += this_count # Write df_var with all years for this predictor to txt file this_bn = '%s_%s_kernelstats.txt' % (xy_txt_bn[:-4], year) this_txt = os.path.join(out_dir, this_bn) df_yr.to_csv(this_txt, sep='\t') df_xy[df_vars.index.tolist()] = df_yr[df_vars.index.tolist()] print 'Time for year %s: %.1f minutes\n\n' % (year, ( (time.time() - t1) / 60)) mosaic_ds = None # Write the dataframe to a text file #if out_dir: out_cols = [col for col in df_xy.columns if 'file' not in col] import pdb pdb.set_trace() out_bn = xy_txt_bn.replace('.txt', '_predictors.txt') out_txt = os.path.join(out_dir, out_bn) #df_out = df_xy.drop(['tile_id', 'row', 'col', 'x', 'y'], axis=1)# May want to keep row/col # Remove any rows where anything is null null_mask = df_xy.isnull().any(axis=1) df_out = df_xy.ix[~null_mask, out_cols] df_out.to_csv(out_txt, sep='\t') df_xy.ix[null_mask, out_cols].to_csv(out_txt.replace('.txt', '_dropped.txt'), sep='\t') print 'Dataframe written to:\n', out_txt print 'Total extraction time: %.1f minutes' % ((time.time() - t0) / 60)
# %% # for y in np.arange(ymax,ymax-nrows,-1): # # print("yRichtung:",y) # # print("ArraypositionY:",arraycount_y) # for x in np.arange(xmin,xmin+ncols,1): # if vegtathoehe_array[x][y]>= # vegsliceverh_array = vegslicecountout_array/vegcountout_array #verhaeltnis von veg im obersten meter zu gesamte vegPunkte # %% #------------------------------- #OutRaster schreiben: #------------------------------- #Boden Raster driver = gdal.GetDriverByName("GTiff") dataset = driver.Create("Export/BodenCount_1000.tif", ncols, nrows, 1, gdal.GDT_Float32) dataset.SetGeoTransform((xmin, 1, 0, ymax, 0, -1)) #Koordinatensystem definieren: dstSRS = osr.SpatialReference() dstSRS.ImportFromEPSG(32632) dest_wkt = dstSRS.ExportToWkt() dataset.SetProjection(dest_wkt) #Raster ausgeben: bandout = dataset.GetRasterBand(1).WriteArray(bodencountout_array) dataset.FlushCache()
from osgeo import gdal from osgeo import ogr from osgeo import osr # Avoid the regressing from https://github.com/conda-forge/gdal-feedstock/pull/129 # See https://github.com/conda-forge/gdal-feedstock/issues/131 from osgeo.gdal_array import * drivers = [ 'netCDF', 'HDF4', 'HDF5', 'GTiff', 'PNG', 'JPEG', 'GPKG', 'KEA', 'JP2OpenJPEG', 'WCS', 'PDF', 'FITS', 'TileDB' ] for driver in drivers: print(driver) assert gdal.GetDriverByName(driver) drivers = ['GML', 'XLS', 'KML', 'SQLite', 'PostgreSQL'] for driver in drivers: print(driver) assert ogr.GetDriverByName(driver) def has_geos(): pnt1 = ogr.CreateGeometryFromWkt('POINT(10 20)') pnt2 = ogr.CreateGeometryFromWkt('POINT(30 20)') ogrex = ogr.GetUseExceptions() ogr.DontUseExceptions() hasgeos = pnt1.Union(pnt2) is not None if ogrex: ogr.UseExceptions()
def getTimeseries(productcode, subproductcode, version, mapsetcode, wkt, start_date, end_date, aggregate): # Extract timeseries from a list of files and return as JSON object # It applies to a single dataset (prod/sprod/version/mapset) and between 2 dates # Several types of aggregation foreseen: # # mean : Sum(Xi)/N(Xi) -> min/max not considered e.g. Rain # cumulate: Sum(Xi) -> min/max not considered e.g. Fire # # count: N(Xi where min < Xi < max) e.g. Vegetation anomalies # surface: count * PixelArea e.g. Water Bodies # percent: count/Ntot e.g. Vegetation anomalies # # History: 1.0 : Initial release - since 2.0.1 -> now renamed '_green' from greenwich package # 1.1 : Since Feb. 2017, it is based on a different approach (gdal.RasterizeLayer instead of greenwich) # in order to solve the issue with MULTIPOLYGON # # Convert the wkt into a geometry ogr.UseExceptions() theGeomWkt = ' '.join(wkt.strip().split()) geom = Geometry(wkt=str(theGeomWkt), srs=4326) # Get Mapset Info mapset_info = querydb.get_mapset(mapsetcode=mapsetcode) # Prepare for computing conversion to area: the pixel size at Lat=0 is computed # The correction to the actual latitude (on AVERAGE value - will be computed below) const_d2km = 12364.35 area_km_equator = abs(mapset_info.pixel_shift_lat) * abs( mapset_info.pixel_shift_long) * const_d2km # Get Product Info product_info = querydb.get_product_out_info(productcode=productcode, subproductcode=subproductcode, version=version) if product_info.__len__() > 0: # Get info from product_info scale_factor = 0 scale_offset = 0 nodata = 0 date_format = '' for row in product_info: scale_factor = row.scale_factor scale_offset = row.scale_offset nodata = row.nodata date_format = row.date_format date_type = row.data_type_id # Create an output/temp shapefile, for managing the output layer (really mandatory ?? Can be simplified ???) try: tmpdir = tempfile.mkdtemp(prefix=__name__, suffix='_getTimeseries', dir=es_constants.base_tmp_dir) except: logger.error('Cannot create temporary dir ' + es_constants.base_tmp_dir + '. Exit') raise NameError('Error in creating tmpdir') out_shape = tmpdir + os.path.sep + "output_shape.shp" outDriver = ogr.GetDriverByName('ESRI Shapefile') # Create the output shapefile outDataSource = outDriver.CreateDataSource(out_shape) dest_srs = ogr.osr.SpatialReference() dest_srs.ImportFromEPSG(4326) outLayer = outDataSource.CreateLayer("Layer", dest_srs) # outLayer = outDataSource.CreateLayer("Layer") idField = ogr.FieldDefn("id", ogr.OFTInteger) outLayer.CreateField(idField) featureDefn = outLayer.GetLayerDefn() feature = ogr.Feature(featureDefn) feature.SetGeometry(geom) feature.SetField("id", 1) outLayer.CreateFeature(feature) feature = None [list_files, dates_list] = getFilesList(productcode, subproductcode, version, mapsetcode, date_format, start_date, end_date) # Built a dictionary with filenames/dates dates_to_files_dict = dict(zip(dates_list, list_files)) # Generate unique list of files unique_list = set(list_files) uniqueFilesValues = [] geo_mask_created = False for infile in unique_list: single_result = { 'filename': '', 'meanvalue_noscaling': nodata, 'meanvalue': None } if infile.strip() != '' and os.path.isfile(infile): # try: # Open input file orig_ds = gdal.Open(infile, gdal.GA_ReadOnly) orig_cs = osr.SpatialReference() orig_cs.ImportFromWkt(orig_ds.GetProjectionRef()) orig_geoT = orig_ds.GetGeoTransform() x_origin = orig_geoT[0] y_origin = orig_geoT[3] pixel_size_x = orig_geoT[1] pixel_size_y = -orig_geoT[5] in_data_type_gdal = conv_data_type_to_gdal(date_type) # Create a mask from the geometry, with the same georef as the input file[s] if not geo_mask_created: # Read polygon extent and round to raster resolution x_min, x_max, y_min, y_max = outLayer.GetExtent() x_min_round = int((x_min - x_origin) / pixel_size_x) * pixel_size_x + x_origin x_max_round = (int( (x_max - x_origin) / (pixel_size_x)) + 1) * pixel_size_x + x_origin y_min_round = (int( (y_min - y_origin) / (pixel_size_y)) - 1) * pixel_size_y + y_origin y_max_round = int((y_max - y_origin) / (pixel_size_y)) * pixel_size_y + y_origin # # # Create the destination data source x_res = int( round((x_max_round - x_min_round) / pixel_size_x)) y_res = int( round((y_max_round - y_min_round) / pixel_size_y)) # # # Create mask in memory mem_driver = gdal.GetDriverByName('MEM') mem_ds = mem_driver.Create('', x_res, y_res, 1, in_data_type_gdal) mask_geoT = [ x_min_round, pixel_size_x, 0, y_max_round, 0, -pixel_size_y ] mem_ds.SetGeoTransform(mask_geoT) mem_ds.SetProjection(orig_cs.ExportToWkt()) # # # Create a Layer with '1' for the pixels to be selected gdal.RasterizeLayer(mem_ds, [1], outLayer, burn_values=[1]) # gdal.RasterizeLayer(mem_ds, [1], outLayer, None, None, [1]) # Read the polygon-mask band = mem_ds.GetRasterBand(1) geo_values = mem_ds.ReadAsArray() # Create a mask from geo_values (mask-out the '0's) geo_mask = ma.make_mask(geo_values == 0) geo_mask_created = True # # # Clean/Close objects mem_ds = None mem_driver = None outDriver = None outLayer = None # Read data from input file x_offset = int((x_min - x_origin) / pixel_size_x) y_offset = int((y_origin - y_max) / pixel_size_y) band_in = orig_ds.GetRasterBand(1) data = band_in.ReadAsArray(x_offset, y_offset, x_res, y_res) # Catch the Error ES2-105 (polygon not included in Mapset) if data is None: logger.error( 'ERROR: polygon extends out of file mapset for file: %s' % infile) return [] # Create a masked array from the data (considering Nodata) masked_data = ma.masked_equal(data, nodata) # Apply on top of it the geo mask mxnodata = ma.masked_where(geo_mask, masked_data) # mxnodata = masked_data # TEMP !!!! # Test ONLY # write_ds_to_geotiff(mem_ds, '/data/processing/exchange/Tests/mem_ds.tif') if aggregate['aggregation_type'] == 'count' or aggregate[ 'aggregation_type'] == 'percent' or aggregate[ 'aggregation_type'] == 'surface': if mxnodata.count() == 0: meanResult = None else: mxrange = mxnodata min_val = aggregate['aggregation_min'] max_val = aggregate['aggregation_max'] if min_val is not None: min_val_scaled = (min_val - scale_offset) / scale_factor mxrange = ma.masked_less(mxnodata, min_val_scaled) # See ES2-271 if max_val is not None: # Scale threshold from physical to digital value max_val_scaled = (max_val - scale_offset) / scale_factor mxrange = ma.masked_greater( mxrange, max_val_scaled) elif max_val is not None: # Scale threshold from physical to digital value max_val_scaled = (max_val - scale_offset) / scale_factor mxrange = ma.masked_greater( mxnodata, max_val_scaled) if aggregate['aggregation_type'] == 'percent': # 'percent' meanResult = float(mxrange.count()) / float( mxnodata.count()) * 100 elif aggregate['aggregation_type'] == 'surface': # 'surface' # Estimate 'average' Latitude y_avg = (y_min + y_max) / 2.0 pixelAvgArea = area_km_equator * math.cos( y_avg / 180 * math.pi) meanResult = float(mxrange.count()) * pixelAvgArea else: # 'count' meanResult = float(mxrange.count()) # Both results are equal finalvalue = meanResult else: # if aggregate['type'] == 'mean' or if aggregate['type'] == 'cumulate': if mxnodata.count() == 0: finalvalue = None meanResult = None else: if aggregate['aggregation_type'] == 'mean': # 'mean' meanResult = mxnodata.mean() else: # 'cumulate' meanResult = mxnodata.sum() finalvalue = (meanResult * scale_factor + scale_offset) # Assign results single_result['filename'] = infile single_result['meanvalue_noscaling'] = meanResult single_result['meanvalue'] = finalvalue else: logger.debug('ERROR: raster file does not exist - %s' % infile) uniqueFilesValues.append(single_result) # Define a dictionary to associate filenames/values files_to_values_dict = dict( (x['filename'], x['meanvalue']) for x in uniqueFilesValues) # Prepare array for result resultDatesValues = [] # Returns a list of 'filenames', 'dates', 'values' for mydate in dates_list: my_result = {'date': datetime.date.today(), 'meanvalue': nodata} # Assign the date my_result['date'] = mydate # Assign the filename my_filename = dates_to_files_dict[mydate] # Map from array of Values my_result['meanvalue'] = files_to_values_dict[my_filename] # Map from array of dates resultDatesValues.append(my_result) try: shutil.rmtree(tmpdir) except: logger.debug('ERROR: Error in deleting tmpdir. Exit') # Return result return resultDatesValues else: logger.debug( 'ERROR: product not registered in the products table! - %s %s %s' % (productcode, subproductcode, version)) return []