def main(argv):
i = 1
output_format = None
in_filename = None
out_filename = None
ovr_level = None
while i < len(argv):
if argv[i] == "-f":
output_format = argv[i + 1]
i = i + 1
elif argv[i] == "-ovr":
ovr_level = int(argv[i + 1])
i = i + 1
elif argv[i][0] == '-':
return Usage()
elif in_filename is None:
in_filename = argv[i]
elif out_filename is None:
out_filename = argv[i]
else:
return Usage()
i = i + 1
if out_filename is None:
return Usage()
if output_format is None:
output_format = GetOutputDriverFor(out_filename, is_raster=False)
src_ds = gdal.Open(in_filename)
out_ds = gdal.GetDriverByName(output_format).Create(
out_filename, 0, 0, 0, gdal.GDT_Unknown)
first_band = src_ds.GetRasterBand(1)
main_gt = src_ds.GetGeoTransform()
for i in ([ovr_level] if ovr_level is not None else
range(1 + first_band.GetOverviewCount())):
src_band = first_band if i == 0 else first_band.GetOverview(i - 1)
out_lyr = out_ds.CreateLayer('main_image' if i == 0 else
('overview_%d' % i),
geom_type=ogr.wkbPolygon,
srs=src_ds.GetSpatialRef())
blockxsize, blockysize = src_band.GetBlockSize()
nxblocks = (src_band.XSize + blockxsize - 1) // blockxsize
nyblocks = (src_band.YSize + blockysize - 1) // blockysize
gt = [
main_gt[0], main_gt[1] * first_band.XSize / src_band.XSize, 0,
main_gt[3], 0, main_gt[5] * first_band.YSize / src_band.YSize
]
for y in range(nyblocks):
ymax = gt[3] + y * blockysize * gt[5]
ymin = ymax + blockysize * gt[5]
for x in range(nxblocks):
xmin = gt[0] + x * blockxsize * gt[1]
xmax = xmin + blockxsize * gt[1]
f = ogr.Feature(out_lyr.GetLayerDefn())
wkt = 'POLYGON((%.18g %.18g,%.18g %.18g,%.18g %.18g,%.18g %.18g,%.18g %.18g))' % (
xmin, ymin, xmin, ymax, xmax, ymax, xmax, ymin, xmin, ymin)
f.SetGeometryDirectly(ogr.CreateGeometryFromWkt(wkt))
out_lyr.CreateFeature(f)
out_ds = None
return 0
def get_src_ds(self):
'get src dataset, convert to RGB(A) if required'
#----------------------------
self.src_path = self.src
if os.path.exists(self.src):
self.src_path = os.path.abspath(self.src)
#~ pf('')
ld('self.src_path', self.src_path, self.src)
# check for source raster type
src_ds = gdal.Open(self.src_path, GA_ReadOnly)
self.src_ds = src_ds
self.description = self.src_ds.GetMetadataItem('DESCRIPTION')
# source is successfully opened, then create destination dir
os.makedirs(self.dest)
src_geotr = src_ds.GetGeoTransform()
src_proj = txt2proj4(src_ds.GetProjection())
gcps = src_ds.GetGCPs()
if gcps:
ld('src GCPsToGeoTransform', gdal.GCPsToGeoTransform(gcps))
if not src_proj and gcps:
src_proj = txt2proj4(src_ds.GetGCPProjection())
override_srs = self.options.srs
if override_srs is not None:
src_proj = txt2proj4(override_srs)
ld('src_proj', src_proj, 'src geotr', src_geotr)
assert src_proj, 'The source does not have a spatial reference system assigned'
src_bands = src_ds.RasterCount
band1 = src_ds.GetRasterBand(1)
if src_bands == 1 and band1.GetColorInterpretation(
) == GCI_PaletteIndex: # source is a paletted raster
transparency = None
if self.base_resampling == 'NearestNeighbour' and self.resampling == Image.NEAREST:
# check if src can be rendered in paletted mode
color_table = band1.GetColorTable()
ncolors = color_table.GetCount()
palette = [
color_table.GetColorEntry(i) for i in range(ncolors)
]
r, g, b, a = zip(*palette)
pil_palette = flatten(zip(
r, g, b)) # PIL doesn't support RGBA palettes
if self.options.dst_nodata is not None:
transparency = int(self.options.dst_nodata.split(',')[0])
elif min(a) == 0:
transparency = a.index(0)
elif ncolors < 256:
pil_palette += [0, 0, 0
] # the last color index is a transparency
transparency = len(pil_palette) / 3 - 1
ld('transparency', transparency)
if transparency is not None: # render in paletted mode
self.transparency = transparency
self.palette = pil_palette
ld('self.palette', self.palette)
else: # convert src to rgb VRT
if not src_geotr or src_geotr == (0.0, 1.0, 0.0, 0.0, 0.0,
1.0):
geotr_txt = ''
else:
geotr_txt = geotr_templ % src_geotr
gcplst_txt = ''
if gcps:
gcp_lst = '\n'.join(
(gcp_templ %
(g.Id, g.GCPPixel, g.GCPLine, g.GCPX, g.GCPY, g.GCPZ)
for g in gcps))
gcp_proj = txt2proj4(src_ds.GetGCPProjection(
)) if override_srs is None else src_proj
gcplst_txt = gcplst_templ % (gcp_proj, gcp_lst)
metadata = src_ds.GetMetadata()
ld('metadata', metadata)
if metadata:
mtd_lst = [
xml_txt('MDI', metadata[mdkey], 4, key=mdkey)
for mdkey in metadata
]
meta_txt = meta_templ % '\n'.join(mtd_lst)
else:
meta_txt = ''
xsize, ysize = (src_ds.RasterXSize, src_ds.RasterYSize)
blxsize, blysize = band1.GetBlockSize()
band_lst = ''.join(
(band_templ % {
'band': band,
'color': color,
'src': cgi.escape(self.src_path, quote=True),
'srcband': 1,
'xsize': xsize,
'ysize': ysize,
'blxsize': blxsize,
'blysize': blysize,
} for band, color in ((1, 'Red'), (2, 'Green'), (3,
'Blue'))))
vrt_txt = vrt_templ % {
'xsize': xsize,
'ysize': ysize,
'metadata': meta_txt,
'srs': (srs_templ % src_proj) if src_proj else '',
'geotr': geotr_txt,
'gcp_list': gcplst_txt,
'band_list': band_lst,
}
src_vrt = os.path.abspath(
os.path.join(self.dest,
self.base + '.src.vrt')) # auxilary VRT file
self.temp_files.append(src_vrt)
self.src_path = src_vrt
with open(src_vrt, 'w') as f:
f.write(vrt_txt.encode('utf-8'))
self.src_ds = gdal.Open(src_vrt, GA_ReadOnly)
return # rgb VRT created
# finished with a paletted raster
if override_srs is not None: # src SRS needs to be relpaced
src_vrt = os.path.join(self.dest,
self.base + '.src.vrt') # auxilary VRT file
self.temp_files.append(src_vrt)
self.src_path = src_vrt
vrt_drv = gdal.GetDriverByName('VRT')
self.src_ds = vrt_drv.CreateCopy(src_vrt,
src_ds) # replace src dataset
ld('override_srs', override_srs, 'txt2wkt(override_srs)',
txt2wkt(override_srs))
self.src_ds.SetProjection(
txt2wkt(override_srs)) # replace source SRS
gcps = self.src_ds.GetGCPs()
if gcps:
self.src_ds.SetGCPs(gcps, txt2wkt(override_srs))
def dn2ref(out_dir, zip_file):
# zip所在父目录
zip_dir = os.path.dirname(zip_file)
zip_name = os.path.splitext(os.path.basename(zip_file))[0]
zip_file_name = zip_name
temp_dir = os.path.normpath(os.path.join(out_dir, zip_name + '_un_zip'))
if not os.path.isdir(temp_dir):
os.mkdir(temp_dir)
zip_value = un_zip(zip_file, temp_dir)
if zip_value == 0:
shutil.rmtree(temp_dir)
return
# 增加用于兼容不同网站下载的Sen2数据
tag = zip_name[0:3]
if tag == 'L1C':
zip_name = list(os.walk(temp_dir))[0][1][0][0:-5]
# 使用Sen2Cor计算地表反射率
safe_dir = os.path.join(temp_dir, '%s.SAFE' % zip_name)
# 定义返回内容的字典
ext_info = {}
# 返回影像时间
date_time_str = zip_name.split('_')[2][0:8]
if not os.path.isdir(safe_dir):
sys.exit('No %s.SAFE dir' % zip_name)
subprocess.call('L2A_Process.bat --resolution 10 --refresh %s' % safe_dir)
# os.system('/home/zhaoshaoshuai/S2/Sen2Cor/bin/L2A_Process --refresh %s' % safe_dir)
L2_dir_list = list(zip_name.split('_'))
L2_dir_list[1] = 'MSIL2A'
L2_dir_name = '_'.join(L2_dir_list)
L2_dir = os.path.join(temp_dir, '%s.SAFE' % L2_dir_name)
L2_data_dir = os.path.join(L2_dir, 'GRANULE')
xml_files = search_file(L2_data_dir, '.xml')
for xml_file in xml_files:
xml_dir = os.path.dirname(xml_file)
jp2_files = search_file(xml_dir, '.jp2')
if jp2_files == []:
continue
xml_name = os.path.basename(xml_dir)
jp2_10_files = get_10_jp2(jp2_files)
if jp2_10_files == []:
continue
jp2_20_files = get_20_jp2(jp2_files)
if jp2_20_files == []:
continue
# 近红外波段重采样
# resam_nir_name_list = list(os.path.splitext(os.path.basename(jp2_20_files[0]))[0].split('_'))
# resam_nir_name_list[2] = 'B08'
# resam_nir_file = os.path.join(os.path.dirname(jp2_20_files[0]), '%s.jp2' % '_'.join(resam_nir_name_list))
# reproj_resample(jp2_10_files[-1], jp2_20_files[0], resam_nir_file)
# jp2_20_files.insert(-3, jp2_10_files[-1])
vrt_10_file = os.path.join(safe_dir, '%s_10m.vrt' % xml_name)
vrt_20_file = os.path.join(safe_dir, '%s_20m.vrt' % xml_name)
vrt_10_dataset = gdal.BuildVRT(
vrt_10_file, jp2_10_files, separate=True)
vrt_20_dataset = gdal.BuildVRT(vrt_20_file, jp2_20_files[:-1], resolution='user', xRes=20, yRes=20,
separate=True,
options=['-r', 'bilinear'])
vrt_10_dataset.FlushCache()
vrt_20_dataset.FlushCache()
vrt_10_dataset = None
vrt_20_dataset = None
isub_ref_dir = os.path.join(out_dir, zip_file_name)
if not os.path.isdir(isub_ref_dir):
os.mkdir(isub_ref_dir)
out_driver = gdal.GetDriverByName('GTiff')
out_10_file = os.path.join(isub_ref_dir, '%s_ref_10m.tif' % xml_name)
out_20_file = os.path.join(isub_ref_dir, '%s_ref_20m.tif' % xml_name)
class_file = os.path.join(isub_ref_dir, '%s_SCL_20m.tif' % xml_name)
# 输出处理后的结果
print("Start exporting images at 10 meters resolution")
out_10_sds = out_driver.CreateCopy(
out_10_file, gdal.Open(vrt_10_file), callback=progress)
print("Start exporting images at 20 meters resolution")
out_20_sds = out_driver.CreateCopy(
out_20_file, gdal.Open(vrt_20_file), callback=progress)
out_class_sds = out_driver.CreateCopy(
class_file, gdal.Open(jp2_20_files[-1]), callback=progress)
# 输出投影信息
PRJ_str = out_10_sds.GetProjection()
oSRC = osr.SpatialReference(PRJ_str)
PRJ = oSRC.GetAttrValue('PROJCS')
out_10_sds = None
out_20_sds = None
out_class_sds = None
shutil.rmtree(temp_dir)
# 返回辅助信息
ext_info["name"] = zip_name
ext_info["date_time"] = date_time_str
ext_info["PRJ"] = PRJ
ext_info["resolution_m"] = ['10', '20', '20']
ext_info["resolution_n"] = ['10', '20', '20']
return ext_info
def zonal_stats(vector_path, raster_path, nodata_value=None, global_src_extent=False):
rds = gdal.Open(raster_path, GA_ReadOnly)
assert(rds)
rb = rds.GetRasterBand(1)
rgt = rds.GetGeoTransform()
if nodata_value:
nodata_value = float(nodata_value)
rb.SetNoDataValue(nodata_value)
vds = ogr.Open(vector_path, GA_ReadOnly) # TODO maybe open update if we want to write stats
assert(vds)
vlyr = vds.GetLayer(0)
# create an in-memory numpy array of the source raster data
# covering the whole extent of the vector layer
if global_src_extent:
# use global source extent
# useful only when disk IO or raster scanning inefficiencies are your limiting factor
# advantage: reads raster data in one pass
# disadvantage: large vector extents may have big memory requirements
src_offset = bbox_to_pixel_offsets(rgt, vlyr.GetExtent())
src_array = rb.ReadAsArray(*src_offset)
# calculate new geotransform of the layer subset
new_gt = (
(rgt[0] + (src_offset[0] * rgt[1])),
rgt[1],
0.0,
(rgt[3] + (src_offset[1] * rgt[5])),
0.0,
rgt[5]
)
mem_drv = ogr.GetDriverByName('Memory')
driver = gdal.GetDriverByName('MEM')
# Loop through vectors
stats = []
feat = vlyr.GetNextFeature()
while feat is not None:
if not global_src_extent:
# use local source extent
# fastest option when you have fast disks and well indexed raster (ie tiled Geotiff)
# advantage: each feature uses the smallest raster chunk
# disadvantage: lots of reads on the source raster
src_offset = bbox_to_pixel_offsets(rgt, feat.geometry().GetEnvelope())
src_array = rb.ReadAsArray(*src_offset)
# calculate new geotransform of the feature subset
new_gt = (
(rgt[0] + (src_offset[0] * rgt[1])),
rgt[1],
0.0,
(rgt[3] + (src_offset[1] * rgt[5])),
0.0,
rgt[5]
)
# Create a temporary vector layer in memory
mem_ds = mem_drv.CreateDataSource('out')
mem_layer = mem_ds.CreateLayer('poly', None, ogr.wkbPolygon)
mem_layer.CreateFeature(feat.Clone())
# Rasterize it
rvds = driver.Create('', src_offset[2], src_offset[3], 1, gdal.GDT_Byte)
rvds.SetGeoTransform(new_gt)
gdal.RasterizeLayer(rvds, [1], mem_layer, burn_values=[1])
rv_array = rvds.ReadAsArray()
# Mask the source data array with our current feature
# we take the logical_not to flip 0<->1 to get the correct mask effect
# we also mask out nodata values explictly
masked = np.ma.MaskedArray(
src_array,
mask=np.logical_or(
src_array == nodata_value,
np.logical_not(rv_array)
)
)
feature_stats = {
'min': float(masked.min()),
'mean': float(masked.mean()),
'max': float(masked.max()),
'std': float(masked.std()),
'sum': float(masked.sum()),
'count': int(masked.count()),
'fid': int(feat.GetFID())}
stats.append(feature_stats)
rvds = None
mem_ds = None
feat = vlyr.GetNextFeature()
vds = Noney
rds = None
return stats
# Create dummy shapefile to story features geometry in (necessary for rasterizing)
drv_mem = ogr.GetDriverByName('Memory')
ds = drv_mem.CreateDataSource("")
ds_lyr = ds.CreateLayer("", SpatialReferenceFromRaster(dem),
ogr.wkbPolygon)
featureDefn = ds_lyr.GetLayerDefn()
out_feat = ogr.Feature(featureDefn)
out_feat.SetGeometry(p_geom_trans)
ds_lyr.CreateFeature(out_feat)
out_feat = None
# CopySHPDisk(ds_lyr, "tryout.shp") #If you wish to check the shp
# Create the destination data source
x_res = math.ceil((x_max - x_min) / gt[1])
y_res = math.ceil((y_max - y_min) / gt[1])
target_ds = gdal.GetDriverByName('MEM').Create('', x_res, y_res,
gdal.GDT_Byte)
target_ds.GetRasterBand(1).SetNoDataValue(-9999)
target_ds.SetProjection(pr)
target_ds.SetGeoTransform((x_min, gt[1], 0, y_max, 0, gt[5]))
# Rasterization
gdal.RasterizeLayer(
target_ds, [1], ds_lyr, burn_values=[1], options=[
'ALL_TOUCHED=TRUE'
]) # ['ALL_TOUCHED=TRUE'] command necessary, however does not work yet
target_array = target_ds.ReadAsArray()
# target_ds = None
# Convert data from the DEM to the extent of the envelope of the polygon (to array)
inv_gt = gdal.InvGeoTransform(gt)
offsets_ul = gdal.ApplyGeoTransform(inv_gt, x_min, y_max)
def main(params,
out_dir=None,
xy_txt=None,
kernel=False,
resolution=30,
tile_id_field=None):
t0 = time.time()
# Read params. Make variables from each line of the 1-line variables
inputs, df_vars = read_params(params)
for var in inputs:
exec("{0} = str({1})").format(var, inputs[var])
columns = df_vars.columns
expected_cols = [
'var_name', 'data_type', 'data_band', 'search_str', 'basepath',
'by_tile', 'nodata', 'path_filter'
]
#missing_cols =
print '\nExtracting variable values for samples:\n%s\n' % xy_txt
if 'out_dir' not in inputs:
out_dir = os.path.dirname(xy_txt)
if not os.path.isdir(out_dir):
#raise IOError('Output directory does not exist: %s' % out_dir)
print 'Output directory does not exist. Creating new directory at', out_dir
os.mkdir(out_dir)
shutil.copy2(params, out_dir)
if not os.path.exists(mosaic_path):
raise IOError('mosaic_path does not exist: %s' % mosaic_path)
if 'years' in locals():
years = [int(yr) for yr in years.split(',')]
else:
try:
year_start = int(year_start)
year_end = int(year_end)
years = range(year_start, year_end + 1)
except NameError:
raise NameError('No list of years or year_start/year_end specified in' +\
' param file:\n%s\n. Re-run script with either of these' +\
' parameters given.' % params)
if 'kernel' in inputs:
if inputs['kernel'].lower() == 'true':
kernel = True
# Get the TSA mosaic as an array
#if (df_vars.by_tsa == 1).any():
print 'Reading mosaic dataset...%s\n' % time.ctime(time.time())
try:
mosaic_ds = ogr.Open(mosaic_path)
if 'resolution' not in inputs:
warnings.warn(
'Resolution not specified. Assuming default of 30...\n')
mosaic = mosaic_ds.GetLayer()
min_x, max_x, min_y, max_y = mosaic.GetExtent()
xsize = int((max_x - min_x) / resolution)
ysize = int((max_y - min_y) / resolution)
prj = mosaic.GetSpatialRef().ExportToWkt()
driver = gdal.GetDriverByName('envi')
x_res = resolution
y_res = -resolution
mosaic_tx = min_x, x_res, 0, max_y, 0, y_res
except:
exc_type, exc_msg, _ = sys.exc_info()
try:
mosaic_ds = gdal.Open(mosaic_path)
exc_type_str = str(exc_type).split('.')[-1].replace("'",
'').replace(
'>', '')
msg = ('Could not open mosaic_path as vector : dataset %s.\n%s: %s' +\
'\n\nAttempting to open as raster...\n') % (mosaic_path, exc_type_str, exc_msg)
except:
exc_type, exc_msg, _ = sys.exc_info()
exc_type_str = str(exc_type).split('.')[-1].replace("'",
'').replace(
'>', '')
msg = 'Could not open mosaic_path as vector or raster: dataset %s.\n%s: %s' % (
mosaic_path, exc_type_str, exc_msg)
raise IOError(msg)
mosaic_tx = mosaic_ds.GetGeoTransform()
mosaic = mosaic_ds.ReadAsArray()
#tile_ids = np.unique(df_tile.tile_id) # Assumes there are coords in all TSAs'''
# Only need to do this once per xy sample set
# Get the TSA at each xy location and the row and col
print 'Extracting tile IDs... %s\n' % time.ctime(time.time())
df_xy = pd.read_csv(xy_txt, sep='\t', index_col='obs_id')
if np.any(df_vars.by_tile):
extract_at_xy(df_xy, mosaic, mosaic_tx,
tile_id_field) # Gets val inplace
df_mosaic = attributes_to_df(mosaic_path)
df_xy = df_xy[
~df_xy.tile_fid.isnull()] #drop samples that weren't inside a tile
df_xy['tile_id'] = df_mosaic.ix[df_xy['tile_fid'],
tile_id_field].tolist()
#tile_id_field = 'tile_id'
ul_x, x_res, x_rot, ul_y, y_rot, y_res = mosaic_tx
df_xy['row'] = [int((y - ul_y) / y_res) for y in df_xy.y]
df_xy['col'] = [int((x - ul_x) / x_res) for x in df_xy.x]
#df_xy = df_xy[df_xy.tile_id > 0]
#df_xy[tile_id_field] = 0
tile_ids = df_xy['tile_id'].unique()
if 'tile_txt' in inputs:
df_tile = pd.read_csv(tile_txt, sep='\t', dtype={'tsa_str': object})
df_tile['tile_str'] = df_tile['tsa_str']
else:
df_tile = pd.DataFrame({
'tile_id': tile_ids,
'tile_str': tile_ids
}) # id string is to ensure compatibility with TSAs
# For each year, do extractions
c = 1 #For keeping count of files processed
n_years = len(years)
#n_vars = len(df_vars)
#import pdb; pdb.set_trace()
n_files = (len(df_tile) * len(df_vars[df_vars.by_tile == 1])) * n_years +\
n_years * (len(df_vars[df_vars.data_band < 0]) + len(df_vars[df_vars.data_band > 0]))
xy_txt_bn = os.path.basename(xy_txt)
#out_dir = os.path.join(out_dir, xy_txt_bn.split('.')[0])
#if not os.path.exists(out_dir): os.mkdir(out_dir)
last_file = 1
for year in years:
t1 = time.time()
df_yr = pd.DataFrame()
for var_name, var_row in df_vars.iterrows(): #var_name is index col
# Get variables from row
search_str = var_row.search_str
basepath = var_row.basepath
path_filter = var_row.path_filter
data_type = var_row.data_type
by_tile = var_row.by_tile
path_filter = ''
#data_band = var_row.data_band
if np.isnan(var_row.nodata):
nodata = None
else:
nodata = int(var_row.nodata)
if int(var_row.data_band) < 0:
data_band = year - 1984 + 1
else:
data_band = int(var_row.data_band)
df_var, last_file = extract_var(year, var_name, by_tile, data_band,
data_type, df_tile, df_xy,
basepath, search_str, path_filter,
mosaic_tx, last_file, n_files,
nodata, kernel)
df_yr[df_var.columns] = df_var
#last_file += this_count
# Write df_var with all years for this predictor to txt file
this_bn = '%s_%s_kernelstats.txt' % (xy_txt_bn[:-4], year)
this_txt = os.path.join(out_dir, this_bn)
df_yr.to_csv(this_txt, sep='\t')
df_xy[df_vars.index.tolist()] = df_yr[df_vars.index.tolist()]
print 'Time for year %s: %.1f minutes\n\n' % (year, (
(time.time() - t1) / 60))
mosaic_ds = None
# Write the dataframe to a text file
#if out_dir:
out_cols = [col for col in df_xy.columns if 'file' not in col]
import pdb
pdb.set_trace()
out_bn = xy_txt_bn.replace('.txt', '_predictors.txt')
out_txt = os.path.join(out_dir, out_bn)
#df_out = df_xy.drop(['tile_id', 'row', 'col', 'x', 'y'], axis=1)# May want to keep row/col
# Remove any rows where anything is null
null_mask = df_xy.isnull().any(axis=1)
df_out = df_xy.ix[~null_mask, out_cols]
df_out.to_csv(out_txt, sep='\t')
df_xy.ix[null_mask,
out_cols].to_csv(out_txt.replace('.txt', '_dropped.txt'),
sep='\t')
print 'Dataframe written to:\n', out_txt
print 'Total extraction time: %.1f minutes' % ((time.time() - t0) / 60)
# %%
# for y in np.arange(ymax,ymax-nrows,-1):
# # print("yRichtung:",y)
# # print("ArraypositionY:",arraycount_y)
# for x in np.arange(xmin,xmin+ncols,1):
# if vegtathoehe_array[x][y]>=
# vegsliceverh_array = vegslicecountout_array/vegcountout_array #verhaeltnis von veg im obersten meter zu gesamte vegPunkte
# %%
#-------------------------------
#OutRaster schreiben:
#-------------------------------
#Boden Raster
driver = gdal.GetDriverByName("GTiff")
dataset = driver.Create("Export/BodenCount_1000.tif", ncols, nrows, 1,
gdal.GDT_Float32)
dataset.SetGeoTransform((xmin, 1, 0, ymax, 0, -1))
#Koordinatensystem definieren:
dstSRS = osr.SpatialReference()
dstSRS.ImportFromEPSG(32632)
dest_wkt = dstSRS.ExportToWkt()
dataset.SetProjection(dest_wkt)
#Raster ausgeben:
bandout = dataset.GetRasterBand(1).WriteArray(bodencountout_array)
dataset.FlushCache()
from osgeo import gdal
from osgeo import ogr
from osgeo import osr
# Avoid the regressing from https://github.com/conda-forge/gdal-feedstock/pull/129
# See https://github.com/conda-forge/gdal-feedstock/issues/131
from osgeo.gdal_array import *
drivers = [
'netCDF', 'HDF4', 'HDF5', 'GTiff', 'PNG', 'JPEG', 'GPKG', 'KEA',
'JP2OpenJPEG', 'WCS', 'PDF', 'FITS', 'TileDB'
]
for driver in drivers:
print(driver)
assert gdal.GetDriverByName(driver)
drivers = ['GML', 'XLS', 'KML', 'SQLite', 'PostgreSQL']
for driver in drivers:
print(driver)
assert ogr.GetDriverByName(driver)
def has_geos():
pnt1 = ogr.CreateGeometryFromWkt('POINT(10 20)')
pnt2 = ogr.CreateGeometryFromWkt('POINT(30 20)')
ogrex = ogr.GetUseExceptions()
ogr.DontUseExceptions()
hasgeos = pnt1.Union(pnt2) is not None
if ogrex:
ogr.UseExceptions()
def getTimeseries(productcode, subproductcode, version, mapsetcode, wkt,
start_date, end_date, aggregate):
# Extract timeseries from a list of files and return as JSON object
# It applies to a single dataset (prod/sprod/version/mapset) and between 2 dates
# Several types of aggregation foreseen:
#
# mean : Sum(Xi)/N(Xi) -> min/max not considered e.g. Rain
# cumulate: Sum(Xi) -> min/max not considered e.g. Fire
#
# count: N(Xi where min < Xi < max) e.g. Vegetation anomalies
# surface: count * PixelArea e.g. Water Bodies
# percent: count/Ntot e.g. Vegetation anomalies
#
# History: 1.0 : Initial release - since 2.0.1 -> now renamed '_green' from greenwich package
# 1.1 : Since Feb. 2017, it is based on a different approach (gdal.RasterizeLayer instead of greenwich)
# in order to solve the issue with MULTIPOLYGON
#
# Convert the wkt into a geometry
ogr.UseExceptions()
theGeomWkt = ' '.join(wkt.strip().split())
geom = Geometry(wkt=str(theGeomWkt), srs=4326)
# Get Mapset Info
mapset_info = querydb.get_mapset(mapsetcode=mapsetcode)
# Prepare for computing conversion to area: the pixel size at Lat=0 is computed
# The correction to the actual latitude (on AVERAGE value - will be computed below)
const_d2km = 12364.35
area_km_equator = abs(mapset_info.pixel_shift_lat) * abs(
mapset_info.pixel_shift_long) * const_d2km
# Get Product Info
product_info = querydb.get_product_out_info(productcode=productcode,
subproductcode=subproductcode,
version=version)
if product_info.__len__() > 0:
# Get info from product_info
scale_factor = 0
scale_offset = 0
nodata = 0
date_format = ''
for row in product_info:
scale_factor = row.scale_factor
scale_offset = row.scale_offset
nodata = row.nodata
date_format = row.date_format
date_type = row.data_type_id
# Create an output/temp shapefile, for managing the output layer (really mandatory ?? Can be simplified ???)
try:
tmpdir = tempfile.mkdtemp(prefix=__name__,
suffix='_getTimeseries',
dir=es_constants.base_tmp_dir)
except:
logger.error('Cannot create temporary dir ' +
es_constants.base_tmp_dir + '. Exit')
raise NameError('Error in creating tmpdir')
out_shape = tmpdir + os.path.sep + "output_shape.shp"
outDriver = ogr.GetDriverByName('ESRI Shapefile')
# Create the output shapefile
outDataSource = outDriver.CreateDataSource(out_shape)
dest_srs = ogr.osr.SpatialReference()
dest_srs.ImportFromEPSG(4326)
outLayer = outDataSource.CreateLayer("Layer", dest_srs)
# outLayer = outDataSource.CreateLayer("Layer")
idField = ogr.FieldDefn("id", ogr.OFTInteger)
outLayer.CreateField(idField)
featureDefn = outLayer.GetLayerDefn()
feature = ogr.Feature(featureDefn)
feature.SetGeometry(geom)
feature.SetField("id", 1)
outLayer.CreateFeature(feature)
feature = None
[list_files,
dates_list] = getFilesList(productcode, subproductcode, version,
mapsetcode, date_format, start_date,
end_date)
# Built a dictionary with filenames/dates
dates_to_files_dict = dict(zip(dates_list, list_files))
# Generate unique list of files
unique_list = set(list_files)
uniqueFilesValues = []
geo_mask_created = False
for infile in unique_list:
single_result = {
'filename': '',
'meanvalue_noscaling': nodata,
'meanvalue': None
}
if infile.strip() != '' and os.path.isfile(infile):
# try:
# Open input file
orig_ds = gdal.Open(infile, gdal.GA_ReadOnly)
orig_cs = osr.SpatialReference()
orig_cs.ImportFromWkt(orig_ds.GetProjectionRef())
orig_geoT = orig_ds.GetGeoTransform()
x_origin = orig_geoT[0]
y_origin = orig_geoT[3]
pixel_size_x = orig_geoT[1]
pixel_size_y = -orig_geoT[5]
in_data_type_gdal = conv_data_type_to_gdal(date_type)
# Create a mask from the geometry, with the same georef as the input file[s]
if not geo_mask_created:
# Read polygon extent and round to raster resolution
x_min, x_max, y_min, y_max = outLayer.GetExtent()
x_min_round = int((x_min - x_origin) /
pixel_size_x) * pixel_size_x + x_origin
x_max_round = (int(
(x_max - x_origin) /
(pixel_size_x)) + 1) * pixel_size_x + x_origin
y_min_round = (int(
(y_min - y_origin) /
(pixel_size_y)) - 1) * pixel_size_y + y_origin
y_max_round = int((y_max - y_origin) /
(pixel_size_y)) * pixel_size_y + y_origin
#
# # Create the destination data source
x_res = int(
round((x_max_round - x_min_round) / pixel_size_x))
y_res = int(
round((y_max_round - y_min_round) / pixel_size_y))
#
# # Create mask in memory
mem_driver = gdal.GetDriverByName('MEM')
mem_ds = mem_driver.Create('', x_res, y_res, 1,
in_data_type_gdal)
mask_geoT = [
x_min_round, pixel_size_x, 0, y_max_round, 0,
-pixel_size_y
]
mem_ds.SetGeoTransform(mask_geoT)
mem_ds.SetProjection(orig_cs.ExportToWkt())
#
# # Create a Layer with '1' for the pixels to be selected
gdal.RasterizeLayer(mem_ds, [1], outLayer, burn_values=[1])
# gdal.RasterizeLayer(mem_ds, [1], outLayer, None, None, [1])
# Read the polygon-mask
band = mem_ds.GetRasterBand(1)
geo_values = mem_ds.ReadAsArray()
# Create a mask from geo_values (mask-out the '0's)
geo_mask = ma.make_mask(geo_values == 0)
geo_mask_created = True
#
# # Clean/Close objects
mem_ds = None
mem_driver = None
outDriver = None
outLayer = None
# Read data from input file
x_offset = int((x_min - x_origin) / pixel_size_x)
y_offset = int((y_origin - y_max) / pixel_size_y)
band_in = orig_ds.GetRasterBand(1)
data = band_in.ReadAsArray(x_offset, y_offset, x_res, y_res)
# Catch the Error ES2-105 (polygon not included in Mapset)
if data is None:
logger.error(
'ERROR: polygon extends out of file mapset for file: %s'
% infile)
return []
# Create a masked array from the data (considering Nodata)
masked_data = ma.masked_equal(data, nodata)
# Apply on top of it the geo mask
mxnodata = ma.masked_where(geo_mask, masked_data)
# mxnodata = masked_data # TEMP !!!!
# Test ONLY
# write_ds_to_geotiff(mem_ds, '/data/processing/exchange/Tests/mem_ds.tif')
if aggregate['aggregation_type'] == 'count' or aggregate[
'aggregation_type'] == 'percent' or aggregate[
'aggregation_type'] == 'surface':
if mxnodata.count() == 0:
meanResult = None
else:
mxrange = mxnodata
min_val = aggregate['aggregation_min']
max_val = aggregate['aggregation_max']
if min_val is not None:
min_val_scaled = (min_val -
scale_offset) / scale_factor
mxrange = ma.masked_less(mxnodata, min_val_scaled)
# See ES2-271
if max_val is not None:
# Scale threshold from physical to digital value
max_val_scaled = (max_val -
scale_offset) / scale_factor
mxrange = ma.masked_greater(
mxrange, max_val_scaled)
elif max_val is not None:
# Scale threshold from physical to digital value
max_val_scaled = (max_val -
scale_offset) / scale_factor
mxrange = ma.masked_greater(
mxnodata, max_val_scaled)
if aggregate['aggregation_type'] == 'percent':
# 'percent'
meanResult = float(mxrange.count()) / float(
mxnodata.count()) * 100
elif aggregate['aggregation_type'] == 'surface':
# 'surface'
# Estimate 'average' Latitude
y_avg = (y_min + y_max) / 2.0
pixelAvgArea = area_km_equator * math.cos(
y_avg / 180 * math.pi)
meanResult = float(mxrange.count()) * pixelAvgArea
else:
# 'count'
meanResult = float(mxrange.count())
# Both results are equal
finalvalue = meanResult
else: # if aggregate['type'] == 'mean' or if aggregate['type'] == 'cumulate':
if mxnodata.count() == 0:
finalvalue = None
meanResult = None
else:
if aggregate['aggregation_type'] == 'mean':
# 'mean'
meanResult = mxnodata.mean()
else:
# 'cumulate'
meanResult = mxnodata.sum()
finalvalue = (meanResult * scale_factor + scale_offset)
# Assign results
single_result['filename'] = infile
single_result['meanvalue_noscaling'] = meanResult
single_result['meanvalue'] = finalvalue
else:
logger.debug('ERROR: raster file does not exist - %s' % infile)
uniqueFilesValues.append(single_result)
# Define a dictionary to associate filenames/values
files_to_values_dict = dict(
(x['filename'], x['meanvalue']) for x in uniqueFilesValues)
# Prepare array for result
resultDatesValues = []
# Returns a list of 'filenames', 'dates', 'values'
for mydate in dates_list:
my_result = {'date': datetime.date.today(), 'meanvalue': nodata}
# Assign the date
my_result['date'] = mydate
# Assign the filename
my_filename = dates_to_files_dict[mydate]
# Map from array of Values
my_result['meanvalue'] = files_to_values_dict[my_filename]
# Map from array of dates
resultDatesValues.append(my_result)
try:
shutil.rmtree(tmpdir)
except:
logger.debug('ERROR: Error in deleting tmpdir. Exit')
# Return result
return resultDatesValues
else:
logger.debug(
'ERROR: product not registered in the products table! - %s %s %s' %
(productcode, subproductcode, version))
return []
