def preview_cloudfree_mosaic(x,y,start,end, bands, satellite,iterations=0,xsize=2000,ysize=2000,file_format="GTiff",data_type=gdal.GDT_CInt16): def resize_array(arr,size): r = numpy.array(arr).astype(numpy.int16) i = Image.fromarray(r) i2 = i.resize(size,Image.NEAREST) r2 = numpy.array(i2) del i2 del i del r return r2 StartDate = start EndDate = end best_data = {} band_str = "+".join([band.name for band in bands]) sat_str = "+".join([sat.name for sat in satellite]) cache_id = ["preview",str(x),str(y),str(start),str(end),band_str,sat_str,str(xsize),str(ysize),file_format,str(iterations)] f_name = "_".join(cache_id) f_name = f_name.replace(" ","_") c_name = f_name cached_res = cache.get(c_name) if cached_res: return str(cached_res) f_name = os.path.join("/tilestore/tile_cache",f_name) tiles = list_tiles(x=[x], y=[y],acq_min=StartDate,acq_max=EndDate,satellites=satellite,dataset_types=[DatasetType.ARG25,DatasetType.PQ25], sort=SortType.ASC) tile_metadata = None tile_count = 0 tile_filled = False for tile in tiles: if tile_filled: break print "merging on tile "+str(tile.x)+", "+str(tile.y) tile_count+=1 dataset = DatasetType.ARG25 in tile.datasets and tile.datasets[DatasetType.ARG25] or None if dataset is None: print "No dataset availible" tile_count-=1 continue tile_metadata = get_dataset_metadata(dataset) if tile_metadata is None: print "NO METADATA" tile_count-=1 continue pqa = DatasetType.PQ25 in tile.datasets and tile.datasets[DatasetType.PQ25] or None mask = None mask = get_mask_pqa(pqa,[PqaMask.PQ_MASK_CLEAR],mask=mask) band_data = get_dataset_data_masked(dataset, mask=mask,bands=bands) swap_arr = None for band in band_data: if not band in best_data: print "Adding "+band.name bd = resize_array(band_data[band],(2000,2000)) print bd best_data[band]=bd del bd else: best = resize_array(best_data[band],(2000,2000)) swap_arr=numpy.in1d(best.ravel(),-999).reshape(best.shape) b_data = numpy.array(band_data[band]) best[swap_arr]=b_data[swap_arr] best_data[band]=numpy.copy(best) del b_data del best del swap_arr if iterations > 0: if tile_count>iterations: print "Exiting after "+str(iterations)+" iterations" break numberOfBands=len(bands) if numberOfBands == 0: return "None" if bands[0] not in best_data: print "No data was merged for "+str(x)+", "+str(y) return "None" numberOfPixelsInXDirection=len(best_data[bands[0]]) numberOfPixelsInYDirection=len(best_data[bands[0]][0]) if tile_count <1: print "No tiles found for "+str(x)+", "+str(y) return "None" driver = gdal.GetDriverByName(file_format) if driver is None: print "No driver found for "+file_format return "None" print f_name+'.tif' raster = driver.Create(f_name+'.tif', numberOfPixelsInXDirection, numberOfPixelsInYDirection, numberOfBands, data_type, options=["BIGTIFF=YES", "INTERLEAVE=BAND"]) gt = tile_metadata.transform gt2 = (gt[0],gt[1]*2.0,gt[2],gt[3],gt[4],gt[5]*2.0) tile_metadata.transform = gt2 raster.SetGeoTransform(tile_metadata.transform) print tile_metadata.transform raster.SetProjection(tile_metadata.projection) index = 1 for band in bands: stack_band = raster.GetRasterBand(index) stack_band.SetNoDataValue(-999) stack_band.WriteArray(best_data[band]) stack_band.ComputeStatistics(True) index+=1 stack_band.FlushCache() del stack_band raster.FlushCache() del raster cache.set(c_name,f_name+".tif") return f_name+".tif"
def obtain_file_from_math(input_file,expressions_list,file_format="GTiff",data_type=gdal.GDT_CFloat32,input_ndv=-999,output_ndv=-999): """ ex. band4,band3, (band4-band3)/(band4+band3) AKA NDVI """ """ Read in file """ if input_file == "None": return "None" driver = gdal.GetDriverByName(file_format) ds = gdal.Open(input_file,0) if ds is None: return "None" arrays = [] band_count = ds.RasterCount xsize = ds.RasterXSize ysize = ds.RasterYSize gt = ds.GetGeoTransform() proj = ds.GetProjection() exp_str = "_".join(expressions_list) cache_id = [os.path.splitext(os.path.basename(input_file))[0],exp_str,str(xsize),str(ysize),file_format] f_name = "_".join(cache_id) f_name = hashlib.sha512(f_name).hexdigest()[0:32] c_name = f_name cached_res = cache.get(c_name) if cached_res: return cached_res f_name = os.path.join("/tilestore/tile_cache",f_name) for i in range(band_count): RB = ds.GetRasterBand(i+1) arrays.append(RB.ReadAsArray(0,0,xsize,ysize).astype(numpy.float32)) del RB var_identifier = "A"+''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) #test if we've used this id in this scope var_test = var_identifier+"_band1" while var_test in globals(): var_identifier = "A"+''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10)) var_test = var_identifier+"_band1" for band_num in range(len(arrays)): globals()[var_identifier+'_band'+str(band_num+1)]=arrays[band_num] results = [] expressions = [expression.replace("band",var_identifier+"_band") for expression in expressions_list] for expression in expressions: results.append(ne.evaluate(expression)) raster = driver.Create(f_name+'.tif', xsize, ysize, len(expressions_list), data_type, options=["BIGTIFF=YES", "INTERLEAVE=BAND"]) raster.SetGeoTransform(gt) raster.SetProjection(proj) index = 1 for band in results: stack_band = raster.GetRasterBand(index) stack_band.SetNoDataValue(output_ndv) stack_band.WriteArray(band) stack_band.ComputeStatistics(True) index+=1 stack_band.FlushCache() del stack_band raster.FlushCache() del raster del ds del results cache.set(c_name,f_name+".tif") return f_name+".tif"
def obtain_cloudfree_mosaic(x,y,start,end, bands, satellite,iterations=0,xsize=4000,ysize=4000,file_format="GTiff",data_type=gdal.GDT_CInt16,months=None): StartDate = start EndDate = end best_data = {} band_str = "+".join([band.name for band in bands]) sat_str = "+".join([sat.name for sat in satellite]) cache_id = [str(x),str(y),str(start),str(end),band_str,sat_str,str(xsize),str(ysize),file_format,str(iterations)] f_name = "_".join(cache_id) f_name = f_name.replace(" ","_") c_name = f_name cached_res = cache.get(c_name) if cached_res: return str(cached_res) f_name = os.path.join("/tilestore/tile_cache",f_name) tiles = list_tiles(x=[x], y=[y],acq_min=StartDate,acq_max=EndDate,satellites=satellite,dataset_types=[DatasetType.ARG25,DatasetType.PQ25], sort=SortType.ASC) tile_metadata = None tile_count = 0 tile_filled = False stats_file = open(f_name+'.csv','w+') total_ins = 0 for tile in tiles: if tile_filled: break if months: print tile.start_datetime.month if not tile.start_datetime.month in months: continue #print "merging on tile "+str(tile.x)+", "+str(tile.y) tile_count+=1 dataset = DatasetType.ARG25 in tile.datasets and tile.datasets[DatasetType.ARG25] or None if dataset is None: print "No dataset availible" tile_count-=1 continue tile_metadata = get_dataset_metadata(dataset) if tile_metadata is None: print "NO METADATA" tile_count-=1 continue pqa = DatasetType.PQ25 in tile.datasets and tile.datasets[DatasetType.PQ25] or None mask = None mask = get_mask_pqa(pqa,[PqaMask.PQ_MASK_CLEAR],mask=mask) band_data = get_dataset_data_masked(dataset, mask=mask,bands=bands) swap_arr = None best = None good_ins = None for band in band_data: if not band in best_data: #print "Adding "+band.name #print band_data[band] best_data[band]=band_data[band] best = numpy.array(best_data[band]) swap_arr=numpy.in1d(best.ravel(),-999).reshape(best.shape) good_ins = len(numpy.where(best[swap_arr]!=-999)[0]) else: best = numpy.array(best_data[band]) swap_arr=numpy.in1d(best.ravel(),-999).reshape(best.shape) b_data = numpy.array(band_data[band]) best[swap_arr]=b_data[swap_arr] best_data[band]=numpy.copy(best) good_ins = len(numpy.where(b_data[swap_arr]!=-999)[0]) del b_data total_ins+=good_ins stats_file.write(str(tile.x)+','+str(tile.y)+','+str(tile.start_datetime.year)+','+str(tile.start_datetime.month)+','+str(len(best[swap_arr]))+','+str(good_ins)+','+str(total_ins)+','+str(tile.dataset)+"\n") del swap_arr del best del good_ins if iterations > 0: if tile_count>iterations: print "Exiting after "+str(iterations)+" iterations" break numberOfBands=len(bands) if numberOfBands == 0: return "None" if bands[0] not in best_data: print "No data was merged for "+str(x)+", "+str(y) return "None" numberOfPixelsInXDirection=len(best_data[bands[0]]) print numberOfPixelsInXDirection numberOfPixelsInYDirection=len(best_data[bands[0]][0]) print numberOfPixelsInYDirection pixels = numberOfPixelsInXDirection if numberOfPixelsInYDirection > numberOfPixelsInXDirection: pixels = numberOfPixelsInYDirection if tile_count <1: print "No tiles found for "+str(x)+", "+str(y) return "None" driver = gdal.GetDriverByName(file_format) if driver is None: print "No driver found for "+file_format return "None" #print f_name+'.tif' raster = driver.Create(f_name+'.tif', pixels, pixels, numberOfBands, data_type, options=["BIGTIFF=YES", "INTERLEAVE=BAND"]) raster.SetGeoTransform(tile_metadata.transform) raster.SetProjection(tile_metadata.projection) index = 1 stats_file.close() for band in bands: stack_band = raster.GetRasterBand(index) stack_band.SetNoDataValue(-999) stack_band.WriteArray(best_data[band]) stack_band.ComputeStatistics(True) index+=1 stack_band.FlushCache() del stack_band raster.FlushCache() del raster cache.set(c_name,f_name+".tif") return f_name+".tif"