def preview_cloudfree_mosaic(x,y,start,end, bands, satellite,iterations=0,xsize=2000,ysize=2000,file_format="GTiff",data_type=gdal.GDT_CInt16):
def resize_array(arr,size):
r = numpy.array(arr).astype(numpy.int16)
i = Image.fromarray(r)
i2 = i.resize(size,Image.NEAREST)
r2 = numpy.array(i2)
del i2
del i
del r
return r2
StartDate = start
EndDate = end
best_data = {}
band_str = "+".join([band.name for band in bands])
sat_str = "+".join([sat.name for sat in satellite])
cache_id = ["preview",str(x),str(y),str(start),str(end),band_str,sat_str,str(xsize),str(ysize),file_format,str(iterations)]
f_name = "_".join(cache_id)
f_name = f_name.replace(" ","_")
c_name = f_name
cached_res = cache.get(c_name)
if cached_res:
return str(cached_res)
f_name = os.path.join("/tilestore/tile_cache",f_name)
tiles = list_tiles(x=[x], y=[y],acq_min=StartDate,acq_max=EndDate,satellites=satellite,dataset_types=[DatasetType.ARG25,DatasetType.PQ25], sort=SortType.ASC)
tile_metadata = None
tile_count = 0
tile_filled = False
for tile in tiles:
if tile_filled:
break
print "merging on tile "+str(tile.x)+", "+str(tile.y)
tile_count+=1
dataset = DatasetType.ARG25 in tile.datasets and tile.datasets[DatasetType.ARG25] or None
if dataset is None:
print "No dataset availible"
tile_count-=1
continue
tile_metadata = get_dataset_metadata(dataset)
if tile_metadata is None:
print "NO METADATA"
tile_count-=1
continue
pqa = DatasetType.PQ25 in tile.datasets and tile.datasets[DatasetType.PQ25] or None
mask = None
mask = get_mask_pqa(pqa,[PqaMask.PQ_MASK_CLEAR],mask=mask)
band_data = get_dataset_data_masked(dataset, mask=mask,bands=bands)
swap_arr = None
for band in band_data:
if not band in best_data:
print "Adding "+band.name
bd = resize_array(band_data[band],(2000,2000))
print bd
best_data[band]=bd
del bd
else:
best = resize_array(best_data[band],(2000,2000))
swap_arr=numpy.in1d(best.ravel(),-999).reshape(best.shape)
b_data = numpy.array(band_data[band])
best[swap_arr]=b_data[swap_arr]
best_data[band]=numpy.copy(best)
del b_data
del best
del swap_arr
if iterations > 0:
if tile_count>iterations:
print "Exiting after "+str(iterations)+" iterations"
break
numberOfBands=len(bands)
if numberOfBands == 0:
return "None"
if bands[0] not in best_data:
print "No data was merged for "+str(x)+", "+str(y)
return "None"
numberOfPixelsInXDirection=len(best_data[bands[0]])
numberOfPixelsInYDirection=len(best_data[bands[0]][0])
if tile_count <1:
print "No tiles found for "+str(x)+", "+str(y)
return "None"
driver = gdal.GetDriverByName(file_format)
if driver is None:
print "No driver found for "+file_format
return "None"
print f_name+'.tif'
raster = driver.Create(f_name+'.tif', numberOfPixelsInXDirection, numberOfPixelsInYDirection, numberOfBands, data_type, options=["BIGTIFF=YES", "INTERLEAVE=BAND"])
gt = tile_metadata.transform
gt2 = (gt[0],gt[1]*2.0,gt[2],gt[3],gt[4],gt[5]*2.0)
tile_metadata.transform = gt2
raster.SetGeoTransform(tile_metadata.transform)
print tile_metadata.transform
raster.SetProjection(tile_metadata.projection)
index = 1
for band in bands:
stack_band = raster.GetRasterBand(index)
stack_band.SetNoDataValue(-999)
stack_band.WriteArray(best_data[band])
stack_band.ComputeStatistics(True)
index+=1
stack_band.FlushCache()
del stack_band
raster.FlushCache()
del raster
cache.set(c_name,f_name+".tif")
return f_name+".tif"
def obtain_file_from_math(input_file,expressions_list,file_format="GTiff",data_type=gdal.GDT_CFloat32,input_ndv=-999,output_ndv=-999):
"""
ex. band4,band3, (band4-band3)/(band4+band3) AKA NDVI
"""
"""
Read in file
"""
if input_file == "None":
return "None"
driver = gdal.GetDriverByName(file_format)
ds = gdal.Open(input_file,0)
if ds is None:
return "None"
arrays = []
band_count = ds.RasterCount
xsize = ds.RasterXSize
ysize = ds.RasterYSize
gt = ds.GetGeoTransform()
proj = ds.GetProjection()
exp_str = "_".join(expressions_list)
cache_id = [os.path.splitext(os.path.basename(input_file))[0],exp_str,str(xsize),str(ysize),file_format]
f_name = "_".join(cache_id)
f_name = hashlib.sha512(f_name).hexdigest()[0:32]
c_name = f_name
cached_res = cache.get(c_name)
if cached_res:
return cached_res
f_name = os.path.join("/tilestore/tile_cache",f_name)
for i in range(band_count):
RB = ds.GetRasterBand(i+1)
arrays.append(RB.ReadAsArray(0,0,xsize,ysize).astype(numpy.float32))
del RB
var_identifier = "A"+''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
#test if we've used this id in this scope
var_test = var_identifier+"_band1"
while var_test in globals():
var_identifier = "A"+''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
var_test = var_identifier+"_band1"
for band_num in range(len(arrays)):
globals()[var_identifier+'_band'+str(band_num+1)]=arrays[band_num]
results = []
expressions = [expression.replace("band",var_identifier+"_band") for expression in expressions_list]
for expression in expressions:
results.append(ne.evaluate(expression))
raster = driver.Create(f_name+'.tif', xsize, ysize, len(expressions_list), data_type, options=["BIGTIFF=YES", "INTERLEAVE=BAND"])
raster.SetGeoTransform(gt)
raster.SetProjection(proj)
index = 1
for band in results:
stack_band = raster.GetRasterBand(index)
stack_band.SetNoDataValue(output_ndv)
stack_band.WriteArray(band)
stack_band.ComputeStatistics(True)
index+=1
stack_band.FlushCache()
del stack_band
raster.FlushCache()
del raster
del ds
del results
cache.set(c_name,f_name+".tif")
return f_name+".tif"
def obtain_cloudfree_mosaic(x,y,start,end, bands, satellite,iterations=0,xsize=4000,ysize=4000,file_format="GTiff",data_type=gdal.GDT_CInt16,months=None):
StartDate = start
EndDate = end
best_data = {}
band_str = "+".join([band.name for band in bands])
sat_str = "+".join([sat.name for sat in satellite])
cache_id = [str(x),str(y),str(start),str(end),band_str,sat_str,str(xsize),str(ysize),file_format,str(iterations)]
f_name = "_".join(cache_id)
f_name = f_name.replace(" ","_")
c_name = f_name
cached_res = cache.get(c_name)
if cached_res:
return str(cached_res)
f_name = os.path.join("/tilestore/tile_cache",f_name)
tiles = list_tiles(x=[x], y=[y],acq_min=StartDate,acq_max=EndDate,satellites=satellite,dataset_types=[DatasetType.ARG25,DatasetType.PQ25], sort=SortType.ASC)
tile_metadata = None
tile_count = 0
tile_filled = False
stats_file = open(f_name+'.csv','w+')
total_ins = 0
for tile in tiles:
if tile_filled:
break
if months:
print tile.start_datetime.month
if not tile.start_datetime.month in months:
continue
#print "merging on tile "+str(tile.x)+", "+str(tile.y)
tile_count+=1
dataset = DatasetType.ARG25 in tile.datasets and tile.datasets[DatasetType.ARG25] or None
if dataset is None:
print "No dataset availible"
tile_count-=1
continue
tile_metadata = get_dataset_metadata(dataset)
if tile_metadata is None:
print "NO METADATA"
tile_count-=1
continue
pqa = DatasetType.PQ25 in tile.datasets and tile.datasets[DatasetType.PQ25] or None
mask = None
mask = get_mask_pqa(pqa,[PqaMask.PQ_MASK_CLEAR],mask=mask)
band_data = get_dataset_data_masked(dataset, mask=mask,bands=bands)
swap_arr = None
best = None
good_ins = None
for band in band_data:
if not band in best_data:
#print "Adding "+band.name
#print band_data[band]
best_data[band]=band_data[band]
best = numpy.array(best_data[band])
swap_arr=numpy.in1d(best.ravel(),-999).reshape(best.shape)
good_ins = len(numpy.where(best[swap_arr]!=-999)[0])
else:
best = numpy.array(best_data[band])
swap_arr=numpy.in1d(best.ravel(),-999).reshape(best.shape)
b_data = numpy.array(band_data[band])
best[swap_arr]=b_data[swap_arr]
best_data[band]=numpy.copy(best)
good_ins = len(numpy.where(b_data[swap_arr]!=-999)[0])
del b_data
total_ins+=good_ins
stats_file.write(str(tile.x)+','+str(tile.y)+','+str(tile.start_datetime.year)+','+str(tile.start_datetime.month)+','+str(len(best[swap_arr]))+','+str(good_ins)+','+str(total_ins)+','+str(tile.dataset)+"\n")
del swap_arr
del best
del good_ins
if iterations > 0:
if tile_count>iterations:
print "Exiting after "+str(iterations)+" iterations"
break
numberOfBands=len(bands)
if numberOfBands == 0:
return "None"
if bands[0] not in best_data:
print "No data was merged for "+str(x)+", "+str(y)
return "None"
numberOfPixelsInXDirection=len(best_data[bands[0]])
print numberOfPixelsInXDirection
numberOfPixelsInYDirection=len(best_data[bands[0]][0])
print numberOfPixelsInYDirection
pixels = numberOfPixelsInXDirection
if numberOfPixelsInYDirection > numberOfPixelsInXDirection:
pixels = numberOfPixelsInYDirection
if tile_count <1:
print "No tiles found for "+str(x)+", "+str(y)
return "None"
driver = gdal.GetDriverByName(file_format)
if driver is None:
print "No driver found for "+file_format
return "None"
#print f_name+'.tif'
raster = driver.Create(f_name+'.tif', pixels, pixels, numberOfBands, data_type, options=["BIGTIFF=YES", "INTERLEAVE=BAND"])
raster.SetGeoTransform(tile_metadata.transform)
raster.SetProjection(tile_metadata.projection)
index = 1
stats_file.close()
for band in bands:
stack_band = raster.GetRasterBand(index)
stack_band.SetNoDataValue(-999)
stack_band.WriteArray(best_data[band])
stack_band.ComputeStatistics(True)
index+=1
stack_band.FlushCache()
del stack_band
raster.FlushCache()
del raster
cache.set(c_name,f_name+".tif")
return f_name+".tif"
