def cloud_ident_temp_composite(path_pattern):
dateiListe = glob.glob(path_pattern); dateiListe.sort()
print "File count: " + str(len(dateiListe))
# Get domain dimensions
xsize = lats.shape[0]
ysize = lats.shape[1]
# create empty 2d-array with exactly the same dimensions as the GeoTiffs to store minimum composite values
Composite_039 = empty((xsize,ysize))
Composite_108 = empty((xsize,ysize))
# fill the empty array with "infinity" values
Composite_039.fill(0)
Composite_108.fill(0)
for datei in dateiListe:
start = time.time()
print "Processing file " + datei + "... "
scene, time_slot, error = loadCalibratedDataForDomain(datei,tempDir,xRITDecompressToolPath,lats,lons,correct=False,hrv_slices = [], channels = ['IR_039','IR_108'])
Composite_039 = maximum(scene["IR_039"].data,Composite_039)
Composite_108 = maximum(scene["IR_108"].data,Composite_108)
end = time.time()
print "\r done (Time elapsed: " + "{:3.2f}".format(end - start) + ')'
writeToGeoTiff(Composite_039, "/home/sebastian/Desktop/composite_039_30tage.tif", xsize, ysize)
writeToGeoTiff(Composite_108, "/home/sebastian/Desktop/composite_108_30tage.tif", xsize, ysize)
return
def reprojWorldClimDEM():
dem = zeros((3712, 3712))
dem.fill(-9999)
demWorldclim = gdal.Open("/home/sebastian/Documents/reproj.tif",
GA_ReadOnly)
demWorldclim_band = demWorldclim.GetRasterBand(1)
demWorldclim_data = demWorldclim_band.ReadAsArray(0, 0,
demWorldclim.RasterXSize,
demWorldclim.RasterYSize)
proj = demWorldclim.GetProjection()
xsize = demWorldclim.RasterXSize
ysize = demWorldclim.RasterYSize
print proj
print xsize
print ysize
for i in range(3477):
print i
for j in range(3622):
dem[i + 52, j + 46] = demWorldclim_data[i, j]
PathToRawMSGData = "/home/sebastian/Documents/MSG_tests/MSG_testdaten/2013/08/08/MSG3-SEVI-MSG15-0100-NA-20130808125744.305000000Z-1081780-1.tar"
scene, time_slot, error = loadCalibratedDataForDomain(
PathToRawMSGData,
tempDir,
xRITDecompressToolPath,
correct=False,
areaBorders=(-5570248.4773392612, -5567248.074173444,
5567248.074173444, 5570248.4773392612),
slices=[1, 2, 3, 4, 5, 6, 7, 8],
channels=['IR_108'])
# Save result as GeoTiff:
driver = gdal.GetDriverByName('GTiff')
dsO = driver.Create("/home/sebastian/Documents/dem_final.tif", 3712, 3712,
1, gdal.GDT_Int16)
dsO.SetProjection(proj)
dsO.GetRasterBand(1).WriteArray(int16(dem))
dsO.FlushCache() # Write to disk.
write_sceneJO(scene, dem, "/home/sebastian/Documents/test/")
#plot2dArray(dem,title="dem",show=True, outputPath="/home/sebastian/Documents/dem_tests.png")
return
def generatePlotsAndHistogramFromHRITFiles(paths, outputFolder):
files = glob.glob(paths)
files.sort()
if len(files) == 0:
print "No Files DUUDE!"
return
for file in files:
print "Processing file: " + file
scene, time_slot, error = loadCalibratedDataForDomain(
file,
tempDir,
xRITDecompressToolPath,
None,
None,
area_name="LCRSProducts",
correct=False)
generatePlotsAndHistogramFromHRITScene(scene, time_slot)
return
Created on Jan 6, 2016
@author: sebastian
'''
import datetime
import glob
import os
from plot.plot_advanced import plotSceneToNightOverviewPNG
from util.load_raw_data import loadCalibratedDataForDomain
xRITDecompressToolPath = "../misc/HRIT/2.06/xRITDecompress/xRITDecompress" # Path to XRIT-Decompress-Tool
tempDir = "/tmp/cloudmask/"
offset_left = -5570248.4773392612
offset_down = -5567248.074173444
domainBuffer = 50
left = 1589-domainBuffer; down = 664+domainBuffer; right = 2356+domainBuffer; up = 154-domainBuffer # LCRS-Domain with buffer
pixelsize = 3000.403165817
areaBorders = (offset_left+left*pixelsize, offset_down+(3712-down)*pixelsize, offset_left+right*pixelsize, offset_down+(3712-up)*pixelsize)
files = glob.glob('/media/sebastian/8203f938-ff46-4cba-a16f-ee346037cf0f/2008/06/*/*.tar'); files.sort()
for datei in files:
print "Processing file " + datei
scene, time_slot, error = loadCalibratedDataForDomain(datei,tempDir,xRITDecompressToolPath,correct=True,areaBorders=areaBorders)
outputpath = "/home/sebastian/Documents/test/"+datetime.datetime.strftime(time_slot,"%Y/%m/%d/")
if not os.path.exists(outputpath):
os.makedirs(outputpath)
plotSceneToNightOverviewPNG(scene,outputpath+datetime.datetime.strftime(time_slot,"%Y%m%d_%H%M") + ".png")
def convertHRITToGeoTiffs(hritPath,outputFolder,channels=[]):
scene, time_slot, error = loadCalibratedDataForDomain(hritPath,tempDir,xRITDecompressToolPath,lats,lons,correct=False,hrv_slices = [], channels=channels)
for channel in channels:
writeToGeoTiff(scene[channel].data,outputFolder+time_slot.strftime("%Y%m%d_%H%M")+"_"+channel+".tif",lats.shape[0], lats.shape[1])
return
def cloudmask(PathToRawMSGData,
use_gdal_msg_loader=False,
write_scenes_to_geotiff=False,
print_progress=False):
start = time.time()
error = None
if print_progress:
print("Processing file " + PathToRawMSGData + "... ")
simpleProfiler.start("extract_raw_data_from_tar")
# Extrahieren des Datums und der Rohdaten aus der übergebenen tar-Datei:
time_slot = xrit_date_from_within_tar(PathToRawMSGData)
extractChannelsFromTarFile(PathToRawMSGData,
tempDir,
slices=xrit_slices,
hrv_slices=xrit_hrv_slices,
channels=xrit_channels)
simpleProfiler.stop("extract_raw_data_from_tar")
block_end = time.time()
if print_progress:
print("UNTAR Done (Time elapsed: " +
"{:3.2f}".format(block_end - start) + ')\n\n')
### This block generates the output folders
time_slot_plot_path = plotDir + time_slot.strftime("%Y%m%d_%H%M/")
if createPlots:
if not os.path.exists(time_slot_plot_path):
os.makedirs(time_slot_plot_path)
time_slot_scene_path = scene_dir + time_slot.strftime('%Y/%m/%d/')
if not os.path.exists(time_slot_scene_path):
os.makedirs(time_slot_scene_path)
time_slot_ircomposite_path = ircomposite_dir + time_slot.strftime(
'%Y/%m/%d/')
if not os.path.exists(time_slot_ircomposite_path):
os.makedirs(time_slot_ircomposite_path)
# reading data
block_start = time.time()
simpleProfiler.start("read_data")
### This block generates either a MSgScene or a Pytroll scene...
if use_gdal_msg_loader:
gdal_scene = GdalMsgLoader(tempDir,
prefixes=["."],
simpleProfiler=simpleProfiler).load_scene(
time_slot,
channel_names=xrit_channels,
geos_area=geos_area,
pixel_area=(left - 1, up - 1, right - 1,
down - 1),
overwrite_dataset_geos_area=True)
#print("gdal_scene", gdal_scene, gdal_scene.geos_area, gdal_scene.pixel_area)
calibrated_gdal_scene = msg_calibrator.calibrate_scene(
gdal_scene,
rad_bands=[],
rad_suffix='',
refl_suffix='',
temp_suffix='',
co2_correct_suffix='')
#print("calibrated_gdal_scene", calibrated_gdal_scene)
scene = calibrated_gdal_scene
sza = calibrated_gdal_scene['zenith'].data
else:
### this block loads data using pytroll and generates sza !!!DANGER!!!: delete_all_files_in_folder will REMOVE ALL FILES from the xrit folder. USE THIS ONLY WHEN FILES ARE FROM TAR!!!
# TODO: unpack in a different folder?
scene, time_slot, error = loadCalibratedDataForDomain(
tempDir,
xRITDecompressToolPath,
areaBorders=areaBorders,
correct=True,
delete_all_files_in_folder=True,
simpleProfiler=simpleProfiler
) #, area_name="met09globeFull", slices=[1,9])
#print "Scene size: " + str(len(list(scene.loaded_channels())[0].data[0])) + "," + str(len(list(scene.loaded_channels())[0].data))
if not error == None:
print error
if print_progress:
print("Calculating sza...")
sza = sza_opencl(scene, time_slot, simpleProfiler=simpleProfiler)
simpleProfiler.stop("read_data")
block_end = time.time()
if print_progress:
print("LOAD Done (Time elapsed: " +
"{:3.2f}".format(block_end - block_start) + ')\n\n')
### This block writes the loaded channels + additional data to disk
if write_scenes_to_geotiff:
if use_gdal_msg_loader:
print "Writing MSgScene to GeoTiff..."
msg_scene_writer.write_scene(scene,
channel_list=scene.channels,
gdal_type=gdal.GDT_Float32)
else:
print "Writing channels to GeoTiffs..."
write_scene(scene, time_slot_scene_path)
print "wrting sza"
writeDataToGeoTiff(sza,
out_dir + time_slot.strftime('%Y%m%d_%H%M') +
"_sza" + ".tif",
area=list(scene.loaded_channels())[0].area)
### This block generates the sza based lightning masks and combines it with the preprocessed elevation mask (aka land/sea mask)
if print_progress:
print("Generating light masks...")
simpleProfiler.start("create_masks")
light_masks = create_light_unmasks(sza, ["D", "N", "B"])
if print_progress:
print("Generating combined masks...")
combined_masks = create_combined_unmasks(light_masks, elevation_masks)
simpleProfiler.stop("create_masks")
### This block starts cloud pixel detection
simpleProfiler.start("cloud_pixel_detection")
if print_progress:
print("Identifying cloud pixels...")
# set interpolate_and_replace_outlayers = True to interpolate and replace block values by the 5x5 mean if value > 2*std
# set algorithm_version=2 for the "old" version" algorithm_version=8 for the new version
simpleProfiler.start("cloud_pixel_identification")
all_day__large_night, small_night_pre = cloud_identification(
scene,
combined_masks,
plot_path=time_slot_plot_path,
mask_names=["DLPC", "DSMC", "NLPC", "NSMC"],
createPlots=createPlots,
interpolate_and_replace_outlayers=True,
algoritm_version=8,
use_opencl_histogram_generation=True,
cl_ctx=cl_ctx,
cl_queue=cl_queue,
print_progress=print_progress,
simpleProfiler=simpleProfiler,
thread_pool=thread_pool)
small_night = small_night_pre & ~combined_masks["N"]
simpleProfiler.stop("cloud_pixel_identification")
if print_progress:
print("Identifying snow pixels...")
simpleProfiler.start("snow_identification")
snow = snow_identification(
scene, plot_path=time_slot_plot_path,
createPlots=createPlots) #mask_names=["DLPC", "DSMC", "NLPC", "NSMC"]
all_day__large_night__filtered = ~snow & all_day__large_night
small_night__filtered = ~snow & small_night
simpleProfiler.stop("snow_identification")
if print_progress:
print("Calculating small droplet proxy...")
simpleProfiler.start("small_droplet_proxy")
small_day__filtered = small_droplet_proxy(scene,
all_day__large_night__filtered,
combined_masks["DLPC"],
combined_masks["D"],
plot_path=time_slot_plot_path,
createPlots=createPlots)
simpleProfiler.stop("small_droplet_proxy")
if print_progress:
print("Merging results...")
simpleProfiler.start("merging_results")
small_drops = small_day__filtered | small_night__filtered
all_drops = all_day__large_night__filtered | small_night__filtered
large_drops = all_drops & ~small_drops
simpleProfiler.stop("merging_results")
if print_progress:
print("Calculating cloud phase...")
simpleProfiler.start("cloud_phase")
liquid = cloud_phase(scene,
small_drops,
plot_path=time_slot_plot_path,
createPlots=createPlots)
simpleProfiler.stop("cloud_phase")
if print_progress:
print("Calculating stratiformity...")
simpleProfiler.start("stratiformity_opencl")
stratiform = stratiformity_opencl(scene,
liquid,
plot_path=time_slot_plot_path,
createPlots=createPlots,
cl_ctx=cl_ctx,
cl_queue=cl_queue)
simpleProfiler.stop("stratiformity_opencl")
simpleProfiler.stop("cloud_pixel_detection")
simpleProfiler.start("write_data")
if print_progress:
print("Generating output...")
#plotOverview(scene,time_slot,[all_day__large_night,small_night,snow,small_drops,large_drops,all_drops,liquid,stratiform],plotDir)
#plotResult(scene,time_slot,stratiform,plotDir)
simpleProfiler.start("writeDataToGeoTiff")
writeDataToGeoTiff(stratiform,
path=time_slot_scene_path +
time_slot.strftime("pyt_%Y%m%d_%H%M_FLS.tif"),
buffr=domainBuffer)
simpleProfiler.stop("writeDataToGeoTiff")
#plotSceneToNightOverviewPNG(scene,time_slot_ircomposite_path+time_slot.strftime("%Y%m%d_%H%M_ircomposite.png"),buffr=domainBuffer)
simpleProfiler.stop("write_data")
if print_progress:
print("Memory usage:",
psutil.virtual_memory()[3] / (1024 * 1024), "MB")
end = time.time()
if print_progress:
print("Done (Time elapsed: " + "{:3.2f}".format(end - start) + ')\n\n')
return