def process(mydir, scene, s3_bucket, s3_folder, zoom, bbox): global verbose, force fullName = os.path.join(mydir, scene+".tif") if not os.path.exists(fullName): print "File does not exist", fullName sys.exit(-1) if verbose: print "Processing", fullName geojsonDir = os.path.join(mydir,"geojson") if not os.path.exists(geojsonDir): os.makedirs(geojsonDir) levelsDir = os.path.join(mydir,"levels") if not os.path.exists(levelsDir): os.makedirs(levelsDir) #shpDir = os.path.join(mydir,"shp") #cmd = "rm -rf " + shpDir #execute(cmd) #os.makedirs(shpDir) merge_filename = os.path.join(geojsonDir, "%s.geojson" % scene) topojson_filename = os.path.join(mydir, "%s.topojson" % scene) browse_filename = os.path.join(mydir, "%s_browse.tif" % scene) subset_filename = os.path.join(mydir, "%s_small_browse.tif" % scene) osm_bg_image = os.path.join(mydir, "..", "osm_bg.png") sw_osm_image = os.path.join(mydir, "%s_thn.jpg" % scene) #shapefile_gz = os.path.join(mydir, "%s.shp.gz" % scene) levels = [2,1] # From http://colorbrewer2.org/ #hexColors = ["#ffffe5", "#feb24c","#f03b20"] hexColors = ["#feb24c","#f03b20"] ds = gdal.Open( fullName ) band = ds.GetRasterBand(1) data = band.ReadAsArray(0, 0, ds.RasterXSize, ds.RasterYSize ) geotransform = ds.GetGeoTransform() xorg = geotransform[0] yorg = geotransform[3] xmax = xorg + geotransform[1]* ds.RasterXSize ymax = yorg + geotransform[5]* ds.RasterYSize #print ymax, xorg, yorg, xmax if force or not os.path.exists(topojson_filename+".gz"): for l in levels: fileName = os.path.join(levelsDir, scene+"_level_%d.tif"%l) CreateLevel(l, geojsonDir, fileName, ds, data, "landslide_nowcast", force,verbose) jsonDict = dict(type='FeatureCollection', features=[]) for l in reversed(levels): fileName = os.path.join(geojsonDir, "landslide_nowcast_level_%d.geojson"%l) if os.path.exists(fileName): if verbose: print "merge", fileName with open(fileName) as data_file: data = json.load(data_file) if 'features' in data: for f in data['features']: jsonDict['features'].append(f) with open(merge_filename, 'w') as outfile: json.dump(jsonDict, outfile) # Convert to topojson cmd = "topojson -p -o "+ topojson_filename + " " + merge_filename + " > /dev/null 2>&1" execute(cmd) cmd = "gzip -f --keep "+ topojson_filename execute(cmd) # Convert to shapefile #if force or not os.path.exists(shpDir) and os.path.exists(merge_filename): # cmd= "ogr2ogr -f 'ESRI Shapefile' %s %s" % ( shpDir, merge_filename) # execute(cmd) #if force or not os.path.exists(shapefile_gz): # cmd = "cd %s; tar -cvzf %s shp" %(mydir, shapefile_gz) # execute(cmd) if not os.path.exists(osm_bg_image): ullat = bbox[3] ullon = bbox[0] lrlat = bbox[1] lrlon = bbox[2] print "wms", ullat, ullon, lrlat, lrlon wms(ullat, ullon, lrlat, lrlon, osm_bg_image) if force or not os.path.exists(sw_osm_image): MakeBrowseImage(ds, browse_filename, subset_filename, osm_bg_image, sw_osm_image,levels, hexColors, force, verbose, zoom) ds = None file_list = [ sw_osm_image, topojson_filename+".gz", fullName] CopyToS3( s3_bucket, s3_folder, file_list, force, verbose ) if not verbose: cmd = "rm -f %s %s %s %s" %( subset_filename, subset_filename+".aux.xml", browse_filename,topojson_filename ) execute(cmd) #cmd = "rm -rf "+shpDir #execute(cmd) cmd = "rm -rf "+levelsDir execute(cmd) cmd = "rm -rf "+geojsonDir execute(cmd) fpath = os.path.join(config.data_dir,"landslide_nowcast", region, ymd) cmd = "rm -rf " + os.path.join(fpath,"iabr*") execute(cmd) cmd = "rm -rf " + os.path.join(fpath,"rr_*") execute(cmd) cmd = "rm -rf " + os.path.join(fpath,"step_*") execute(cmd)
def process(gpm_dir, gis_file_day, region, s3_bucket, s3_folder, ymd):
# subset the file for that region
bbox = region['bbox']
gis_file = os.path.join(BASE_DIR, gpm_dir, gis_file_day)
subset_file = os.path.join(BASE_DIR, gpm_dir, "gpm_24.%s.tif" % ymd)
if force or not os.path.exists(subset_file):
cmd = "gdalwarp -overwrite -q -te %f %f %f %f %s %s" % (
bbox[0], bbox[1], bbox[2], bbox[3], gis_file, subset_file)
execute(cmd)
geojsonDir = os.path.join(gpm_dir, "geojson")
if not os.path.exists(geojsonDir):
os.makedirs(geojsonDir)
levelsDir = os.path.join(gpm_dir, "levels")
if not os.path.exists(levelsDir):
os.makedirs(levelsDir)
merge_filename = os.path.join(geojsonDir, "gpm_24.%s.geojson" % ymd)
topojson_filename = os.path.join(geojsonDir, "..",
"gpm_24.%s.topojson" % ymd)
browse_filename = os.path.join(geojsonDir, "..",
"gpm_24.%s_browse.tif" % ymd)
subset_filename = os.path.join(geojsonDir, "..",
"gpm_24.%s_small_browse.tif" % ymd)
osm_bg_image = os.path.join(geojsonDir, "..", "osm_bg.png")
sw_osm_image = os.path.join(geojsonDir, "..", "gpm_24.%s_thn.jpg" % ymd)
levels = [144, 89, 55, 34, 21, 13, 8, 5, 3, 2, 1]
# From http://colorbrewer2.org/
hexColors = [
"#f7fcf0", "#e0f3db", "#ccebc5", "#a8ddb5", "#7bccc4", "#4eb3d3",
"#2b8cbe", "#0868ac", "#084081", "#810F7C", "#4D004A"
]
ds = gdal.Open(subset_file)
band = ds.GetRasterBand(1)
data = band.ReadAsArray(0, 0, ds.RasterXSize, ds.RasterYSize)
data /= 10 # back to mm
if force or not os.path.exists(topojson_filename + ".gz"):
for l in levels:
fileName = os.path.join(levelsDir, ymd + "_level_%d.tif" % l)
CreateLevel(l, geojsonDir, fileName, ds, data,
"daily_precipitation", force, verbose)
jsonDict = dict(type='FeatureCollection', features=[])
for l in reversed(levels):
fileName = os.path.join(geojsonDir,
"daily_precipitation_level_%d.geojson" % l)
if os.path.exists(fileName):
print "merge", fileName
with open(fileName) as data_file:
data = json.load(data_file)
if 'features' in data:
for f in data['features']:
jsonDict['features'].append(f)
with open(merge_filename, 'w') as outfile:
json.dump(jsonDict, outfile)
# Convert to topojson
cmd = "topojson -p -o " + topojson_filename + " " + merge_filename
execute(cmd)
cmd = "gzip --keep " + topojson_filename
execute(cmd)
if force or not os.path.exists(sw_osm_image):
# problem is that we need to scale it or adjust the levels for coloring (easier)
adjusted_levels = [1440, 890, 550, 340, 210, 130, 80, 50, 30, 20, 10]
MakeBrowseImage(ds, browse_filename, subset_filename, osm_bg_image,
sw_osm_image, adjusted_levels, hexColors, force,
verbose)
ds = None
file_list = [
sw_osm_image, topojson_filename, topojson_filename + ".gz", subset_file
]
CopyToS3(s3_bucket, s3_folder, file_list, force, verbose)
def process(swe_dir, scene, s3_bucket, s3_folder): fullName = os.path.join(swe_dir, scene+".tif") if not os.path.exists(fullName): print "File does not exist", fullName sys.exit(-1) if verbose: print "Processing", fullName geojsonDir = os.path.join(swe_dir,"geojson") if not os.path.exists(geojsonDir): os.makedirs(geojsonDir) levelsDir = os.path.join(swe_dir,"levels") if not os.path.exists(levelsDir): os.makedirs(levelsDir) merge_filename = os.path.join(geojsonDir, "%s_levels.geojson" % scene) topojson_filename = os.path.join(geojsonDir, "..", "%s_levels.topojson" % scene) browse_filename = os.path.join(geojsonDir, "..", "%s_browse.tif" % scene) subset_filename = os.path.join(geojsonDir, "..", "%s_small_browse.tif" % scene) osm_bg_image = os.path.join(geojsonDir, "..", "osm_bg.png") sw_osm_image = os.path.join(geojsonDir, "..", "%s_thn.jpg" % scene) levels = [144, 89, 55, 34, 21, 13, 8, 5, 3] # From http://colorbrewer2.org/ hexColors = ["#d73027","#f46d43","#fdae61", "#fee08b", "#ffffbf","#d9ef8b","#a6d96a","#66bd63","#1a9850"] ds = gdal.Open( fullName ) band = ds.GetRasterBand(1) data = band.ReadAsArray(0, 0, ds.RasterXSize, ds.RasterYSize ) if force or not os.path.exists(topojson_filename+".gz"): for l in levels: fileName = os.path.join(levelsDir, scene+"_level_%d.tif"%l) CreateLevel(l, geojsonDir, fileName, ds, data, "soil_moisture", force,verbose) jsonDict = dict(type='FeatureCollection', features=[]) for l in reversed(levels): fileName = os.path.join(geojsonDir, "soil_moisture_level_%d.geojson"%l) if os.path.exists(fileName): print "merge", fileName with open(fileName) as data_file: data = json.load(data_file) if 'features' in data: for f in data['features']: jsonDict['features'].append(f) with open(merge_filename, 'w') as outfile: json.dump(jsonDict, outfile) # Convert to topojson cmd = "topojson -p -o "+ topojson_filename + " " + merge_filename execute(cmd) cmd = "gzip --keep "+ topojson_filename execute(cmd) if force or not os.path.exists(sw_osm_image): MakeBrowseImage(ds, browse_filename, subset_filename, osm_bg_image, sw_osm_image,levels, hexColors, force, verbose) ds = None file_list = [ sw_osm_image, topojson_filename, topojson_filename+".gz", fullName ] CopyToS3( s3_bucket, s3_folder, file_list, force, verbose )
def process(mydir, gis_file, regionName, region, subfolder, s3_bucket,
s3_folder, ymd):
# subset the file for that region
bbox = region['bbox']
subset_file = os.path.join(mydir, "%s.%s.tif" % (subfolder, ymd))
print "subset_file", subset_file
if force or not os.path.exists(subset_file):
cmd = "gdalwarp -overwrite -q -te %f %f %f %f %s %s" % (
bbox[0], bbox[1], bbox[2], bbox[3], gis_file, subset_file)
execute(cmd)
geojsonDir = os.path.join(mydir, "geojson")
if not os.path.exists(geojsonDir):
os.makedirs(geojsonDir)
levelsDir = os.path.join(mydir, "levels")
if not os.path.exists(levelsDir):
os.makedirs(levelsDir)
merge_filename = os.path.join(geojsonDir,
"%s.%s.geojson" % (subfolder, ymd))
topojson_filename = os.path.join(geojsonDir, "..",
"%s.%s.topojson" % (subfolder, ymd))
browse_filename = os.path.join(geojsonDir, "..",
"%s.%s_browse.tif" % (subfolder, ymd))
subset_filename = os.path.join(geojsonDir, "..",
"%s.%s_small_browse.tif" % (subfolder, ymd))
osm_bg_image = os.path.join(geojsonDir, "..", "osm_bg.png")
sw_osm_image = os.path.join(geojsonDir, "..",
"%s.%s_thn.jpg" % (subfolder, ymd))
ds = gdal.Open(subset_file)
band = ds.GetRasterBand(1)
data = band.ReadAsArray(0, 0, ds.RasterXSize, ds.RasterYSize)
geotransform = ds.GetGeoTransform()
xorg = geotransform[0]
yorg = geotransform[3]
pres = geotransform[1]
xmax = xorg + geotransform[1] * ds.RasterXSize
ymax = yorg - geotransform[1] * ds.RasterYSize
if force or not os.path.exists(topojson_filename + ".gz"):
for l in levels:
fileName = os.path.join(levelsDir, ymd + "_level_%d.tif" % l)
CreateLevel(l, geojsonDir, fileName, ds, data, "precip", force,
verbose)
jsonDict = dict(type='FeatureCollection', features=[])
for l in reversed(levels):
fileName = os.path.join(geojsonDir, "precip_level_%d.geojson" % l)
if os.path.exists(fileName):
print "merge", fileName
with open(fileName) as data_file:
data = json.load(data_file)
if 'features' in data:
for f in data['features']:
jsonDict['features'].append(f)
with open(merge_filename, 'w') as outfile:
json.dump(jsonDict, outfile)
# Convert to topojson
cmd = "topojson -p -o " + topojson_filename + " " + merge_filename
execute(cmd)
cmd = "gzip --keep " + topojson_filename
execute(cmd)
if not os.path.exists(osm_bg_image):
ullat = yorg
ullon = xorg
lrlat = ymax
lrlon = xmax
wms(ullat, ullon, lrlat, lrlon, osm_bg_image)
if force or not os.path.exists(sw_osm_image):
zoom = region['thn_zoom']
MakeBrowseImage(ds, browse_filename, subset_filename, osm_bg_image,
sw_osm_image, levels, hexColors, force, verbose, zoom)
ds = None
file_list = [
sw_osm_image, topojson_filename, topojson_filename + ".gz", subset_file
]
CopyToS3(s3_bucket, s3_folder, file_list, force, verbose)
def process(mydir, lsFile, regionName, region, s3_bucket, s3_folder):
scene = regionName
subsetFileName = os.path.join(mydir, "ls.2011_subset.tif")
if force or not os.path.exists(subsetFileName):
bbox = region['bbox']
print region['name'], region['bbox']
warpOptions = "-q -overwrite -co COMPRESS=DEFLATE -t_srs EPSG:4326 -te %s %s %s %s " % (
bbox[0], bbox[1], bbox[2], bbox[3])
warpCmd = 'gdalwarp ' + warpOptions + lsFile + ' ' + subsetFileName
execute(warpCmd)
if verbose:
print "LS Subset", subsetFileName
if verbose:
print "Processing", subsetFileName
geojsonDir = os.path.join(mydir, "geojson")
if not os.path.exists(geojsonDir):
os.makedirs(geojsonDir)
levelsDir = os.path.join(mydir, "levels")
if not os.path.exists(levelsDir):
os.makedirs(levelsDir)
merge_filename = os.path.join(geojsonDir, "%s_levels.geojson" % scene)
topojson_filename = os.path.join(geojsonDir, "..", "ls.2011.topojson")
browse_filename = os.path.join(geojsonDir, "..", "ls.2011_browse.tif")
subset_filename = os.path.join(geojsonDir, "..",
"ls.2011_small_browse.tif")
osm_bg_image = os.path.join(geojsonDir, "..", "osm_bg.png")
sw_osm_image = os.path.join(geojsonDir, "..", "ls.2011_thn.jpg")
levels = [5500, 3400, 2100, 1300, 800, 500, 300, 200, 100]
# From http://colorbrewer2.org/
hexColors = [
"#f7f4f9", "#e7e1ef", "#d4b9da", "#c994c7", "#df65b0", "#e7298a",
"#ce1256", "#980043", "#67001f"
]
ds = gdal.Open(subsetFileName)
band = ds.GetRasterBand(1)
data = band.ReadAsArray(0, 0, ds.RasterXSize, ds.RasterYSize)
if force or not os.path.exists(topojson_filename + ".gz"):
for l in levels:
fileName = os.path.join(levelsDir, scene + "_level_%d.tif" % l)
CreateLevel(l, geojsonDir, fileName, ds, data, "population", force,
verbose)
jsonDict = dict(type='FeatureCollection', features=[])
for l in reversed(levels):
fileName = os.path.join(geojsonDir,
"population_level_%d.geojson" % l)
if os.path.exists(fileName):
print "merge", fileName
with open(fileName) as data_file:
data = json.load(data_file)
if 'features' in data:
for f in data['features']:
jsonDict['features'].append(f)
with open(merge_filename, 'w') as outfile:
json.dump(jsonDict, outfile)
# Convert to topojson
cmd = "topojson -p -o " + topojson_filename + " " + merge_filename
execute(cmd)
cmd = "gzip --keep " + topojson_filename
execute(cmd)
if not os.path.exists(osm_bg_image):
geotransform = ds.GetGeoTransform()
xorg = geotransform[0]
yorg = geotransform[3]
xmax = xorg + geotransform[1] * ds.RasterXSize
ymax = yorg + geotransform[5] * ds.RasterYSize
ullat = yorg
ullon = xorg
lrlat = ymax
lrlon = xmax
print "wms", ullat, ullon, lrlat, lrlon
wms(ullat, ullon, lrlat, lrlon, osm_bg_image)
if force or not os.path.exists(sw_osm_image):
zoom = region['thn_zoom']
MakeBrowseImage(ds, browse_filename, subset_filename, osm_bg_image,
sw_osm_image, levels, hexColors, force, verbose, zoom)
ds = None
file_list = [
sw_osm_image, topojson_filename, topojson_filename + ".gz",
subsetFileName
]
CopyToS3(s3_bucket, s3_folder, file_list, force, verbose)
def process_file(mydir, filename, s3_bucket, s3_folder):
print "Processing", filename
geojsonDir = os.path.join(mydir, "geojson")
if not os.path.exists(geojsonDir):
os.makedirs(geojsonDir)
levelsDir = os.path.join(mydir, "levels")
if not os.path.exists(levelsDir):
os.makedirs(levelsDir)
#shpDir = os.path.join(mydir,"shp")
#if not os.path.exists(shpDir):
# os.makedirs(shpDir)
super_subset_file = os.path.join(mydir, "geos5_precip_super.%s.tif" % ymd)
merge_filename = os.path.join(geojsonDir, "geos5_precip.%s.geojson" % ymd)
topojson_filename = os.path.join(geojsonDir, "..",
"geos5_precip.%s.topojson" % ymd)
browse_filename = os.path.join(geojsonDir, "..",
"geos5_precip.%s_browse.tif" % ymd)
subset_filename = os.path.join(geojsonDir, "..",
"geos5_precip.%s_small_browse.tif" % ymd)
subset_aux_filename = os.path.join(
geojsonDir, "..", "geos5_precip.%s_small_browse.tif.aux.xml" % ymd)
osm_bg_image = os.path.join(mydir, "../..", "osm_bg.png")
sw_osm_image = os.path.join(geojsonDir, "..",
"geos5_precip.%s_thn.jpg" % ymd)
json_filename = os.path.join(geojsonDir, "geos5_precip.%s.json" % (ymd))
#shp_filename = os.path.join(mydir, "geos5_precip.%s.shp.gz" % (ymd))
#shp_zip_file = os.path.join(mydir, "geos5_precip.%s.shp.zip" % (ymd))
#if force or not os.path.exists(subset_file):
# cmd = "gdalwarp -overwrite -q -te %f %f %f %f %s %s" % (bbox[0], bbox[1], bbox[2], bbox[3], filename, subset_file)
# execute(cmd)
ds = gdal.Open(filename)
geotransform = ds.GetGeoTransform()
px = geotransform[1] / 2
py = geotransform[5] / 2
xorg = geotransform[0]
yorg = geotransform[3]
xmax = xorg + geotransform[1] * ds.RasterXSize
ymax = yorg + geotransform[5] * ds.RasterYSize
#print ymax, xorg, yorg, xmax
ds = None
# upsample and convolve
if force or not os.path.exists(super_subset_file):
# we need to have square pixels
cmd = "gdalwarp -overwrite -q -r cubicspline -tr %s %s -co COMPRESS=LZW %s %s" % (
str(px), str(px), filename, super_subset_file)
execute(cmd)
levels = [377, 233, 144, 89, 55, 34, 21, 13, 8, 5, 3]
# http://hclwizard.org/hcl-color-scheme/
# http://vis4.net/blog/posts/avoid-equidistant-hsv-colors/
# from http://tristen.ca/hcl-picker/#/hlc/12/1/241824/55FEFF
# This is in inverse order from levels
#hexColors = [ "#56F6FC","#58DEEE","#5BC6DE","#5EAFCC","#5E99B8","#5D84A3","#596F8D","#535B77","#4A4861","#3F374B","#322737","#241824"]
# GPM palette
hexColors = [
"#f7fcf0", "#e0f3db", "#ccebc5", "#a8ddb5", "#7bccc4", "#4eb3d3",
"#2b8cbe", "#0868ac", "#084081", "#810F7C", "#4D004A"
]
ds = gdal.Open(super_subset_file)
band = ds.GetRasterBand(1)
data = band.ReadAsArray(0, 0, ds.RasterXSize, ds.RasterYSize)
if force or not os.path.exists(topojson_filename + ".gz"):
for l in levels:
fileName = os.path.join(levelsDir, ymd + "_level_%d.tif" % l)
CreateLevel(l, geojsonDir, fileName, ds, data, "geos5_precip",
force, verbose)
jsonDict = dict(type='FeatureCollection', features=[])
for l in reversed(levels):
fileName = os.path.join(geojsonDir,
"geos5_precip_level_%d.geojson" % l)
if os.path.exists(fileName):
if verbose:
print "merge", fileName
with open(fileName) as data_file:
data = json.load(data_file)
if 'features' in data:
for f in data['features']:
jsonDict['features'].append(f)
with open(merge_filename, 'w') as outfile:
json.dump(jsonDict, outfile)
# Convert to topojson
quiet = "> /dev/null 2>&1"
if verbose:
quiet = " "
cmd = "topojson -p --bbox --simplify-proportion 0.5 -o " + topojson_filename + " " + merge_filename + quiet
execute(cmd)
cmd = "gzip -f " + topojson_filename
execute(cmd)
# Create shapefile gz
#if force or not os.path.exists(shp_filename):
# # Convert simplified topojson to geojson
# cmd = "topojson-geojson --precision 4 %s -o %s" % (topojson_filename, geojsonDir)
# execute(cmd)
# cmd = "ogr2ogr -f 'ESRI Shapefile' %s %s" % (shpDir, json_filename)
# execute(cmd)
#cmd = "cd %s; tar -zcvf %s %s" % (mydir, shp_filename, shpDir)
# cmd = "cd %s; zip %s shp/*" %(mydir, shp_zip_file)
# execute(cmd)
if not os.path.exists(osm_bg_image):
#print "wms", ymax, xorg, yorg, xmax, osm_bg_image
wms(90, -180, -90, 180, osm_bg_image)
if force or not os.path.exists(sw_osm_image):
zoom = 1
scale = 1
rColors = list(reversed(hexColors))
MakeBrowseImage(ds, browse_filename, subset_filename, osm_bg_image,
sw_osm_image, levels, rColors, force, verbose, zoom,
scale)
ds = None
file_list = [sw_osm_image, topojson_filename + ".gz", filename]
CopyToS3(s3_bucket, s3_folder, file_list, 1, 1)
if not verbose: # Cleanup
if config.USING_AWS_S3_FOR_STORAGE:
cmd = "rm -rf %s " % (mydir)
execute(cmd)
else:
cmd = "rm -rf %s %s %s %s %s %s %s %s" % (
merge_filename, browse_filename, topojson_filename,
subset_filename, super_subset_file, subset_aux_filename,
geojsonDir, levelsDir)
execute(cmd)
def process_lowres(self):
name = "flood_14km"
input_fname = "Flood_byStor_%s%02d00.bin" % (ym, day)
input_fullname = os.path.join(self.inpath, "gfms", ymd, input_fname)
output_fname = "%s.%s%02d.tif" % (name, ym, day)
output_fullname = os.path.join(self.inpath, "gfms", ymd, output_fname)
super_fname = "%s.%s%02d.x2.tif" % (name, ym, day)
super_fullname = os.path.join(self.inpath, "gfms", ymd, super_fname)
super_fname_rgb = "%s.%s%02d.x2.rgb.tif" % (name, ym, day)
super_fullname_rgb = os.path.join(self.inpath, "gfms", ymd,
super_fname_rgb)
geojson_fname = "%s.%s%02d.geojson" % (name, ym, day)
geojson_fullname = os.path.join(self.inpath, "gfms", ymd,
geojson_fname)
topojson_fname = "%s.%s%02d.topojson" % (name, ym, day)
topojson_fullname = os.path.join(self.inpath, "gfms", ymd,
topojson_fname)
topojson_fullname_gz = topojson_fullname + ".gz"
shp_gz_file = os.path.join(self.inpath, "gfms", ymd,
"%s.%s%02d.shp.gz" % (name, ym, day))
shp_zip_file = os.path.join(self.inpath, "gfms", ymd,
"%s.%s%02d.shp.zip" % (name, ym, day))
output_rgb_fname = "%s.%s%02d_rgb.tif" % (name, ym, day)
output_rgb_fullname = os.path.join(self.inpath, "gfms", ymd,
output_rgb_fname)
color_file = os.path.join("cluts", "gfms_colors.txt")
flood_dir = os.path.join(self.inpath, "gfms", ymd)
geojsonDir = os.path.join(flood_dir, "geojson")
if not os.path.exists(geojsonDir):
os.makedirs(geojsonDir)
levelsDir = os.path.join(flood_dir, "levels")
if not os.path.exists(levelsDir):
os.makedirs(levelsDir)
merge_filename = os.path.join(geojsonDir, "%s_levels.geojson" % ymd)
browse_filename = os.path.join(geojsonDir, "..", "%s_browse.tif" % ymd)
browse_aux_filename = os.path.join(geojsonDir, "..",
"%s_small_browse.tif.aux.xml" % ymd)
subset_filename = os.path.join(geojsonDir, "..",
"%s_small_browse.tif" % ymd)
osm_bg_image = os.path.join(flood_dir, "..", "osm_bg.png")
sw_osm_image = os.path.join(geojsonDir, "..",
"%s.%s%02d_thn.jpg" % (name, ym, day))
x = -127.25
y = 50
res = 0.125
if self.force or not os.path.exists(output_fullname):
rows = 800
cols = 2458
size = rows * cols
if verbose:
print "gfms processing:", input_fullname
fd = open(input_fullname, 'rb')
shape = (rows, cols)
data = numpy.fromfile(file=fd, dtype=numpy.float32,
count=size).reshape(shape)
data[data < 0] = 0 #PGC
#print "stats:", data.size, data.min(), data.mean(), data.max(), data.std()
# Create gtif
driver = gdal.GetDriverByName("GTiff")
#dst_ds = driver.Create(output_fullname, cols, rows, 1, gdal.GDT_Float32)
dst_ds = driver.Create(output_fullname, cols, rows, 1,
gdal.GDT_Byte)
# top left x, w-e pixel resolution, rotation, top left y, rotation, n-s pixel resolution
dst_ds.SetGeoTransform([x, res, 0, y, 0, -res])
# set the reference info
srs = osr.SpatialReference()
srs.ImportFromEPSG(4326)
dst_ds.SetProjection(srs.ExportToWkt())
# write the band
band = dst_ds.GetRasterBand(1)
#band.SetNoDataValue(-9999)
band.WriteArray(data)
dst_ds = None
# Supersample it
scale = 2
if self.force or not os.path.exists(super_fullname):
cmd = "gdalwarp -overwrite -q -tr %f %f -r mode %s %s" % (
res / scale, res / scale, output_fullname, super_fullname)
self.execute(cmd)
# Create RGB
if self.verbose and (self.force
or not os.path.exists(output_rgb_fullname)):
cmd = "gdaldem color-relief -q -alpha " + output_fullname + " " + color_file + " " + output_rgb_fullname
self.execute(cmd)
if self.verbose and (self.force
or not os.path.exists(super_fullname_rgb)):
cmd = "gdaldem color-relief -q -alpha " + super_fullname + " " + color_file + " " + super_fullname_rgb
self.execute(cmd)
# json
#if self.force or not os.path.exists(geojson_fullname):
# cmd = "makesurface vectorize --classfile gmfs_classes.csv --outfile %s --outvar flood %s " %( geojson_fullname, super_fullname)
# self.execute(cmd)
# topojson
#if self.force or not os.path.exists(topojson_fullname):
# cmd = "topojson --simplify-proportion 0.75 --bbox -p risk -o %s -- flood_24hr_forecast=%s" % (topojson_fullname, geojson_fullname )
# self.execute(cmd)
#if self.force or not os.path.exists(topojson_fullname_gz):
# cmd = "gzip --keep %s" % (topojson_fullname)
# self.execute(cmd)
levels = [200, 100, 50, 20, 10, 1]
hexColors = [
"#FF0000", "#FFA500", "#FFD700", "#0000FF", "#00BFFF", "#00FF00"
]
ds = gdal.Open(super_fullname)
band = ds.GetRasterBand(1)
data = band.ReadAsArray(0, 0, ds.RasterXSize, ds.RasterYSize)
if self.force or not os.path.exists(topojson_fullname + ".gz"):
for l in levels:
fileName = os.path.join(levelsDir, ymd + "_level_%d.tif" % l)
CreateLevel(l, geojsonDir, fileName, ds, data, "flood", force,
verbose)
jsonDict = dict(type='FeatureCollection', features=[])
for l in reversed(levels):
fileName = os.path.join(geojsonDir,
"flood_level_%d.geojson" % l)
if os.path.exists(fileName):
# print "merge", fileName
with open(fileName) as data_file:
data = json.load(data_file)
if 'features' in data:
for f in data['features']:
jsonDict['features'].append(f)
with open(merge_filename, 'w') as outfile:
json.dump(jsonDict, outfile)
# Convert to topojson
quiet = " > /dev/null 2>&1"
if verbose:
quiet = " "
cmd = "topojson -p -o " + topojson_fullname + " " + merge_filename + quiet
self.execute(cmd)
if verbose:
cmd = "gzip -f --keep " + topojson_fullname
else:
cmd = "gzip -f " + topojson_fullname
self.execute(cmd)
#cmd= "ogr2ogr -f 'ESRI Shapefile' %s %s" % ( shpDir, merge_filename)
#self.execute(cmd)
#if force or not os.path.exists(shp_zip_file):
# mydir = os.path.join(self.inpath, "gfms", ymd)
# #cmd = "cd %s; tar -cvzf %s shp" %(mydir, shp_gz_file)
# cmd = "cd %s; zip %s shp/*" %(mydir, shp_zip_file)
# self.execute(cmd)
if not os.path.exists(osm_bg_image):
wms(y, x, -y, 180, osm_bg_image)
if self.force or not os.path.exists(sw_osm_image):
MakeBrowseImage(ds,
browse_filename,
subset_filename,
osm_bg_image,
sw_osm_image,
levels,
hexColors,
force,
verbose,
zoom=2)
file_list = [sw_osm_image, topojson_fullname_gz, output_fullname]
CopyToS3(s3_bucket, s3_folder, file_list, force, verbose)
if not self.verbose:
cmd = "rm -rf %s %s %s %s %s %s %s %s" % (
browse_filename, input_fullname, subset_filename,
super_fullname, output_rgb_fullname, browse_aux_filename,
levelsDir, geojsonDir)
self.execute(cmd)
def process_file( mydir, filename, s3_bucket, s3_folder, bbox, regionName): global force, verbose print "Processing", filename geojsonDir = os.path.join(mydir,"geojson") if not os.path.exists(geojsonDir): os.makedirs(geojsonDir) levelsDir = os.path.join(mydir,"levels") if not os.path.exists(levelsDir): os.makedirs(levelsDir) super_subset_file = os.path.join(mydir, "geos5_sat_super.%s.tif" % ymd) merge_filename = os.path.join(geojsonDir, "geos5_sat.%s.geojson" % ymd) topojson_filename = os.path.join(geojsonDir, "..", "geos5_sat.%s.topojson" % ymd) browse_filename = os.path.join(geojsonDir, "..", "geos5_sat.%s_browse.tif" % ymd) subset_filename = os.path.join(geojsonDir, "..", "geos5_sat.%s_small_browse.tif" % ymd) subset_aux_filename = os.path.join(geojsonDir, "..", "geos5_sat.%s_small_browse.tif.aux.xml" % ymd) osm_bg_image = os.path.join(mydir, "..", "osm_bg_%s.png" % regionName ) sw_osm_image = os.path.join(geojsonDir, "..", "geos5_sat.%s_thn.jpg" % ymd) json_filename = os.path.join(geojsonDir, "geos5_sat.%s.json" % (ymd)) ds = gdal.Open( filename ) geotransform = ds.GetGeoTransform() px = geotransform[1] / 10 py = geotransform[5] / 10 xorg = geotransform[0] yorg = geotransform[3] xmax = xorg + geotransform[1]* ds.RasterXSize ymax = yorg + geotransform[5]* ds.RasterYSize #print ymax, xorg, yorg, xmax ds = None # upsample and convolve if force or not os.path.exists(super_subset_file): # we need to have square pixels cmd = "gdalwarp -overwrite -q -r cubic -tr %s %s -co COMPRESS=DEFLATE %s %s" % (str(px), str(py), filename, super_subset_file) execute(cmd) levels = [5, 4, 3, 2] hexColors = [ "ff9a00", "ff0000", "ff99cc", "cc00cc" ] ds = gdal.Open( super_subset_file ) band = ds.GetRasterBand(1) data = band.ReadAsArray(0, 0, ds.RasterXSize, ds.RasterYSize ) if force or not os.path.exists(topojson_filename+".gz"): for l in levels: fileName = os.path.join(levelsDir, ymd+"_level_%d.tif"%l) CreateLevel(l, geojsonDir, fileName, ds, data, "geos5_sat", force, verbose) jsonDict = dict(type='FeatureCollection', features=[]) for l in reversed(levels): fileName = os.path.join(geojsonDir, "geos5_sat_level_%d.geojson"%l) if os.path.exists(fileName): if verbose: print "merge", fileName with open(fileName) as data_file: data = json.load(data_file) if 'features' in data: for f in data['features']: jsonDict['features'].append(f) with open(merge_filename, 'w') as outfile: json.dump(jsonDict, outfile) # Convert to topojson quiet = "> /dev/null 2>&1" if verbose: quiet = " " cmd = "topojson -p --bbox --simplify-proportion 0.1 -o "+ topojson_filename + " " + merge_filename + quiet execute(cmd) cmd = "gzip -f "+ topojson_filename execute(cmd) if not os.path.exists(osm_bg_image): ullat = bbox[3] ullon = bbox[0] lrlat = bbox[1] lrlon = bbox[2] wms(ullat, ullon, lrlat, lrlon, osm_bg_image) if force or not os.path.exists(sw_osm_image): zoom = 1 scale = 1 rColors = list(reversed(hexColors)) MakeBrowseImage(ds, browse_filename, subset_filename, osm_bg_image, sw_osm_image, levels, rColors, force, verbose, zoom, scale) ds = None file_list = [ sw_osm_image, topojson_filename+".gz", filename ] CopyToS3( s3_bucket, s3_folder, file_list, force, verbose ) if not verbose: # Cleanup cmd = "rm -rf %s %s %s %s %s %s %s %s" % ( merge_filename, browse_filename, topojson_filename, subset_filename, super_subset_file, subset_aux_filename, geojsonDir, levelsDir) execute(cmd)
def process_file(mydir, filename, s3_bucket, s3_folder):
print "Processing", filename
geojsonDir = os.path.join(mydir, "geojson")
if not os.path.exists(geojsonDir):
os.makedirs(geojsonDir)
levelsDir = os.path.join(mydir, "levels")
if not os.path.exists(levelsDir):
os.makedirs(levelsDir)
shpDir = os.path.join(mydir, "shp")
if not os.path.exists(shpDir):
os.makedirs(shpDir)
super_subset_file = os.path.join(mydir, "geos5_precip_super.%s.tif" % ymd)
merge_filename = os.path.join(geojsonDir, "geos5_precip.%s.geojson" % ymd)
topojson_filename = os.path.join(geojsonDir, "..",
"geos5_precip.%s.topojson" % ymd)
browse_filename = os.path.join(geojsonDir, "..",
"geos5_precip.%s_browse.tif" % ymd)
subset_filename = os.path.join(geojsonDir, "..",
"geos5_precip.%s_small_browse.tif" % ymd)
subset_aux_filename = os.path.join(
geojsonDir, "..", "geos5_precip.%s_small_browse.tif.aux.xml" % ymd)
osm_bg_image = os.path.join(geojsonDir, "..", "osm_bg.png")
sw_osm_image = os.path.join(geojsonDir, "..",
"geos5_precip.%s_thn.jpg" % ymd)
shp_filename = os.path.join(mydir, "geos5_precip.%s.shp.gz" % (ymd))
json_filename = os.path.join(geojsonDir, "geos5_precip.%s.json" % (ymd))
shp_zip_file = os.path.join(mydir, "geos5_precip.%s.shp.zip" % (ymd))
#if force or not os.path.exists(subset_file):
# cmd = "gdalwarp -overwrite -q -te %f %f %f %f %s %s" % (bbox[0], bbox[1], bbox[2], bbox[3], filename, subset_file)
# execute(cmd)
ds = gdal.Open(filename)
geotransform = ds.GetGeoTransform()
px = geotransform[1] / 5
py = geotransform[5] / 5
ds = None
# upsample and convolve
if force or not os.path.exists(super_subset_file):
# we need to have square pixels
cmd = "gdalwarp -overwrite -q -r cubicspline -tr %s %s -co COMPRESS=LZW %s %s" % (
str(px), str(px), filename, super_subset_file)
execute(cmd)
levels = [377, 233, 144, 89, 55, 34, 21, 13, 8, 5, 3, 2]
# http://hclwizard.org/hcl-color-scheme/
# http://vis4.net/blog/posts/avoid-equidistant-hsv-colors/
# from http://tristen.ca/hcl-picker/#/hlc/12/1/241824/55FEFF
hexColors = [
"#56F6FC", "#58DEEE", "#5BC6DE", "#5EAFCC", "#5E99B8", "#5D84A3",
"#596F8D", "#535B77", "#4A4861", "#3F374B", "#322737", "#241824"
]
ds = gdal.Open(super_subset_file)
band = ds.GetRasterBand(1)
data = band.ReadAsArray(0, 0, ds.RasterXSize, ds.RasterYSize)
if force or not os.path.exists(topojson_filename + ".gz"):
for l in levels:
fileName = os.path.join(levelsDir, ymd + "_level_%d.tif" % l)
CreateLevel(l, geojsonDir, fileName, ds, data, "geos5_precip",
force, verbose)
jsonDict = dict(type='FeatureCollection', features=[])
for l in reversed(levels):
fileName = os.path.join(geojsonDir,
"geos5_precip_level_%d.geojson" % l)
if os.path.exists(fileName):
if verbose:
print "merge", fileName
with open(fileName) as data_file:
data = json.load(data_file)
if 'features' in data:
for f in data['features']:
jsonDict['features'].append(f)
with open(merge_filename, 'w') as outfile:
json.dump(jsonDict, outfile)
# Convert to topojson
cmd = "topojson -p --bbox --simplify-proportion 0.5 -o " + topojson_filename + " " + merge_filename
execute(cmd)
cmd = "gzip --keep " + topojson_filename
execute(cmd)
# Create shapefile gz
if force or not os.path.exists(shp_filename):
# Convert simplified topojson to geojson
cmd = "topojson-geojson --precision 4 %s -o %s" % (topojson_filename,
geojsonDir)
execute(cmd)
cmd = "ogr2ogr -f 'ESRI Shapefile' %s %s" % (shpDir, json_filename)
execute(cmd)
#cmd = "cd %s; tar -zcvf %s %s" % (mydir, shp_filename, shpDir)
cmd = "cd %s; zip %s shp/*" % (mydir, shp_zip_file)
execute(cmd)
if force or not os.path.exists(sw_osm_image):
zoom = 1
scale = 1
MakeBrowseImage(ds, browse_filename, subset_filename, osm_bg_image,
sw_osm_image, levels, hexColors, force, verbose, zoom,
scale)
ds = None
file_list = [
sw_osm_image, topojson_filename, topojson_filename + ".gz", filename,
shp_zip_file
]
CopyToS3(s3_bucket, s3_folder, file_list, 1, 1)
if not verbose: # Cleanup
cmd = "rm -rf %s %s %s %s %s %s %s %s %s %s" % (
merge_filename, browse_filename, topojson_filename,
subset_filename, super_subset_file, osm_bg_image,
subset_aux_filename, geojsonDir, levelsDir, shpDir)
execute(cmd)
def process(gpm_dir, name, gis_file_day, ymd, regionName, region, s3_bucket,
s3_folder):
# subset the file for that region
bbox = region['bbox']
gis_file = os.path.join(gpm_dir, gis_file_day)
if not os.path.exists(gis_file):
print "gis file does not exist", gis_file
sys.exit(-1)
region_dir = os.path.join(gpm_dir, regionName)
if not os.path.exists(region_dir):
os.makedirs(region_dir)
subset_file = os.path.join(region_dir, "%s.%s.tif" % (name, ymd))
if force or not os.path.exists(subset_file):
cmd = "gdalwarp -overwrite -q -te %f %f %f %f %s %s" % (
bbox[0], bbox[1], bbox[2], bbox[3], gis_file, subset_file)
execute(cmd)
geojsonDir = os.path.join(region_dir, "geojson")
if not os.path.exists(geojsonDir):
os.makedirs(geojsonDir)
levelsDir = os.path.join(region_dir, "levels")
if not os.path.exists(levelsDir):
os.makedirs(levelsDir)
merge_filename = os.path.join(geojsonDir, "%s.%s.geojson" % (name, ymd))
topojson_filename = os.path.join(geojsonDir, "..",
"%s.%s.topojson" % (name, ymd))
browse_filename = os.path.join(geojsonDir, "..",
"%s.%s_browse.tif" % (name, ymd))
subset_filename = os.path.join(geojsonDir, "..",
"%s.%s_small_browse.tif" % (name, ymd))
subset_aux_filename = os.path.join(
geojsonDir, "..", "%s.%s_small_browse.tif.aux" % (name, ymd))
osm_bg_image = os.path.join(geojsonDir, "..", "osm_bg.png")
sw_osm_image = os.path.join(geojsonDir, "..",
"%s.%s_thn.jpg" % (name, ymd))
levels = [377, 233, 144, 89, 55, 34, 21, 13, 8, 5, 3]
# From http://colorbrewer2.org/
hexColors = [
"#f7fcf0", "#e0f3db", "#ccebc5", "#a8ddb5", "#7bccc4", "#4eb3d3",
"#2b8cbe", "#0868ac", "#084081", "#810F7C", "#4D004A"
]
ds = gdal.Open(subset_file)
band = ds.GetRasterBand(1)
data = band.ReadAsArray(0, 0, ds.RasterXSize, ds.RasterYSize)
geotransform = ds.GetGeoTransform()
xorg = geotransform[0]
yorg = geotransform[3]
pres = geotransform[1]
xmax = xorg + geotransform[1] * ds.RasterXSize
ymax = yorg - geotransform[1] * ds.RasterYSize
data /= 10 # back to mm
if force or not os.path.exists(topojson_filename + ".gz"):
for l in levels:
fileName = os.path.join(levelsDir, ymd + "_level_%d.tif" % l)
CreateLevel(l, geojsonDir, fileName, ds, data, "precip", force,
verbose)
jsonDict = dict(type='FeatureCollection', features=[])
for l in reversed(levels):
fileName = os.path.join(geojsonDir, "precip_level_%d.geojson" % l)
if os.path.exists(fileName):
if verbose:
print "merge", fileName
with open(fileName) as data_file:
data = json.load(data_file)
if 'features' in data:
for f in data['features']:
jsonDict['features'].append(f)
with open(merge_filename, 'w') as outfile:
json.dump(jsonDict, outfile)
if verbose:
output = " "
else:
output = " > /dev/null 2>&1"
# Convert to topojson
cmd = "topojson -p -o " + topojson_filename + " " + merge_filename + output
execute(cmd)
cmd = "gzip -f " + topojson_filename
execute(cmd)
# problem is that we need to scale it or adjust the levels for coloring (easier)
adjusted_levels = [3770, 2330, 1440, 890, 550, 340, 210, 130, 80, 50, 30]
zoom = region['thn_zoom']
if not os.path.exists(osm_bg_image):
ullat = yorg
ullon = xorg
lrlat = ymax
lrlon = xmax
wms(ullat, ullon, lrlat, lrlon, osm_bg_image)
if force or not os.path.exists(sw_osm_image):
rColors = list(reversed(hexColors))
MakeBrowseImage(ds, browse_filename, subset_filename, osm_bg_image,
sw_osm_image, adjusted_levels, rColors, force, verbose,
zoom)
ds = None
file_list = [sw_osm_image, topojson_filename + ".gz", subset_file]
CopyToS3(s3_bucket, s3_folder, file_list, force, verbose)
if not verbose: # Cleanup
cmd = "rm -rf %s %s %s %s %s %s" % (
osm_bg_image, browse_filename, subset_filename,
subset_aux_filename, geojsonDir, levelsDir)
execute(cmd)
def process_viirs_chla_file( mydir, regionName, viirs_filename, s3_bucket, s3_folder): print "Processing", viirs_filename+":chlor_a" region = config.regions[regionName] bbox = region['bbox'] rdir = os.path.join(mydir, regionName) if not os.path.exists(rdir): os.makedirs(rdir) geojsonDir = os.path.join(rdir,"geojson") if not os.path.exists(geojsonDir): os.makedirs(geojsonDir) levelsDir = os.path.join(rdir,"levels") if not os.path.exists(levelsDir): os.makedirs(levelsDir) shpDir = os.path.join(rdir,"shp") if not os.path.exists(shpDir): os.makedirs(shpDir) subset_file = os.path.join(rdir, "viirs_chla.%s.tif" % ymd) super_subset_file = os.path.join(rdir, "viirs_chla_super.%s.tif" % ymd) merge_filename = os.path.join(geojsonDir, "viirs_chla.%s.geojson" % ymd) topojson_filename = os.path.join(geojsonDir, "..", "viirs_chla.%s.topojson" % ymd) browse_filename = os.path.join(geojsonDir, "..", "viirs_chla.%s_browse.tif" % ymd) subset_filename = os.path.join(geojsonDir, "..", "viirs_chla.%s_small_browse.tif" % ymd) osm_bg_image = os.path.join(geojsonDir, "..", "osm_bg.png") sw_osm_image = os.path.join(geojsonDir, "..", "viirs_chla.%s_thn.jpg" % ymd) shp_filename = os.path.join(rdir, "viirs_chla.%s.shp.gz" % (ymd)) json_filename = os.path.join(geojsonDir, "viirs_chla.%s.json" % (ymd)) if force or not os.path.exists(subset_file): cmd = "gdalwarp -overwrite -q -te %f %f %f %f %s %s" % (bbox[0], bbox[1], bbox[2], bbox[3], viirs_filename, subset_file) execute(cmd) ds = gdal.Open( subset_file ) geotransform = ds.GetGeoTransform() px = geotransform[1] / 10 py = geotransform[5] / 10 ds = None # upsample and convolve if force or not os.path.exists(super_subset_file): cmd = "gdalwarp -overwrite -q -r cubicspline -tr %s %s -te %f %f %f %f -co COMPRESS=LZW %s %s" % (str(px), str(py), bbox[0], bbox[1], bbox[2], bbox[3], subset_file, super_subset_file) execute(cmd) levels = [350, 100, 50, 30, 20, 15, 10, 5, 3, 1] # From http://colorbrewer2.org/ hexColors = ["#5e4fa2", "#3288bd", "#66c2a5", "#abdda4", "#e6f598", "#fee08b", "#fdae61", "#f46d43", "#d53e4f", "#9e0142"] ds = gdal.Open( super_subset_file ) band = ds.GetRasterBand(1) data = band.ReadAsArray(0, 0, ds.RasterXSize, ds.RasterYSize ) data *= 100 if force or not os.path.exists(topojson_filename+".gz"): for l in levels: fileName = os.path.join(levelsDir, ymd+"_level_%d.tif"%l) CreateLevel(l, geojsonDir, fileName, ds, data, "viirs_chla", force, verbose) jsonDict = dict(type='FeatureCollection', features=[]) for l in reversed(levels): fileName = os.path.join(geojsonDir, "viirs_chla_level_%d.geojson"%l) if os.path.exists(fileName): print "merge", fileName with open(fileName) as data_file: data = json.load(data_file) if 'features' in data: for f in data['features']: jsonDict['features'].append(f) with open(merge_filename, 'w') as outfile: json.dump(jsonDict, outfile) # Convert to topojson cmd = "topojson -p -o "+ topojson_filename + " " + merge_filename execute(cmd) cmd = "gzip --keep "+ topojson_filename execute(cmd) # Create shapefile gz if force or not os.path.exists(shp_filename): # Convert simplified topojson to geojson cmd = "topojson-geojson --precision 5 %s -o %s" % (topojson_filename, geojsonDir) execute(cmd) cmd = "ogr2ogr -f 'ESRI Shapefile' %s %s" % (shpDir, json_filename) execute(cmd) cmd = "cd %s; tar -zcvf %s %s" % (rdir, shp_filename, shpDir) execute(cmd) if force or not .path.exists(sw_osm_image): zoom = region['thn_zoom'] scale = 100 MakeBrowseImage(ds, browse_filename, subset_filename, osm_bg_image, sw_osm_image, levels, hexColors, force, verbose, zoom, scale) ds = None file_list = [ sw_osm_image, topojson_filename, topojson_filename+".gz", subset_file, shp_filename ] CopyToS3( s3_bucket, s3_folder, file_list, force, verbose )
def process_file(filename, product, variable, wrf_mode, dt, region, hours,
s3_bucket, s3_folder):
if verbose:
print "processing: " + filename + " product:", product, " variable:" + variable
zoom = region['thn_zoom']
fpathname = os.path.dirname(filename)
fdir = os.path.join(fpathname, dt, product)
if not os.path.exists(fdir):
os.makedirs(fdir)
if force:
cmd = "rm -rf " + fdir + "/*"
execute(cmd)
geojsonDir = os.path.join(fdir, "geojson")
if not os.path.exists(geojsonDir):
os.makedirs(geojsonDir)
levelsDir = os.path.join(fdir, "levels")
if not os.path.exists(levelsDir):
os.makedirs(levelsDir)
levels = [377, 233, 144, 89, 55, 34, 21, 13, 8, 5, 3]
colors = [
"#4D004A", "#810F7C", "#084081", "#0868ac", "#2b8cbe", "#4eb3d3",
"#7bccc4", "#a8ddb5", "#ccebc5", "#e0f3db", "#f7fcf0"
]
output_file = os.path.join(fdir, "%s.%s_MERC.tif" % (product, dt))
flipped_file = os.path.join(fdir, "%s.%s_MERC_flipped.tif" % (product, dt))
supersamp_file = os.path.join(fdir,
"%s.%s_MERC_flipped_100.tif" % (product, dt))
reproj_file = os.path.join(fdir, "%s.%s.tif" % (product, dt))
reproj_rgb_file = os.path.join(fdir, "%s.%s_rgb.tif" % (product, dt))
color_file = os.path.join("cluts", "wrf_colors.txt")
merge_filename = os.path.join(geojsonDir, "%s.%s.geojson" % (product, dt))
topojson_filename = os.path.join(geojsonDir, "..",
"%s.%s.topojson" % (product, dt))
browse_filename = os.path.join(geojsonDir, "..",
"%s.%s_browse.tif" % (product, dt))
subset_filename = os.path.join(geojsonDir, "..",
"%s.%s_small_browse.tif" % (product, dt))
subset_aux_filename = os.path.join(
geojsonDir, "..", "%s.%s_small_browse.tif.aux" % (product, dt))
osm_bg_image = os.path.join(fdir, "..", "osm_bg.png")
sw_osm_image = os.path.join(geojsonDir, "..",
"%s.%s_thn.jpg" % (product, dt))
if force or not os.path.exists(output_file):
grbs = pygrib.open(filename)
grbvars = grbs.select(name=variable)
count = 0
total = []
for grbvar in grbvars:
grbstr = "%s" % grbvar
arr = grbstr.split(':')
# 1090:Total Precipitation:kg m**-2 (instant):mercator:surface:level 0:fcst time 1 hrs:from 201604241200
id = arr[0]
name = arr[1]
rate = arr[2]
fcst = arr[6]
dt = arr[7].replace('from ', '')
if (rate.find('accum') > 0):
count += 1
print id, name, rate, fcst, dt
if count == 1:
total = grbvar['values']
else:
total += grbvar['values']
#lats, lons = p.latlons()
if count > hours:
break
grbs.close()
# Get critical info from GRIB file
ds = gdal.Open(filename)
ncols = ds.RasterXSize
nrows = ds.RasterYSize
geotransform = ds.GetGeoTransform()
projection = ds.GetProjection()
# Create gtif
driver = gdal.GetDriverByName("GTiff")
dst_ds = driver.Create(output_file, ncols, nrows, 1, gdal.GDT_Float32)
band = dst_ds.GetRasterBand(1)
band.WriteArray(total, 0, 0)
dst_ds.SetGeoTransform(geotransform)
dst_ds.SetProjection(projection)
dst_ds = None
ds = None
# Flip it since it is bottom up
if force or not os.path.exists(flipped_file):
cmd = "flip_raster.py -o %s %s" % (flipped_file, output_file)
execute(cmd)
# Reproject to EPSG:4326
if force or not os.path.exists(reproj_file):
cmd = "gdalwarp -q -t_srs EPSG:4326 " + flipped_file + " " + reproj_file
execute(cmd)
ds = gdal.Open(reproj_file)
geotransform = ds.GetGeoTransform()
pixelsize = geotransform[1]
ds = None
# Supersample and interpolate to make it output smoother
if force or not os.path.exists(supersamp_file):
cmd = "gdalwarp -q -multi -tr %f %f -r cubicspline %s %s" % (
pixelsize / 10, pixelsize / 10, reproj_file, supersamp_file)
execute(cmd)
# Color it using colormap
if verbose and not os.path.exists(reproj_rgb_file):
cmd = "gdaldem color-relief -q -alpha " + supersamp_file + " " + color_file + " " + reproj_rgb_file
execute(cmd)
ds = gdal.Open(supersamp_file)
band = ds.GetRasterBand(1)
data = band.ReadAsArray(0, 0, ds.RasterXSize, ds.RasterYSize)
geotransform = ds.GetGeoTransform()
xorg = geotransform[0]
yorg = geotransform[3]
pres = geotransform[1]
xmax = xorg + geotransform[1] * ds.RasterXSize
ymax = yorg - geotransform[1] * ds.RasterYSize
if force or not os.path.exists(topojson_filename + ".gz"):
for l in levels:
fileName = os.path.join(levelsDir, dt + "_level_%d.tif" % l)
CreateLevel(l, geojsonDir, fileName, ds, data, "precip", force,
verbose)
jsonDict = dict(type='FeatureCollection', features=[])
for l in reversed(levels):
fileName = os.path.join(geojsonDir, "precip_level_%d.geojson" % l)
if os.path.exists(fileName):
if verbose:
print "merge", fileName
with open(fileName) as data_file:
data = json.load(data_file)
if 'features' in data:
for f in data['features']:
jsonDict['features'].append(f)
with open(merge_filename, 'w') as outfile:
json.dump(jsonDict, outfile)
if verbose:
output = " "
else:
output = " > /dev/null 2>&1"
# Convert to topojson
cmd = "topojson -p -o " + topojson_filename + " " + merge_filename + output
execute(cmd)
if verbose:
cmd = "gzip --keep -f " + topojson_filename
else:
cmd = "gzip -f " + topojson_filename
execute(cmd)
if not os.path.exists(osm_bg_image):
ullat = yorg
ullon = xorg
lrlat = ymax
lrlon = xmax
wms(ullat, ullon, lrlat, lrlon, osm_bg_image)
if force or not os.path.exists(sw_osm_image):
scale = 2
MakeBrowseImage(ds, browse_filename, supersamp_file, osm_bg_image,
sw_osm_image, levels, colors, force, verbose, zoom,
scale)
ds = None
file_list = [sw_osm_image, topojson_filename + ".gz", reproj_file]
CopyToS3(s3_bucket, s3_folder, file_list, force, verbose)
if not verbose: # Cleanup
cmd = "rm -rf %s %s %s %s %s %s %s %s" % (
flipped_file, output_file, supersamp_file, browse_filename,
subset_filename, subset_aux_filename, geojsonDir, levelsDir)
execute(cmd)
def process(mydir, scene, s3_bucket, s3_folder): fullName = os.path.join(mydir, scene+".tif") if not os.path.exists(fullName): print "File does not exist", fullName sys.exit(-1) # Flood inundation map for Namibia has to large of an extent [[10,-30,30,-10]] # we can trim it [15, -20, 20, -10] subsetFileName = os.path.join(mydir, "%s_subset.tif" % scene) if force or not os.path.exists(subsetFileName): bbox = [15, -20, 20, -12] warpOptions = "-q -overwrite -co COMPRESS=DEFLATE -t_srs EPSG:4326 -te %s %s %s %s " % (bbox[0], bbox[1], bbox[2], bbox[3]) warpCmd = 'gdalwarp ' + warpOptions + fullName + ' ' + subsetFileName execute( warpCmd ) #sys.exit(-1) geojsonDir = os.path.join(mydir,"geojson") if not os.path.exists(geojsonDir): os.makedirs(geojsonDir) levelsDir = os.path.join(mydir,"levels") if not os.path.exists(levelsDir): os.makedirs(levelsDir) merge_filename = os.path.join(geojsonDir, "ef5.%s.geojson" % scene) topojson_filename = os.path.join(geojsonDir, "..", "ef5.%s.topojson" % scene) browse_filename = os.path.join(geojsonDir, "..", "ef5.%s_browse.tif" % scene) small_filename = os.path.join(geojsonDir, "..", "ef5.%s_small_browse.tif" % scene) osm_bg_image = os.path.join(geojsonDir, "..", "osm_bg.png") sw_osm_image = os.path.join(geojsonDir, "..", "ef5.%s_thn.jpg" % scene) ds = gdal.Open( subsetFileName ) band = ds.GetRasterBand(1) data = band.ReadAsArray(0, 0, ds.RasterXSize, ds.RasterYSize ) levels = [ 21, 13, 8, 5, 3, 2, 1] hexColors = [ "#fee5d9", "#fcbba1", "#fc9272", "#fb6a4a", "#ef3b2c", "#cb181d", "#99000d"] if force or not os.path.exists(topojson_filename+".gz"): if verbose: print "Processing", subsetFileName for l in levels: fileName = os.path.join(levelsDir, scene+"_level_%d.tif"%l) CreateLevel(l, geojsonDir, fileName, ds, data, "height", force, verbose) jsonDict = dict(type='FeatureCollection', features=[]) for l in reversed(levels): fileName = os.path.join(geojsonDir, "height_level_%d.geojson"%l) if os.path.exists(fileName): print "merge", fileName with open(fileName) as data_file: data = json.load(data_file) if 'features' in data: for f in data['features']: jsonDict['features'].append(f) with open(merge_filename, 'w') as outfile: json.dump(jsonDict, outfile) # Convert to topojson cmd = "topojson -p -o "+ topojson_filename + " " + merge_filename execute(cmd) cmd = "gzip --keep "+ topojson_filename execute(cmd) if force or not os.path.exists(sw_osm_image): MakeBrowseImage(ds, browse_filename, subsetFileName, osm_bg_image, sw_osm_image, levels, hexColors, force, verbose, 6) # we could remove geojsonDir and levelsDir #cmd = "rm -rf %s %s" %(geojsonDir, levelsDir) ds = None file_list = [ sw_osm_image, topojson_filename, topojson_filename+".gz", fullName ] CopyToS3( s3_bucket, s3_folder, file_list, force, verbose )