def process_url(mydir, url, ymd, bbox, zoom, s3_bucket, s3_folder):
csv_filename = os.path.join(os.path.join(mydir,
"modis_af." + ymd + '.csv'))
geojson_filename = os.path.join(
os.path.join(mydir, "modis_af." + ymd + '.geojson'))
geojsongz_filename = os.path.join(
os.path.join(mydir, "modis_af." + ymd + '.geojson.gz'))
tif_filename = os.path.join(os.path.join(mydir,
"modis_af." + ymd + '.tif'))
osm_bg_image = os.path.join(os.path.join(mydir, "osm_bg_image.tif"))
thn_image = os.path.join(
os.path.join(mydir, "modis_af." + ymd + '_thn.jpg'))
if force or not os.path.exists(csv_filename):
urllib.urlretrieve(url, csv_filename)
if force or not os.path.exists(geojson_filename):
csv_to_geojson(csv_filename, geojson_filename, bbox)
if force or not os.path.exists(geojsongz_filename):
cmd = 'gzip < %s > %s' % (geojson_filename, geojsongz_filename)
execute(cmd)
#url = "https://firms.modaps.eosdis.nasa.gov/wms/?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&LAYERS=fires24&width=400&height=250&BBOX=54,5.5,102,40"
#if force or not os.path.exists(tif_filename):
# urllib.urlretrieve(url, tif_filename)
# print "retrieved ", tif_filename
centerlat = (bbox[1] + bbox[3]) / 2
centerlon = (bbox[0] + bbox[2]) / 2
rasterXSize = 400
rasterYSize = 250
mapbox_image(centerlat, centerlon, zoom, rasterXSize, rasterYSize,
osm_bg_image)
ullon, ullat, lrlon, lrlat = browseimage.Gen_bbox(centerlat, centerlon,
zoom, rasterXSize,
rasterYSize)
#print ullon, lrlat, lrlon, ullat
url = "https://firms.modaps.eosdis.nasa.gov/wms/?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&LAYERS=fires24&width=400&height=250&BBOX="
url += str(ullon) + "," + str(lrlat) + "," + str(lrlon) + "," + str(ullat)
if force or not os.path.exists(tif_filename):
urllib.urlretrieve(url, tif_filename)
#print "retrieved ", tif_filename
# superimpose the suface water over map background
if force or not os.path.isfile(thn_image):
cmd = str.format("composite -gravity center {0} {1} {2}", tif_filename,
osm_bg_image, thn_image)
execute(cmd)
file_list = [tif_filename, geojson_filename, geojsongz_filename, thn_image]
CopyToS3(s3_bucket, s3_folder, file_list, force, verbose)
def process_url( mydir, url, ymd, bbox, zoom, s3_bucket, s3_folder ):
csv_filename = os.path.join(os.path.join(mydir, "modis_af." + ymd + '.csv'))
geojson_filename = os.path.join(os.path.join(mydir, "modis_af." + ymd + '.geojson'))
geojsongz_filename = os.path.join(os.path.join(mydir, "modis_af." + ymd + '.geojson.gz'))
tif_filename = os.path.join(os.path.join(mydir, "modis_af." + ymd + '.tif'))
osm_bg_image = os.path.join(os.path.join(mydir, "osm_bg_image.tif"))
thn_image = os.path.join(os.path.join(mydir, "modis_af." + ymd + '_thn.jpg'))
if force or not os.path.exists(csv_filename):
print "retrieving:", url
urllib.urlretrieve(url, csv_filename)
#request = urllib2.Request(url, csv_filename)
#data = urllib2.urlopen(request).read()
#data = requests.get(url, verify=False)
print data
sys.exit(-1)
if force or not os.path.exists(geojson_filename):
csv_to_geojson(csv_filename, geojson_filename, bbox)
if force or not os.path.exists(geojsongz_filename):
cmd = 'gzip < %s > %s' %( geojson_filename, geojsongz_filename)
execute(cmd)
#url = "https://firms.modaps.eosdis.nasa.gov/wms/?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&LAYERS=fires24&width=400&height=250&BBOX=54,5.5,102,40"
#if force or not os.path.exists(tif_filename):
# urllib.urlretrieve(url, tif_filename)
# print "retrieved ", tif_filename
centerlat = (bbox[1]+bbox[3])/2
centerlon = (bbox[0]+bbox[2])/2
rasterXSize = 400
rasterYSize = 250
mapbox_image(centerlat, centerlon, zoom, rasterXSize, rasterYSize, osm_bg_image)
ullon, ullat, lrlon, lrlat = browseimage.bbox(centerlat, centerlon, zoom, rasterXSize, rasterYSize)
print ullon, lrlat, lrlon, ullat
url = "https://firms.modaps.eosdis.nasa.gov/wms/?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&LAYERS=fires24&width=400&height=250&BBOX="
url += str(ullon) + ","+ str(lrlat) + "," + str(lrlon) + "," + str(ullat)
if 1 or force or not os.path.exists(tif_filename):
urllib.urlretrieve(url, tif_filename)
print "retrieved ", tif_filename
# superimpose the suface water over map background
#if force or not os.path.isfile(sw_osm_image):
if force or not os.path.isfile(thn_image):
cmd = str.format("composite -gravity center {0} {1} {2}", tif_filename, osm_bg_image, thn_image)
execute(cmd)
file_list = [ tif_filename, geojson_filename, geojsongz_filename, thn_image ]
CopyToS3( s3_bucket, s3_folder, file_list, force, verbose )
def process_file( mydir, fullName, ymd, bbox, zoom, s3_bucket, s3_folder ):
geojson_filename = os.path.join(os.path.join(mydir, "modis_af." + ymd + '.geojson'))
geojson_filename_subset = os.path.join(os.path.join(mydir, "modis_af_subset." + ymd + '.geojson'))
geojsongz_filename = os.path.join(os.path.join(mydir, "modis_af." + ymd + '.geojson.gz'))
tif_filename = os.path.join(os.path.join(mydir, "modis_af." + ymd + '.tif'))
osm_bg_image = os.path.join(os.path.join(mydir, "osm_bg_image.tif"))
thn_image = os.path.join(os.path.join(mydir, "modis_af." + ymd + '_thn.jpg'))
# create geojson form shapefile
if force or not os.path.exits(geojson_filename):
cmd = "ogr2ogr -f GeoJSON %s %s" % (geojson_filename, fullName )
execute(cmd)
#if force or not os.path.exists(geojson_filename):
# geojson_to_geojson(geojson_filename, geojson_filename_subset, bbox)
if force or not os.path.exists(geojsongz_filename):
cmd = 'gzip < %s > %s' %( geojson_filename, geojsongz_filename)
execute(cmd)
centerlat = (bbox[1]+bbox[3])/2
centerlon = (bbox[0]+bbox[2])/2
rasterXSize = 400
rasterYSize = 250
mapbox_image(centerlat, centerlon, zoom, rasterXSize, rasterYSize, osm_bg_image)
ullon, ullat, lrlon, lrlat = browseimage.bbox(centerlat, centerlon, zoom, rasterXSize, rasterYSize)
print ullon, lrlat, lrlon, ullat
url = "https://firms.modaps.eosdis.nasa.gov/wms/?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&LAYERS=fires24&width=400&height=250&BBOX="
url += str(ullon) + ","+ str(lrlat) + "," + str(lrlon) + "," + str(ullat)
if 1 or force or not os.path.exists(tif_filename):
urllib.urlretrieve(url, tif_filename)
print "retrieved ", tif_filename
# superimpose the suface water over map background
#if force or not os.path.isfile(sw_osm_image):
if force or not os.path.isfile(thn_image):
cmd = str.format("composite -gravity center {0} {1} {2}", tif_filename, osm_bg_image, thn_image)
execute(cmd)
file_list = [ tif_filename, geojson_filename, geojsongz_filename, thn_image ]
CopyToS3( s3_bucket, s3_folder, file_list, force, verbose )
def process_url( mydir, url, ymd, bbox, zoom, s3_bucket, s3_folder ):
csv_filename = os.path.join(os.path.join(mydir, "viirs_af." + ymd + '.csv'))
geojson_filename = os.path.join(os.path.join(mydir, "viirs_af." + ymd + '.geojson'))
geojsongz_filename = os.path.join(os.path.join(mydir, "viirs_af." + ymd + '.geojson.gz'))
tif_filename = os.path.join(os.path.join(mydir, "viirs_af." + ymd + '.tif'))
osm_bg_image = os.path.join(os.path.join(mydir, "osm_bg_image.tif"))
thn_image = os.path.join(os.path.join(mydir, "viirs_af." + ymd + '_thn.jpg'))
if verbose:
print "sys version", sys.version_info[0], sys.version_info[1], sys.version_info[2]
if (sys.version_info[0] == 2) and (sys.version_info[1] >= 7) and (sys.version_info[2] > 9):
context = ssl._create_unverified_context()
else:
context = None
if force or not os.path.exists(csv_filename):
if verbose:
print "retrieve", url, csv_filename
if context:
urllib.urlretrieve(url, csv_filename, context=context)
else:
urllib.urlretrieve(url, csv_filename)
if force or not os.path.exists(geojson_filename):
csv_to_geojson(csv_filename, geojson_filename, bbox)
if force or not os.path.exists(geojsongz_filename):
cmd = 'gzip -f < %s > %s' %( geojson_filename, geojsongz_filename)
execute(cmd)
#url = "https://firms.modaps.eosdis.nasa.gov/wms/?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&LAYERS=fires24&width=400&height=250&BBOX=54,5.5,102,40"
#if force or not os.path.exists(tif_filename):
# urllib.urlretrieve(url, tif_filename)
# print "retrieved ", tif_filename
centerlat = (bbox[1]+bbox[3])/2
centerlon = (bbox[0]+bbox[2])/2
rasterXSize = 400
rasterYSize = 250
mapbox_image(centerlat, centerlon, zoom, rasterXSize, rasterYSize, osm_bg_image)
ullon, ullat, lrlon, lrlat = browseimage.Gen_bbox(centerlat, centerlon, zoom, rasterXSize, rasterYSize)
#print ullon, lrlat, lrlon, ullat
url = "https://firms.modaps.eosdis.nasa.gov/wms/?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&LAYERS=fires24&width=400&height=250&BBOX="
url += str(ullon) + ","+ str(lrlat) + "," + str(lrlon) + "," + str(ullat)
if force or not os.path.exists(tif_filename):
if verbose:
print "retrieve", tif_filename
if context:
urllib.urlretrieve(url, tif_filename, context=context)
else:
urllib.urlretrieve(url, tif_filename)
#print "retrieved ", tif_filename
# superimpose the suface water over map background
if force or not os.path.isfile(thn_image):
cmd = str.format("composite -quiet -gravity center {0} {1} {2}", tif_filename, osm_bg_image, thn_image)
execute(cmd)
file_list = [ tif_filename, geojson_filename, geojsongz_filename, thn_image ]
CopyToS3( s3_bucket, s3_folder, file_list, force, verbose )
logger.info(
"gpm_movie.py starTime: %s endTime: %s zoom: %s, lat: %f lon: %f outputDir: %s",
startTime, endTime, z, centerlat, centerlon, outputDir)
goldenratio = 1.618
height = 610
width = 1024
if not os.path.exists(outputDir):
os.mkdir(outputDir)
osm_bg_image = os.path.join(outputDir, "mapbox_map.png")
# Let's generate a map background
if force or not os.path.exists(osm_bg_image):
mapbox_image(centerlat, centerlon, z, width, height, osm_bg_image)
ullon, ullat, lrlon, lrlat = Gen_bbox(centerlat, centerlon, z, width,
height)
#
# Let's make sure we have all the files one day at a time
#
done = 0
logger.info("Checking GPM files availability...")
while not done:
get_30mn_files(outputDir, year, month, day, ymd)
startTime += timedelta(days=1)
if (startTime > endTime):
done = 1
else:
def process_url( mydir, url, ymd, bbox, zoom, s3_bucket, s3_folder ):
orig_filename = os.path.join(os.path.join(mydir, "..", "quakes." + ymd + '.geojson'))
csv_filename = os.path.join(os.path.join(mydir, "quakes." + ymd + '.csv'))
geojson_filename = os.path.join(os.path.join(mydir, "quakes." + ymd + '.geojson'))
geojsongz_filename = os.path.join(os.path.join(mydir, "quakes." + ymd + '.geojson.gz'))
tif_filename = os.path.join(os.path.join(mydir, "quakes." + ymd + '.tif'))
osm_bg_image = os.path.join(os.path.join(mydir, "osm_bg_image.tif"))
thn_image = os.path.join(os.path.join(mydir, "quakes." + ymd + '_thn.jpg'))
if force or not os.path.exists(orig_filename):
if verbose:
print "retrieving:", orig_filename
urllib.urlretrieve(url, orig_filename)
json_data = open(orig_filename).read()
data = json.loads(json_data)
results = {}
results['type'] = "FeatureCollection"
results['bbox'] = data['bbox']
results['metadata'] = data['metadata']
results['features'] = []
for f in data['features']:
coords = f['geometry']['coordinates']
lon = coords[0]
lat = coords[1]
if inbbox(bbox, lat, lon):
qtime = int(f['properties']['time'])/1000
f['properties']['date'] = time.ctime(qtime)
newprops = {
'type': f['properties']['type'],
'title': f['properties']['title'],
'place': f['properties']['place'],
'mag': f['properties']['mag'],
'rms': f['properties']['rms'],
'status': f['properties']['status'],
'date': f['properties']['date'],
'detail': f['properties']['detail'],
'url': f['properties']['url']
}
f['properties'] = newprops
results['features'].append(f)
if verbose:
print "found", len(results['features'])
with open(geojson_filename, 'w') as outfile:
json.dump(results, outfile)
if force or not os.path.exists(geojsongz_filename):
cmd = 'gzip < %s > %s' %( geojson_filename, geojsongz_filename)
execute(cmd)
centerlat = (bbox[1]+bbox[3])/2
centerlon = (bbox[0]+bbox[2])/2
rasterXSize = 400
rasterYSize = 250
mapbox_image(centerlat, centerlon, zoom, rasterXSize, rasterYSize, osm_bg_image)
ullon, ullat, lrlon, lrlat = browseimage.Gen_bbox(centerlat, centerlon, zoom, rasterXSize, rasterYSize)
dx = (lrlon-ullon)/rasterXSize
dy = (ullat-lrlat)/rasterXSize
#print "org:", ullon, ullat, dx, dy
im = Image.open(osm_bg_image)
draw = ImageDraw.Draw(im)
for f in results['features']:
coords = f['geometry']['coordinates']
lon = coords[0]
lat = coords[1]
x = int((lon-ullon)/dx)
y = int((ullat-lat)/dx)
#print lon, lat, x, y
draw.ellipse( [(x-1,y-1),(x+1,y+1)])
im.save(tif_filename, "PNG")
# superimpose the suface water over map background
#if force or not os.path.isfile(sw_osm_image):
if force or not os.path.isfile(thn_image):
#cmd = str.format("composite -gravity center {0} {1} {2}", tif_filename, osm_bg_image, thn_image)
cmd = "cp %s %s" % (tif_filename, thn_image)
execute(cmd)
file_list = [ geojson_filename, geojsongz_filename, thn_image ]
CopyToS3( s3_bucket, s3_folder, file_list, force, verbose )
if not verbose:
cmd = "rm -rf %s %s" % (tif_filename, osm_bg_image)
execute(cmd)
outputDir = options.outputDir
logger.info ("gpm_movie.py starTime: %s endTime: %s zoom: %s, lat: %f lon: %f outputDir: %s", startTime , endTime, z, centerlat, centerlon, outputDir)
goldenratio = 1.618
height = 610
width = 1024
if not os.path.exists(outputDir):
os.mkdir(outputDir)
osm_bg_image = os.path.join(outputDir, "mapbox_map.png")
# Let's generate a map background
if force or not os.path.exists(osm_bg_image):
mapbox_image(centerlat, centerlon, z, width, height, osm_bg_image)
ullon, ullat, lrlon, lrlat = Gen_bbox(centerlat, centerlon, z, width, height)
#
# Let's make sure we have all the files one day at a time
#
done = 0
logger.info("Checking GPM files availability...")
while not done:
get_30mn_files(outputDir, year, month, day, ymd)
startTime += timedelta(days=1)
if( startTime > endTime):
done = 1
else:
year = startTime.year
def process_url(mydir, url, ymd, bbox, zoom, s3_bucket, s3_folder):
orig_filename = os.path.join(os.path.join(mydir, "..", "quakes." + ymd + ".geojson"))
csv_filename = os.path.join(os.path.join(mydir, "quakes." + ymd + ".csv"))
geojson_filename = os.path.join(os.path.join(mydir, "quakes." + ymd + ".geojson"))
geojsongz_filename = os.path.join(os.path.join(mydir, "quakes." + ymd + ".geojson.gz"))
tif_filename = os.path.join(os.path.join(mydir, "quakes." + ymd + ".tif"))
osm_bg_image = os.path.join(os.path.join(mydir, "osm_bg_image.tif"))
thn_image = os.path.join(os.path.join(mydir, "quakes." + ymd + "_thn.jpg"))
if force or not os.path.exists(orig_filename):
if verbose:
print "retrieving:", orig_filename
urllib.urlretrieve(url, orig_filename)
json_data = open(orig_filename).read()
data = json.loads(json_data)
results = {}
results["type"] = "FeatureCollection"
results["bbox"] = data["bbox"]
results["metadata"] = data["metadata"]
results["features"] = []
for f in data["features"]:
coords = f["geometry"]["coordinates"]
lon = coords[0]
lat = coords[1]
if inbbox(bbox, lat, lon):
qtime = int(f["properties"]["time"]) / 1000
newp = {}
newp["title"] = f["properties"]["title"]
newp["mag"] = f["properties"]["mag"]
newp["date"] = time.ctime(qtime)
newp["place"] = f["properties"]["place"]
# f['properties']['date'] = time.ctime(qtime)
f["properties"] = newp
results["features"].append(f)
print "found", len(results["features"])
with open(geojson_filename, "w") as outfile:
json.dump(results, outfile)
if force or not os.path.exists(geojsongz_filename):
cmd = "gzip < %s > %s" % (geojson_filename, geojsongz_filename)
execute(cmd)
centerlat = (bbox[1] + bbox[3]) / 2
centerlon = (bbox[0] + bbox[2]) / 2
rasterXSize = 400
rasterYSize = 250
mapbox_image(centerlat, centerlon, zoom, rasterXSize, rasterYSize, osm_bg_image)
ullon, ullat, lrlon, lrlat = browseimage.bbox(centerlat, centerlon, zoom, rasterXSize, rasterYSize)
dx = (lrlon - ullon) / rasterXSize
dy = (ullat - lrlat) / rasterXSize
# print "org:", ullon, ullat, dx, dy
im = Image.open(osm_bg_image)
draw = ImageDraw.Draw(im)
for f in results["features"]:
coords = f["geometry"]["coordinates"]
lon = coords[0]
lat = coords[1]
x = int((lon - ullon) / dx)
y = int((ullat - lat) / dx)
# print lon, lat, x, y
draw.ellipse([(x - 1, y - 1), (x + 1, y + 1)])
im.save(tif_filename, "PNG")
# superimpose the suface water over map background
# if force or not os.path.isfile(sw_osm_image):
if force or not os.path.isfile(thn_image):
# cmd = str.format("composite -gravity center {0} {1} {2}", tif_filename, osm_bg_image, thn_image)
cmd = "cp %s %s" % (tif_filename, thn_image)
execute(cmd)
file_list = [geojson_filename, geojsongz_filename, thn_image]
CopyToS3(s3_bucket, s3_folder, file_list, force, verbose)
with open(geojson_filename, 'w') as outfile: json.dump(entries, outfile) if force or not os.path.exists(geojsongz_filename): cmd = 'gzip < %s > %s' %( geojson_filename, geojsongz_filename) execute(cmd) if force or not os.path.isfile(thn_image): centerlat = (bbox[1]+bbox[3])/2 centerlon = (bbox[0]+bbox[2])/2 rasterXSize = 400 rasterYSize = 250 mapbox_image(centerlat, centerlon, zoom, rasterXSize, rasterYSize, osm_bg_image) ullon, ullat, lrlon, lrlat = browseimage.Gen_bbox(centerlat, centerlon, zoom, rasterXSize, rasterYSize) dx = (lrlon-ullon)/rasterXSize dy = (ullat-lrlat)/rasterXSize im = Image.open(osm_bg_image) draw = ImageDraw.Draw(im) for f in features : coordinates = f['geometry']['coordinates'] lon = coordinates[0] lat = coordinates[1] x = int((lon-ullon)/dx)