def process_url( mydir, url, ymd, bbox, zoom, s3_bucket, s3_folder ): csv_filename = os.path.join(os.path.join(mydir, "modis_af." + ymd + '.csv')) geojson_filename = os.path.join(os.path.join(mydir, "modis_af." + ymd + '.geojson')) geojsongz_filename = os.path.join(os.path.join(mydir, "modis_af." + ymd + '.geojson.gz')) tif_filename = os.path.join(os.path.join(mydir, "modis_af." + ymd + '.tif')) osm_bg_image = os.path.join(os.path.join(mydir, "osm_bg_image.tif")) thn_image = os.path.join(os.path.join(mydir, "modis_af." + ymd + '_thn.jpg')) if force or not os.path.exists(csv_filename): print "retrieving:", url urllib.urlretrieve(url, csv_filename) #request = urllib2.Request(url, csv_filename) #data = urllib2.urlopen(request).read() #data = requests.get(url, verify=False) print data sys.exit(-1) if force or not os.path.exists(geojson_filename): csv_to_geojson(csv_filename, geojson_filename, bbox) if force or not os.path.exists(geojsongz_filename): cmd = 'gzip < %s > %s' %( geojson_filename, geojsongz_filename) execute(cmd) #url = "https://firms.modaps.eosdis.nasa.gov/wms/?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&LAYERS=fires24&width=400&height=250&BBOX=54,5.5,102,40" #if force or not os.path.exists(tif_filename): # urllib.urlretrieve(url, tif_filename) # print "retrieved ", tif_filename centerlat = (bbox[1]+bbox[3])/2 centerlon = (bbox[0]+bbox[2])/2 rasterXSize = 400 rasterYSize = 250 mapbox_image(centerlat, centerlon, zoom, rasterXSize, rasterYSize, osm_bg_image) ullon, ullat, lrlon, lrlat = browseimage.bbox(centerlat, centerlon, zoom, rasterXSize, rasterYSize) print ullon, lrlat, lrlon, ullat url = "https://firms.modaps.eosdis.nasa.gov/wms/?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&LAYERS=fires24&width=400&height=250&BBOX=" url += str(ullon) + ","+ str(lrlat) + "," + str(lrlon) + "," + str(ullat) if 1 or force or not os.path.exists(tif_filename): urllib.urlretrieve(url, tif_filename) print "retrieved ", tif_filename # superimpose the suface water over map background #if force or not os.path.isfile(sw_osm_image): if force or not os.path.isfile(thn_image): cmd = str.format("composite -gravity center {0} {1} {2}", tif_filename, osm_bg_image, thn_image) execute(cmd) file_list = [ tif_filename, geojson_filename, geojsongz_filename, thn_image ] CopyToS3( s3_bucket, s3_folder, file_list, force, verbose )
def process_file(mydir, fullName, ymd, bbox, zoom, s3_bucket, s3_folder): geojson_filename = os.path.join( os.path.join(mydir, "modis_af." + ymd + '.geojson')) geojson_filename_subset = os.path.join( os.path.join(mydir, "modis_af_subset." + ymd + '.geojson')) geojsongz_filename = os.path.join( os.path.join(mydir, "modis_af." + ymd + '.geojson.gz')) tif_filename = os.path.join(os.path.join(mydir, "modis_af." + ymd + '.tif')) osm_bg_image = os.path.join(os.path.join(mydir, "osm_bg_image.tif")) thn_image = os.path.join( os.path.join(mydir, "modis_af." + ymd + '_thn.jpg')) # create geojson form shapefile if force or not os.path.exits(geojson_filename): cmd = "ogr2ogr -f GeoJSON %s %s" % (geojson_filename, fullName) execute(cmd) #if force or not os.path.exists(geojson_filename): # geojson_to_geojson(geojson_filename, geojson_filename_subset, bbox) if force or not os.path.exists(geojsongz_filename): cmd = 'gzip < %s > %s' % (geojson_filename, geojsongz_filename) execute(cmd) centerlat = (bbox[1] + bbox[3]) / 2 centerlon = (bbox[0] + bbox[2]) / 2 rasterXSize = 400 rasterYSize = 250 mapbox_image(centerlat, centerlon, zoom, rasterXSize, rasterYSize, osm_bg_image) ullon, ullat, lrlon, lrlat = browseimage.bbox(centerlat, centerlon, zoom, rasterXSize, rasterYSize) print ullon, lrlat, lrlon, ullat url = "https://firms.modaps.eosdis.nasa.gov/wms/?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&LAYERS=fires24&width=400&height=250&BBOX=" url += str(ullon) + "," + str(lrlat) + "," + str(lrlon) + "," + str(ullat) if 1 or force or not os.path.exists(tif_filename): urllib.urlretrieve(url, tif_filename) print "retrieved ", tif_filename # superimpose the suface water over map background #if force or not os.path.isfile(sw_osm_image): if force or not os.path.isfile(thn_image): cmd = str.format("composite -gravity center {0} {1} {2}", tif_filename, osm_bg_image, thn_image) execute(cmd) file_list = [tif_filename, geojson_filename, geojsongz_filename, thn_image] CopyToS3(s3_bucket, s3_folder, file_list, force, verbose)
def process_file( mydir, fullName, ymd, bbox, zoom, s3_bucket, s3_folder ): geojson_filename = os.path.join(os.path.join(mydir, "modis_af." + ymd + '.geojson')) geojson_filename_subset = os.path.join(os.path.join(mydir, "modis_af_subset." + ymd + '.geojson')) geojsongz_filename = os.path.join(os.path.join(mydir, "modis_af." + ymd + '.geojson.gz')) tif_filename = os.path.join(os.path.join(mydir, "modis_af." + ymd + '.tif')) osm_bg_image = os.path.join(os.path.join(mydir, "osm_bg_image.tif")) thn_image = os.path.join(os.path.join(mydir, "modis_af." + ymd + '_thn.jpg')) # create geojson form shapefile if force or not os.path.exits(geojson_filename): cmd = "ogr2ogr -f GeoJSON %s %s" % (geojson_filename, fullName ) execute(cmd) #if force or not os.path.exists(geojson_filename): # geojson_to_geojson(geojson_filename, geojson_filename_subset, bbox) if force or not os.path.exists(geojsongz_filename): cmd = 'gzip < %s > %s' %( geojson_filename, geojsongz_filename) execute(cmd) centerlat = (bbox[1]+bbox[3])/2 centerlon = (bbox[0]+bbox[2])/2 rasterXSize = 400 rasterYSize = 250 mapbox_image(centerlat, centerlon, zoom, rasterXSize, rasterYSize, osm_bg_image) ullon, ullat, lrlon, lrlat = browseimage.bbox(centerlat, centerlon, zoom, rasterXSize, rasterYSize) print ullon, lrlat, lrlon, ullat url = "https://firms.modaps.eosdis.nasa.gov/wms/?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&LAYERS=fires24&width=400&height=250&BBOX=" url += str(ullon) + ","+ str(lrlat) + "," + str(lrlon) + "," + str(ullat) if 1 or force or not os.path.exists(tif_filename): urllib.urlretrieve(url, tif_filename) print "retrieved ", tif_filename # superimpose the suface water over map background #if force or not os.path.isfile(sw_osm_image): if force or not os.path.isfile(thn_image): cmd = str.format("composite -gravity center {0} {1} {2}", tif_filename, osm_bg_image, thn_image) execute(cmd) file_list = [ tif_filename, geojson_filename, geojsongz_filename, thn_image ] CopyToS3( s3_bucket, s3_folder, file_list, force, verbose )
def process_url(mydir, url, ymd, bbox, zoom, s3_bucket, s3_folder): orig_filename = os.path.join(os.path.join(mydir, "..", "quakes." + ymd + ".geojson")) csv_filename = os.path.join(os.path.join(mydir, "quakes." + ymd + ".csv")) geojson_filename = os.path.join(os.path.join(mydir, "quakes." + ymd + ".geojson")) geojsongz_filename = os.path.join(os.path.join(mydir, "quakes." + ymd + ".geojson.gz")) tif_filename = os.path.join(os.path.join(mydir, "quakes." + ymd + ".tif")) osm_bg_image = os.path.join(os.path.join(mydir, "osm_bg_image.tif")) thn_image = os.path.join(os.path.join(mydir, "quakes." + ymd + "_thn.jpg")) if force or not os.path.exists(orig_filename): if verbose: print "retrieving:", orig_filename urllib.urlretrieve(url, orig_filename) json_data = open(orig_filename).read() data = json.loads(json_data) results = {} results["type"] = "FeatureCollection" results["bbox"] = data["bbox"] results["metadata"] = data["metadata"] results["features"] = [] for f in data["features"]: coords = f["geometry"]["coordinates"] lon = coords[0] lat = coords[1] if inbbox(bbox, lat, lon): qtime = int(f["properties"]["time"]) / 1000 newp = {} newp["title"] = f["properties"]["title"] newp["mag"] = f["properties"]["mag"] newp["date"] = time.ctime(qtime) newp["place"] = f["properties"]["place"] # f['properties']['date'] = time.ctime(qtime) f["properties"] = newp results["features"].append(f) print "found", len(results["features"]) with open(geojson_filename, "w") as outfile: json.dump(results, outfile) if force or not os.path.exists(geojsongz_filename): cmd = "gzip < %s > %s" % (geojson_filename, geojsongz_filename) execute(cmd) centerlat = (bbox[1] + bbox[3]) / 2 centerlon = (bbox[0] + bbox[2]) / 2 rasterXSize = 400 rasterYSize = 250 mapbox_image(centerlat, centerlon, zoom, rasterXSize, rasterYSize, osm_bg_image) ullon, ullat, lrlon, lrlat = browseimage.bbox(centerlat, centerlon, zoom, rasterXSize, rasterYSize) dx = (lrlon - ullon) / rasterXSize dy = (ullat - lrlat) / rasterXSize # print "org:", ullon, ullat, dx, dy im = Image.open(osm_bg_image) draw = ImageDraw.Draw(im) for f in results["features"]: coords = f["geometry"]["coordinates"] lon = coords[0] lat = coords[1] x = int((lon - ullon) / dx) y = int((ullat - lat) / dx) # print lon, lat, x, y draw.ellipse([(x - 1, y - 1), (x + 1, y + 1)]) im.save(tif_filename, "PNG") # superimpose the suface water over map background # if force or not os.path.isfile(sw_osm_image): if force or not os.path.isfile(thn_image): # cmd = str.format("composite -gravity center {0} {1} {2}", tif_filename, osm_bg_image, thn_image) cmd = "cp %s %s" % (tif_filename, thn_image) execute(cmd) file_list = [geojson_filename, geojsongz_filename, thn_image] CopyToS3(s3_bucket, s3_folder, file_list, force, verbose)
def process_url(mydir, url, ymd, bbox, zoom, s3_bucket, s3_folder): csv_filename = os.path.join(os.path.join(mydir, "modis_af." + ymd + '.csv')) geojson_filename = os.path.join( os.path.join(mydir, "modis_af." + ymd + '.geojson')) geojsongz_filename = os.path.join( os.path.join(mydir, "modis_af." + ymd + '.geojson.gz')) tif_filename = os.path.join(os.path.join(mydir, "modis_af." + ymd + '.tif')) osm_bg_image = os.path.join(os.path.join(mydir, "osm_bg_image.tif")) thn_image = os.path.join( os.path.join(mydir, "modis_af." + ymd + '_thn.jpg')) if force or not os.path.exists(csv_filename): print "retrieving:", url urllib.urlretrieve(url, csv_filename) #request = urllib2.Request(url, csv_filename) #data = urllib2.urlopen(request).read() #data = requests.get(url, verify=False) print data sys.exit(-1) if force or not os.path.exists(geojson_filename): csv_to_geojson(csv_filename, geojson_filename, bbox) if force or not os.path.exists(geojsongz_filename): cmd = 'gzip < %s > %s' % (geojson_filename, geojsongz_filename) execute(cmd) #url = "https://firms.modaps.eosdis.nasa.gov/wms/?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&LAYERS=fires24&width=400&height=250&BBOX=54,5.5,102,40" #if force or not os.path.exists(tif_filename): # urllib.urlretrieve(url, tif_filename) # print "retrieved ", tif_filename centerlat = (bbox[1] + bbox[3]) / 2 centerlon = (bbox[0] + bbox[2]) / 2 rasterXSize = 400 rasterYSize = 250 mapbox_image(centerlat, centerlon, zoom, rasterXSize, rasterYSize, osm_bg_image) ullon, ullat, lrlon, lrlat = browseimage.bbox(centerlat, centerlon, zoom, rasterXSize, rasterYSize) print ullon, lrlat, lrlon, ullat url = "https://firms.modaps.eosdis.nasa.gov/wms/?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&LAYERS=fires24&width=400&height=250&BBOX=" url += str(ullon) + "," + str(lrlat) + "," + str(lrlon) + "," + str(ullat) if 1 or force or not os.path.exists(tif_filename): urllib.urlretrieve(url, tif_filename) print "retrieved ", tif_filename # superimpose the suface water over map background #if force or not os.path.isfile(sw_osm_image): if force or not os.path.isfile(thn_image): cmd = str.format("composite -gravity center {0} {1} {2}", tif_filename, osm_bg_image, thn_image) execute(cmd) file_list = [tif_filename, geojson_filename, geojsongz_filename, thn_image] CopyToS3(s3_bucket, s3_folder, file_list, force, verbose)