def process_url(mydir, url, ymd, bbox, zoom, s3_bucket, s3_folder):

    csv_filename = os.path.join(os.path.join(mydir,
                                             "modis_af." + ymd + '.csv'))
    geojson_filename = os.path.join(
        os.path.join(mydir, "modis_af." + ymd + '.geojson'))
    geojsongz_filename = os.path.join(
        os.path.join(mydir, "modis_af." + ymd + '.geojson.gz'))
    tif_filename = os.path.join(os.path.join(mydir,
                                             "modis_af." + ymd + '.tif'))
    osm_bg_image = os.path.join(os.path.join(mydir, "osm_bg_image.tif"))
    thn_image = os.path.join(
        os.path.join(mydir, "modis_af." + ymd + '_thn.jpg'))

    if force or not os.path.exists(csv_filename):
        urllib.urlretrieve(url, csv_filename)

    if force or not os.path.exists(geojson_filename):
        csv_to_geojson(csv_filename, geojson_filename, bbox)

    if force or not os.path.exists(geojsongz_filename):
        cmd = 'gzip < %s > %s' % (geojson_filename, geojsongz_filename)
        execute(cmd)

    #url = "https://firms.modaps.eosdis.nasa.gov/wms/?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&LAYERS=fires24&width=400&height=250&BBOX=54,5.5,102,40"

    #if force or not os.path.exists(tif_filename):
    #	urllib.urlretrieve(url, tif_filename)
    #	print "retrieved ", tif_filename

    centerlat = (bbox[1] + bbox[3]) / 2
    centerlon = (bbox[0] + bbox[2]) / 2
    rasterXSize = 400
    rasterYSize = 250

    mapbox_image(centerlat, centerlon, zoom, rasterXSize, rasterYSize,
                 osm_bg_image)

    ullon, ullat, lrlon, lrlat = browseimage.Gen_bbox(centerlat, centerlon,
                                                      zoom, rasterXSize,
                                                      rasterYSize)
    #print ullon, lrlat, lrlon, ullat

    url = "https://firms.modaps.eosdis.nasa.gov/wms/?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&LAYERS=fires24&width=400&height=250&BBOX="
    url += str(ullon) + "," + str(lrlat) + "," + str(lrlon) + "," + str(ullat)

    if force or not os.path.exists(tif_filename):
        urllib.urlretrieve(url, tif_filename)
        #print "retrieved ", tif_filename

    # superimpose the suface water over map background
    if force or not os.path.isfile(thn_image):
        cmd = str.format("composite -gravity center {0} {1} {2}", tif_filename,
                         osm_bg_image, thn_image)
        execute(cmd)

    file_list = [tif_filename, geojson_filename, geojsongz_filename, thn_image]
    CopyToS3(s3_bucket, s3_folder, file_list, force, verbose)
def process_url( mydir, url, ymd, bbox, zoom, s3_bucket, s3_folder ):

	csv_filename		= os.path.join(os.path.join(mydir,  "modis_af." + ymd + '.csv'))
	geojson_filename	= os.path.join(os.path.join(mydir,  "modis_af." + ymd + '.geojson'))
	geojsongz_filename	= os.path.join(os.path.join(mydir,  "modis_af." + ymd + '.geojson.gz'))
	tif_filename		= os.path.join(os.path.join(mydir,  "modis_af." + ymd + '.tif'))
	osm_bg_image		= os.path.join(os.path.join(mydir,  "osm_bg_image.tif"))
	thn_image			= os.path.join(os.path.join(mydir,  "modis_af." + ymd + '_thn.jpg'))
	
	if force or not os.path.exists(csv_filename):
		print "retrieving:", url
		urllib.urlretrieve(url, csv_filename)
		#request 	= urllib2.Request(url, csv_filename)
		#data	 	= urllib2.urlopen(request).read()
		#data = requests.get(url, verify=False)
		
		
		print data
		sys.exit(-1)

	if force or not os.path.exists(geojson_filename):
		csv_to_geojson(csv_filename, geojson_filename, bbox)
		
	if force or not os.path.exists(geojsongz_filename):
		cmd = 'gzip < %s > %s' %( geojson_filename, geojsongz_filename)
		execute(cmd)

	#url = "https://firms.modaps.eosdis.nasa.gov/wms/?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&LAYERS=fires24&width=400&height=250&BBOX=54,5.5,102,40"
	
	#if force or not os.path.exists(tif_filename):
	#	urllib.urlretrieve(url, tif_filename)
	#	print "retrieved ", tif_filename
	
	centerlat 	= (bbox[1]+bbox[3])/2
	centerlon	= (bbox[0]+bbox[2])/2
	rasterXSize	= 400
	rasterYSize	= 250
	
	mapbox_image(centerlat, centerlon, zoom, rasterXSize, rasterYSize, osm_bg_image)
	
	ullon, ullat, lrlon, lrlat = browseimage.bbox(centerlat, centerlon, zoom, rasterXSize, rasterYSize)
	print ullon, lrlat, lrlon, ullat
	
	url = "https://firms.modaps.eosdis.nasa.gov/wms/?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&LAYERS=fires24&width=400&height=250&BBOX="
	url += str(ullon) + ","+ str(lrlat) + "," + str(lrlon) + "," + str(ullat)
	
	if 1 or force or not os.path.exists(tif_filename):
		urllib.urlretrieve(url, tif_filename)
		print "retrieved ", tif_filename
	
	# superimpose the suface water over map background
	#if force or not os.path.isfile(sw_osm_image):	
	if force or not os.path.isfile(thn_image):	
		cmd = str.format("composite -gravity center {0} {1} {2}", tif_filename, osm_bg_image, thn_image)
		execute(cmd)
		
	file_list = [ tif_filename, geojson_filename, geojsongz_filename, thn_image ]
	CopyToS3( s3_bucket, s3_folder, file_list, force, verbose )
def process_file( mydir, fullName, ymd, bbox, zoom, s3_bucket, s3_folder ):

	geojson_filename		= os.path.join(os.path.join(mydir,  "modis_af." + ymd + '.geojson'))
	geojson_filename_subset	= os.path.join(os.path.join(mydir,  "modis_af_subset." + ymd + '.geojson'))
	geojsongz_filename		= os.path.join(os.path.join(mydir,  "modis_af." + ymd + '.geojson.gz'))
	tif_filename			= os.path.join(os.path.join(mydir,  "modis_af." + ymd + '.tif'))
	osm_bg_image			= os.path.join(os.path.join(mydir,  "osm_bg_image.tif"))
	thn_image				= os.path.join(os.path.join(mydir,  "modis_af." + ymd + '_thn.jpg'))
	
	# create geojson form shapefile
	if force or not os.path.exits(geojson_filename):
		cmd = "ogr2ogr -f GeoJSON %s %s" % (geojson_filename, fullName )
		execute(cmd)
	
	#if force or not os.path.exists(geojson_filename):
	#	geojson_to_geojson(geojson_filename, geojson_filename_subset, bbox)
	
	if force or not os.path.exists(geojsongz_filename):
		cmd = 'gzip < %s > %s' %( geojson_filename, geojsongz_filename)
		execute(cmd)
	
	centerlat 	= (bbox[1]+bbox[3])/2
	centerlon	= (bbox[0]+bbox[2])/2
	rasterXSize	= 400
	rasterYSize	= 250
	
	mapbox_image(centerlat, centerlon, zoom, rasterXSize, rasterYSize, osm_bg_image)
	
	ullon, ullat, lrlon, lrlat = browseimage.bbox(centerlat, centerlon, zoom, rasterXSize, rasterYSize)
	print ullon, lrlat, lrlon, ullat
	
	url = "https://firms.modaps.eosdis.nasa.gov/wms/?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&LAYERS=fires24&width=400&height=250&BBOX="
	url += str(ullon) + ","+ str(lrlat) + "," + str(lrlon) + "," + str(ullat)
	
	if 1 or force or not os.path.exists(tif_filename):
		urllib.urlretrieve(url, tif_filename)
		print "retrieved ", tif_filename
	
	# superimpose the suface water over map background
	#if force or not os.path.isfile(sw_osm_image):	
	if force or not os.path.isfile(thn_image):	
		cmd = str.format("composite -gravity center {0} {1} {2}", tif_filename, osm_bg_image, thn_image)
		execute(cmd)
		
	file_list = [ tif_filename, geojson_filename, geojsongz_filename, thn_image ]
	CopyToS3( s3_bucket, s3_folder, file_list, force, verbose )
def process_url( mydir, url, ymd, bbox, zoom, s3_bucket, s3_folder ):

	csv_filename		= os.path.join(os.path.join(mydir,  "viirs_af." + ymd + '.csv'))
	geojson_filename	= os.path.join(os.path.join(mydir,  "viirs_af." + ymd + '.geojson'))
	geojsongz_filename	= os.path.join(os.path.join(mydir,  "viirs_af." + ymd + '.geojson.gz'))
	tif_filename		= os.path.join(os.path.join(mydir,  "viirs_af." + ymd + '.tif'))
	osm_bg_image		= os.path.join(os.path.join(mydir,  "osm_bg_image.tif"))
	thn_image			= os.path.join(os.path.join(mydir,  "viirs_af." + ymd + '_thn.jpg'))
	
	if verbose:
		print "sys version", sys.version_info[0], sys.version_info[1], sys.version_info[2]
	
	if (sys.version_info[0] == 2) and (sys.version_info[1] >= 7) and (sys.version_info[2] > 9):
		context 			= ssl._create_unverified_context()
	else:
		context				= None
	
	if force or not os.path.exists(csv_filename):
		if verbose:
			print "retrieve", url, csv_filename
		
		if context:
			urllib.urlretrieve(url, csv_filename, context=context)
		else:
			urllib.urlretrieve(url, csv_filename)
		
	if force or not os.path.exists(geojson_filename):
		csv_to_geojson(csv_filename, geojson_filename, bbox)
		
	if force or not os.path.exists(geojsongz_filename):
		cmd = 'gzip -f < %s > %s' %( geojson_filename, geojsongz_filename)
		execute(cmd)

	#url = "https://firms.modaps.eosdis.nasa.gov/wms/?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&LAYERS=fires24&width=400&height=250&BBOX=54,5.5,102,40"
	
	#if force or not os.path.exists(tif_filename):
	#	urllib.urlretrieve(url, tif_filename)
	#	print "retrieved ", tif_filename
	
	centerlat 	= (bbox[1]+bbox[3])/2
	centerlon	= (bbox[0]+bbox[2])/2
	rasterXSize	= 400
	rasterYSize	= 250
	
	mapbox_image(centerlat, centerlon, zoom, rasterXSize, rasterYSize, osm_bg_image)
	
	ullon, ullat, lrlon, lrlat = browseimage.Gen_bbox(centerlat, centerlon, zoom, rasterXSize, rasterYSize)
	#print ullon, lrlat, lrlon, ullat
	
	url = "https://firms.modaps.eosdis.nasa.gov/wms/?SERVICE=WMS&VERSION=1.1.1&REQUEST=GetMap&LAYERS=fires24&width=400&height=250&BBOX="
	url += str(ullon) + ","+ str(lrlat) + "," + str(lrlon) + "," + str(ullat)
	
	if force or not os.path.exists(tif_filename):
		if verbose:
			print "retrieve", tif_filename
		
		if context:
			urllib.urlretrieve(url, tif_filename, context=context)
		else:
			urllib.urlretrieve(url, tif_filename)
		#print "retrieved ", tif_filename
	
	# superimpose the suface water over map background
	if force or not os.path.isfile(thn_image):	
		cmd = str.format("composite -quiet -gravity center {0} {1} {2}", tif_filename, osm_bg_image, thn_image)
		execute(cmd)
		
	file_list = [ tif_filename, geojson_filename, geojsongz_filename, thn_image ]
	CopyToS3( s3_bucket, s3_folder, file_list, force, verbose )
Exemple #5
0
    logger.info(
        "gpm_movie.py starTime: %s endTime: %s zoom: %s, lat: %f lon: %f outputDir: %s",
        startTime, endTime, z, centerlat, centerlon, outputDir)

    goldenratio = 1.618
    height = 610
    width = 1024

    if not os.path.exists(outputDir):
        os.mkdir(outputDir)

    osm_bg_image = os.path.join(outputDir, "mapbox_map.png")

    # Let's generate a map background
    if force or not os.path.exists(osm_bg_image):
        mapbox_image(centerlat, centerlon, z, width, height, osm_bg_image)

    ullon, ullat, lrlon, lrlat = Gen_bbox(centerlat, centerlon, z, width,
                                          height)

    #
    # Let's make sure we have all the files one day at a time
    #
    done = 0
    logger.info("Checking GPM files availability...")
    while not done:
        get_30mn_files(outputDir, year, month, day, ymd)
        startTime += timedelta(days=1)
        if (startTime > endTime):
            done = 1
        else:
Exemple #6
0
def process_url( mydir, url, ymd, bbox, zoom, s3_bucket, s3_folder ):
	orig_filename		= os.path.join(os.path.join(mydir,  "..", "quakes." + ymd + '.geojson'))

	csv_filename		= os.path.join(os.path.join(mydir,  "quakes." + ymd + '.csv'))
	geojson_filename	= os.path.join(os.path.join(mydir,  "quakes." + ymd + '.geojson'))
	geojsongz_filename	= os.path.join(os.path.join(mydir,  "quakes." + ymd + '.geojson.gz'))
	tif_filename		= os.path.join(os.path.join(mydir,  "quakes." + ymd + '.tif'))
	osm_bg_image		= os.path.join(os.path.join(mydir,  "osm_bg_image.tif"))
	thn_image			= os.path.join(os.path.join(mydir,  "quakes." + ymd + '_thn.jpg'))
		
	if force or not os.path.exists(orig_filename):
		if verbose:
			print "retrieving:", orig_filename
		urllib.urlretrieve(url, orig_filename)

	json_data 	= open(orig_filename).read()
	data 		= json.loads(json_data)
	
	results = {}
	results['type'] 	= "FeatureCollection"
	results['bbox'] 	= data['bbox']
	results['metadata'] = data['metadata']
	results['features']	= []
	
	for f in data['features']:
		coords = f['geometry']['coordinates']
		lon		= coords[0]
		lat		= coords[1]
		if inbbox(bbox, lat, lon):
			qtime = int(f['properties']['time'])/1000
			f['properties']['date'] = time.ctime(qtime)
			
			newprops = {
				'type': f['properties']['type'],
				'title': f['properties']['title'],
				'place': f['properties']['place'],
				'mag': f['properties']['mag'],
				'rms': f['properties']['rms'],
				'status': f['properties']['status'],
				'date': f['properties']['date'],
				'detail': f['properties']['detail'],
				'url': f['properties']['url']
			}
			
			f['properties'] = newprops
			
			results['features'].append(f)
			
	if verbose:
		print "found", len(results['features'])
		
	with open(geojson_filename, 'w') as outfile:
	    json.dump(results, outfile)
		
	if force or not os.path.exists(geojsongz_filename):
		cmd = 'gzip < %s > %s' %( geojson_filename, geojsongz_filename)
		execute(cmd)

	centerlat 	= (bbox[1]+bbox[3])/2
	centerlon	= (bbox[0]+bbox[2])/2
	rasterXSize	= 400
	rasterYSize	= 250
	
	mapbox_image(centerlat, centerlon, zoom, rasterXSize, rasterYSize, osm_bg_image)
	
	ullon, ullat, lrlon, lrlat = browseimage.Gen_bbox(centerlat, centerlon, zoom, rasterXSize, rasterYSize)
	dx = (lrlon-ullon)/rasterXSize
	dy = (ullat-lrlat)/rasterXSize

	#print "org:", ullon, ullat, dx, dy
	
	im 		= Image.open(osm_bg_image)
	draw 	= ImageDraw.Draw(im)
	for f in results['features']:
		coords = f['geometry']['coordinates']
		lon		= coords[0]
		lat		= coords[1]
		x		= int((lon-ullon)/dx)
		y		= int((ullat-lat)/dx)
		
		#print lon, lat, x, y
		draw.ellipse( [(x-1,y-1),(x+1,y+1)])
	
	im.save(tif_filename, "PNG")
	
	# superimpose the suface water over map background
	#if force or not os.path.isfile(sw_osm_image):	
	if force or not os.path.isfile(thn_image):	
		#cmd = str.format("composite -gravity center {0} {1} {2}", tif_filename, osm_bg_image, thn_image)
		cmd = "cp %s %s" % (tif_filename, thn_image)
		execute(cmd)
		
	file_list = [ geojson_filename, geojsongz_filename, thn_image ]
	CopyToS3( s3_bucket, s3_folder, file_list, force, verbose )
	
	if not verbose:
		cmd = "rm -rf %s %s" % (tif_filename, osm_bg_image)
		execute(cmd)
Exemple #7
0
	outputDir		= options.outputDir
		
	logger.info ("gpm_movie.py starTime: %s endTime: %s zoom: %s, lat: %f lon: %f outputDir: %s", startTime , endTime, z, centerlat, centerlon, outputDir)
			
	goldenratio		= 1.618
	height			= 610
	width			= 1024
		
	if not os.path.exists(outputDir):
		os.mkdir(outputDir)
		
	osm_bg_image	= os.path.join(outputDir, "mapbox_map.png")
		
	# Let's generate a map background
	if force or not os.path.exists(osm_bg_image):
		mapbox_image(centerlat, centerlon, z, width, height, osm_bg_image)
		
	ullon, ullat, lrlon, lrlat = Gen_bbox(centerlat, centerlon, z, width, height)
		
	#
	# Let's make sure we have all the files one day at a time
	#
	done = 0
	logger.info("Checking GPM files availability...")
	while not done:
		get_30mn_files(outputDir, year, month, day, ymd)
		startTime += timedelta(days=1)
		if( startTime > endTime):
			done = 1
		else:
			year		= startTime.year
Exemple #8
0
def process_url(mydir, url, ymd, bbox, zoom, s3_bucket, s3_folder):
    orig_filename = os.path.join(os.path.join(mydir, "..", "quakes." + ymd + ".geojson"))

    csv_filename = os.path.join(os.path.join(mydir, "quakes." + ymd + ".csv"))
    geojson_filename = os.path.join(os.path.join(mydir, "quakes." + ymd + ".geojson"))
    geojsongz_filename = os.path.join(os.path.join(mydir, "quakes." + ymd + ".geojson.gz"))
    tif_filename = os.path.join(os.path.join(mydir, "quakes." + ymd + ".tif"))
    osm_bg_image = os.path.join(os.path.join(mydir, "osm_bg_image.tif"))
    thn_image = os.path.join(os.path.join(mydir, "quakes." + ymd + "_thn.jpg"))

    if force or not os.path.exists(orig_filename):
        if verbose:
            print "retrieving:", orig_filename
        urllib.urlretrieve(url, orig_filename)

    json_data = open(orig_filename).read()
    data = json.loads(json_data)

    results = {}
    results["type"] = "FeatureCollection"
    results["bbox"] = data["bbox"]
    results["metadata"] = data["metadata"]
    results["features"] = []

    for f in data["features"]:
        coords = f["geometry"]["coordinates"]
        lon = coords[0]
        lat = coords[1]
        if inbbox(bbox, lat, lon):
            qtime = int(f["properties"]["time"]) / 1000

            newp = {}
            newp["title"] = f["properties"]["title"]
            newp["mag"] = f["properties"]["mag"]
            newp["date"] = time.ctime(qtime)
            newp["place"] = f["properties"]["place"]

            # f['properties']['date'] = time.ctime(qtime)
            f["properties"] = newp
            results["features"].append(f)

    print "found", len(results["features"])
    with open(geojson_filename, "w") as outfile:
        json.dump(results, outfile)

    if force or not os.path.exists(geojsongz_filename):
        cmd = "gzip < %s > %s" % (geojson_filename, geojsongz_filename)
        execute(cmd)

    centerlat = (bbox[1] + bbox[3]) / 2
    centerlon = (bbox[0] + bbox[2]) / 2
    rasterXSize = 400
    rasterYSize = 250

    mapbox_image(centerlat, centerlon, zoom, rasterXSize, rasterYSize, osm_bg_image)

    ullon, ullat, lrlon, lrlat = browseimage.bbox(centerlat, centerlon, zoom, rasterXSize, rasterYSize)
    dx = (lrlon - ullon) / rasterXSize
    dy = (ullat - lrlat) / rasterXSize

    # print "org:", ullon, ullat, dx, dy

    im = Image.open(osm_bg_image)
    draw = ImageDraw.Draw(im)
    for f in results["features"]:
        coords = f["geometry"]["coordinates"]
        lon = coords[0]
        lat = coords[1]
        x = int((lon - ullon) / dx)
        y = int((ullat - lat) / dx)

        # print lon, lat, x, y
        draw.ellipse([(x - 1, y - 1), (x + 1, y + 1)])

    im.save(tif_filename, "PNG")

    # superimpose the suface water over map background
    # if force or not os.path.isfile(sw_osm_image):
    if force or not os.path.isfile(thn_image):
        # cmd = str.format("composite -gravity center {0} {1} {2}", tif_filename, osm_bg_image, thn_image)
        cmd = "cp %s %s" % (tif_filename, thn_image)
        execute(cmd)

    file_list = [geojson_filename, geojsongz_filename, thn_image]
    CopyToS3(s3_bucket, s3_folder, file_list, force, verbose)
Exemple #9
0
		with open(geojson_filename, 'w') as outfile:
			json.dump(entries, outfile)
		
		if force or not os.path.exists(geojsongz_filename):
			cmd = 'gzip < %s > %s' %( geojson_filename, geojsongz_filename)
			execute(cmd)
	
		
	if force or not os.path.isfile(thn_image):	
		centerlat 	= (bbox[1]+bbox[3])/2
		centerlon	= (bbox[0]+bbox[2])/2
		rasterXSize	= 400
		rasterYSize	= 250

		mapbox_image(centerlat, centerlon, zoom, rasterXSize, rasterYSize, osm_bg_image)

		ullon, ullat, lrlon, lrlat = browseimage.Gen_bbox(centerlat, centerlon, zoom, rasterXSize, rasterYSize)
		dx = (lrlon-ullon)/rasterXSize
		dy = (ullat-lrlat)/rasterXSize


		im 		= Image.open(osm_bg_image)
		draw 	= ImageDraw.Draw(im)

		for f in features :
			coordinates = f['geometry']['coordinates']
			lon		= coordinates[0]
			lat		= coordinates[1]
	
			x		= int((lon-ullon)/dx)