def download(dbname, dts, bbox): """Downloads the PRISM data products for a set of dates *dt* and imports them into the PostGIS database *dbname*.""" url = "jsimpson.pps.eosdis.nasa.gov" ftp = FTP(url) # FIXME: Change to RHEAS-specific password ftp.login('*****@*****.**', '*****@*****.**') ftp.cwd("data/imerg/gis") outpath = tempfile.mkdtemp() for dt in [dts[0] + timedelta(t) for t in range((dts[-1] - dts[0]).days+1)]: try: ftp.cwd("/data/imerg/gis/{0}/{1:02d}".format(dt.year, dt.month)) filenames = [f for f in ftp.nlst() if re.match(r"3B.*{0}.*S000000.*1day\.tif.*".format(dt.strftime("%Y%m%d")), f) is not None] if len(filenames) > 0: fname = filenames[0] with open("{0}/{1}".format(outpath, fname), 'wb') as f: ftp.retrbinary("RETR {0}".format(fname), f.write) with open("{0}/{1}".format(outpath, fname.replace("tif", "tfw")), 'wb') as f: ftp.retrbinary("RETR {0}".format(fname.replace("tif", "tfw")), f.write) tfname = fname.replace("tif", "tfw") fname = datasets.uncompress(fname, outpath) datasets.uncompress(tfname, outpath) subprocess.call(["gdalwarp", "-t_srs", "epsg:4326", "{0}/{1}".format(outpath, fname), "{0}/prec.tif".format(outpath)]) if bbox is not None: subprocess.call(["gdal_translate", "-a_srs", "epsg:4326", "-projwin", "{0}".format(bbox[0]), "{0}".format(bbox[3]), "{0}".format(bbox[2]), "{0}".format(bbox[1]), "{0}/prec.tif".format(outpath), "{0}/prec1.tif".format(outpath)]) else: subprocess.call(["gdal_translate", "-a_srs", "epsg:4326", "{0}/prec.tif".format(outpath), "{0}/prec1.tif".format(outpath)]) # multiply by 0.1 to get mm/hr and 24 to get mm/day cmd = " ".join(["gdal_calc.py", "-A", "{0}/prec1.tif".format(outpath), "--outfile={0}/prec2.tif".format(outpath), "--calc=\"0.1*A\""]) subprocess.call(cmd, shell=True) dbio.ingest(dbname, "{0}/prec2.tif".format(outpath), dt, table, False) except: print("WARNING! No data were available to import into {0} for {1}.".format(table, dt.strftime("%Y-%m-%d")))
def download(dbname, dts, bbox): """Downloads the PRISM data products for a set of dates *dt* and imports them into the PostGIS database *dbname*.""" log = logging.getLogger(__name__) url = "jsimpson.pps.eosdis.nasa.gov" ftp = FTP(url) # FIXME: Change to RHEAS-specific password ftp.login('*****@*****.**', '*****@*****.**') ftp.cwd("data/imerg/gis") outpath = tempfile.mkdtemp() for dt in [dts[0] + timedelta(t) for t in range((dts[-1] - dts[0]).days+1)]: try: if dt.year < datetime.today().year: ftp.cwd("/data/imerg/gis/{0}/{1:02d}".format(dt.year, dt.month)) else: ftp.cwd("/data/imerg/gis/{0:02d}".format(dt.month)) filenames = [f for f in ftp.nlst() if re.match("3B.*{0}.*E235959.*1day.tif".format(dt.strftime("%Y%m%d")), f) is not None] if len(filenames) > 0: fname = filenames[0] with open("{0}/{1}".format(outpath, fname), 'wb') as f: ftp.retrbinary("RETR {0}".format(fname), f.write) with open("{0}/{1}".format(outpath, fname.replace("tif", "tfw")), 'wb') as f: ftp.retrbinary("RETR {0}".format(fname.replace("tif", "tfw")), f.write) tfname = fname.replace("tif", "tfw") fname = datasets.uncompress(fname, outpath) datasets.uncompress(tfname, outpath) proc = subprocess.Popen(["gdalwarp", "-srcnodata", "29999", "-dstnodata", "-9999", "-overwrite", "-t_srs", "epsg:4326", "-ot", "Float32", "{0}/{1}".format(outpath, fname), "{0}/prec.tif".format(outpath)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) out, err = proc.communicate() log.debug(out) if bbox is not None: proc = subprocess.Popen(["gdal_translate", "-ot", "Float32", "-a_srs", "epsg:4326", "-projwin", "{0}".format(bbox[0]), "{0}".format(bbox[3]), "{0}".format(bbox[2]), "{0}".format(bbox[1]), "{0}/prec.tif".format(outpath), "{0}/prec1.tif".format(outpath)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) out, err = proc.communicate() log.debug(out) else: proc = subprocess.Popen(["gdal_translate", "-a_srs", "epsg:4326", "{0}/prec.tif".format(outpath), "{0}/prec1.tif".format(outpath)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) out, err = proc.communicate() log.debug(out) # multiply by 0.1 to get mm/hr and 24 to get mm/day proc = subprocess.Popen(["gdal_calc.py", "--NoDataValue=-9999", "-A", "{0}/prec1.tif".format(outpath), "--outfile={0}/prec2.tif".format(outpath), "--calc=0.1*A"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) out, err = proc.communicate() log.debug(out) dbio.ingest(dbname, "{0}/prec2.tif".format(outpath), dt, table, True) except: log.warning("No data were available to import into {0} for {1}.".format(table, dt.strftime("%Y-%m-%d")))
def wrapper(*args, **kwargs): outpath, filename, bbox, dt = fetch(*args, **kwargs) if filename is not None: lfilename = datasets.uncompress(filename, outpath) f = gdal.Open("{0}/{1}".format(outpath, lfilename)) xul, xres, _, yul, _, yres = f.GetGeoTransform() data = f.ReadAsArray() nr, nc = data.shape lat = np.arange(yul + yres / 2.0, yul + yres * nr, yres) lon = np.arange(xul + xres / 2.0, xul + xres * nc, xres) i1, i2, j1, j2 = datasets.spatialSubset(lat, lon, xres, bbox) data = data[i1:i2, j1:j2] lat = lat[i1:i2] lon = lon[j1:j2] shutil.rmtree(outpath) else: data = lat = lon = None return data, lat, lon, dt
def wrapper(*args, **kwargs): outpath, filename, bbox, dt = fetch(*args, **kwargs) if filename is not None: lfilename = datasets.uncompress(filename, outpath) f = gdal.Open("{0}/{1}".format(outpath, lfilename)) xul, xres, _, yul, _, yres = f.GetGeoTransform() data = f.ReadAsArray() nr, nc = data.shape lat = np.arange(yul + yres/2.0, yul + yres * nr, yres) lon = np.arange(xul + xres/2.0, xul + xres * nc, xres) i1, i2, j1, j2 = datasets.spatialSubset(lat, lon, xres, bbox) data = data[i1:i2, j1:j2] lat = lat[i1:i2] lon = lon[j1:j2] shutil.rmtree(outpath) else: data = lat = lon = None return data, lat, lon, dt
def download(dbname, dts, bbox): """Downloads the PRISM data products for a set of dates *dt* and imports them into the PostGIS database *dbname*.""" url = "jsimpson.pps.eosdis.nasa.gov" ftp = FTP(url) # FIXME: Change to RHEAS-specific password ftp.login('*****@*****.**', '*****@*****.**') ftp.cwd("data/imerg/gis") outpath = tempfile.mkdtemp() for dt in [ dts[0] + timedelta(t) for t in range((dts[-1] - dts[0]).days + 1) ]: try: ftp.cwd("/data/imerg/gis/{0}/{1:02d}".format(dt.year, dt.month)) filenames = [ f for f in ftp.nlst() if re.match( r"3B.*{0}.*S000000.*1day\.tif.*".format( dt.strftime("%Y%m%d")), f) is not None ] if len(filenames) > 0: fname = filenames[0] with open("{0}/{1}".format(outpath, fname), 'wb') as f: ftp.retrbinary("RETR {0}".format(fname), f.write) with open( "{0}/{1}".format(outpath, fname.replace("tif", "tfw")), 'wb') as f: ftp.retrbinary( "RETR {0}".format(fname.replace("tif", "tfw")), f.write) tfname = fname.replace("tif", "tfw") fname = datasets.uncompress(fname, outpath) datasets.uncompress(tfname, outpath) subprocess.call([ "gdalwarp", "-t_srs", "epsg:4326", "{0}/{1}".format(outpath, fname), "{0}/prec.tif".format(outpath) ]) if bbox is not None: subprocess.call([ "gdal_translate", "-a_srs", "epsg:4326", "-projwin", "{0}".format(bbox[0]), "{0}".format(bbox[3]), "{0}".format(bbox[2]), "{0}".format(bbox[1]), "{0}/prec.tif".format(outpath), "{0}/prec1.tif".format(outpath) ]) else: subprocess.call([ "gdal_translate", "-a_srs", "epsg:4326", "{0}/prec.tif".format(outpath), "{0}/prec1.tif".format(outpath) ]) # multiply by 0.1 to get mm/hr and 24 to get mm/day cmd = " ".join([ "gdal_calc.py", "-A", "{0}/prec1.tif".format(outpath), "--outfile={0}/prec2.tif".format(outpath), "--calc=\"0.1*A\"" ]) subprocess.call(cmd, shell=True) dbio.ingest(dbname, "{0}/prec2.tif".format(outpath), dt, table, False) except: print( "WARNING! No data were available to import into {0} for {1}.". format(table, dt.strftime("%Y-%m-%d")))