def download(dbname, dt, bbox=None): """Downloads SMOS soil mositure data for a set of dates *dt* and imports them into the PostGIS database *dbname*. Optionally uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat].""" res = 0.25 url = "http://*****:*****@cp34-bec.cmima.csic.es/thredds/dodsC/NRTSM001D025A_ALL" f = netcdf.Dataset(url) lat = f.variables['lat'][::-1] # swap latitude orientation to northwards lon = f.variables['lon'][:] i1, i2, j1, j2 = datasets.spatialSubset(lat, lon, res, bbox) smi1 = len(lat) - i2 - 1 smi2 = len(lat) - i1 - 1 lat = lat[i1:i2] lon = lon[j1:j2] t0 = datetime(2010, 1, 12) # initial date of SMOS data t1 = (dt[0] - t0).days t2 = (dt[-1] - t0).days + 1 ti = range(t1, t2) sm = f.variables['SM'][ti, smi1:smi2, j1:j2] # FIXME: Use spatially variable observation error # smv = f.variables['VARIANCE_SM'][ti, i1:i2, j1:j2] for tj in range(sm.shape[0]): filename = dbio.writeGeotif(lat, lon, res, sm[tj, :, :]) t = t0 + timedelta(ti[tj]) dbio.ingest(dbname, filename, t, table, False) print("Imported SMOS {0}".format(tj)) os.remove(filename)
def download(dbname, dts, bbox=None): """Downloads SMAP soil mositure data for a set of dates *dt* and imports them into the PostGIS database *dbname*. Optionally uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat].""" res = 0.36 url = "n5eil01u.ecs.nsidc.org" ftp = FTP(url) ftp.login() for dt in [dts[0] + timedelta(tt) for tt in range((dts[1] - dts[0]).days + 1)]: r = ftp.cwd("/pub/SAN/SMAP/SPL3SMP.002/{0}".format(dt.strftime("%Y.%m.%d"))) if r.find("successful") > 0: outpath = tempfile.mkdtemp() fname = [f for f in ftp.nlst() if f.find("h5") > 0][0] with open("{0}/{1}".format(outpath, fname), 'wb') as f: ftp.retrbinary("RETR {0}".format(fname), f.write) f = h5py.File("{0}/{1}".format(outpath, fname)) lat = f['Soil_Moisture_Retrieval_Data']['latitude'][:, 0] lon = f['Soil_Moisture_Retrieval_Data']['longitude'][0, :] lon[lon > 180] -= 360.0 # FIXME: Need to add reprojection from EASE grid i1, i2, j1, j2 = datasets.spatialSubset(np.sort(lat)[::-1], np.sort(lon), res, bbox) lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] sm = np.zeros((len(lati), len(loni))) for i in range(len(lati)): for j in range(len(loni)): sm[i, j] = f['Soil_Moisture_Retrieval_Data']['soil_moisture'][i, j] # FIXME: Use spatially variable observation error # sme = f['Soil_Moisture_Retrieval_Data']['soil_moisture_error'][i1:i2, j1:j2] lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] filename = dbio.writeGeotif(lat, lon, res, sm) dbio.ingest(dbname, filename, dt, table, False) else: print("No SMAP data available for {0}.".format(dt.strftime("%Y-%m-%d")))
def ingest(dbname, table, data, lat, lon, res, t): """Import data into RHEAS database.""" for tj in range(data.shape[0]): filename = dbio.writeGeotif(lat, lon, res, data[tj, :, :]) dbio.ingest(dbname, filename, t[tj], table) print("Imported {0} in {1}".format(t[tj].strftime("%Y-%m-%d"), table)) os.remove(filename)
def download(dbname, dt, bbox=None): """Downloads SMOS soil mositure data for a set of dates *dt* and imports them into the PostGIS database *dbname*. Optionally uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat].""" res = 0.25 url = "http://*****:*****@cp34-bec.cmima.csic.es/thredds/dodsC/NRTSM001D025A_ALL" f = netcdf.Dataset(url) lat = f.variables['lat'][:] lon = f.variables['lon'][:] if bbox is not None: i = np.where(np.logical_and(lat > bbox[1], lat < bbox[3]))[0] j = np.where(np.logical_and(lon > bbox[0], lon < bbox[2]))[0] lat = lat[i] lon = lon[j] else: i = range(len(lat)) j = range(len(lon)) t0 = datetime(2010, 1, 12) # initial date of SMOS data t1 = (dt[0] - t0).days t2 = (dt[1] - t0).days + 1 ti = range(t1, t2) sm = f.variables['SM'][ti, i, j] # FIXME: Use spatially variable observation error # smv = f.variables['VARIANCE_SM'][ti, i, j] for tj in range(sm.shape[0]): filename = dbio.writeGeotif(lat, lon, res, sm[tj, :, :]) t = t0 + timedelta(ti[tj]) dbio.ingest(dbname, filename, t, table, False) print("Imported SMOS {0}".format(tj)) os.remove(filename)
def download(dbname, dt, bbox=None): """Downloads SMAP soil mositure data for a set of dates *dt* and imports them into the PostGIS database *dbname*. Optionally uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat].""" res = 0.36 url = "ftp://n5eil01u.ecs.nsidc.org" ftp = FTP(url) ftp.login() ftp.cwd("SAN/SMAP/SPL3SMP.002") days = ftp.nlst() datadir = dt.strftime("%Y.%m.%d") if datadir in days: outpath = tempfile.mkdtemp() ftp.cwd(datadir) fname = [f for f in ftp.nlst() if f.find("h5") > 0][0] with open("{0}/{1}".format(outpath, fname), "wb") as f: ftp.retrbinary("RETR {0}".format(fname), f.write) f = h5py.File("{0}/{1}".format(outpath, fname)) lat = f["Soil_Moisture_Retrieval_Data"]["latitude"][:, 0] lon = f["Soil_Moisture_Retrieval_Data"]["longitude"][0, :] if bbox is not None: i = np.where(np.logical_and(lat > bbox[1], lat < bbox[3]))[0] j = np.where(np.logical_and(lon > bbox[0], lon < bbox[2]))[0] lat = lat[i] lon = lon[j] else: i = range(len(lat)) j = range(len(lon)) sm = f["Soil_Moisture_Retrieval_Data"]["soil_moisture"][i[0] : i[-1] + 1, j[0] : j[-1] + 1] # FIXME: Use spatially variable observation error # sme = f['Soil_Moisture_Retrieval_Data']['soil_moisture_error'][i[0]:i[-1]+1, j[0]:j[-1]+1] filename = dbio.writeGeotif(lat, lon, res, sm) dbio.ingest(dbname, filename, dt, table, False)
def _downloadVariable(varname, dbname, dts, bbox): """Download specific variable from the MERRA Reanalysis dataset.""" # FIXME: Grid is not rectangular, but 0.5 x 0.625 degrees res = 0.5 for ts in [dts[0] + timedelta(dti) for dti in range((dts[1] - dts[0]).days + 1)]: try: runid = _merraRunid(ts.year) url = "http://goldsmr4.sci.gsfc.nasa.gov:80/opendap/MERRA2/M2T1NXSLV.5.12.4/{1}/{2:02d}/MERRA2_{0}.tavg1_2d_slv_Nx.{1:04d}{2:02d}{3:02d}.nc4".format(runid, ts.year, ts.month, ts.day) ds = netcdf.Dataset(url) lat = ds.variables["lat"][:] lon = ds.variables["lon"][:] lon[lon > 180] -= 360.0 i1, i2, j1, j2 = datasets.spatialSubset(np.sort(lat)[::-1], np.sort(lon), res, bbox) data = np.zeros((i2-i1, j2-j1)) lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] if varname == "tmax": hdata = ds.variables["T2M"][:, lati, loni] data = np.amax(hdata, axis=0) - 273.15 elif varname == "tmin": hdata = ds.variables["T2M"][:, lati, loni] data = np.amin(hdata, axis=0) - 273.15 elif varname in ["wind"]: hdata = np.sqrt(ds.variables["U10M"][:, lati, loni]**2 + ds.variables["V10M"][:, lati, loni]**2) data = np.mean(hdata, axis=0) lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] filename = dbio.writeGeotif(lat, lon, res, data) dbio.ingest(dbname, filename, ts, "{0}.merra".format(varname)) os.remove(filename) except: print("Cannot import MERRA dataset for {0}!".format(ts.strftime("%Y-%m-%d")))
def download(dbname, dts, bbox=None): """Downloads IRI forecast tercile probability data from the IRI data server, and imports them into the database *db*. Optionally uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat].""" leadtime = 3 res = 2.5 baseurl = "http://iridl.ldeo.columbia.edu/SOURCES/.IRI/.FD/.Seasonal_Forecast/.{0}/.prob/dods" table = {"Precipitation": "precip.iri", "Temperature": "tmax.iri"} for varname in ["Precipitation", "Temperature"]: purl = baseurl.format(varname) pds = netcdf.Dataset(purl) lat = pds.variables["Y"][:] lon = pds.variables["X"][:] if bbox is not None: i = np.where(np.logical_and(lat > bbox[1], lat < bbox[3]))[0] j = np.where(np.logical_and(lon > bbox[0], lon < bbox[2]))[0] lat = lat[i] lon = lon[j] else: i = range(len(lat)) j = range(len(lon)) t = pds.variables["F"][:] ti = [tt for tt in range(len(t)) if t[tt] >= ((dts[0].year - 1960) * 12 + dts[0].month - 0.5) and t[tt] <= ((dts[1].year - 1960) * 12 + dts[1].month - 0.5)] for tt in ti: dt = date(1960, 1, 1) + relativedelta(months=int(t[tt])) for m in range(leadtime): for ci, c in enumerate(["below", "normal", "above"]): data = pds.variables["prob"][tt, m, i, j, ci] filename = dbio.writeGeotif(lat, lon, res, data) ingest(dbname, filename, dt, m + 1, c, table[varname]) os.remove(filename)
def _downloadVariable(varname, dbname, dt, bbox): """Download specific variable from the MERRA Reanalysis dataset.""" # FIXME: Grid is not rectangular, but 0.5 x 0.625 degrees res = 0.5 try: url = "http://goldsmr4.sci.gsfc.nasa.gov:80/dods/M2T1NXSLV" ds = netcdf.Dataset(url) lat = ds.variables["lat"][:] lon = ds.variables["lon"][:] lon[lon > 180] -= 360.0 i1, i2, j1, j2 = datasets.spatialSubset(np.sort(lat)[::-1], np.sort(lon), res, bbox) data = np.zeros((i2-i1, j2-j1)) lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] t = ds.variables["time"] tt = netcdf.num2date(t[:], units=t.units) ti = np.where(tt == dt)[0][0] if varname == "tmax": hdata = ds.variables["t2m"][ti:ti+24, lati, loni] data = np.amax(hdata, axis=0) - 273.15 elif varname == "tmin": hdata = ds.variables["t2m"][ti:ti+24, lati, loni] data = np.amin(hdata, axis=0) - 273.15 elif varname in ["wind"]: hdata = np.sqrt(ds.variables["u10m"][ti:ti+24, lati, loni]**2 + ds.variables["v10m"][ti:ti+24, lati, loni]**2) data = np.mean(hdata, axis=0) lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] filename = dbio.writeGeotif(lat, lon, res, data) table = "{0}.merra".format(varname) dbio.ingest(dbname, filename, dt, table) print("Imported {0} in {1}".format(tt[ti].strftime("%Y-%m-%d"), table)) os.remove(filename) except: print("Cannot import MERRA dataset for {0}!".format(dt.strftime("%Y-%m-%d")))
def download(dbname, dt, bbox=None): """Downloads SMOS soil mositure data for a set of dates *dt* and imports them into the PostGIS database *dbname*. Optionally uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat].""" log = logging.getLogger(__name__) res = 0.25 url = "http://*****:*****@cp34-bec.cmima.csic.es/thredds/dodsC/NRTSM001D025A_ALL" f = netcdf.Dataset(url) lat = f.variables['lat'][::-1] # swap latitude orientation to northwards lon = f.variables['lon'][:] i1, i2, j1, j2 = datasets.spatialSubset(lat, lon, res, bbox) smi1 = len(lat) - i2 - 1 smi2 = len(lat) - i1 - 1 lat = lat[i1:i2] lon = lon[j1:j2] t0 = datetime(2010, 1, 12) # initial date of SMOS data t1 = (dt[0] - t0).days if t1 < 0: log.warning("Reseting start date to {0}".format(t0.strftime("%Y-%m-%d"))) t1 = 0 t2 = (dt[-1] - t0).days + 1 nt, _, _ = f.variables['SM'].shape if t2 > nt: t2 = nt log.warning("Reseting end date to {0}".format((t0 + timedelta(t2)).strftime("%Y-%m-%d"))) ti = range(t1, t2) sm = f.variables['SM'][ti, smi1:smi2, j1:j2] # FIXME: Use spatially variable observation error # smv = f.variables['VARIANCE_SM'][ti, i1:i2, j1:j2] for tj in range(sm.shape[0]): filename = dbio.writeGeotif(lat, lon, res, sm[tj, :, :]) t = t0 + timedelta(ti[tj]) dbio.ingest(dbname, filename, t, table, False) log.info("Imported SMOS {0}".format(tj)) os.remove(filename)
def download(dbname, dts, bbox=None): """Downloads IRI forecast tercile probability data from the IRI data server, and imports them into the database *dbname*. Optionally uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat].""" leadtime = 3 res = 2.5 baseurl = "http://iridl.ldeo.columbia.edu/SOURCES/.IRI/.FD/.Seasonal_Forecast/.{0}/.prob/dods" table = {"Precipitation": "precip.iri", "Temperature": "tmax.iri"} for varname in ["Precipitation", "Temperature"]: purl = baseurl.format(varname) pds = netcdf.Dataset(purl) lat = pds.variables["Y"][:] lon = pds.variables["X"][:] lon[lon > 180] -= 360.0 i1, i2, j1, j2 = datasets.spatialSubset(np.sort(lat)[::-1], np.sort(lon), res, bbox) lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] t = pds.variables["F"][:] ti = [tt for tt in range(len(t)) if t[tt] >= ((dts[0].year - 1960) * 12 + dts[0].month - 0.5) and t[tt] <= ((dts[1].year - 1960) * 12 + dts[1].month - 0.5)] for tt in ti: dt = date(1960, 1, 1) + relativedelta(months=int(t[tt])) for m in range(leadtime): for ci, c in enumerate(["below", "normal", "above"]): data = pds.variables["prob"][tt, m, lati, loni, ci] filename = dbio.writeGeotif(lat, lon, res, data) ingest(dbname, filename, dt, m + 1, c, table[varname]) os.remove(filename)
def _downloadVariable(varname, dbname, dts, bbox): """Download specific variable from the MERRA Reanalysis dataset.""" # FIXME: Grid is not rectangular, but 0.5 x 0.625 degrees res = 0.5 for ts in [dts[0] + timedelta(dti) for dti in range((dts[1] - dts[0]).days + 1)]: try: url = "http://goldsmr4.sci.gsfc.nasa.gov:80/opendap/MERRA2/M2T1NXSLV.5.12.4/{0}/{1:02d}/MERRA2_400.tavg1_2d_slv_Nx.{0:04d}{1:02d}{2:02d}.nc4".format(ts.year, ts.month, ts.day) ds = netcdf.Dataset(url) lat = ds.variables["lat"][:] lon = ds.variables["lon"][:] lon[lon > 180] -= 360.0 if bbox is not None: i = np.where(np.logical_and(lat > bbox[1], lat < bbox[3]))[0] j = np.where(np.logical_and(lon > bbox[0], lon < bbox[2]))[0] lat = lat[i] lon = lon[j] else: i = range(len(lat)) j = range(len(lon)) data = np.zeros((len(i), len(j))) if varname == "tmax": hdata = ds.variables["T2M"][:, i, j] data = np.amax(hdata, axis=0) - 273.15 elif varname == "tmin": hdata = ds.variables["T2M"][:, i, j] data = np.amin(hdata, axis=0) - 273.15 elif varname in ["wind"]: hdata = np.sqrt(ds.variables["U10M"][:, i, j]**2 + ds.variables["V10M"][:, i, j]**2) data = np.mean(hdata, axis=0) filename = dbio.writeGeotif(lat, lon, res, data) dbio.ingest(dbname, filename, ts, "{0}.merra".format(varname)) os.remove(filename) except: print("Cannot import MERRA dataset for {0}!".format(ts.strftime("%Y-%m-%d")))
def _downloadVariable(varname, dbname, dt, bbox=None): """Download specific variable from the NCEP Reanalysis dataset.""" log = logging.getLogger(__name__) res = 1.875 baseurl = "http://www.esrl.noaa.gov/psd/thredds/dodsC/Datasets/ncep.reanalysis.dailyavgs/surface_gauss" if varname == "tmax": urls = ["{0}/tmax.2m.gauss.{1}.nc".format(baseurl, dt[0].year)] dsvar = ["tmax"] elif varname == "tmin": urls = ["{0}/tmin.2m.gauss.{1}.nc".format(baseurl, dt[0].year)] dsvar = ["tmin"] else: urls = [ "{0}/uwnd.10m.gauss.{1}.nc".format(baseurl, dt[0].year), "{0}/vwnd.10m.gauss.{1}.nc".format(baseurl, dt[0].year) ] dsvar = ["uwnd", "vwnd"] data = None for ui, url in enumerate(urls): pds = netcdf.Dataset(url) lat = pds.variables["lat"][:] lon = pds.variables["lon"][:] lon[lon > 180] -= 360.0 i1, i2, j1, j2 = datasets.spatialSubset( np.sort(lat)[::-1], np.sort(lon), res, bbox) t = pds.variables["time"] tt = netcdf.num2date(t[:], units=t.units) ti = [ tj for tj in range(len(tt)) if resetDatetime(tt[tj]) >= dt[0] and resetDatetime(tt[tj]) <= dt[1] ] if len(ti) > 0: lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] if data is None: data = pds.variables[dsvar[ui]][ti, lati, loni] else: data = np.sqrt(data**2.0 + pds.variables[dsvar[ui]][ti, lati, loni]**2.0) if "temp" in dsvar: data -= 273.15 lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] table = "{0}.ncep".format(varname) for t in range(len(ti)): filename = dbio.writeGeotif(lat, lon, res, data[t, :, :]) dbio.ingest(dbname, filename, tt[ti[t]], table) os.remove(filename) for dtt in [ dt[0] + timedelta(days=tj) for tj in range((dt[-1] - dt[0]).days + 1) ]: if dtt not in tt: log.warning( "NCEP data not available for {0}. Skipping download!".format( dtt.strftime("%Y-%m-%d")))
def _downloadVariable(varname, dbname, dt, bbox=None): """Download specific variable from the NCEP Reanalysis dataset.""" res = 1.875 if varname == "tmax": urls = [ "http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.maximum/.temp/dods" ] dsvar = ["temp"] elif varname == "tmin": urls = [ "http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.minimum/.temp/dods" ] dsvar = ["temp"] else: urls = [ "http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.u/dods", "http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.v/dods" ] dsvar = ["u", "v"] data = None for ui, url in enumerate(urls): pds = netcdf.Dataset(url) lat = pds.variables["Y"][:] lon = pds.variables["X"][:] lon[lon > 180] -= 360.0 i1, i2, j1, j2 = datasets.spatialSubset( np.sort(lat)[::-1], np.sort(lon), res, bbox) t = pds.variables["T"] tt = netcdf.num2date(t[:], units=t.units) # ti = [tj for tj in range(len(tt)) if tt[tj] >= dt] ti = [ tj for tj in range(len(tt)) if resetDatetime(tt[tj]) >= dt[0] and resetDatetime(tt[tj]) <= dt[1] ] if len(ti) > 0: lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] if data is None: data = pds.variables[dsvar[ui]][ti, 0, lati, loni] else: data = np.sqrt(data**2.0 + pds.variables[dsvar[ui]][ti, 0, lati, loni]**2.0) if "temp" in dsvar: data -= 273.15 lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] table = "{0}.ncep".format(varname) for t in range(len(ti)): filename = dbio.writeGeotif(lat, lon, res, data[t, :, :]) dbio.ingest(dbname, filename, tt[ti[t]], table) print("Imported {0} in {1}".format(tt[ti[0]].strftime("%Y-%m-%d"), table)) os.remove(filename)
def ingest(dbname, table, data, lat, lon, res, t, resample=True, overwrite=True): """Import data into RHEAS database.""" log = logging.getLogger(__name__) if data is not None: if len(data.shape) > 2: data = data[0, :, :] filename = dbio.writeGeotif(lat, lon, res, data) dbio.ingest(dbname, filename, t, table, resample, overwrite) os.remove(filename) else: log.warning("No data were available to import into {0} for {1}.".format(table, t.strftime("%Y-%m-%d")))
def ingest(dbname, table, data, lat, lon, res, t, resample=True, overwrite=True): """Import data into RHEAS database.""" log = logging.getLogger(__name__) if data is not None: if len(data.shape) > 2: data = data[0, :, :] filename = dbio.writeGeotif(lat, lon, res, data) dbio.ingest(dbname, filename, t, table, resample, overwrite) log.info("Imported {0} in {1}".format(t.strftime("%Y-%m-%d"), table)) os.remove(filename) else: log.warning("No data were available to import into {0} for {1}.".format(table, t.strftime("%Y-%m-%d")))
def download(dbname, dts, bbox=None, enhanced=False): """Downloads SMAP soil mositure data for a set of dates *dt* and imports them into the PostGIS database *dbname*. Optionally uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat].""" log = logging.getLogger(__name__) if enhanced: res = 0.09 url = "https://n5eil01u.ecs.nsidc.org/SMAP/SPL3SMP_E.001" else: res = 0.36 url = "https://n5eil01u.ecs.nsidc.org/DP4/SMAP/SPL3SMP.004" for dt in [ dts[0] + timedelta(tt) for tt in range((dts[-1] - dts[0]).days + 1) ]: try: outpath, fname = earthdata.download( "{0}/{1}".format(url, dt.strftime("%Y.%m.%d")), "SMAP_L3_SM_P_\S*.h5") f = h5py.File("{0}/{1}".format(outpath, fname)) varname = None for v in f.keys(): if "latitude" in f[v] and "longitude" in f[v]: varname = v assert varname is not None lat = f[varname]['latitude'][:, 0] lon = f[varname]['longitude'][0, :] lon[lon > 180] -= 360.0 # FIXME: Need to add reprojection from EASE grid i1, i2, j1, j2 = datasets.spatialSubset( np.sort(lat)[::-1], np.sort(lon), res, bbox) lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] sm = np.zeros((len(lati), len(loni))) for i in range(len(lati)): for j in range(len(loni)): sm[i, j] = f[varname]['soil_moisture'][i, j] # FIXME: Use spatially variable observation error # sme = f[varname]['soil_moisture_error'][i1:i2, j1:j2] lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] filename = dbio.writeGeotif(lat, lon, res, sm) dbio.ingest(dbname, filename, dt, table, False) except: log.warning("No SMAP data available for {0}.".format( dt.strftime("%Y-%m-%d")))
def _downloadVariable(varname, dbname, dt, bbox=None): """Download specific variable from the NCEP Reanalysis dataset.""" log = logging.getLogger(__name__) res = 1.875 baseurl = "http://www.esrl.noaa.gov/psd/thredds/dodsC/Datasets/ncep.reanalysis.dailyavgs/surface_gauss" if varname == "tmax": urls = ["{0}/tmax.2m.gauss.{1}.nc".format(baseurl, dt[0].year)] dsvar = ["tmax"] elif varname == "tmin": urls = ["{0}/tmin.2m.gauss.{1}.nc".format(baseurl, dt[0].year)] dsvar = ["tmin"] else: urls = ["{0}/uwnd.10m.gauss.{1}.nc".format(baseurl, dt[0].year), "{0}/vwnd.10m.gauss.{1}.nc".format(baseurl, dt[0].year)] dsvar = ["uwnd", "vwnd"] data = None for ui, url in enumerate(urls): pds = netcdf.Dataset(url) lat = pds.variables["lat"][:] lon = pds.variables["lon"][:] lon[lon > 180] -= 360.0 i1, i2, j1, j2 = datasets.spatialSubset(np.sort(lat)[::-1], np.sort(lon), res, bbox) t = pds.variables["time"] tt = netcdf.num2date(t[:], units=t.units) ti = [tj for tj in range(len(tt)) if resetDatetime(tt[tj]) >= dt[0] and resetDatetime(tt[tj]) <= dt[1]] if len(ti) > 0: lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] if data is None: data = pds.variables[dsvar[ui]][ti, lati, loni] else: data = np.sqrt( data ** 2.0 + pds.variables[dsvar[ui]][ti, lati, loni] ** 2.0) if "temp" in dsvar: data -= 273.15 lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] table = "{0}.ncep".format(varname) for t in range(len(ti)): filename = dbio.writeGeotif(lat, lon, res, data[t, :, :]) dbio.ingest(dbname, filename, tt[ti[t]], table) os.remove(filename) for dtt in [dt[0] + timedelta(days=tj) for tj in range((dt[-1]-dt[0]).days + 1)]: if dtt not in tt: log.warning("NCEP data not available for {0}. Skipping download!".format( dtt.strftime("%Y-%m-%d")))
def download(dbname, dts, bbox=None): """Downloads SMAP soil mositure data for a set of dates *dt* and imports them into the PostGIS database *dbname*. Optionally uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat].""" res = 0.36 url = "n5eil01u.ecs.nsidc.org" ftp = FTP(url) ftp.login() for dt in [ dts[0] + timedelta(tt) for tt in range((dts[-1] - dts[0]).days + 1) ]: r = ftp.cwd("/pub/SAN/SMAP/SPL3SMP.003/{0}".format( dt.strftime("%Y.%m.%d"))) if r.find("successful") > 0: outpath = tempfile.mkdtemp() fname = [f for f in ftp.nlst() if f.find("h5") > 0][0] with open("{0}/{1}".format(outpath, fname), 'wb') as f: ftp.retrbinary("RETR {0}".format(fname), f.write) f = h5py.File("{0}/{1}".format(outpath, fname)) lat = f['Soil_Moisture_Retrieval_Data']['latitude'][:, 0] lon = f['Soil_Moisture_Retrieval_Data']['longitude'][0, :] lon[lon > 180] -= 360.0 # FIXME: Need to add reprojection from EASE grid i1, i2, j1, j2 = datasets.spatialSubset( np.sort(lat)[::-1], np.sort(lon), res, bbox) lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] sm = np.zeros((len(lati), len(loni))) for i in range(len(lati)): for j in range(len(loni)): sm[i, j] = f['Soil_Moisture_Retrieval_Data']['soil_moisture'][ i, j] # FIXME: Use spatially variable observation error # sme = f['Soil_Moisture_Retrieval_Data']['soil_moisture_error'][i1:i2, j1:j2] lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] filename = dbio.writeGeotif(lat, lon, res, sm) dbio.ingest(dbname, filename, dt, table, False) else: print("No SMAP data available for {0}.".format( dt.strftime("%Y-%m-%d")))
def _downloadVariable(varname, dbname, dt, bbox=None): """Download specific variable from the NCEP Reanalysis dataset.""" res = 1.875 if varname == "tmax": urls = ["http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.maximum/.temp/dods"] dsvar = ["temp"] elif varname == "tmin": urls = ["http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.minimum/.temp/dods"] dsvar = ["temp"] else: urls = ["http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.u/dods", "http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.v/dods"] dsvar = ["u", "v"] data = None for ui, url in enumerate(urls): pds = netcdf.Dataset(url) lat = pds.variables["Y"][:] lon = pds.variables["X"][:] lon[lon > 180] -= 360.0 if bbox is not None: i = np.where(np.logical_and(lat > bbox[1], lat < bbox[3]))[0] j = np.where(np.logical_and(lon > bbox[0], lon < bbox[2]))[0] lat = lat[i] lon = lon[j] else: i = range(len(lat)) j = range(len(lon)) t = pds.variables["T"] tt = netcdf.num2date(t[:], units=t.units) ti = [tj for tj in range(len(tt)) if tt[tj] >= dt[ 0] and tt[tj] <= dt[1]] if data is None: data = pds.variables[dsvar[ui]][ti, 0, i, j] else: data = np.sqrt( data ** 2.0 + pds.variables[dsvar[ui]][ti, 0, i, j] ** 2.0) if "temp" in dsvar: data -= 273.15 for tj in range(data.shape[0]): filename = dbio.writeGeotif(lat, lon, res, data[tj, :, :]) dbio.ingest(dbname, filename, tt[ti[tj]], "{0}.ncep".format(varname)) os.remove(filename)
def _downloadVariable(varname, dbname, dt, bbox=None): """Download specific variable from the NCEP Reanalysis dataset.""" res = 1.875 if varname == "tmax": urls = ["http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.maximum/.temp/dods"] dsvar = ["temp"] elif varname == "tmin": urls = ["http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.minimum/.temp/dods"] dsvar = ["temp"] else: urls = ["http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.u/dods", "http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.v/dods"] dsvar = ["u", "v"] data = None for ui, url in enumerate(urls): pds = netcdf.Dataset(url) lat = pds.variables["Y"][:] lon = pds.variables["X"][:] lon[lon > 180] -= 360.0 i1, i2, j1, j2 = datasets.spatialSubset(np.sort(lat)[::-1], np.sort(lon), res, bbox) t = pds.variables["T"] tt = netcdf.num2date(t[:], units=t.units) # ti = [tj for tj in range(len(tt)) if tt[tj] >= dt] ti = [tj for tj in range(len(tt)) if resetDatetime(tt[tj]) >= dt[0] and resetDatetime(tt[tj]) <= dt[1]] if len(ti) > 0: lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] if data is None: data = pds.variables[dsvar[ui]][ti, 0, lati, loni] else: data = np.sqrt( data ** 2.0 + pds.variables[dsvar[ui]][ti, 0, lati, loni] ** 2.0) if "temp" in dsvar: data -= 273.15 lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] table = "{0}.ncep".format(varname) for t in range(len(ti)): filename = dbio.writeGeotif(lat, lon, res, data[t, :, :]) dbio.ingest(dbname, filename, tt[ti[t]], table) print("Imported {0} in {1}".format(tt[ti[0]].strftime("%Y-%m-%d"), table)) os.remove(filename)
def ingest(dbname, table, data, lat, lon, res, t, resample=True, overwrite=True): """Import data into RHEAS database.""" if data is not None: if len(data.shape) > 2: data = data[0, :, :] filename = dbio.writeGeotif(lat, lon, res, data) dbio.ingest(dbname, filename, t, table, resample, overwrite) print("Imported {0} in {1}".format(t.strftime("%Y-%m-%d"), table)) os.remove(filename) else: print("WARNING! No data were available to import into {0} for {1}.". format(table, t.strftime("%Y-%m-%d")))
def _downloadVariable(varname, dbname, dt, bbox): """Download specific variable from the MERRA Reanalysis dataset.""" # FIXME: Grid is not rectangular, but 0.5 x 0.625 degrees log = logging.getLogger(__name__) res = 0.5 try: url = "http://goldsmr4.sci.gsfc.nasa.gov:80/dods/M2T1NXSLV" ds = netcdf.Dataset(url) lat = ds.variables["lat"][:] lon = ds.variables["lon"][:] lon[lon > 180] -= 360.0 i1, i2, j1, j2 = datasets.spatialSubset( np.sort(lat)[::-1], np.sort(lon), res, bbox) data = np.zeros((i2 - i1, j2 - j1)) lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] t = ds.variables["time"] tt = netcdf.num2date(t[:], units=t.units) ti = np.where(tt == dt)[0][0] if varname == "tmax": hdata = ds.variables["t2m"][ti:ti + 24, lati, loni] data = np.amax(hdata, axis=0) - 273.15 elif varname == "tmin": hdata = ds.variables["t2m"][ti:ti + 24, lati, loni] data = np.amin(hdata, axis=0) - 273.15 elif varname in ["wind"]: hdata = np.sqrt(ds.variables["u10m"][ti:ti + 24, lati, loni]**2 + ds.variables["v10m"][ti:ti + 24, lati, loni]**2) data = np.mean(hdata, axis=0) lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] filename = dbio.writeGeotif(lat, lon, res, data) table = "{0}.merra".format(varname) dbio.ingest(dbname, filename, dt, table) log.info("Imported {0} in {1}".format(tt[ti].strftime("%Y-%m-%d"), table)) os.remove(filename) except: log.warning("Cannot import MERRA dataset for {0}!".format( dt.strftime("%Y-%m-%d")))
def download(dbname, dt, bbox=None): """Downloads SMOS soil mositure data for a set of dates *dt* and imports them into the PostGIS database *dbname*. Optionally uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat].""" log = logging.getLogger(__name__) res = 0.25 url = "http://*****:*****@cp34-bec.cmima.csic.es/thredds/dodsC/NRTSM001D025A_ALL" f = netcdf.Dataset(url) lat = f.variables['lat'][::-1] # swap latitude orientation to northwards lon = f.variables['lon'][:] i1, i2, j1, j2 = datasets.spatialSubset(lat, lon, res, bbox) smi1 = len(lat) - i2 - 1 smi2 = len(lat) - i1 - 1 lat = lat[i1:i2] lon = lon[j1:j2] t0 = datetime(2010, 1, 12) # initial date of SMOS data t1 = (dt[0] - t0).days if t1 < 0: log.warning("Reseting start date to {0}".format( t0.strftime("%Y-%m-%d"))) t1 = 0 t2 = (dt[-1] - t0).days + 1 nt, _, _ = f.variables['SM'].shape if t2 > nt: t2 = nt log.warning("Reseting end date to {0}".format( (t0 + timedelta(t2)).strftime("%Y-%m-%d"))) ti = range(t1, t2) sm = f.variables['SM'][ti, smi1:smi2, j1:j2] sm = sm[:, ::-1, :] # flip latitude dimension in data array # FIXME: Use spatially variable observation error # smv = f.variables['VARIANCE_SM'][ti, i1:i2, j1:j2][:, ::-1, :] pos, smlat, smlon = regridNearestNeighbor(lat, lon, res) for tj in range(sm.shape[0]): smdata = sm[tj, :, :].ravel()[pos].reshape((len(smlat), len(smlon))) filename = dbio.writeGeotif(smlat, smlon, res, smdata) t = t0 + timedelta(ti[tj]) dbio.ingest(dbname, filename, t, table, False) os.remove(filename)
def download(dbname, dts, bbox=None, enhanced=False): """Downloads SMAP soil mositure data for a set of dates *dt* and imports them into the PostGIS database *dbname*. Optionally uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat].""" log = logging.getLogger(__name__) if enhanced: res = 0.09 url = "https://n5eil01u.ecs.nsidc.org/SMAP/SPL3SMP_E.001" else: res = 0.36 url = "https://n5eil01u.ecs.nsidc.org/DP4/SMAP/SPL3SMP.004" for dt in [dts[0] + timedelta(tt) for tt in range((dts[-1] - dts[0]).days + 1)]: try: outpath, fname = earthdata.download("{0}/{1}".format(url, dt.strftime("%Y.%m.%d")), "SMAP_L3_SM_P_\S*.h5") f = h5py.File("{0}/{1}".format(outpath, fname)) varname = None for v in f.keys(): if "latitude" in f[v] and "longitude" in f[v]: varname = v assert varname is not None lat = f[varname]['latitude'][:, 0] lon = f[varname]['longitude'][0, :] lon[lon > 180] -= 360.0 # FIXME: Need to add reprojection from EASE grid i1, i2, j1, j2 = datasets.spatialSubset(np.sort(lat)[::-1], np.sort(lon), res, bbox) lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] sm = np.zeros((len(lati), len(loni))) for i in range(len(lati)): for j in range(len(loni)): sm[i, j] = f[varname]['soil_moisture'][i, j] # FIXME: Use spatially variable observation error # sme = f[varname]['soil_moisture_error'][i1:i2, j1:j2] lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] filename = dbio.writeGeotif(lat, lon, res, sm) dbio.ingest(dbname, filename, dt, table, False) except: log.warning("No SMAP data available for {0}.".format(dt.strftime("%Y-%m-%d")))