def wrapper(*args, **kwargs): url, varname, bbox, dt = fetch(*args, **kwargs) ds = netcdf4.Dataset(url) for var in ds.variables: if var.lower().startswith("lon") or var.lower() == "x": lonvar = var if var.lower().startswith("lat") or var.lower() == "y": latvar = var if var.lower().startswith("time") or var.lower() == "t": timevar = var lat = ds.variables[latvar][:] lon = ds.variables[lonvar][:] lon[lon > 180] -= 360 res = abs(lat[0] - lat[1]) # assume rectangular grid i1, i2, j1, j2 = datasets.spatialSubset( np.sort(lat)[::-1], np.sort(lon), res, bbox) t = ds.variables[timevar] tt = netcdf4.num2date(t[:], units=t.units) ti = [ tj for tj in range(len(tt)) if resetDatetime(tt[tj]) >= dt[0] and resetDatetime(tt[tj]) <= dt[1] ] if len(ti) > 0: lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] data = ds.variables[varname][ti, lati, loni] dt = tt[ti] else: data = None dt = None lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] return data, lat, lon, dt
def download(dbname, dts, bbox=None): """Downloads IRI forecast tercile probability data from the IRI data server, and imports them into the database *dbname*. Optionally uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat].""" leadtime = 3 res = 2.5 baseurl = "http://iridl.ldeo.columbia.edu/SOURCES/.IRI/.FD/.Seasonal_Forecast/.{0}/.prob/dods" table = {"Precipitation": "precip.iri", "Temperature": "tmax.iri"} for varname in ["Precipitation", "Temperature"]: purl = baseurl.format(varname) pds = netcdf.Dataset(purl) lat = pds.variables["Y"][:] lon = pds.variables["X"][:] lon[lon > 180] -= 360.0 i1, i2, j1, j2 = datasets.spatialSubset(np.sort(lat)[::-1], np.sort(lon), res, bbox) lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] t = pds.variables["F"][:] ti = [tt for tt in range(len(t)) if t[tt] >= ((dts[0].year - 1960) * 12 + dts[0].month - 0.5) and t[tt] <= ((dts[1].year - 1960) * 12 + dts[1].month - 0.5)] for tt in ti: dt = date(1960, 1, 1) + relativedelta(months=int(t[tt])) for m in range(leadtime): for ci, c in enumerate(["below", "normal", "above"]): data = pds.variables["prob"][tt, m, lati, loni, ci] filename = dbio.writeGeotif(lat, lon, res, data) ingest(dbname, filename, dt, m + 1, c, table[varname]) os.remove(filename)
def _downloadVariable(varname, dbname, dts, bbox): """Download specific variable from the MERRA Reanalysis dataset.""" # FIXME: Grid is not rectangular, but 0.5 x 0.625 degrees res = 0.5 for ts in [dts[0] + timedelta(dti) for dti in range((dts[1] - dts[0]).days + 1)]: try: runid = _merraRunid(ts.year) url = "http://goldsmr4.sci.gsfc.nasa.gov:80/opendap/MERRA2/M2T1NXSLV.5.12.4/{1}/{2:02d}/MERRA2_{0}.tavg1_2d_slv_Nx.{1:04d}{2:02d}{3:02d}.nc4".format(runid, ts.year, ts.month, ts.day) ds = netcdf.Dataset(url) lat = ds.variables["lat"][:] lon = ds.variables["lon"][:] lon[lon > 180] -= 360.0 i1, i2, j1, j2 = datasets.spatialSubset(np.sort(lat)[::-1], np.sort(lon), res, bbox) data = np.zeros((i2-i1, j2-j1)) lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] if varname == "tmax": hdata = ds.variables["T2M"][:, lati, loni] data = np.amax(hdata, axis=0) - 273.15 elif varname == "tmin": hdata = ds.variables["T2M"][:, lati, loni] data = np.amin(hdata, axis=0) - 273.15 elif varname in ["wind"]: hdata = np.sqrt(ds.variables["U10M"][:, lati, loni]**2 + ds.variables["V10M"][:, lati, loni]**2) data = np.mean(hdata, axis=0) lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] filename = dbio.writeGeotif(lat, lon, res, data) dbio.ingest(dbname, filename, ts, "{0}.merra".format(varname)) os.remove(filename) except: print("Cannot import MERRA dataset for {0}!".format(ts.strftime("%Y-%m-%d")))
def wrapper(*args, **kwargs): url, varname, bbox, dt = fetch(*args, **kwargs) ds = netcdf4.Dataset(url) for var in ds.variables: if var.lower().startswith("lon") or var.lower() == "x": lonvar = var if var.lower().startswith("lat") or var.lower() == "y": latvar = var if var.lower().startswith("time") or var.lower() == "t": timevar = var lat = ds.variables[latvar][:] lon = ds.variables[lonvar][:] lon[lon > 180] -= 360 res = abs(lat[0]-lat[1]) # assume rectangular grid i1, i2, j1, j2 = datasets.spatialSubset(np.sort(lat)[::-1], np.sort(lon), res, bbox) t = ds.variables[timevar] tt = netcdf4.num2date(t[:], units=t.units) ti = [tj for tj in range(len(tt)) if resetDatetime(tt[tj]) >= dt[0] and resetDatetime(tt[tj]) <= dt[1]] if len(ti) > 0: lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] data = ds.variables[varname][ti, lati, loni] dt = tt[ti] else: data = None dt = None lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] return data, lat, lon, dt
def download(dbname, dt, bbox=None): """Downloads SMOS soil mositure data for a set of dates *dt* and imports them into the PostGIS database *dbname*. Optionally uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat].""" log = logging.getLogger(__name__) res = 0.25 url = "http://*****:*****@cp34-bec.cmima.csic.es/thredds/dodsC/NRTSM001D025A_ALL" f = netcdf.Dataset(url) lat = f.variables['lat'][::-1] # swap latitude orientation to northwards lon = f.variables['lon'][:] i1, i2, j1, j2 = datasets.spatialSubset(lat, lon, res, bbox) smi1 = len(lat) - i2 - 1 smi2 = len(lat) - i1 - 1 lat = lat[i1:i2] lon = lon[j1:j2] t0 = datetime(2010, 1, 12) # initial date of SMOS data t1 = (dt[0] - t0).days if t1 < 0: log.warning("Reseting start date to {0}".format(t0.strftime("%Y-%m-%d"))) t1 = 0 t2 = (dt[-1] - t0).days + 1 nt, _, _ = f.variables['SM'].shape if t2 > nt: t2 = nt log.warning("Reseting end date to {0}".format((t0 + timedelta(t2)).strftime("%Y-%m-%d"))) ti = range(t1, t2) sm = f.variables['SM'][ti, smi1:smi2, j1:j2] # FIXME: Use spatially variable observation error # smv = f.variables['VARIANCE_SM'][ti, i1:i2, j1:j2] for tj in range(sm.shape[0]): filename = dbio.writeGeotif(lat, lon, res, sm[tj, :, :]) t = t0 + timedelta(ti[tj]) dbio.ingest(dbname, filename, t, table, False) log.info("Imported SMOS {0}".format(tj)) os.remove(filename)
def download(dbname, dts, bbox=None): """Downloads SMAP soil mositure data for a set of dates *dt* and imports them into the PostGIS database *dbname*. Optionally uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat].""" res = 0.36 url = "n5eil01u.ecs.nsidc.org" ftp = FTP(url) ftp.login() for dt in [dts[0] + timedelta(tt) for tt in range((dts[1] - dts[0]).days + 1)]: r = ftp.cwd("/pub/SAN/SMAP/SPL3SMP.002/{0}".format(dt.strftime("%Y.%m.%d"))) if r.find("successful") > 0: outpath = tempfile.mkdtemp() fname = [f for f in ftp.nlst() if f.find("h5") > 0][0] with open("{0}/{1}".format(outpath, fname), 'wb') as f: ftp.retrbinary("RETR {0}".format(fname), f.write) f = h5py.File("{0}/{1}".format(outpath, fname)) lat = f['Soil_Moisture_Retrieval_Data']['latitude'][:, 0] lon = f['Soil_Moisture_Retrieval_Data']['longitude'][0, :] lon[lon > 180] -= 360.0 # FIXME: Need to add reprojection from EASE grid i1, i2, j1, j2 = datasets.spatialSubset(np.sort(lat)[::-1], np.sort(lon), res, bbox) lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] sm = np.zeros((len(lati), len(loni))) for i in range(len(lati)): for j in range(len(loni)): sm[i, j] = f['Soil_Moisture_Retrieval_Data']['soil_moisture'][i, j] # FIXME: Use spatially variable observation error # sme = f['Soil_Moisture_Retrieval_Data']['soil_moisture_error'][i1:i2, j1:j2] lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] filename = dbio.writeGeotif(lat, lon, res, sm) dbio.ingest(dbname, filename, dt, table, False) else: print("No SMAP data available for {0}.".format(dt.strftime("%Y-%m-%d")))
def _downloadVariable(varname, dbname, dt, bbox): """Download specific variable from the MERRA Reanalysis dataset.""" # FIXME: Grid is not rectangular, but 0.5 x 0.625 degrees res = 0.5 try: url = "http://goldsmr4.sci.gsfc.nasa.gov:80/dods/M2T1NXSLV" ds = netcdf.Dataset(url) lat = ds.variables["lat"][:] lon = ds.variables["lon"][:] lon[lon > 180] -= 360.0 i1, i2, j1, j2 = datasets.spatialSubset(np.sort(lat)[::-1], np.sort(lon), res, bbox) data = np.zeros((i2-i1, j2-j1)) lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] t = ds.variables["time"] tt = netcdf.num2date(t[:], units=t.units) ti = np.where(tt == dt)[0][0] if varname == "tmax": hdata = ds.variables["t2m"][ti:ti+24, lati, loni] data = np.amax(hdata, axis=0) - 273.15 elif varname == "tmin": hdata = ds.variables["t2m"][ti:ti+24, lati, loni] data = np.amin(hdata, axis=0) - 273.15 elif varname in ["wind"]: hdata = np.sqrt(ds.variables["u10m"][ti:ti+24, lati, loni]**2 + ds.variables["v10m"][ti:ti+24, lati, loni]**2) data = np.mean(hdata, axis=0) lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] filename = dbio.writeGeotif(lat, lon, res, data) table = "{0}.merra".format(varname) dbio.ingest(dbname, filename, dt, table) print("Imported {0} in {1}".format(tt[ti].strftime("%Y-%m-%d"), table)) os.remove(filename) except: print("Cannot import MERRA dataset for {0}!".format(dt.strftime("%Y-%m-%d")))
def download(dbname, dt, bbox=None): """Downloads SMOS soil mositure data for a set of dates *dt* and imports them into the PostGIS database *dbname*. Optionally uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat].""" res = 0.25 url = "http://*****:*****@cp34-bec.cmima.csic.es/thredds/dodsC/NRTSM001D025A_ALL" f = netcdf.Dataset(url) lat = f.variables['lat'][::-1] # swap latitude orientation to northwards lon = f.variables['lon'][:] i1, i2, j1, j2 = datasets.spatialSubset(lat, lon, res, bbox) smi1 = len(lat) - i2 - 1 smi2 = len(lat) - i1 - 1 lat = lat[i1:i2] lon = lon[j1:j2] t0 = datetime(2010, 1, 12) # initial date of SMOS data t1 = (dt[0] - t0).days t2 = (dt[-1] - t0).days + 1 ti = range(t1, t2) sm = f.variables['SM'][ti, smi1:smi2, j1:j2] # FIXME: Use spatially variable observation error # smv = f.variables['VARIANCE_SM'][ti, i1:i2, j1:j2] for tj in range(sm.shape[0]): filename = dbio.writeGeotif(lat, lon, res, sm[tj, :, :]) t = t0 + timedelta(ti[tj]) dbio.ingest(dbname, filename, t, table, False) print("Imported SMOS {0}".format(tj)) os.remove(filename)
def _downloadVariable(varname, dbname, dt, bbox=None): """Download specific variable from the NCEP Reanalysis dataset.""" log = logging.getLogger(__name__) res = 1.875 baseurl = "http://www.esrl.noaa.gov/psd/thredds/dodsC/Datasets/ncep.reanalysis.dailyavgs/surface_gauss" if varname == "tmax": urls = ["{0}/tmax.2m.gauss.{1}.nc".format(baseurl, dt[0].year)] dsvar = ["tmax"] elif varname == "tmin": urls = ["{0}/tmin.2m.gauss.{1}.nc".format(baseurl, dt[0].year)] dsvar = ["tmin"] else: urls = [ "{0}/uwnd.10m.gauss.{1}.nc".format(baseurl, dt[0].year), "{0}/vwnd.10m.gauss.{1}.nc".format(baseurl, dt[0].year) ] dsvar = ["uwnd", "vwnd"] data = None for ui, url in enumerate(urls): pds = netcdf.Dataset(url) lat = pds.variables["lat"][:] lon = pds.variables["lon"][:] lon[lon > 180] -= 360.0 i1, i2, j1, j2 = datasets.spatialSubset( np.sort(lat)[::-1], np.sort(lon), res, bbox) t = pds.variables["time"] tt = netcdf.num2date(t[:], units=t.units) ti = [ tj for tj in range(len(tt)) if resetDatetime(tt[tj]) >= dt[0] and resetDatetime(tt[tj]) <= dt[1] ] if len(ti) > 0: lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] if data is None: data = pds.variables[dsvar[ui]][ti, lati, loni] else: data = np.sqrt(data**2.0 + pds.variables[dsvar[ui]][ti, lati, loni]**2.0) if "temp" in dsvar: data -= 273.15 lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] table = "{0}.ncep".format(varname) for t in range(len(ti)): filename = dbio.writeGeotif(lat, lon, res, data[t, :, :]) dbio.ingest(dbname, filename, tt[ti[t]], table) os.remove(filename) for dtt in [ dt[0] + timedelta(days=tj) for tj in range((dt[-1] - dt[0]).days + 1) ]: if dtt not in tt: log.warning( "NCEP data not available for {0}. Skipping download!".format( dtt.strftime("%Y-%m-%d")))
def _downloadVariable(varname, dbname, dt, bbox=None): """Download specific variable from the NCEP Reanalysis dataset.""" res = 1.875 if varname == "tmax": urls = [ "http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.maximum/.temp/dods" ] dsvar = ["temp"] elif varname == "tmin": urls = [ "http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.minimum/.temp/dods" ] dsvar = ["temp"] else: urls = [ "http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.u/dods", "http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.v/dods" ] dsvar = ["u", "v"] data = None for ui, url in enumerate(urls): pds = netcdf.Dataset(url) lat = pds.variables["Y"][:] lon = pds.variables["X"][:] lon[lon > 180] -= 360.0 i1, i2, j1, j2 = datasets.spatialSubset( np.sort(lat)[::-1], np.sort(lon), res, bbox) t = pds.variables["T"] tt = netcdf.num2date(t[:], units=t.units) # ti = [tj for tj in range(len(tt)) if tt[tj] >= dt] ti = [ tj for tj in range(len(tt)) if resetDatetime(tt[tj]) >= dt[0] and resetDatetime(tt[tj]) <= dt[1] ] if len(ti) > 0: lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] if data is None: data = pds.variables[dsvar[ui]][ti, 0, lati, loni] else: data = np.sqrt(data**2.0 + pds.variables[dsvar[ui]][ti, 0, lati, loni]**2.0) if "temp" in dsvar: data -= 273.15 lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] table = "{0}.ncep".format(varname) for t in range(len(ti)): filename = dbio.writeGeotif(lat, lon, res, data[t, :, :]) dbio.ingest(dbname, filename, tt[ti[t]], table) print("Imported {0} in {1}".format(tt[ti[0]].strftime("%Y-%m-%d"), table)) os.remove(filename)
def download(dbname, dts, bbox=None, enhanced=False): """Downloads SMAP soil mositure data for a set of dates *dt* and imports them into the PostGIS database *dbname*. Optionally uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat].""" log = logging.getLogger(__name__) if enhanced: res = 0.09 url = "https://n5eil01u.ecs.nsidc.org/SMAP/SPL3SMP_E.001" else: res = 0.36 url = "https://n5eil01u.ecs.nsidc.org/DP4/SMAP/SPL3SMP.004" for dt in [ dts[0] + timedelta(tt) for tt in range((dts[-1] - dts[0]).days + 1) ]: try: outpath, fname = earthdata.download( "{0}/{1}".format(url, dt.strftime("%Y.%m.%d")), "SMAP_L3_SM_P_\S*.h5") f = h5py.File("{0}/{1}".format(outpath, fname)) varname = None for v in f.keys(): if "latitude" in f[v] and "longitude" in f[v]: varname = v assert varname is not None lat = f[varname]['latitude'][:, 0] lon = f[varname]['longitude'][0, :] lon[lon > 180] -= 360.0 # FIXME: Need to add reprojection from EASE grid i1, i2, j1, j2 = datasets.spatialSubset( np.sort(lat)[::-1], np.sort(lon), res, bbox) lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] sm = np.zeros((len(lati), len(loni))) for i in range(len(lati)): for j in range(len(loni)): sm[i, j] = f[varname]['soil_moisture'][i, j] # FIXME: Use spatially variable observation error # sme = f[varname]['soil_moisture_error'][i1:i2, j1:j2] lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] filename = dbio.writeGeotif(lat, lon, res, sm) dbio.ingest(dbname, filename, dt, table, False) except: log.warning("No SMAP data available for {0}.".format( dt.strftime("%Y-%m-%d")))
def _downloadVariable(varname, dbname, dt, bbox=None): """Download specific variable from the NCEP Reanalysis dataset.""" log = logging.getLogger(__name__) res = 1.875 baseurl = "http://www.esrl.noaa.gov/psd/thredds/dodsC/Datasets/ncep.reanalysis.dailyavgs/surface_gauss" if varname == "tmax": urls = ["{0}/tmax.2m.gauss.{1}.nc".format(baseurl, dt[0].year)] dsvar = ["tmax"] elif varname == "tmin": urls = ["{0}/tmin.2m.gauss.{1}.nc".format(baseurl, dt[0].year)] dsvar = ["tmin"] else: urls = ["{0}/uwnd.10m.gauss.{1}.nc".format(baseurl, dt[0].year), "{0}/vwnd.10m.gauss.{1}.nc".format(baseurl, dt[0].year)] dsvar = ["uwnd", "vwnd"] data = None for ui, url in enumerate(urls): pds = netcdf.Dataset(url) lat = pds.variables["lat"][:] lon = pds.variables["lon"][:] lon[lon > 180] -= 360.0 i1, i2, j1, j2 = datasets.spatialSubset(np.sort(lat)[::-1], np.sort(lon), res, bbox) t = pds.variables["time"] tt = netcdf.num2date(t[:], units=t.units) ti = [tj for tj in range(len(tt)) if resetDatetime(tt[tj]) >= dt[0] and resetDatetime(tt[tj]) <= dt[1]] if len(ti) > 0: lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] if data is None: data = pds.variables[dsvar[ui]][ti, lati, loni] else: data = np.sqrt( data ** 2.0 + pds.variables[dsvar[ui]][ti, lati, loni] ** 2.0) if "temp" in dsvar: data -= 273.15 lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] table = "{0}.ncep".format(varname) for t in range(len(ti)): filename = dbio.writeGeotif(lat, lon, res, data[t, :, :]) dbio.ingest(dbname, filename, tt[ti[t]], table) os.remove(filename) for dtt in [dt[0] + timedelta(days=tj) for tj in range((dt[-1]-dt[0]).days + 1)]: if dtt not in tt: log.warning("NCEP data not available for {0}. Skipping download!".format( dtt.strftime("%Y-%m-%d")))
def wrapper(*args, **kwargs): outpath, filename, bbox, dt = fetch(*args, **kwargs) if filename is not None: lfilename = datasets.uncompress(filename, outpath) f = gdal.Open("{0}/{1}".format(outpath, lfilename)) xul, xres, _, yul, _, yres = f.GetGeoTransform() data = f.ReadAsArray() nr, nc = data.shape lat = np.arange(yul + yres/2.0, yul + yres * nr, yres) lon = np.arange(xul + xres/2.0, xul + xres * nc, xres) i1, i2, j1, j2 = datasets.spatialSubset(lat, lon, xres, bbox) data = data[i1:i2, j1:j2] lat = lat[i1:i2] lon = lon[j1:j2] shutil.rmtree(outpath) else: data = lat = lon = None return data, lat, lon, dt
def wrapper(*args, **kwargs): outpath, filename, bbox, dt = fetch(*args, **kwargs) if filename is not None: lfilename = datasets.uncompress(filename, outpath) f = gdal.Open("{0}/{1}".format(outpath, lfilename)) xul, xres, _, yul, _, yres = f.GetGeoTransform() data = f.ReadAsArray() nr, nc = data.shape lat = np.arange(yul + yres / 2.0, yul + yres * nr, yres) lon = np.arange(xul + xres / 2.0, xul + xres * nc, xres) i1, i2, j1, j2 = datasets.spatialSubset(lat, lon, xres, bbox) data = data[i1:i2, j1:j2] lat = lat[i1:i2] lon = lon[j1:j2] shutil.rmtree(outpath) else: data = lat = lon = None return data, lat, lon, dt
def download(dbname, dts, bbox=None): """Downloads SMAP soil mositure data for a set of dates *dt* and imports them into the PostGIS database *dbname*. Optionally uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat].""" res = 0.36 url = "n5eil01u.ecs.nsidc.org" ftp = FTP(url) ftp.login() for dt in [ dts[0] + timedelta(tt) for tt in range((dts[-1] - dts[0]).days + 1) ]: r = ftp.cwd("/pub/SAN/SMAP/SPL3SMP.003/{0}".format( dt.strftime("%Y.%m.%d"))) if r.find("successful") > 0: outpath = tempfile.mkdtemp() fname = [f for f in ftp.nlst() if f.find("h5") > 0][0] with open("{0}/{1}".format(outpath, fname), 'wb') as f: ftp.retrbinary("RETR {0}".format(fname), f.write) f = h5py.File("{0}/{1}".format(outpath, fname)) lat = f['Soil_Moisture_Retrieval_Data']['latitude'][:, 0] lon = f['Soil_Moisture_Retrieval_Data']['longitude'][0, :] lon[lon > 180] -= 360.0 # FIXME: Need to add reprojection from EASE grid i1, i2, j1, j2 = datasets.spatialSubset( np.sort(lat)[::-1], np.sort(lon), res, bbox) lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] sm = np.zeros((len(lati), len(loni))) for i in range(len(lati)): for j in range(len(loni)): sm[i, j] = f['Soil_Moisture_Retrieval_Data']['soil_moisture'][ i, j] # FIXME: Use spatially variable observation error # sme = f['Soil_Moisture_Retrieval_Data']['soil_moisture_error'][i1:i2, j1:j2] lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] filename = dbio.writeGeotif(lat, lon, res, sm) dbio.ingest(dbname, filename, dt, table, False) else: print("No SMAP data available for {0}.".format( dt.strftime("%Y-%m-%d")))
def _downloadVariable(varname, dbname, dt, bbox=None): """Download specific variable from the NCEP Reanalysis dataset.""" res = 1.875 if varname == "tmax": urls = ["http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.maximum/.temp/dods"] dsvar = ["temp"] elif varname == "tmin": urls = ["http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.minimum/.temp/dods"] dsvar = ["temp"] else: urls = ["http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.u/dods", "http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.v/dods"] dsvar = ["u", "v"] data = None for ui, url in enumerate(urls): pds = netcdf.Dataset(url) lat = pds.variables["Y"][:] lon = pds.variables["X"][:] lon[lon > 180] -= 360.0 i1, i2, j1, j2 = datasets.spatialSubset(np.sort(lat)[::-1], np.sort(lon), res, bbox) t = pds.variables["T"] tt = netcdf.num2date(t[:], units=t.units) # ti = [tj for tj in range(len(tt)) if tt[tj] >= dt] ti = [tj for tj in range(len(tt)) if resetDatetime(tt[tj]) >= dt[0] and resetDatetime(tt[tj]) <= dt[1]] if len(ti) > 0: lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] if data is None: data = pds.variables[dsvar[ui]][ti, 0, lati, loni] else: data = np.sqrt( data ** 2.0 + pds.variables[dsvar[ui]][ti, 0, lati, loni] ** 2.0) if "temp" in dsvar: data -= 273.15 lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] table = "{0}.ncep".format(varname) for t in range(len(ti)): filename = dbio.writeGeotif(lat, lon, res, data[t, :, :]) dbio.ingest(dbname, filename, tt[ti[t]], table) print("Imported {0} in {1}".format(tt[ti[0]].strftime("%Y-%m-%d"), table)) os.remove(filename)
def _downloadVariable(varname, dbname, dt, bbox): """Download specific variable from the MERRA Reanalysis dataset.""" # FIXME: Grid is not rectangular, but 0.5 x 0.625 degrees log = logging.getLogger(__name__) res = 0.5 try: url = "http://goldsmr4.sci.gsfc.nasa.gov:80/dods/M2T1NXSLV" ds = netcdf.Dataset(url) lat = ds.variables["lat"][:] lon = ds.variables["lon"][:] lon[lon > 180] -= 360.0 i1, i2, j1, j2 = datasets.spatialSubset( np.sort(lat)[::-1], np.sort(lon), res, bbox) data = np.zeros((i2 - i1, j2 - j1)) lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] t = ds.variables["time"] tt = netcdf.num2date(t[:], units=t.units) ti = np.where(tt == dt)[0][0] if varname == "tmax": hdata = ds.variables["t2m"][ti:ti + 24, lati, loni] data = np.amax(hdata, axis=0) - 273.15 elif varname == "tmin": hdata = ds.variables["t2m"][ti:ti + 24, lati, loni] data = np.amin(hdata, axis=0) - 273.15 elif varname in ["wind"]: hdata = np.sqrt(ds.variables["u10m"][ti:ti + 24, lati, loni]**2 + ds.variables["v10m"][ti:ti + 24, lati, loni]**2) data = np.mean(hdata, axis=0) lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] filename = dbio.writeGeotif(lat, lon, res, data) table = "{0}.merra".format(varname) dbio.ingest(dbname, filename, dt, table) log.info("Imported {0} in {1}".format(tt[ti].strftime("%Y-%m-%d"), table)) os.remove(filename) except: log.warning("Cannot import MERRA dataset for {0}!".format( dt.strftime("%Y-%m-%d")))
def download(dbname, dt, bbox=None): """Downloads SMOS soil mositure data for a set of dates *dt* and imports them into the PostGIS database *dbname*. Optionally uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat].""" log = logging.getLogger(__name__) res = 0.25 url = "http://*****:*****@cp34-bec.cmima.csic.es/thredds/dodsC/NRTSM001D025A_ALL" f = netcdf.Dataset(url) lat = f.variables['lat'][::-1] # swap latitude orientation to northwards lon = f.variables['lon'][:] i1, i2, j1, j2 = datasets.spatialSubset(lat, lon, res, bbox) smi1 = len(lat) - i2 - 1 smi2 = len(lat) - i1 - 1 lat = lat[i1:i2] lon = lon[j1:j2] t0 = datetime(2010, 1, 12) # initial date of SMOS data t1 = (dt[0] - t0).days if t1 < 0: log.warning("Reseting start date to {0}".format( t0.strftime("%Y-%m-%d"))) t1 = 0 t2 = (dt[-1] - t0).days + 1 nt, _, _ = f.variables['SM'].shape if t2 > nt: t2 = nt log.warning("Reseting end date to {0}".format( (t0 + timedelta(t2)).strftime("%Y-%m-%d"))) ti = range(t1, t2) sm = f.variables['SM'][ti, smi1:smi2, j1:j2] sm = sm[:, ::-1, :] # flip latitude dimension in data array # FIXME: Use spatially variable observation error # smv = f.variables['VARIANCE_SM'][ti, i1:i2, j1:j2][:, ::-1, :] pos, smlat, smlon = regridNearestNeighbor(lat, lon, res) for tj in range(sm.shape[0]): smdata = sm[tj, :, :].ravel()[pos].reshape((len(smlat), len(smlon))) filename = dbio.writeGeotif(smlat, smlon, res, smdata) t = t0 + timedelta(ti[tj]) dbio.ingest(dbname, filename, t, table, False) os.remove(filename)
def download(dbname, dts, bbox=None, enhanced=False): """Downloads SMAP soil mositure data for a set of dates *dt* and imports them into the PostGIS database *dbname*. Optionally uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat].""" log = logging.getLogger(__name__) if enhanced: res = 0.09 url = "https://n5eil01u.ecs.nsidc.org/SMAP/SPL3SMP_E.001" else: res = 0.36 url = "https://n5eil01u.ecs.nsidc.org/DP4/SMAP/SPL3SMP.004" for dt in [dts[0] + timedelta(tt) for tt in range((dts[-1] - dts[0]).days + 1)]: try: outpath, fname = earthdata.download("{0}/{1}".format(url, dt.strftime("%Y.%m.%d")), "SMAP_L3_SM_P_\S*.h5") f = h5py.File("{0}/{1}".format(outpath, fname)) varname = None for v in f.keys(): if "latitude" in f[v] and "longitude" in f[v]: varname = v assert varname is not None lat = f[varname]['latitude'][:, 0] lon = f[varname]['longitude'][0, :] lon[lon > 180] -= 360.0 # FIXME: Need to add reprojection from EASE grid i1, i2, j1, j2 = datasets.spatialSubset(np.sort(lat)[::-1], np.sort(lon), res, bbox) lati = np.argsort(lat)[::-1][i1:i2] loni = np.argsort(lon)[j1:j2] sm = np.zeros((len(lati), len(loni))) for i in range(len(lati)): for j in range(len(loni)): sm[i, j] = f[varname]['soil_moisture'][i, j] # FIXME: Use spatially variable observation error # sme = f[varname]['soil_moisture_error'][i1:i2, j1:j2] lat = np.sort(lat)[::-1][i1:i2] lon = np.sort(lon)[j1:j2] filename = dbio.writeGeotif(lat, lon, res, sm) dbio.ingest(dbname, filename, dt, table, False) except: log.warning("No SMAP data available for {0}.".format(dt.strftime("%Y-%m-%d")))