def download(dbname, dt, bbox=None):
"""Downloads SMOS soil mositure data for a set of dates *dt*
and imports them into the PostGIS database *dbname*. Optionally
uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat]."""
res = 0.25
url = "http://*****:*****@cp34-bec.cmima.csic.es/thredds/dodsC/NRTSM001D025A_ALL"
f = netcdf.Dataset(url)
lat = f.variables['lat'][::-1] # swap latitude orientation to northwards
lon = f.variables['lon'][:]
i1, i2, j1, j2 = datasets.spatialSubset(lat, lon, res, bbox)
smi1 = len(lat) - i2 - 1
smi2 = len(lat) - i1 - 1
lat = lat[i1:i2]
lon = lon[j1:j2]
t0 = datetime(2010, 1, 12) # initial date of SMOS data
t1 = (dt[0] - t0).days
t2 = (dt[-1] - t0).days + 1
ti = range(t1, t2)
sm = f.variables['SM'][ti, smi1:smi2, j1:j2]
# FIXME: Use spatially variable observation error
# smv = f.variables['VARIANCE_SM'][ti, i1:i2, j1:j2]
for tj in range(sm.shape[0]):
filename = dbio.writeGeotif(lat, lon, res, sm[tj, :, :])
t = t0 + timedelta(ti[tj])
dbio.ingest(dbname, filename, t, table, False)
print("Imported SMOS {0}".format(tj))
os.remove(filename)
def download(dbname, dts, bbox=None):
"""Downloads SMAP soil mositure data for a set of dates *dt*
and imports them into the PostGIS database *dbname*. Optionally
uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat]."""
res = 0.36
url = "n5eil01u.ecs.nsidc.org"
ftp = FTP(url)
ftp.login()
for dt in [dts[0] + timedelta(tt) for tt in range((dts[1] - dts[0]).days + 1)]:
r = ftp.cwd("/pub/SAN/SMAP/SPL3SMP.002/{0}".format(dt.strftime("%Y.%m.%d")))
if r.find("successful") > 0:
outpath = tempfile.mkdtemp()
fname = [f for f in ftp.nlst() if f.find("h5") > 0][0]
with open("{0}/{1}".format(outpath, fname), 'wb') as f:
ftp.retrbinary("RETR {0}".format(fname), f.write)
f = h5py.File("{0}/{1}".format(outpath, fname))
lat = f['Soil_Moisture_Retrieval_Data']['latitude'][:, 0]
lon = f['Soil_Moisture_Retrieval_Data']['longitude'][0, :]
lon[lon > 180] -= 360.0
# FIXME: Need to add reprojection from EASE grid
i1, i2, j1, j2 = datasets.spatialSubset(np.sort(lat)[::-1], np.sort(lon), res, bbox)
lati = np.argsort(lat)[::-1][i1:i2]
loni = np.argsort(lon)[j1:j2]
sm = np.zeros((len(lati), len(loni)))
for i in range(len(lati)):
for j in range(len(loni)):
sm[i, j] = f['Soil_Moisture_Retrieval_Data']['soil_moisture'][i, j]
# FIXME: Use spatially variable observation error
# sme = f['Soil_Moisture_Retrieval_Data']['soil_moisture_error'][i1:i2, j1:j2]
lat = np.sort(lat)[::-1][i1:i2]
lon = np.sort(lon)[j1:j2]
filename = dbio.writeGeotif(lat, lon, res, sm)
dbio.ingest(dbname, filename, dt, table, False)
else:
print("No SMAP data available for {0}.".format(dt.strftime("%Y-%m-%d")))
def ingest(dbname, table, data, lat, lon, res, t):
"""Import data into RHEAS database."""
for tj in range(data.shape[0]):
filename = dbio.writeGeotif(lat, lon, res, data[tj, :, :])
dbio.ingest(dbname, filename, t[tj], table)
print("Imported {0} in {1}".format(t[tj].strftime("%Y-%m-%d"), table))
os.remove(filename)
def download(dbname, dt, bbox=None):
"""Downloads SMOS soil mositure data for a set of dates *dt*
and imports them into the PostGIS database *dbname*. Optionally
uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat]."""
res = 0.25
url = "http://*****:*****@cp34-bec.cmima.csic.es/thredds/dodsC/NRTSM001D025A_ALL"
f = netcdf.Dataset(url)
lat = f.variables['lat'][:]
lon = f.variables['lon'][:]
if bbox is not None:
i = np.where(np.logical_and(lat > bbox[1], lat < bbox[3]))[0]
j = np.where(np.logical_and(lon > bbox[0], lon < bbox[2]))[0]
lat = lat[i]
lon = lon[j]
else:
i = range(len(lat))
j = range(len(lon))
t0 = datetime(2010, 1, 12) # initial date of SMOS data
t1 = (dt[0] - t0).days
t2 = (dt[1] - t0).days + 1
ti = range(t1, t2)
sm = f.variables['SM'][ti, i, j]
# FIXME: Use spatially variable observation error
# smv = f.variables['VARIANCE_SM'][ti, i, j]
for tj in range(sm.shape[0]):
filename = dbio.writeGeotif(lat, lon, res, sm[tj, :, :])
t = t0 + timedelta(ti[tj])
dbio.ingest(dbname, filename, t, table, False)
print("Imported SMOS {0}".format(tj))
os.remove(filename)
def download(dbname, dt, bbox=None):
"""Downloads SMAP soil mositure data for a set of dates *dt*
and imports them into the PostGIS database *dbname*. Optionally
uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat]."""
res = 0.36
url = "ftp://n5eil01u.ecs.nsidc.org"
ftp = FTP(url)
ftp.login()
ftp.cwd("SAN/SMAP/SPL3SMP.002")
days = ftp.nlst()
datadir = dt.strftime("%Y.%m.%d")
if datadir in days:
outpath = tempfile.mkdtemp()
ftp.cwd(datadir)
fname = [f for f in ftp.nlst() if f.find("h5") > 0][0]
with open("{0}/{1}".format(outpath, fname), "wb") as f:
ftp.retrbinary("RETR {0}".format(fname), f.write)
f = h5py.File("{0}/{1}".format(outpath, fname))
lat = f["Soil_Moisture_Retrieval_Data"]["latitude"][:, 0]
lon = f["Soil_Moisture_Retrieval_Data"]["longitude"][0, :]
if bbox is not None:
i = np.where(np.logical_and(lat > bbox[1], lat < bbox[3]))[0]
j = np.where(np.logical_and(lon > bbox[0], lon < bbox[2]))[0]
lat = lat[i]
lon = lon[j]
else:
i = range(len(lat))
j = range(len(lon))
sm = f["Soil_Moisture_Retrieval_Data"]["soil_moisture"][i[0] : i[-1] + 1, j[0] : j[-1] + 1]
# FIXME: Use spatially variable observation error
# sme = f['Soil_Moisture_Retrieval_Data']['soil_moisture_error'][i[0]:i[-1]+1, j[0]:j[-1]+1]
filename = dbio.writeGeotif(lat, lon, res, sm)
dbio.ingest(dbname, filename, dt, table, False)
def _downloadVariable(varname, dbname, dts, bbox):
"""Download specific variable from the MERRA Reanalysis dataset."""
# FIXME: Grid is not rectangular, but 0.5 x 0.625 degrees
res = 0.5
for ts in [dts[0] + timedelta(dti) for dti in range((dts[1] - dts[0]).days + 1)]:
try:
runid = _merraRunid(ts.year)
url = "http://goldsmr4.sci.gsfc.nasa.gov:80/opendap/MERRA2/M2T1NXSLV.5.12.4/{1}/{2:02d}/MERRA2_{0}.tavg1_2d_slv_Nx.{1:04d}{2:02d}{3:02d}.nc4".format(runid, ts.year, ts.month, ts.day)
ds = netcdf.Dataset(url)
lat = ds.variables["lat"][:]
lon = ds.variables["lon"][:]
lon[lon > 180] -= 360.0
i1, i2, j1, j2 = datasets.spatialSubset(np.sort(lat)[::-1], np.sort(lon), res, bbox)
data = np.zeros((i2-i1, j2-j1))
lati = np.argsort(lat)[::-1][i1:i2]
loni = np.argsort(lon)[j1:j2]
if varname == "tmax":
hdata = ds.variables["T2M"][:, lati, loni]
data = np.amax(hdata, axis=0) - 273.15
elif varname == "tmin":
hdata = ds.variables["T2M"][:, lati, loni]
data = np.amin(hdata, axis=0) - 273.15
elif varname in ["wind"]:
hdata = np.sqrt(ds.variables["U10M"][:, lati, loni]**2 + ds.variables["V10M"][:, lati, loni]**2)
data = np.mean(hdata, axis=0)
lat = np.sort(lat)[::-1][i1:i2]
lon = np.sort(lon)[j1:j2]
filename = dbio.writeGeotif(lat, lon, res, data)
dbio.ingest(dbname, filename, ts, "{0}.merra".format(varname))
os.remove(filename)
except:
print("Cannot import MERRA dataset for {0}!".format(ts.strftime("%Y-%m-%d")))
def download(dbname, dts, bbox=None):
"""Downloads IRI forecast tercile probability data from the IRI data server,
and imports them into the database *db*. Optionally uses a bounding box to
limit the region with [minlon, minlat, maxlon, maxlat]."""
leadtime = 3
res = 2.5
baseurl = "http://iridl.ldeo.columbia.edu/SOURCES/.IRI/.FD/.Seasonal_Forecast/.{0}/.prob/dods"
table = {"Precipitation": "precip.iri", "Temperature": "tmax.iri"}
for varname in ["Precipitation", "Temperature"]:
purl = baseurl.format(varname)
pds = netcdf.Dataset(purl)
lat = pds.variables["Y"][:]
lon = pds.variables["X"][:]
if bbox is not None:
i = np.where(np.logical_and(lat > bbox[1], lat < bbox[3]))[0]
j = np.where(np.logical_and(lon > bbox[0], lon < bbox[2]))[0]
lat = lat[i]
lon = lon[j]
else:
i = range(len(lat))
j = range(len(lon))
t = pds.variables["F"][:]
ti = [tt for tt in range(len(t)) if t[tt] >= ((dts[0].year - 1960) * 12 + dts[0].month - 0.5) and t[tt] <= ((dts[1].year - 1960) * 12 + dts[1].month - 0.5)]
for tt in ti:
dt = date(1960, 1, 1) + relativedelta(months=int(t[tt]))
for m in range(leadtime):
for ci, c in enumerate(["below", "normal", "above"]):
data = pds.variables["prob"][tt, m, i, j, ci]
filename = dbio.writeGeotif(lat, lon, res, data)
ingest(dbname, filename, dt, m + 1, c, table[varname])
os.remove(filename)
def _downloadVariable(varname, dbname, dt, bbox):
"""Download specific variable from the MERRA Reanalysis dataset."""
# FIXME: Grid is not rectangular, but 0.5 x 0.625 degrees
res = 0.5
try:
url = "http://goldsmr4.sci.gsfc.nasa.gov:80/dods/M2T1NXSLV"
ds = netcdf.Dataset(url)
lat = ds.variables["lat"][:]
lon = ds.variables["lon"][:]
lon[lon > 180] -= 360.0
i1, i2, j1, j2 = datasets.spatialSubset(np.sort(lat)[::-1], np.sort(lon), res, bbox)
data = np.zeros((i2-i1, j2-j1))
lati = np.argsort(lat)[::-1][i1:i2]
loni = np.argsort(lon)[j1:j2]
t = ds.variables["time"]
tt = netcdf.num2date(t[:], units=t.units)
ti = np.where(tt == dt)[0][0]
if varname == "tmax":
hdata = ds.variables["t2m"][ti:ti+24, lati, loni]
data = np.amax(hdata, axis=0) - 273.15
elif varname == "tmin":
hdata = ds.variables["t2m"][ti:ti+24, lati, loni]
data = np.amin(hdata, axis=0) - 273.15
elif varname in ["wind"]:
hdata = np.sqrt(ds.variables["u10m"][ti:ti+24, lati, loni]**2 + ds.variables["v10m"][ti:ti+24, lati, loni]**2)
data = np.mean(hdata, axis=0)
lat = np.sort(lat)[::-1][i1:i2]
lon = np.sort(lon)[j1:j2]
filename = dbio.writeGeotif(lat, lon, res, data)
table = "{0}.merra".format(varname)
dbio.ingest(dbname, filename, dt, table)
print("Imported {0} in {1}".format(tt[ti].strftime("%Y-%m-%d"), table))
os.remove(filename)
except:
print("Cannot import MERRA dataset for {0}!".format(dt.strftime("%Y-%m-%d")))
def download(dbname, dt, bbox=None):
"""Downloads SMOS soil mositure data for a set of dates *dt*
and imports them into the PostGIS database *dbname*. Optionally
uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat]."""
log = logging.getLogger(__name__)
res = 0.25
url = "http://*****:*****@cp34-bec.cmima.csic.es/thredds/dodsC/NRTSM001D025A_ALL"
f = netcdf.Dataset(url)
lat = f.variables['lat'][::-1] # swap latitude orientation to northwards
lon = f.variables['lon'][:]
i1, i2, j1, j2 = datasets.spatialSubset(lat, lon, res, bbox)
smi1 = len(lat) - i2 - 1
smi2 = len(lat) - i1 - 1
lat = lat[i1:i2]
lon = lon[j1:j2]
t0 = datetime(2010, 1, 12) # initial date of SMOS data
t1 = (dt[0] - t0).days
if t1 < 0:
log.warning("Reseting start date to {0}".format(t0.strftime("%Y-%m-%d")))
t1 = 0
t2 = (dt[-1] - t0).days + 1
nt, _, _ = f.variables['SM'].shape
if t2 > nt:
t2 = nt
log.warning("Reseting end date to {0}".format((t0 + timedelta(t2)).strftime("%Y-%m-%d")))
ti = range(t1, t2)
sm = f.variables['SM'][ti, smi1:smi2, j1:j2]
# FIXME: Use spatially variable observation error
# smv = f.variables['VARIANCE_SM'][ti, i1:i2, j1:j2]
for tj in range(sm.shape[0]):
filename = dbio.writeGeotif(lat, lon, res, sm[tj, :, :])
t = t0 + timedelta(ti[tj])
dbio.ingest(dbname, filename, t, table, False)
log.info("Imported SMOS {0}".format(tj))
os.remove(filename)
def download(dbname, dts, bbox=None):
"""Downloads IRI forecast tercile probability data from the IRI data server,
and imports them into the database *dbname*. Optionally uses a bounding box to
limit the region with [minlon, minlat, maxlon, maxlat]."""
leadtime = 3
res = 2.5
baseurl = "http://iridl.ldeo.columbia.edu/SOURCES/.IRI/.FD/.Seasonal_Forecast/.{0}/.prob/dods"
table = {"Precipitation": "precip.iri", "Temperature": "tmax.iri"}
for varname in ["Precipitation", "Temperature"]:
purl = baseurl.format(varname)
pds = netcdf.Dataset(purl)
lat = pds.variables["Y"][:]
lon = pds.variables["X"][:]
lon[lon > 180] -= 360.0
i1, i2, j1, j2 = datasets.spatialSubset(np.sort(lat)[::-1], np.sort(lon), res, bbox)
lati = np.argsort(lat)[::-1][i1:i2]
loni = np.argsort(lon)[j1:j2]
lat = np.sort(lat)[::-1][i1:i2]
lon = np.sort(lon)[j1:j2]
t = pds.variables["F"][:]
ti = [tt for tt in range(len(t)) if t[tt] >= ((dts[0].year - 1960) * 12 + dts[0].month - 0.5) and t[tt] <= ((dts[1].year - 1960) * 12 + dts[1].month - 0.5)]
for tt in ti:
dt = date(1960, 1, 1) + relativedelta(months=int(t[tt]))
for m in range(leadtime):
for ci, c in enumerate(["below", "normal", "above"]):
data = pds.variables["prob"][tt, m, lati, loni, ci]
filename = dbio.writeGeotif(lat, lon, res, data)
ingest(dbname, filename, dt, m + 1, c, table[varname])
os.remove(filename)
def _downloadVariable(varname, dbname, dts, bbox):
"""Download specific variable from the MERRA Reanalysis dataset."""
# FIXME: Grid is not rectangular, but 0.5 x 0.625 degrees
res = 0.5
for ts in [dts[0] + timedelta(dti) for dti in range((dts[1] - dts[0]).days + 1)]:
try:
url = "http://goldsmr4.sci.gsfc.nasa.gov:80/opendap/MERRA2/M2T1NXSLV.5.12.4/{0}/{1:02d}/MERRA2_400.tavg1_2d_slv_Nx.{0:04d}{1:02d}{2:02d}.nc4".format(ts.year, ts.month, ts.day)
ds = netcdf.Dataset(url)
lat = ds.variables["lat"][:]
lon = ds.variables["lon"][:]
lon[lon > 180] -= 360.0
if bbox is not None:
i = np.where(np.logical_and(lat > bbox[1], lat < bbox[3]))[0]
j = np.where(np.logical_and(lon > bbox[0], lon < bbox[2]))[0]
lat = lat[i]
lon = lon[j]
else:
i = range(len(lat))
j = range(len(lon))
data = np.zeros((len(i), len(j)))
if varname == "tmax":
hdata = ds.variables["T2M"][:, i, j]
data = np.amax(hdata, axis=0) - 273.15
elif varname == "tmin":
hdata = ds.variables["T2M"][:, i, j]
data = np.amin(hdata, axis=0) - 273.15
elif varname in ["wind"]:
hdata = np.sqrt(ds.variables["U10M"][:, i, j]**2 + ds.variables["V10M"][:, i, j]**2)
data = np.mean(hdata, axis=0)
filename = dbio.writeGeotif(lat, lon, res, data)
dbio.ingest(dbname, filename, ts, "{0}.merra".format(varname))
os.remove(filename)
except:
print("Cannot import MERRA dataset for {0}!".format(ts.strftime("%Y-%m-%d")))
def _downloadVariable(varname, dbname, dt, bbox=None):
"""Download specific variable from the NCEP Reanalysis dataset."""
log = logging.getLogger(__name__)
res = 1.875
baseurl = "http://www.esrl.noaa.gov/psd/thredds/dodsC/Datasets/ncep.reanalysis.dailyavgs/surface_gauss"
if varname == "tmax":
urls = ["{0}/tmax.2m.gauss.{1}.nc".format(baseurl, dt[0].year)]
dsvar = ["tmax"]
elif varname == "tmin":
urls = ["{0}/tmin.2m.gauss.{1}.nc".format(baseurl, dt[0].year)]
dsvar = ["tmin"]
else:
urls = [
"{0}/uwnd.10m.gauss.{1}.nc".format(baseurl, dt[0].year),
"{0}/vwnd.10m.gauss.{1}.nc".format(baseurl, dt[0].year)
]
dsvar = ["uwnd", "vwnd"]
data = None
for ui, url in enumerate(urls):
pds = netcdf.Dataset(url)
lat = pds.variables["lat"][:]
lon = pds.variables["lon"][:]
lon[lon > 180] -= 360.0
i1, i2, j1, j2 = datasets.spatialSubset(
np.sort(lat)[::-1], np.sort(lon), res, bbox)
t = pds.variables["time"]
tt = netcdf.num2date(t[:], units=t.units)
ti = [
tj for tj in range(len(tt)) if resetDatetime(tt[tj]) >= dt[0]
and resetDatetime(tt[tj]) <= dt[1]
]
if len(ti) > 0:
lati = np.argsort(lat)[::-1][i1:i2]
loni = np.argsort(lon)[j1:j2]
if data is None:
data = pds.variables[dsvar[ui]][ti, lati, loni]
else:
data = np.sqrt(data**2.0 +
pds.variables[dsvar[ui]][ti, lati, loni]**2.0)
if "temp" in dsvar:
data -= 273.15
lat = np.sort(lat)[::-1][i1:i2]
lon = np.sort(lon)[j1:j2]
table = "{0}.ncep".format(varname)
for t in range(len(ti)):
filename = dbio.writeGeotif(lat, lon, res, data[t, :, :])
dbio.ingest(dbname, filename, tt[ti[t]], table)
os.remove(filename)
for dtt in [
dt[0] + timedelta(days=tj)
for tj in range((dt[-1] - dt[0]).days + 1)
]:
if dtt not in tt:
log.warning(
"NCEP data not available for {0}. Skipping download!".format(
dtt.strftime("%Y-%m-%d")))
def _downloadVariable(varname, dbname, dt, bbox=None):
"""Download specific variable from the NCEP Reanalysis dataset."""
res = 1.875
if varname == "tmax":
urls = [
"http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.maximum/.temp/dods"
]
dsvar = ["temp"]
elif varname == "tmin":
urls = [
"http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.minimum/.temp/dods"
]
dsvar = ["temp"]
else:
urls = [
"http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.u/dods",
"http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.v/dods"
]
dsvar = ["u", "v"]
data = None
for ui, url in enumerate(urls):
pds = netcdf.Dataset(url)
lat = pds.variables["Y"][:]
lon = pds.variables["X"][:]
lon[lon > 180] -= 360.0
i1, i2, j1, j2 = datasets.spatialSubset(
np.sort(lat)[::-1], np.sort(lon), res, bbox)
t = pds.variables["T"]
tt = netcdf.num2date(t[:], units=t.units)
# ti = [tj for tj in range(len(tt)) if tt[tj] >= dt]
ti = [
tj for tj in range(len(tt)) if resetDatetime(tt[tj]) >= dt[0]
and resetDatetime(tt[tj]) <= dt[1]
]
if len(ti) > 0:
lati = np.argsort(lat)[::-1][i1:i2]
loni = np.argsort(lon)[j1:j2]
if data is None:
data = pds.variables[dsvar[ui]][ti, 0, lati, loni]
else:
data = np.sqrt(data**2.0 +
pds.variables[dsvar[ui]][ti, 0, lati,
loni]**2.0)
if "temp" in dsvar:
data -= 273.15
lat = np.sort(lat)[::-1][i1:i2]
lon = np.sort(lon)[j1:j2]
table = "{0}.ncep".format(varname)
for t in range(len(ti)):
filename = dbio.writeGeotif(lat, lon, res, data[t, :, :])
dbio.ingest(dbname, filename, tt[ti[t]], table)
print("Imported {0} in {1}".format(tt[ti[0]].strftime("%Y-%m-%d"),
table))
os.remove(filename)
def ingest(dbname, table, data, lat, lon, res, t, resample=True, overwrite=True):
"""Import data into RHEAS database."""
log = logging.getLogger(__name__)
if data is not None:
if len(data.shape) > 2:
data = data[0, :, :]
filename = dbio.writeGeotif(lat, lon, res, data)
dbio.ingest(dbname, filename, t, table, resample, overwrite)
os.remove(filename)
else:
log.warning("No data were available to import into {0} for {1}.".format(table, t.strftime("%Y-%m-%d")))
def ingest(dbname, table, data, lat, lon, res, t, resample=True, overwrite=True):
"""Import data into RHEAS database."""
log = logging.getLogger(__name__)
if data is not None:
if len(data.shape) > 2:
data = data[0, :, :]
filename = dbio.writeGeotif(lat, lon, res, data)
dbio.ingest(dbname, filename, t, table, resample, overwrite)
log.info("Imported {0} in {1}".format(t.strftime("%Y-%m-%d"), table))
os.remove(filename)
else:
log.warning("No data were available to import into {0} for {1}.".format(table, t.strftime("%Y-%m-%d")))
def download(dbname, dts, bbox=None, enhanced=False):
"""Downloads SMAP soil mositure data for a set of dates *dt*
and imports them into the PostGIS database *dbname*. Optionally
uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat]."""
log = logging.getLogger(__name__)
if enhanced:
res = 0.09
url = "https://n5eil01u.ecs.nsidc.org/SMAP/SPL3SMP_E.001"
else:
res = 0.36
url = "https://n5eil01u.ecs.nsidc.org/DP4/SMAP/SPL3SMP.004"
for dt in [
dts[0] + timedelta(tt) for tt in range((dts[-1] - dts[0]).days + 1)
]:
try:
outpath, fname = earthdata.download(
"{0}/{1}".format(url, dt.strftime("%Y.%m.%d")),
"SMAP_L3_SM_P_\S*.h5")
f = h5py.File("{0}/{1}".format(outpath, fname))
varname = None
for v in f.keys():
if "latitude" in f[v] and "longitude" in f[v]:
varname = v
assert varname is not None
lat = f[varname]['latitude'][:, 0]
lon = f[varname]['longitude'][0, :]
lon[lon > 180] -= 360.0
# FIXME: Need to add reprojection from EASE grid
i1, i2, j1, j2 = datasets.spatialSubset(
np.sort(lat)[::-1], np.sort(lon), res, bbox)
lati = np.argsort(lat)[::-1][i1:i2]
loni = np.argsort(lon)[j1:j2]
sm = np.zeros((len(lati), len(loni)))
for i in range(len(lati)):
for j in range(len(loni)):
sm[i, j] = f[varname]['soil_moisture'][i, j]
# FIXME: Use spatially variable observation error
# sme = f[varname]['soil_moisture_error'][i1:i2, j1:j2]
lat = np.sort(lat)[::-1][i1:i2]
lon = np.sort(lon)[j1:j2]
filename = dbio.writeGeotif(lat, lon, res, sm)
dbio.ingest(dbname, filename, dt, table, False)
except:
log.warning("No SMAP data available for {0}.".format(
dt.strftime("%Y-%m-%d")))
def _downloadVariable(varname, dbname, dt, bbox=None):
"""Download specific variable from the NCEP Reanalysis dataset."""
log = logging.getLogger(__name__)
res = 1.875
baseurl = "http://www.esrl.noaa.gov/psd/thredds/dodsC/Datasets/ncep.reanalysis.dailyavgs/surface_gauss"
if varname == "tmax":
urls = ["{0}/tmax.2m.gauss.{1}.nc".format(baseurl, dt[0].year)]
dsvar = ["tmax"]
elif varname == "tmin":
urls = ["{0}/tmin.2m.gauss.{1}.nc".format(baseurl, dt[0].year)]
dsvar = ["tmin"]
else:
urls = ["{0}/uwnd.10m.gauss.{1}.nc".format(baseurl, dt[0].year), "{0}/vwnd.10m.gauss.{1}.nc".format(baseurl, dt[0].year)]
dsvar = ["uwnd", "vwnd"]
data = None
for ui, url in enumerate(urls):
pds = netcdf.Dataset(url)
lat = pds.variables["lat"][:]
lon = pds.variables["lon"][:]
lon[lon > 180] -= 360.0
i1, i2, j1, j2 = datasets.spatialSubset(np.sort(lat)[::-1], np.sort(lon), res, bbox)
t = pds.variables["time"]
tt = netcdf.num2date(t[:], units=t.units)
ti = [tj for tj in range(len(tt)) if resetDatetime(tt[tj]) >= dt[0] and resetDatetime(tt[tj]) <= dt[1]]
if len(ti) > 0:
lati = np.argsort(lat)[::-1][i1:i2]
loni = np.argsort(lon)[j1:j2]
if data is None:
data = pds.variables[dsvar[ui]][ti, lati, loni]
else:
data = np.sqrt(
data ** 2.0 + pds.variables[dsvar[ui]][ti, lati, loni] ** 2.0)
if "temp" in dsvar:
data -= 273.15
lat = np.sort(lat)[::-1][i1:i2]
lon = np.sort(lon)[j1:j2]
table = "{0}.ncep".format(varname)
for t in range(len(ti)):
filename = dbio.writeGeotif(lat, lon, res, data[t, :, :])
dbio.ingest(dbname, filename, tt[ti[t]], table)
os.remove(filename)
for dtt in [dt[0] + timedelta(days=tj) for tj in range((dt[-1]-dt[0]).days + 1)]:
if dtt not in tt:
log.warning("NCEP data not available for {0}. Skipping download!".format(
dtt.strftime("%Y-%m-%d")))
def download(dbname, dts, bbox=None):
"""Downloads SMAP soil mositure data for a set of dates *dt*
and imports them into the PostGIS database *dbname*. Optionally
uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat]."""
res = 0.36
url = "n5eil01u.ecs.nsidc.org"
ftp = FTP(url)
ftp.login()
for dt in [
dts[0] + timedelta(tt) for tt in range((dts[-1] - dts[0]).days + 1)
]:
r = ftp.cwd("/pub/SAN/SMAP/SPL3SMP.003/{0}".format(
dt.strftime("%Y.%m.%d")))
if r.find("successful") > 0:
outpath = tempfile.mkdtemp()
fname = [f for f in ftp.nlst() if f.find("h5") > 0][0]
with open("{0}/{1}".format(outpath, fname), 'wb') as f:
ftp.retrbinary("RETR {0}".format(fname), f.write)
f = h5py.File("{0}/{1}".format(outpath, fname))
lat = f['Soil_Moisture_Retrieval_Data']['latitude'][:, 0]
lon = f['Soil_Moisture_Retrieval_Data']['longitude'][0, :]
lon[lon > 180] -= 360.0
# FIXME: Need to add reprojection from EASE grid
i1, i2, j1, j2 = datasets.spatialSubset(
np.sort(lat)[::-1], np.sort(lon), res, bbox)
lati = np.argsort(lat)[::-1][i1:i2]
loni = np.argsort(lon)[j1:j2]
sm = np.zeros((len(lati), len(loni)))
for i in range(len(lati)):
for j in range(len(loni)):
sm[i,
j] = f['Soil_Moisture_Retrieval_Data']['soil_moisture'][
i, j]
# FIXME: Use spatially variable observation error
# sme = f['Soil_Moisture_Retrieval_Data']['soil_moisture_error'][i1:i2, j1:j2]
lat = np.sort(lat)[::-1][i1:i2]
lon = np.sort(lon)[j1:j2]
filename = dbio.writeGeotif(lat, lon, res, sm)
dbio.ingest(dbname, filename, dt, table, False)
else:
print("No SMAP data available for {0}.".format(
dt.strftime("%Y-%m-%d")))
def _downloadVariable(varname, dbname, dt, bbox=None):
"""Download specific variable from the NCEP Reanalysis dataset."""
res = 1.875
if varname == "tmax":
urls = ["http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.maximum/.temp/dods"]
dsvar = ["temp"]
elif varname == "tmin":
urls = ["http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.minimum/.temp/dods"]
dsvar = ["temp"]
else:
urls = ["http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.u/dods",
"http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.v/dods"]
dsvar = ["u", "v"]
data = None
for ui, url in enumerate(urls):
pds = netcdf.Dataset(url)
lat = pds.variables["Y"][:]
lon = pds.variables["X"][:]
lon[lon > 180] -= 360.0
if bbox is not None:
i = np.where(np.logical_and(lat > bbox[1], lat < bbox[3]))[0]
j = np.where(np.logical_and(lon > bbox[0], lon < bbox[2]))[0]
lat = lat[i]
lon = lon[j]
else:
i = range(len(lat))
j = range(len(lon))
t = pds.variables["T"]
tt = netcdf.num2date(t[:], units=t.units)
ti = [tj for tj in range(len(tt)) if tt[tj] >= dt[
0] and tt[tj] <= dt[1]]
if data is None:
data = pds.variables[dsvar[ui]][ti, 0, i, j]
else:
data = np.sqrt(
data ** 2.0 + pds.variables[dsvar[ui]][ti, 0, i, j] ** 2.0)
if "temp" in dsvar:
data -= 273.15
for tj in range(data.shape[0]):
filename = dbio.writeGeotif(lat, lon, res, data[tj, :, :])
dbio.ingest(dbname, filename, tt[ti[tj]], "{0}.ncep".format(varname))
os.remove(filename)
def _downloadVariable(varname, dbname, dt, bbox=None):
"""Download specific variable from the NCEP Reanalysis dataset."""
res = 1.875
if varname == "tmax":
urls = ["http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.maximum/.temp/dods"]
dsvar = ["temp"]
elif varname == "tmin":
urls = ["http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.minimum/.temp/dods"]
dsvar = ["temp"]
else:
urls = ["http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.u/dods",
"http://iridl.ldeo.columbia.edu/SOURCES/.NOAA/.NCEP-NCAR/.CDAS-1/.DAILY/.Diagnostic/.above_ground/.v/dods"]
dsvar = ["u", "v"]
data = None
for ui, url in enumerate(urls):
pds = netcdf.Dataset(url)
lat = pds.variables["Y"][:]
lon = pds.variables["X"][:]
lon[lon > 180] -= 360.0
i1, i2, j1, j2 = datasets.spatialSubset(np.sort(lat)[::-1], np.sort(lon), res, bbox)
t = pds.variables["T"]
tt = netcdf.num2date(t[:], units=t.units)
# ti = [tj for tj in range(len(tt)) if tt[tj] >= dt]
ti = [tj for tj in range(len(tt)) if resetDatetime(tt[tj]) >= dt[0] and resetDatetime(tt[tj]) <= dt[1]]
if len(ti) > 0:
lati = np.argsort(lat)[::-1][i1:i2]
loni = np.argsort(lon)[j1:j2]
if data is None:
data = pds.variables[dsvar[ui]][ti, 0, lati, loni]
else:
data = np.sqrt(
data ** 2.0 + pds.variables[dsvar[ui]][ti, 0, lati, loni] ** 2.0)
if "temp" in dsvar:
data -= 273.15
lat = np.sort(lat)[::-1][i1:i2]
lon = np.sort(lon)[j1:j2]
table = "{0}.ncep".format(varname)
for t in range(len(ti)):
filename = dbio.writeGeotif(lat, lon, res, data[t, :, :])
dbio.ingest(dbname, filename, tt[ti[t]], table)
print("Imported {0} in {1}".format(tt[ti[0]].strftime("%Y-%m-%d"), table))
os.remove(filename)
def ingest(dbname,
table,
data,
lat,
lon,
res,
t,
resample=True,
overwrite=True):
"""Import data into RHEAS database."""
if data is not None:
if len(data.shape) > 2:
data = data[0, :, :]
filename = dbio.writeGeotif(lat, lon, res, data)
dbio.ingest(dbname, filename, t, table, resample, overwrite)
print("Imported {0} in {1}".format(t.strftime("%Y-%m-%d"), table))
os.remove(filename)
else:
print("WARNING! No data were available to import into {0} for {1}.".
format(table, t.strftime("%Y-%m-%d")))
def _downloadVariable(varname, dbname, dt, bbox):
"""Download specific variable from the MERRA Reanalysis dataset."""
# FIXME: Grid is not rectangular, but 0.5 x 0.625 degrees
log = logging.getLogger(__name__)
res = 0.5
try:
url = "http://goldsmr4.sci.gsfc.nasa.gov:80/dods/M2T1NXSLV"
ds = netcdf.Dataset(url)
lat = ds.variables["lat"][:]
lon = ds.variables["lon"][:]
lon[lon > 180] -= 360.0
i1, i2, j1, j2 = datasets.spatialSubset(
np.sort(lat)[::-1], np.sort(lon), res, bbox)
data = np.zeros((i2 - i1, j2 - j1))
lati = np.argsort(lat)[::-1][i1:i2]
loni = np.argsort(lon)[j1:j2]
t = ds.variables["time"]
tt = netcdf.num2date(t[:], units=t.units)
ti = np.where(tt == dt)[0][0]
if varname == "tmax":
hdata = ds.variables["t2m"][ti:ti + 24, lati, loni]
data = np.amax(hdata, axis=0) - 273.15
elif varname == "tmin":
hdata = ds.variables["t2m"][ti:ti + 24, lati, loni]
data = np.amin(hdata, axis=0) - 273.15
elif varname in ["wind"]:
hdata = np.sqrt(ds.variables["u10m"][ti:ti + 24, lati, loni]**2 +
ds.variables["v10m"][ti:ti + 24, lati, loni]**2)
data = np.mean(hdata, axis=0)
lat = np.sort(lat)[::-1][i1:i2]
lon = np.sort(lon)[j1:j2]
filename = dbio.writeGeotif(lat, lon, res, data)
table = "{0}.merra".format(varname)
dbio.ingest(dbname, filename, dt, table)
log.info("Imported {0} in {1}".format(tt[ti].strftime("%Y-%m-%d"),
table))
os.remove(filename)
except:
log.warning("Cannot import MERRA dataset for {0}!".format(
dt.strftime("%Y-%m-%d")))
def download(dbname, dt, bbox=None):
"""Downloads SMOS soil mositure data for a set of dates *dt*
and imports them into the PostGIS database *dbname*. Optionally
uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat]."""
log = logging.getLogger(__name__)
res = 0.25
url = "http://*****:*****@cp34-bec.cmima.csic.es/thredds/dodsC/NRTSM001D025A_ALL"
f = netcdf.Dataset(url)
lat = f.variables['lat'][::-1] # swap latitude orientation to northwards
lon = f.variables['lon'][:]
i1, i2, j1, j2 = datasets.spatialSubset(lat, lon, res, bbox)
smi1 = len(lat) - i2 - 1
smi2 = len(lat) - i1 - 1
lat = lat[i1:i2]
lon = lon[j1:j2]
t0 = datetime(2010, 1, 12) # initial date of SMOS data
t1 = (dt[0] - t0).days
if t1 < 0:
log.warning("Reseting start date to {0}".format(
t0.strftime("%Y-%m-%d")))
t1 = 0
t2 = (dt[-1] - t0).days + 1
nt, _, _ = f.variables['SM'].shape
if t2 > nt:
t2 = nt
log.warning("Reseting end date to {0}".format(
(t0 + timedelta(t2)).strftime("%Y-%m-%d")))
ti = range(t1, t2)
sm = f.variables['SM'][ti, smi1:smi2, j1:j2]
sm = sm[:, ::-1, :] # flip latitude dimension in data array
# FIXME: Use spatially variable observation error
# smv = f.variables['VARIANCE_SM'][ti, i1:i2, j1:j2][:, ::-1, :]
pos, smlat, smlon = regridNearestNeighbor(lat, lon, res)
for tj in range(sm.shape[0]):
smdata = sm[tj, :, :].ravel()[pos].reshape((len(smlat), len(smlon)))
filename = dbio.writeGeotif(smlat, smlon, res, smdata)
t = t0 + timedelta(ti[tj])
dbio.ingest(dbname, filename, t, table, False)
os.remove(filename)
def download(dbname, dts, bbox=None, enhanced=False):
"""Downloads SMAP soil mositure data for a set of dates *dt*
and imports them into the PostGIS database *dbname*. Optionally
uses a bounding box to limit the region with [minlon, minlat, maxlon, maxlat]."""
log = logging.getLogger(__name__)
if enhanced:
res = 0.09
url = "https://n5eil01u.ecs.nsidc.org/SMAP/SPL3SMP_E.001"
else:
res = 0.36
url = "https://n5eil01u.ecs.nsidc.org/DP4/SMAP/SPL3SMP.004"
for dt in [dts[0] + timedelta(tt) for tt in range((dts[-1] - dts[0]).days + 1)]:
try:
outpath, fname = earthdata.download("{0}/{1}".format(url, dt.strftime("%Y.%m.%d")), "SMAP_L3_SM_P_\S*.h5")
f = h5py.File("{0}/{1}".format(outpath, fname))
varname = None
for v in f.keys():
if "latitude" in f[v] and "longitude" in f[v]:
varname = v
assert varname is not None
lat = f[varname]['latitude'][:, 0]
lon = f[varname]['longitude'][0, :]
lon[lon > 180] -= 360.0
# FIXME: Need to add reprojection from EASE grid
i1, i2, j1, j2 = datasets.spatialSubset(np.sort(lat)[::-1], np.sort(lon), res, bbox)
lati = np.argsort(lat)[::-1][i1:i2]
loni = np.argsort(lon)[j1:j2]
sm = np.zeros((len(lati), len(loni)))
for i in range(len(lati)):
for j in range(len(loni)):
sm[i, j] = f[varname]['soil_moisture'][i, j]
# FIXME: Use spatially variable observation error
# sme = f[varname]['soil_moisture_error'][i1:i2, j1:j2]
lat = np.sort(lat)[::-1][i1:i2]
lon = np.sort(lon)[j1:j2]
filename = dbio.writeGeotif(lat, lon, res, sm)
dbio.ingest(dbname, filename, dt, table, False)
except:
log.warning("No SMAP data available for {0}.".format(dt.strftime("%Y-%m-%d")))
