def download(dbname, dts, bbox):
"""Downloads the MODIS evapotranspiration data product MOD16 for
a set of dates *dt* and imports them into the PostGIS database *dbname*."""
log = logging.getLogger(__name__)
res = 0.01
urlbase = "http://files.ntsg.umt.edu"
tiles = modis.findTiles(bbox)
if tiles is not None:
for dt in [dts[0] + timedelta(dti) for dti in range((dts[-1] - dts[0]).days + 1)]:
url = "{0}/data/NTSG_Products/MOD16/MOD16A2.105_MERRAGMAO/Y{1}".format(urlbase, dt.year)
resp_year = requests.get(url)
try:
assert resp_year.status_code == 200
days = [link for link in BeautifulSoup(resp_year.text, parse_only=SoupStrainer('a')) if isinstance(link, Tag) and link.text.find(dt.strftime("%j")) >= 0]
assert len(days) > 0
resp_day = requests.get("{0}{1}".format(urlbase, days[0].get('href')))
assert resp_day.status_code == 200
files = [link.get('href') for link in BeautifulSoup(resp_day.text, parse_only=SoupStrainer('a')) if isinstance(link, Tag) and link.text.find("hdf") > 0]
files = [f for f in files if any(f.find("h{0:02d}v{1:02d}".format(t[1], t[0])) > 0 for t in tiles)]
outpath = tempfile.mkdtemp()
for fname in files:
resp_file = requests.get("{0}{1}".format(urlbase, fname))
filename = fname.split("/")[-1]
with open("{0}/{1}".format(outpath, filename), 'wb') as fout:
for chunk in resp_file:
fout.write(chunk)
proc = subprocess.Popen(["gdal_translate", "HDF4_EOS:EOS_GRID:{0}/{1}:MOD_Grid_MOD16A2:ET_1km".format(
outpath, filename), "{0}/{1}".format(outpath, filename).replace("hdf", "tif")], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
tifs = glob.glob("{0}/*.tif".format(outpath))
proc = subprocess.Popen(
["gdal_merge.py", "-o", "{0}/et.tif".format(outpath)] + tifs, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen(["gdal_calc.py", "-A", "{0}/et.tif".format(outpath), "--outfile={0}/et1.tif".format(
outpath), "--NoDataValue=-9999", "--calc=(A<32701)*(0.1*A+9999)-9999"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen(["gdalwarp", "-t_srs", "+proj=latlong +ellps=sphere", "-tr", str(
res), str(-res), "{0}/et1.tif".format(outpath), "{0}/et2.tif".format(outpath)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
if bbox is None:
pstr = []
else:
pstr = ["-projwin", str(bbox[0]), str(bbox[3]), str(bbox[2]), str(bbox[1])]
proc = subprocess.Popen(["gdal_translate"] + pstr + ["-a_srs", "epsg:4326", "{0}/et2.tif".format(outpath), "{0}/et3.tif".format(outpath)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
dbio.ingest(
dbname, "{0}/et3.tif".format(outpath), dt, table, False)
shutil.rmtree(outpath)
except:
log.warning("MOD16 data not available for {0}. Skipping download!".format(
dt.strftime("%Y-%m-%d")))
def download(dbname, dts, bbox):
"""Downloads the combined MODIS LAI data product MCD15 for
a specific date *dt* and imports them into the PostGIS database *dbname*."""
log = logging.getLogger(__name__)
res = 0.01
burl = "http://e4ftl01.cr.usgs.gov/MOTA/MCD15A2.005"
tiles = modis.findTiles(bbox)
if tiles is not None:
for dt in [dts[0] + timedelta(dti) for dti in range((dts[-1] - dts[0]).days + 1)]:
outpath = tempfile.mkdtemp()
url = "{0}/{1:04d}.{2:02d}.{3:02d}".format(
burl, dt.year, dt.month, dt.day)
req = requests.get(url, auth=(username, password))
if req.status_code == 200:
dom = html.fromstring(req.text)
files = [link for link in dom.xpath('//a/@href')]
if len(files) > 0:
filenames = [filter(lambda s: re.findall(r'MCD.*h{0:02d}v{1:02d}.*hdf$'.format(t[1], t[0]), s), files) for t in tiles]
for filename in filenames:
if len(filename) > 0:
filename = filename[0]
proc = subprocess.Popen(["wget", "-L", "--load-cookies", ".cookiefile", "--save-cookies", ".cookiefile", "--user", username, "--password", password, "{0}/{1}".format(url, filename), "-O", "{0}/{1}".format(outpath, filename)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen(["gdal_translate", "HDF4_EOS:EOS_GRID:{0}/{1}:MOD_Grid_MOD15A2:Lai_1km".format(
outpath, filename), "{0}/{1}".format(outpath, filename).replace("hdf", "tif")], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
tifs = glob.glob("{0}/*.tif".format(outpath))
if len(tifs) > 0:
proc = subprocess.Popen(["gdal_merge.py", "-o", "{0}/lai.tif".format(outpath)] + tifs, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen(["gdal_calc.py", "-A", "{0}/lai.tif".format(outpath), "--outfile={0}/lai1.tif".format(outpath), "--NoDataValue=-9999", "--calc=\"(A<101.0)*(0.1*A+9999.0)-9999.0\""], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen(["gdalwarp", "-t_srs", "'+proj=latlong +ellps=sphere'", "-tr", str(res), str(-res), "{0}/lai1.tif".format(outpath), "{0}/lai2.tif".format(outpath)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen(["gdal_translate", "-a_srs", "epsg:4326", "{0}/lai2.tif".format(outpath), "{0}/lai3.tif".format(outpath)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
dbio.ingest(
dbname, "{0}/lai3.tif".format(outpath), dt, table, False)
shutil.rmtree(outpath)
else:
log.warning("MCD15 data not available for {0}. Skipping download!".format(
dt.strftime("%Y-%m-%d")))
def download(dbname, dts, bbox):
"""Downloads the MODSCAG snow cover fraction data product for a specific
date *dt* and imports it into the PostGIS database *dbname*."""
log = logging.getLogger(__name__)
res = 0.01
tiles = modis.findTiles(bbox)
for dt in [dts[0] + timedelta(dti) for dti in range((dts[-1] - dts[0]).days + 1)]:
temppath = tempfile.mkdtemp()
url = "https://snow-data.jpl.nasa.gov/modscag-historic/{0}/{1}".format(dt.year, dt.strftime("%j"))
r = requests.get(url, auth=HTTPDigestAuth(username, password))
if r.status_code == 200:
dom = lxml.html.fromstring(r.text)
links = [link for link in dom.xpath('//a/@href') if link.find("snow_fraction.tif") > 0]
for t in tiles:
filenames = filter(lambda f: f.find("h{0:02d}v{1:02d}".format(t[1], t[0])) > 0, links)
if len(filenames) > 0:
filename = filenames[0]
r = requests.get("{0}/{1}".format(url, filename), auth=HTTPDigestAuth(username, password))
with open("{0}/{1}".format(temppath, filename), 'wb') as fout:
fout.write(r.content)
tifs = glob.glob("{0}/*.tif".format(temppath))
if len(tifs) > 0:
proc = subprocess.Popen(["gdal_merge.py", "-o", "{0}/snow.tif".format(temppath)] + tifs, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen(["gdal_calc.py", "-A", "{0}/snow.tif".format(temppath), "--outfile={0}/snow1.tif".format(
temppath), "--NoDataValue=-9999", "--calc=\"(A<101.0)*(A+9999.0)-9999.0\""], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen(["gdalwarp", "-t_srs", "'+proj=latlong +ellps=sphere'", "-tr", str(
res), str(-res), "{0}/snow1.tif".format(temppath), "{0}/snow2.tif".format(temppath)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
if bbox is None:
pstr = []
else:
pstr = ["-projwin", str(bbox[0]), str(bbox[3]), str(bbox[2]), str(bbox[1])]
proc = subprocess.Popen(["gdal_translate", "-a_srs", "epsg:4326"] + pstr + ["{0}/snow2.tif".format(temppath), "{0}/snow3.tif".format(temppath)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
dbio.ingest(dbname, "{0}/snow3.tif".format(temppath), dt, table, False)
shutil.rmtree(temppath)
else:
log.warning("MODSCAG data not available for {0}. Skipping download!".format(
dt.strftime("%Y-%m-%d")))
def download(dbname, dts, bbox):
"""Downloads the Terra MODIS snow cover fraction data product MOD10 for
a specific date *dt* and imports them into the PostGIS database *dbname*."""
log = logging.getLogger(__name__)
res = 0.005
url = "n5eil01u.ecs.nsidc.org"
tiles = modis.findTiles(bbox)
if tiles is not None:
ftp = FTP(url)
ftp.login()
for dt in [dts[0] + timedelta(dti) for dti in range((dts[-1] - dts[0]).days + 1)]:
try:
ftp.cwd("SAN/MOST/MOD10A1.005/{1:04d}.{2:02d}.{3:02d}".format(url, dt.year, dt.month, dt.day))
files = [f for f in ftp.nlst() if any(
f.find("h{0:02d}v{1:02d}".format(t[1], t[0])) > 0 for t in tiles)]
files = filter(lambda s: s.endswith("hdf"), files)
outpath = tempfile.mkdtemp()
for fname in files:
with open("{0}/{1}".format(outpath, fname), 'wb') as f:
ftp.retrbinary("RETR {0}".format(fname), f.write)
proc = subprocess.Popen(["gdal_translate", "HDF4_EOS:EOS_GRID:{0}/{1}:MOD_Grid_Snow_500m:Fractional_Snow_Cover".format(outpath, fname), "{0}/{1}".format(outpath, fname).replace("hdf", "tif")], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
tifs = glob.glob("{0}/*.tif".format(outpath))
proc = subprocess.Popen(
["gdal_merge.py", "-a_nodata", "-9999", "-o", "{0}/snow.tif".format(outpath)] + tifs, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen(["gdal_calc.py", "-A", "{0}/snow.tif".format(outpath), "--outfile={0}/snow1.tif".format(
outpath), "--NoDataValue=-9999", "--calc=\"(A<101.0)*(A+9999.0)-9999.0\""], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen(["gdalwarp", "-t_srs", "'+proj=latlong +ellps=sphere'", "-tr", str(
res), str(-res), "{0}/snow1.tif".format(outpath), "{0}/snow2.tif".format(outpath)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen(["gdal_translate", "-a_srs", "epsg:4326", "{0}/snow2.tif".format(outpath), "{0}/snow3.tif".format(outpath)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
dbio.ingest(
dbname, "{0}/snow3.tif".format(outpath), dt, table, False)
shutil.rmtree(outpath)
except:
log.warning("MOD10 data not available for {0}. Skipping download!".format(
dt.strftime("%Y-%m-%d")))
def download(dbname, dts, bbox):
"""Downloads the combined MODIS LAI data product MCD15 for
a specific date *dt* and imports them into the PostGIS database *dbname*."""
log = logging.getLogger(__name__)
res = 0.005
burl = "http://e4ftl01.cr.usgs.gov/MOTA/MCD15A2H.006"
tiles = modis.findTiles(bbox)
if tiles is not None:
for dt in [dts[0] + timedelta(dti) for dti in range((dts[-1] - dts[0]).days + 1)]:
outpath = tempfile.mkdtemp()
url = "{0}/{1:04d}.{2:02d}.{3:02d}".format(burl, dt.year, dt.month, dt.day)
filenames = []
for t in tiles:
try:
tmppath, fname = earthdata.download(url, "MCD15A2H.A{0}.h{1:02d}v{2:02d}.006.*.hdf".format(dt.strftime("%Y%j"), t[1], t[0]))
except ConnectionError:
fname = None
if fname:
filenames.append("{0}/{1}".format(tmppath, fname))
for filename in filenames:
proc = subprocess.Popen(["gdal_translate", "HDF4_EOS:EOS_GRID:{0}:MOD_Grid_MOD15A2H:Lai_500m".format(
filename), "{0}/{1}".format(outpath, filename.split("/")[-1]).replace("hdf", "tif")], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
shutil.rmtree("/".join(filename.split("/")[:-1]))
tifs = glob.glob("{0}/*.tif".format(outpath))
if len(tifs) > 0:
proc = subprocess.Popen(["gdal_merge.py", "-o", "{0}/lai.tif".format(outpath)] + tifs, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen(["gdal_calc.py", "-A", "{0}/lai.tif".format(outpath), "--outfile={0}/lai1.tif".format(outpath), "--NoDataValue=-9999", "--calc=(A<101.0)*(0.1*A+9999.0)-9999.0"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen(["gdalwarp", "-t_srs", "+proj=latlong +ellps=sphere", "-tr", str(res), str(-res), "{0}/lai1.tif".format(outpath), "{0}/lai2.tif".format(outpath)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen(["gdal_translate", "-a_srs", "epsg:4326", "{0}/lai2.tif".format(outpath), "{0}/lai3.tif".format(outpath)], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
dbio.ingest(dbname, "{0}/lai3.tif".format(outpath), dt, table, False)
else:
log.warning("MCD15 data not available for {0}. Skipping download!".format(dt.strftime("%Y-%m-%d")))
shutil.rmtree(outpath)
def download(dbname, dts, bbox):
"""Downloads the MODIS evapotranspiration data product MOD16 for
a set of dates *dt* and imports them into the PostGIS database *dbname*."""
res = 0.01
url = "ftp.ntsg.umt.edu"
tiles = modis.findTiles(bbox)
if tiles is not None:
ftp = FTP(url)
ftp.login()
for dt in [dts[0] + timedelta(dti) for dti in range((dts[1] - dts[0]).days + 1)]:
try:
ftp.cwd(
"pub/MODIS/NTSG_Products/MOD16/MOD16A2.105_MERRAGMAO/Y{0}".format(dt.year))
days = ftp.nlst()
datadir = "D{0}".format(dt.strftime("%j"))
if datadir in days:
ftp.cwd(datadir)
files = [f for f in ftp.nlst() if any(
f.find("h{0:02d}v{1:02d}".format(t[1], t[0])) > 0 for t in tiles)]
outpath = tempfile.mkdtemp()
for fname in files:
with open("{0}/{1}".format(outpath, fname), 'wb') as f:
ftp.retrbinary("RETR {0}".format(fname), f.write)
subprocess.call(["gdal_translate", "HDF4_EOS:EOS_GRID:{0}/{1}:MOD_Grid_MOD16A2:ET_1km".format(
outpath, fname), "{0}/{1}".format(outpath, fname).replace("hdf", "tif")])
tifs = " ".join(glob.glob("{0}/*.tif".format(outpath)))
subprocess.call(
["gdal_merge.py", "-o", "{0}/et.tif".format(outpath)] + tifs)
cmd = " ".join(["gdal_calc.py", "-A", "{0}/et.tif".format(outpath), "--outfile={0}/et1.tif".format(
outpath), "--NoDataValue=-9999", "--calc=\"(A<32701)*(0.1*A+9999)-9999\""])
subprocess.call(cmd, shell=True)
cmd = " ".join(["gdalwarp", "-t_srs", "'+proj=latlong +ellps=sphere'", "-tr", str(
res), str(-res), "{0}/et1.tif".format(outpath), "{0}/et2.tif".format(outpath)])
subprocess.call(cmd, shell=True)
subprocess.call(["gdal_translate", "-a_srs", "epsg:4326",
"{0}/et2.tif".format(outpath), "{0}/et3.tif".format(outpath)])
dbio.ingest(
dbname, "{0}/et3.tif".format(outpath), dt, table, False)
shutil.rmtree(outpath)
except:
print("MOD16 data not available for {0}. Skipping download!".format(
dt.strftime("%Y-%m-%d")))
def download(dbname, dts, bbox):
"""Downloads the combined MODIS LAI data product MCD15 for
a specific date *dt* and imports them into the PostGIS database *dbname*."""
res = 0.01
burl = "http://e4ftl01.cr.usgs.gov/MOTA/MCD15A2.005"
tiles = modis.findTiles(bbox)
if tiles is not None:
for dt in [dts[0] + timedelta(dti) for dti in range((dts[1] - dts[0]).days + 1)]:
outpath = tempfile.mkdtemp()
url = "{0}/{1:04d}.{2:02d}.{3:02d}".format(
burl, dt.year, dt.month, dt.day)
connection = urllib.urlopen(url)
dom = html.fromstring(connection.read())
files = [link for link in dom.xpath('//a/@href')]
if len(files) > 0:
filenames = [filter(lambda s: re.findall(
r'MCD.*h{0:02d}v{1:02d}.*hdf$'.format(t[1], t[0]), s), files) for t in tiles]
for filename in filenames:
if len(filename) > 0:
filename = filename[0]
urllib.urlretrieve(
"{0}/{1}".format(url, filename), "{0}/{1}".format(outpath, filename))
subprocess.call(["gdal_translate", "HDF4_EOS:EOS_GRID:{0}/{1}:MOD_Grid_MOD15A2:Lai_1km".format(
outpath, filename), "{0}/{1}".format(outpath, filename).replace("hdf", "tif")])
tifs = glob.glob("{0}/*.tif".format(outpath))
if len(tifs) > 0:
subprocess.call(
["gdal_merge.py", "-o", "{0}/lai.tif".format(outpath)] + tifs)
cmd = " ".join(["gdal_calc.py", "-A", "{0}/lai.tif".format(outpath), "--outfile={0}/lai1.tif".format(
outpath), "--NoDataValue=-9999", "--calc=\"(A<101.0)*(0.1*A+9999.0)-9999.0\""])
subprocess.call(cmd, shell=True)
cmd = " ".join(["gdalwarp", "-t_srs", "'+proj=latlong +ellps=sphere'", "-tr", str(
res), str(-res), "{0}/lai1.tif".format(outpath), "{0}/lai2.tif".format(outpath)])
subprocess.call(cmd, shell=True)
subprocess.call(["gdal_translate", "-a_srs", "epsg:4326",
"{0}/lai2.tif".format(outpath), "{0}/lai3.tif".format(outpath)])
dbio.ingest(
dbname, "{0}/lai3.tif".format(outpath), dt, table, False)
shutil.rmtree(outpath)
else:
print("MCD15 data not available for {0}. Skipping download!".format(
dt.strftime("%Y-%m-%d")))
def download(dbname, dts, bbox):
"""Downloads the MODIS evapotranspiration data product MOD16 for
a set of dates *dt* and imports them into the PostGIS database *dbname*."""
log = logging.getLogger(__name__)
res = 0.01
url = "ftp.ntsg.umt.edu"
tiles = modis.findTiles(bbox)
if tiles is not None:
ftp = FTP(url)
ftp.login()
for dt in [
dts[0] + timedelta(dti)
for dti in range((dts[-1] - dts[0]).days + 1)
]:
try:
ftp.cwd(
"pub/MODIS/NTSG_Products/MOD16/MOD16A2.105_MERRAGMAO/Y{0}".
format(dt.year))
days = ftp.nlst()
datadir = "D{0}".format(dt.strftime("%j"))
if datadir in days:
ftp.cwd(datadir)
files = [
f for f in ftp.nlst() if any(
f.find("h{0:02d}v{1:02d}".format(t[1], t[0])) > 0
for t in tiles)
]
outpath = tempfile.mkdtemp()
for fname in files:
with open("{0}/{1}".format(outpath, fname), 'wb') as f:
ftp.retrbinary("RETR {0}".format(fname), f.write)
proc = subprocess.Popen([
"gdal_translate",
"HDF4_EOS:EOS_GRID:{0}/{1}:MOD_Grid_MOD16A2:ET_1km"
.format(outpath, fname), "{0}/{1}".format(
outpath, fname).replace("hdf", "tif")
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
tifs = glob.glob("{0}/*.tif".format(outpath))
proc = subprocess.Popen(
["gdal_merge.py", "-o", "{0}/et.tif".format(outpath)] +
tifs,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen([
"gdal_calc.py", "-A", "{0}/et.tif".format(outpath),
"--outfile={0}/et1.tif".format(outpath),
"--NoDataValue=-9999",
"--calc=\"(A<32701)*(0.1*A+9999)-9999\""
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen([
"gdalwarp", "-t_srs", "'+proj=latlong +ellps=sphere'",
"-tr",
str(res),
str(-res), "{0}/et1.tif".format(outpath),
"{0}/et2.tif".format(outpath)
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
if bbox is None:
pstr = []
else:
pstr = [
"-projwin",
str(bbox[0]),
str(bbox[3]),
str(bbox[2]),
str(bbox[1])
]
proc = subprocess.Popen(["gdal_translate"] + pstr + [
"-a_srs", "epsg:4326", "{0}/et2.tif".format(outpath),
"{0}/et3.tif".format(outpath)
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
dbio.ingest(dbname, "{0}/et3.tif".format(outpath), dt,
table, False)
shutil.rmtree(outpath)
except:
log.warning(
"MOD16 data not available for {0}. Skipping download!".
format(dt.strftime("%Y-%m-%d")))
def download(dbname, dts, bbox):
"""Downloads the MODIS evapotranspiration data product MOD16 for
a set of dates *dt* and imports them into the PostGIS database *dbname*."""
log = logging.getLogger(__name__)
res = 0.01
url = "ftp.ntsg.umt.edu"
tiles = modis.findTiles(bbox)
if tiles is not None:
ftp = FTP(url)
ftp.login()
for dt in [dts[0] + timedelta(dti) for dti in range((dts[-1] - dts[0]).days + 1)]:
try:
ftp.cwd("pub/MODIS/NTSG_Products/MOD16/MOD16A2.105_MERRAGMAO/Y{0}".format(dt.year))
days = ftp.nlst()
datadir = "D{0}".format(dt.strftime("%j"))
if datadir in days:
ftp.cwd(datadir)
files = [
f for f in ftp.nlst() if any(f.find("h{0:02d}v{1:02d}".format(t[1], t[0])) > 0 for t in tiles)
]
outpath = tempfile.mkdtemp()
for fname in files:
with open("{0}/{1}".format(outpath, fname), "wb") as f:
ftp.retrbinary("RETR {0}".format(fname), f.write)
proc = subprocess.Popen(
[
"gdal_translate",
"HDF4_EOS:EOS_GRID:{0}/{1}:MOD_Grid_MOD16A2:ET_1km".format(outpath, fname),
"{0}/{1}".format(outpath, fname).replace("hdf", "tif"),
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
out, err = proc.communicate()
log.debug(out)
tifs = glob.glob("{0}/*.tif".format(outpath))
proc = subprocess.Popen(
["gdal_merge.py", "-o", "{0}/et.tif".format(outpath)] + tifs,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen(
[
"gdal_calc.py",
"-A",
"{0}/et.tif".format(outpath),
"--outfile={0}/et1.tif".format(outpath),
"--NoDataValue=-9999",
'--calc="(A<32701)*(0.1*A+9999)-9999"',
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen(
[
"gdalwarp",
"-t_srs",
"'+proj=latlong +ellps=sphere'",
"-tr",
str(res),
str(-res),
"{0}/et1.tif".format(outpath),
"{0}/et2.tif".format(outpath),
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
out, err = proc.communicate()
log.debug(out)
if bbox is None:
pstr = []
else:
pstr = ["-projwin", str(bbox[0]), str(bbox[3]), str(bbox[2]), str(bbox[1])]
proc = subprocess.Popen(
["gdal_translate"]
+ pstr
+ ["-a_srs", "epsg:4326", "{0}/et2.tif".format(outpath), "{0}/et3.tif".format(outpath)],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
out, err = proc.communicate()
log.debug(out)
dbio.ingest(dbname, "{0}/et3.tif".format(outpath), dt, table, False)
shutil.rmtree(outpath)
except:
log.warning("MOD16 data not available for {0}. Skipping download!".format(dt.strftime("%Y-%m-%d")))
def download(dbname, dts, bbox):
"""Downloads the MODSCAG snow cover fraction data product for a specific
date *dt* and imports it into the PostGIS database *dbname*."""
log = logging.getLogger(__name__)
res = 0.01
tiles = modis.findTiles(bbox)
for dt in [
dts[0] + timedelta(dti)
for dti in range((dts[-1] - dts[0]).days + 1)
]:
temppath = tempfile.mkdtemp()
url = "https://snow-data.jpl.nasa.gov/modscag-historic/{0}/{1}".format(
dt.year, dt.strftime("%j"))
r = requests.get(url, auth=HTTPDigestAuth(username, password))
if r.status_code == 200:
dom = lxml.html.fromstring(r.text)
links = [
link for link in dom.xpath('//a/@href')
if link.find("snow_fraction.tif") > 0
]
for t in tiles:
filenames = filter(
lambda f: f.find("h{0:02d}v{1:02d}".format(t[1], t[0])) >
0, links)
if len(filenames) > 0:
filename = filenames[0]
r = requests.get("{0}/{1}".format(url, filename),
auth=HTTPDigestAuth(username, password))
with open("{0}/{1}".format(temppath, filename),
'wb') as fout:
fout.write(r.content)
tifs = glob.glob("{0}/*.tif".format(temppath))
if len(tifs) > 0:
proc = subprocess.Popen(
["gdal_merge.py", "-o", "{0}/snow.tif".format(temppath)] +
tifs,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen([
"gdal_calc.py", "-A", "{0}/snow.tif".format(temppath),
"--outfile={0}/snow1.tif".format(temppath),
"--NoDataValue=-9999",
"--calc=\"(A<101.0)*(A+9999.0)-9999.0\""
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen([
"gdalwarp", "-t_srs", "'+proj=latlong +ellps=sphere'",
"-tr",
str(res),
str(-res), "{0}/snow1.tif".format(temppath),
"{0}/snow2.tif".format(temppath)
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
if bbox is None:
pstr = []
else:
pstr = [
"-projwin",
str(bbox[0]),
str(bbox[3]),
str(bbox[2]),
str(bbox[1])
]
proc = subprocess.Popen(
["gdal_translate", "-a_srs", "epsg:4326"] + pstr + [
"{0}/snow2.tif".format(temppath),
"{0}/snow3.tif".format(temppath)
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
dbio.ingest(dbname, "{0}/snow3.tif".format(temppath), dt,
table, False)
shutil.rmtree(temppath)
else:
log.warning(
"MODSCAG data not available for {0}. Skipping download!".
format(dt.strftime("%Y-%m-%d")))
def download(dbname, dts, bbox):
"""Downloads the combined MODIS LAI data product MCD15 for
a specific date *dt* and imports them into the PostGIS database *dbname*."""
res = 0.01
burl = "http://e4ftl01.cr.usgs.gov/MOTA/MCD15A2.005"
tiles = modis.findTiles(bbox)
if tiles is not None:
for dt in [
dts[0] + timedelta(dti)
for dti in range((dts[-1] - dts[0]).days + 1)
]:
outpath = tempfile.mkdtemp()
url = "{0}/{1:04d}.{2:02d}.{3:02d}".format(burl, dt.year, dt.month,
dt.day)
connection = urllib.urlopen(url)
dom = html.fromstring(connection.read())
files = [link for link in dom.xpath('//a/@href')]
if len(files) > 0:
filenames = [
filter(
lambda s: re.findall(
r'MCD.*h{0:02d}v{1:02d}.*hdf$'.format(t[1], t[0]),
s), files) for t in tiles
]
for filename in filenames:
if len(filename) > 0:
filename = filename[0]
urllib.urlretrieve("{0}/{1}".format(url, filename),
"{0}/{1}".format(outpath, filename))
subprocess.call([
"gdal_translate",
"HDF4_EOS:EOS_GRID:{0}/{1}:MOD_Grid_MOD15A2:Lai_1km"
.format(outpath, filename),
"{0}/{1}".format(outpath,
filename).replace("hdf", "tif")
])
tifs = glob.glob("{0}/*.tif".format(outpath))
if len(tifs) > 0:
subprocess.call(
["gdal_merge.py", "-o", "{0}/lai.tif".format(outpath)
] + tifs)
cmd = " ".join([
"gdal_calc.py", "-A", "{0}/lai.tif".format(outpath),
"--outfile={0}/lai1.tif".format(outpath),
"--NoDataValue=-9999",
"--calc=\"(A<101.0)*(0.1*A+9999.0)-9999.0\""
])
subprocess.call(cmd, shell=True)
cmd = " ".join([
"gdalwarp", "-t_srs", "'+proj=latlong +ellps=sphere'",
"-tr",
str(res),
str(-res), "{0}/lai1.tif".format(outpath),
"{0}/lai2.tif".format(outpath)
])
subprocess.call(cmd, shell=True)
subprocess.call([
"gdal_translate", "-a_srs", "epsg:4326",
"{0}/lai2.tif".format(outpath),
"{0}/lai3.tif".format(outpath)
])
dbio.ingest(dbname, "{0}/lai3.tif".format(outpath), dt,
table, False)
shutil.rmtree(outpath)
else:
print("MCD15 data not available for {0}. Skipping download!".
format(dt.strftime("%Y-%m-%d")))
def download(dbname, dts, bbox):
"""Downloads the MODIS evapotranspiration data product MOD16 for
a set of dates *dt* and imports them into the PostGIS database *dbname*."""
log = logging.getLogger(__name__)
res = 0.01
urlbase = "http://files.ntsg.umt.edu"
tiles = modis.findTiles(bbox)
if tiles is not None:
for dt in [
dts[0] + timedelta(dti)
for dti in range((dts[-1] - dts[0]).days + 1)
]:
url = "{0}/data/NTSG_Products/MOD16/MOD16A2.105_MERRAGMAO/Y{1}".format(
urlbase, dt.year)
resp_year = requests.get(url)
try:
assert resp_year.status_code == 200
days = [
link
for link in BeautifulSoup(resp_year.text,
parse_only=SoupStrainer('a'))
if isinstance(link, Tag)
and link.text.find(dt.strftime("%j")) >= 0
]
assert len(days) > 0
resp_day = requests.get("{0}{1}".format(
urlbase, days[0].get('href')))
assert resp_day.status_code == 200
files = [
link.get('href')
for link in BeautifulSoup(resp_day.text,
parse_only=SoupStrainer('a'))
if isinstance(link, Tag) and link.text.find("hdf") > 0
]
files = [
f for f in files if any(
f.find("h{0:02d}v{1:02d}".format(t[1], t[0])) > 0
for t in tiles)
]
outpath = tempfile.mkdtemp()
for fname in files:
resp_file = requests.get("{0}{1}".format(urlbase, fname))
filename = fname.split("/")[-1]
with open("{0}/{1}".format(outpath, filename),
'wb') as fout:
for chunk in resp_file:
fout.write(chunk)
proc = subprocess.Popen([
"gdal_translate",
"HDF4_EOS:EOS_GRID:{0}/{1}:MOD_Grid_MOD16A2:ET_1km".
format(outpath, filename), "{0}/{1}".format(
outpath, filename).replace("hdf", "tif")
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
tifs = glob.glob("{0}/*.tif".format(outpath))
proc = subprocess.Popen(
["gdal_merge.py", "-o", "{0}/et.tif".format(outpath)] +
tifs,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen([
"gdal_calc.py", "-A", "{0}/et.tif".format(outpath),
"--outfile={0}/et1.tif".format(outpath),
"--NoDataValue=-9999", "--calc=(A<32701)*(0.1*A+9999)-9999"
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen([
"gdalwarp", "-t_srs", "+proj=latlong +ellps=sphere", "-tr",
str(res),
str(-res), "{0}/et1.tif".format(outpath),
"{0}/et2.tif".format(outpath)
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
if bbox is None:
pstr = []
else:
pstr = [
"-projwin",
str(bbox[0]),
str(bbox[3]),
str(bbox[2]),
str(bbox[1])
]
proc = subprocess.Popen(["gdal_translate"] + pstr + [
"-a_srs", "epsg:4326", "{0}/et2.tif".format(outpath),
"{0}/et3.tif".format(outpath)
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
dbio.ingest(dbname, "{0}/et3.tif".format(outpath), dt, table,
False)
shutil.rmtree(outpath)
except:
log.warning(
"MOD16 data not available for {0}. Skipping download!".
format(dt.strftime("%Y-%m-%d")))
def download(dbname, dts, bbox):
"""Downloads the combined MODIS LAI data product MCD15 for
a specific date *dt* and imports them into the PostGIS database *dbname*."""
log = logging.getLogger(__name__)
res = 0.005
burl = "http://e4ftl01.cr.usgs.gov/MOTA/MCD15A2H.006"
tiles = modis.findTiles(bbox)
if tiles is not None:
for dt in [
dts[0] + timedelta(dti)
for dti in range((dts[-1] - dts[0]).days + 1)
]:
outpath = tempfile.mkdtemp()
url = "{0}/{1:04d}.{2:02d}.{3:02d}".format(burl, dt.year, dt.month,
dt.day)
filenames = []
for t in tiles:
try:
tmppath, fname = earthdata.download(
url, "MCD15A2H.A{0}.h{1:02d}v{2:02d}.006.*.hdf".format(
dt.strftime("%Y%j"), t[1], t[0]))
except ConnectionError:
fname = None
if fname:
filenames.append("{0}/{1}".format(tmppath, fname))
for filename in filenames:
proc = subprocess.Popen([
"gdal_translate",
"HDF4_EOS:EOS_GRID:{0}:MOD_Grid_MOD15A2H:Lai_500m".format(
filename), "{0}/{1}".format(
outpath,
filename.split("/")[-1]).replace("hdf", "tif")
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
shutil.rmtree("/".join(filename.split("/")[:-1]))
tifs = glob.glob("{0}/*.tif".format(outpath))
if len(tifs) > 0:
proc = subprocess.Popen(
["gdal_merge.py", "-o", "{0}/lai.tif".format(outpath)] +
tifs,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen([
"gdal_calc.py", "-A", "{0}/lai.tif".format(outpath),
"--outfile={0}/lai1.tif".format(outpath),
"--NoDataValue=-9999",
"--calc=(A<101.0)*(0.1*A+9999.0)-9999.0"
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen([
"gdalwarp", "-t_srs", "+proj=latlong +ellps=sphere", "-tr",
str(res),
str(-res), "{0}/lai1.tif".format(outpath),
"{0}/lai2.tif".format(outpath)
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
proc = subprocess.Popen([
"gdal_translate", "-a_srs", "epsg:4326",
"{0}/lai2.tif".format(outpath),
"{0}/lai3.tif".format(outpath)
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
out, err = proc.communicate()
log.debug(out)
dbio.ingest(dbname, "{0}/lai3.tif".format(outpath), dt, table,
False)
else:
log.warning(
"MCD15 data not available for {0}. Skipping download!".
format(dt.strftime("%Y-%m-%d")))
shutil.rmtree(outpath)
