def fetch_Landsat_WELD(product, tiles, years, outdir):
"""
Fetch WELD data from the server at [http://e4ftl01.cr.usgs.gov/WELD].
Weld data is corrected and processed Landsat 5 and 7 data that is distributed in the
MODIS sinusoidal projection and grid format. Read more about WELD data.
https://landsat.usgs.gov/WELD.php
http://globalmonitoring.sdstate.edu/projects/weldglobal/
:param product: WELD product to download such as 'USWK','USMO','USYR'
:param tiles: list of tiles to grab such as ['h11v12','h11v11']
:param years: list of years to grab such as range(2001,2014)
:param outdir: output directory to save downloaded files
:return output_filelist: A list of full filepaths to files fetched be this function
"""
output_filelist = []
# check formats
global dates
tiles = core.enf_list(tiles)
years = core.enf_list(years)
years = [str(year) for year in years]
# create output directories
for tile in tiles:
if not os.path.exists(os.path.join(outdir, tile)):
os.makedirs(os.path.join(outdir, tile))
print('Connecting to servers!')
# Map the contents of the directory
site = 'https://e4ftl01.cr.usgs.gov/WELD/WELD' + product + '.001'
try:
dates = list_http_e4ftl01(site)
except:
print('Could not connect to site! check inputs!')
# find just the folders within the desired year range.
good_dates = []
for date in dates:
try:
y, m, d = date.split(".")
if y in years:
good_dates.append(date)
except:
pass
print("Found {0} days within year range".format(len(good_dates)))
# for all folders within the desired date range, map the subfolder contents.
for good_date in good_dates:
files = list_http_e4ftl01(site + '/' + good_date)
for afile in files:
# only list files with desired tilenames and not preview jpgs
if not '.jpg' in afile:
for tile in tiles:
if tile in afile:
# assemble the address
address = '/'.join([site, good_date, afile])
print("Downloading {0}".format(address))
#download the file.
outname = os.path.join(outdir, tile, afile)
output_filelist.append(outname)
download_url(address, outname)
return
def fetch_MODIS(product,
version,
tiles,
outdir,
start_dto,
end_dto,
force_overwrite=False):
"""
Fetch MODIS Land products from one of two servers. If this function
runs and downloads 0 files, check that your inputs are consistent
with the naming convention at the appropriate server address.
http://e4ftl01.cr.usgs.gov
ftp://n5eil01u.ecs.nsidc.org
:param product: MODIS product to download such as 'MOD10A1' or 'MYD11A1'
:param version: version number, usually '004' or '041' or '005'
:param tiles: list of tiles to grab such as ['h11v12','h11v11']
NOTE: for some MODIS products, the h and v are omitted.
:param outdir : output directory to save downloaded files
:param start_dto: datetime object, the starting date of the range of data to download
:param end_dto: datetime object, the ending date of the range of data to download
:param force_overwrite: will re-download files even if they already exist
:return out_filepaths: a list of filepaths to all files created by this function
"""
out_filepaths = []
# check formats
tiles = core.enf_list(tiles)
# create output directories
if not os.path.exists(outdir):
os.makedirs(outdir)
print("Connecting to servers!")
# obtain the web address, protocol information, and subdirectory where
# this tpe of MODIS data can be found.
site, isftp, Dir = _find_modis_product(product, version)
if Dir:
print("Connected to {0}/{1}".format(site, Dir))
else:
print("Connected to {0}".format(site))
# Depending on the type of connection (ftp vs http) populate the file list
try:
if isftp:
dates, _ = list_ftp(site, False, False, Dir)
else:
dates = list_http_e4ftl01(site)
except:
raise ValueError("Could not connect to {0}/{1}".format(site, Dir))
# refine contents down to just addresses of valid year and j_day
good_dates = []
for date in dates:
try:
date_dto = datetime.strptime(date, "%Y.%m.%d")
if start_dto <= date_dto <= end_dto:
good_dates.append(date)
except:
print("skipping non date folder name {0}".format(date))
print('Found {0} days within range'.format(len(good_dates)))
# for all folders within the desired date range, map the subfolder contents.
for good_date in good_dates:
if isftp:
files, _ = list_ftp(site, False, False, Dir + '/' + good_date)
else:
files = list_http_e4ftl01(site + '/' + good_date)
for afile in files:
# only list files with desired tile names and not preview jpgs
if not '.jpg' in afile:
for tile in tiles:
if tile in afile:
# assemble the address
if isftp:
address = '/'.join(
['ftp://' + site, Dir, good_date, afile])
else:
address = '/'.join([site, good_date, afile])
#download the file
outname = os.path.join(outdir, afile)
out_filepaths.append(outname)
if not os.path.isfile(outname) and not force_overwrite:
download_url(address, outname)
print('Downloaded {0}'.format(address))
print("Finished retrieving MODIS - {0} data!".format(product))
print("Downloaded {0} files".format(len(out_filepaths)))
return out_filepaths
def fetch_Landsat_WELD(product, tiles, years, outdir):
"""
Fetch WELD data from the server at [http://e4ftl01.cr.usgs.gov/WELD]
Weld data is corrected and processed Landsat 5 and 7 data that is distributed in the
MODIS sinusoidal projection and grid format. Read more about WELD data.
https://landsat.usgs.gov/WELD.php
http://globalmonitoring.sdstate.edu/projects/weldglobal/
Inputs:
product WELD product to download such as 'USWK','USMO','USYR'
tiles list of tiles to grab such as ['h11v12','h11v11']
years list of years to grab such as range(2001,2014)
outdir output directory to save downloaded files
"""
# check formats
global dates
tiles = core.enf_list(tiles)
years = core.enf_list(years)
years = [str(year) for year in years]
# create output directories
for tile in tiles:
if not os.path.exists(os.path.join(outdir,tile)):
os.makedirs(os.path.join(outdir,tile))
print '{Fetch_Landsat_WELD} Connecting to servers!'
# Map the contents of the directory
site= 'http://e4ftl01.cr.usgs.gov/WELD/WELD'+product+'.001'
try:
dates = list_http_e4ftl01(site)
except:
print '{Fetch_Landsat_WELD} Could not connect to site! check inputs!'
# find just the folders within the desired year range.
good_dates=[]
for date in dates:
try:
y, m, d = date.split(".")
if y in years:
good_dates.append(date)
except: pass
print 'Found ' + str(len(good_dates)) + ' days within year range'
# for all folders within the desired date range, map the subfolder contents.
for good_date in good_dates:
files = list_http_e4ftl01(site+'/'+good_date)
for afile in files:
# only list files with desired tilenames and not preview jpgs
if not '.jpg' in afile:
for tile in tiles:
if tile in afile:
# assemble the address
address = '/'.join([site,good_date,afile])
print '{Fetch_Landsat_WELD} Downloading' + address
#download the file.
outname = os.path.join(outdir,tile,afile)
download_url(address, outname)
return
def fetch_MODIS(product, version, tiles, outdir, start_dto, end_dto,
force_overwrite = False):
"""
Fetch MODIS Land products from one of two servers. If this function
runs and downloads 0 files, check that your inputs are consistent
with the naming convention at the appropriate server address.
http://e4ftl01.cr.usgs.gov
ftp://n5eil01u.ecs.nsidc.org
:param product: MODIS product to download such as 'MOD10A1' or 'MYD11A1'
:param version: version number, usually '004' or '041' or '005'
:param tiles: list of tiles to grab such as ['h11v12','h11v11']
NOTE: for some MODIS products, the h and v are omitted.
:param outdir : output directory to save downloaded files
:param start_dto: datetime object, the starting date of the range of data to download
:param end_dto: datetime object, the ending date of the range of data to download
:param force_overwrite: will re-download files even if they already exist
:return out_filepaths: a list of filepaths to all files created by this function
"""
out_filepaths = []
# check formats
tiles = core.enf_list(tiles)
# create output directories
if not os.path.exists(outdir):
os.makedirs(outdir)
print("Connecting to servers!")
# obtain the web address, protocol information, and subdirectory where
# this tpe of MODIS data can be found.
site, isftp, Dir = _find_modis_product(product, version)
if Dir:
print("Connected to {0}/{1}".format(site, Dir))
else:
print("Connected to {0}".format(site))
# Depending on the type of connection (ftp vs http) populate the file list
try:
if isftp:
dates,_ = list_ftp(site, False, False, Dir)
else:
dates = list_http_e4ftl01(site)
except:
raise ValueError("Could not connect to {0}/{1}".format(site,Dir))
# refine contents down to just addresses of valid year and j_day
good_dates = []
for date in dates:
try:
date_dto = datetime.strptime(date, "%Y.%m.%d")
if start_dto <= date_dto <= end_dto:
good_dates.append(date)
except:
print("skipping non date folder name {0}".format(date))
print('Found {0} days within range'.format(len(good_dates)))
# for all folders within the desired date range, map the subfolder contents.
for good_date in good_dates:
if isftp:
files,_ = list_ftp(site, False, False, Dir + '/' + good_date)
else:
files = list_http_e4ftl01(site + '/' + good_date)
for afile in files:
# only list files with desired tile names and not preview jpgs
if not '.jpg' in afile:
for tile in tiles:
if tile in afile:
# assemble the address
if isftp:
address='/'.join(['ftp://'+site, Dir, good_date, afile])
else:
address='/'.join([site, good_date, afile])
#download the file
outname = os.path.join(outdir, afile)
out_filepaths.append(outname)
if not os.path.isfile(outname) and not force_overwrite:
download_url(address, outname)
print('Downloaded {0}'.format(address))
print("Finished retrieving MODIS - {0} data!".format(product))
print("Downloaded {0} files".format(len(out_filepaths)))
return out_filepaths
