def fetch_Landsat_WELD(product, tiles, years, outdir): """ Fetch WELD data from the server at [http://e4ftl01.cr.usgs.gov/WELD]. Weld data is corrected and processed Landsat 5 and 7 data that is distributed in the MODIS sinusoidal projection and grid format. Read more about WELD data. https://landsat.usgs.gov/WELD.php http://globalmonitoring.sdstate.edu/projects/weldglobal/ :param product: WELD product to download such as 'USWK','USMO','USYR' :param tiles: list of tiles to grab such as ['h11v12','h11v11'] :param years: list of years to grab such as range(2001,2014) :param outdir: output directory to save downloaded files :return output_filelist: A list of full filepaths to files fetched be this function """ output_filelist = [] # check formats global dates tiles = core.enf_list(tiles) years = core.enf_list(years) years = [str(year) for year in years] # create output directories for tile in tiles: if not os.path.exists(os.path.join(outdir, tile)): os.makedirs(os.path.join(outdir, tile)) print('Connecting to servers!') # Map the contents of the directory site = 'https://e4ftl01.cr.usgs.gov/WELD/WELD' + product + '.001' try: dates = list_http_e4ftl01(site) except: print('Could not connect to site! check inputs!') # find just the folders within the desired year range. good_dates = [] for date in dates: try: y, m, d = date.split(".") if y in years: good_dates.append(date) except: pass print("Found {0} days within year range".format(len(good_dates))) # for all folders within the desired date range, map the subfolder contents. for good_date in good_dates: files = list_http_e4ftl01(site + '/' + good_date) for afile in files: # only list files with desired tilenames and not preview jpgs if not '.jpg' in afile: for tile in tiles: if tile in afile: # assemble the address address = '/'.join([site, good_date, afile]) print("Downloading {0}".format(address)) #download the file. outname = os.path.join(outdir, tile, afile) output_filelist.append(outname) download_url(address, outname) return
def fetch_MODIS(product, version, tiles, outdir, start_dto, end_dto, force_overwrite=False): """ Fetch MODIS Land products from one of two servers. If this function runs and downloads 0 files, check that your inputs are consistent with the naming convention at the appropriate server address. http://e4ftl01.cr.usgs.gov ftp://n5eil01u.ecs.nsidc.org :param product: MODIS product to download such as 'MOD10A1' or 'MYD11A1' :param version: version number, usually '004' or '041' or '005' :param tiles: list of tiles to grab such as ['h11v12','h11v11'] NOTE: for some MODIS products, the h and v are omitted. :param outdir : output directory to save downloaded files :param start_dto: datetime object, the starting date of the range of data to download :param end_dto: datetime object, the ending date of the range of data to download :param force_overwrite: will re-download files even if they already exist :return out_filepaths: a list of filepaths to all files created by this function """ out_filepaths = [] # check formats tiles = core.enf_list(tiles) # create output directories if not os.path.exists(outdir): os.makedirs(outdir) print("Connecting to servers!") # obtain the web address, protocol information, and subdirectory where # this tpe of MODIS data can be found. site, isftp, Dir = _find_modis_product(product, version) if Dir: print("Connected to {0}/{1}".format(site, Dir)) else: print("Connected to {0}".format(site)) # Depending on the type of connection (ftp vs http) populate the file list try: if isftp: dates, _ = list_ftp(site, False, False, Dir) else: dates = list_http_e4ftl01(site) except: raise ValueError("Could not connect to {0}/{1}".format(site, Dir)) # refine contents down to just addresses of valid year and j_day good_dates = [] for date in dates: try: date_dto = datetime.strptime(date, "%Y.%m.%d") if start_dto <= date_dto <= end_dto: good_dates.append(date) except: print("skipping non date folder name {0}".format(date)) print('Found {0} days within range'.format(len(good_dates))) # for all folders within the desired date range, map the subfolder contents. for good_date in good_dates: if isftp: files, _ = list_ftp(site, False, False, Dir + '/' + good_date) else: files = list_http_e4ftl01(site + '/' + good_date) for afile in files: # only list files with desired tile names and not preview jpgs if not '.jpg' in afile: for tile in tiles: if tile in afile: # assemble the address if isftp: address = '/'.join( ['ftp://' + site, Dir, good_date, afile]) else: address = '/'.join([site, good_date, afile]) #download the file outname = os.path.join(outdir, afile) out_filepaths.append(outname) if not os.path.isfile(outname) and not force_overwrite: download_url(address, outname) print('Downloaded {0}'.format(address)) print("Finished retrieving MODIS - {0} data!".format(product)) print("Downloaded {0} files".format(len(out_filepaths))) return out_filepaths
def fetch_Landsat_WELD(product, tiles, years, outdir): """ Fetch WELD data from the server at [http://e4ftl01.cr.usgs.gov/WELD] Weld data is corrected and processed Landsat 5 and 7 data that is distributed in the MODIS sinusoidal projection and grid format. Read more about WELD data. https://landsat.usgs.gov/WELD.php http://globalmonitoring.sdstate.edu/projects/weldglobal/ Inputs: product WELD product to download such as 'USWK','USMO','USYR' tiles list of tiles to grab such as ['h11v12','h11v11'] years list of years to grab such as range(2001,2014) outdir output directory to save downloaded files """ # check formats global dates tiles = core.enf_list(tiles) years = core.enf_list(years) years = [str(year) for year in years] # create output directories for tile in tiles: if not os.path.exists(os.path.join(outdir,tile)): os.makedirs(os.path.join(outdir,tile)) print '{Fetch_Landsat_WELD} Connecting to servers!' # Map the contents of the directory site= 'http://e4ftl01.cr.usgs.gov/WELD/WELD'+product+'.001' try: dates = list_http_e4ftl01(site) except: print '{Fetch_Landsat_WELD} Could not connect to site! check inputs!' # find just the folders within the desired year range. good_dates=[] for date in dates: try: y, m, d = date.split(".") if y in years: good_dates.append(date) except: pass print 'Found ' + str(len(good_dates)) + ' days within year range' # for all folders within the desired date range, map the subfolder contents. for good_date in good_dates: files = list_http_e4ftl01(site+'/'+good_date) for afile in files: # only list files with desired tilenames and not preview jpgs if not '.jpg' in afile: for tile in tiles: if tile in afile: # assemble the address address = '/'.join([site,good_date,afile]) print '{Fetch_Landsat_WELD} Downloading' + address #download the file. outname = os.path.join(outdir,tile,afile) download_url(address, outname) return
def fetch_MODIS(product, version, tiles, outdir, start_dto, end_dto, force_overwrite = False): """ Fetch MODIS Land products from one of two servers. If this function runs and downloads 0 files, check that your inputs are consistent with the naming convention at the appropriate server address. http://e4ftl01.cr.usgs.gov ftp://n5eil01u.ecs.nsidc.org :param product: MODIS product to download such as 'MOD10A1' or 'MYD11A1' :param version: version number, usually '004' or '041' or '005' :param tiles: list of tiles to grab such as ['h11v12','h11v11'] NOTE: for some MODIS products, the h and v are omitted. :param outdir : output directory to save downloaded files :param start_dto: datetime object, the starting date of the range of data to download :param end_dto: datetime object, the ending date of the range of data to download :param force_overwrite: will re-download files even if they already exist :return out_filepaths: a list of filepaths to all files created by this function """ out_filepaths = [] # check formats tiles = core.enf_list(tiles) # create output directories if not os.path.exists(outdir): os.makedirs(outdir) print("Connecting to servers!") # obtain the web address, protocol information, and subdirectory where # this tpe of MODIS data can be found. site, isftp, Dir = _find_modis_product(product, version) if Dir: print("Connected to {0}/{1}".format(site, Dir)) else: print("Connected to {0}".format(site)) # Depending on the type of connection (ftp vs http) populate the file list try: if isftp: dates,_ = list_ftp(site, False, False, Dir) else: dates = list_http_e4ftl01(site) except: raise ValueError("Could not connect to {0}/{1}".format(site,Dir)) # refine contents down to just addresses of valid year and j_day good_dates = [] for date in dates: try: date_dto = datetime.strptime(date, "%Y.%m.%d") if start_dto <= date_dto <= end_dto: good_dates.append(date) except: print("skipping non date folder name {0}".format(date)) print('Found {0} days within range'.format(len(good_dates))) # for all folders within the desired date range, map the subfolder contents. for good_date in good_dates: if isftp: files,_ = list_ftp(site, False, False, Dir + '/' + good_date) else: files = list_http_e4ftl01(site + '/' + good_date) for afile in files: # only list files with desired tile names and not preview jpgs if not '.jpg' in afile: for tile in tiles: if tile in afile: # assemble the address if isftp: address='/'.join(['ftp://'+site, Dir, good_date, afile]) else: address='/'.join([site, good_date, afile]) #download the file outname = os.path.join(outdir, afile) out_filepaths.append(outname) if not os.path.isfile(outname) and not force_overwrite: download_url(address, outname) print('Downloaded {0}'.format(address)) print("Finished retrieving MODIS - {0} data!".format(product)) print("Downloaded {0} files".format(len(out_filepaths))) return out_filepaths