#!/usr/bin/env python ''' Python script to pull all MODIS files for LAI product for given years for given tiles into destination_folder ''' from uclgeog.get_modis_files import get_modis_files from datetime import datetime years = [2017, 2016] tiles = ['h09v05'] destination_folder = 'data' for year in years: ndoys = (datetime(year, 12, 31) - datetime(year, 1, 1)).days + 1 for doy in range(1, ndoys + 1, 1): print(year, ndoys, doy, end=' ') filenames = get_modis_files(doy,year,tiles,base_url='https://n5eil01u.ecs.nsidc.org/MOST',\ version=6, product='MOD10A1') print(filenames)
def mosaic_and_clip(tiles=['h17v03'], doy=1, year=2020, ofolder=None, folder="data/", grid=None, layer="Lai_500m", shpfile=None, country_code=None, product='MCD15A3H', verbose=False, nodata=255, base_url='https://e4ftl01.cr.usgs.gov/MOTA', frmat="MEM"): """ Simple high-level function for downloading MODIS dataset from Earthdata. tiles: list of MODIS tiles to access doy: day of year for dataset. Not that some products are only produced every 4 or 8 days, so requesting a dataset for a day that doesnt exist will fail. year: year of dataset. 1999 to now. folder: folder for storing datasets default: data layer: data layer. See product specification page for more details. https://lpdaac.usgs.gov/products/mcd15a3hv006/ default: Lai_500m product: product id. See product specification page for more details. e.g. https://lpdaac.usgs.gov/products/mcd15a3hv006/ default: MCD15A3H verbose: verbose flag defaultL False country_code: FIPS country code for any masking: https://en.wikipedia.org/wiki/List_of_FIPS_country_codes default: None shpfile: Shapefile to use for data masking default: TM_WORLD_BORDERS-0.3.zip nodata: no data value default: 255 base_url: base URL of datasets default: https://e4ftl01.cr.usgs.gov/MOTA frmat: output file format: MEM, VRT or GTiff default MEM (data array) """ tiles = list(tiles) if ofolder == None: ofolder = folder folder_path = Path(folder) ofolder_path = Path(ofolder) # mkdir folder_path.mkdir(parents=True, exist_ok=True) ofolder_path.mkdir(parents=True, exist_ok=True) if verbose: print(f'Testing for MODIS files on this computer ...') # download files if we need to mfiles = get_modis_files(doy,year,tiles,product=product,version=6,\ destination_folder=folder,verbose=verbose,\ base_url=base_url) # Find all files to mosaic together hdf_files = find_mcdfiles(year, doy, tiles, folder, product=product) if verbose: print(f'files: {hdf_files}') # Create GDAL friendly-names... gdal_filenames = create_gdal_friendly_names(hdf_files, layer, grid=grid, product=product) if verbose: print(f'dataset: {gdal_filenames}') ''' If borders specified: ''' # get borders if needed if country_code == None: if verbose: print(f'output format {frmat}') print(f'No data value: {nodata}') if frmat == "MEM": g = gdal.Warp("", gdal_filenames, format="MEM", dstNodata=nodata) if g: data = g.ReadAsArray() if verbose: print(f'returning data array') return data else: print( f'failed to warp {str(gdal_filenames)} {year}, {doy}, {tiles}, {folder}' ) elif frmat == "VRT": try: geotiff_fnamex = f"{layer:s}_{year:d}_{doy:03d}.vrt" geotiff_fname = ofolder_path / geotiff_fnamex g = gdal.Warp(geotiff_fname.as_posix(), gdal_filenames, format=frmat, dstNodata=nodata) except: pass if g: del g ofile = geotiff_fname.as_posix() if verbose: print(f'returning data in {ofile}') return ofile elif frmat == "GTiff": try: geotiff_fnamex = f"{layer:s}_{year:d}_{doy:03d}.tif" geotiff_fname = ofolder_path / geotiff_fnamex g = gdal.Warp(geotiff_fname.as_posix(), gdal_filenames, format=frmat, dstNodata=nodata) except: pass if g: del g ofile = geotiff_fname.as_posix() if verbose: print(f'returning data in {ofile}') return ofile else: print( f'failed to warp {str(gdal_filenames)} {year}, {doy}, {tiles}, {folder}' ) else: print("Only MEM, VRT or GTiff formats supported!") else: if shpfile == None: shpfile = get_world(data=folder).replace('.zip', '.shp') if verbose: print( f'{shpfile:s} shapefile used to mask FIPS code {country_code:s}' ) if verbose: print(f'output format {frmat}') print(f'No data value: {nodata}') if frmat == "MEM": g = gdal.Warp("", gdal_filenames, format="MEM", dstNodata=nodata, cutlineDSName=shpfile, cutlineWhere=f"FIPS='{country_code:s}'", cropToCutline=True) if g: data = g.ReadAsArray() if verbose: print(f'returning data in array') return data else: print( f'failed to warp {str(gdal_filenames)} {year}, {doy}, {tiles}, {folder}' ) elif frmat == "VRT": try: geotiff_fnamex = f"{layer:s}_{year:d}_{doy:03d}_{country_code:s}.vrt" geotiff_fname = ofolder_path / geotiff_fnamex g = gdal.Warp(geotiff_fname.as_posix(), gdal_filenames, format=frmat, dstNodata=nodata, cutlineDSName=shpfile, cutlineWhere=f"FIPS='{country_code:s}'", cropToCutline=True) except: pass if g: del g ofile = geotiff_fname.as_posix() if verbose: print(f'returning data in {ofile}') return ofile elif frmat == "GTiff": try: geotiff_fnamex = f"{layer:s}_{year:d}_{doy:03d}_{country_code:s}.tif" geotiff_fname = ofolder_path / geotiff_fnamex g = gdal.Warp(geotiff_fname.as_posix(), gdal_filenames, format=frmat, dstNodata=nodata, cutlineDSName=shpfile, cutlineWhere=f"FIPS='{country_code:s}'", cropToCutline=True) except: pass if g: del g ofile = geotiff_fname.as_posix() if verbose: print(f'returning data in {ofile}') return ofile else: print( f'failed to warp {str(gdal_filenames)} {year}, {doy}, {tiles}, {folder}' ) else: print("Only MEM, VRT or GTiff formats supported!")
if ofile.exists(): done = True # try to download it from server if download: done = procure_dataset(fname, verbose=True) from uclgeog.get_modis_files import get_modis_files ''' Get the MODIS LC files from the server to store in data ''' try: url = 'https://e4ftl01.cr.usgs.gov//MODV6_Cmp_C/MOTA/' filename = get_modis_files(1,year,[tiles],base_url=url,\ version=6,verbose=True,\ destination_folder='data',\ product='MCD12Q1')[0] print(filename) except: print('server may be down') from uclgeog.process_timeseries import mosaic_and_clip ''' Extract and clip the dataset ''' lc_data = mosaic_and_clip(tiles, 1, year, folder='data', layer="LC_Type3", shpfile='data/TM_WORLD_BORDERS-0.3.shp',
def procure_dataset(dataset_name, destination_folder="data",verbose=False, locations=["/data/selene/ucfajlg/uclgeog_data/",\ "/data/selene/ucfajlg/uclgeog_data/lai_data/",\ "/archive/rsu_raid_0/plewis/public_html/uclgeog_data"][::-1],\ modis_urls=['https://e4ftl01.cr.usgs.gov/MOTA',\ 'https://e4ftl01.cr.usgs.gov/MOLT',\ 'https://e4ftl01.cr.usgs.gov/MOLA',\ 'https://e4ftl01.cr.usgs.gov//MODV6_Cmp_C/MOTA/',\ 'https://e4ftl01.cr.usgs.gov/VIIRS',\ 'https://n5eil01u.ecs.nsidc.org/MOST/',\ 'https://n5eil01u.ecs.nsidc.org/MOSA/',\ 'https://n5eil01u.ecs.nsidc.org/VIIRS/'],\ urls=["http://www2.geog.ucl.ac.uk/~ucfajlg/uclgeog_data/",\ "http://www2.geog.ucl.ac.uk/~plewis/uclgeog_data/",\ "http://www2.geog.ucl.ac.uk/~plewis/uclgeog_data/lai_files/"][::-1]): """Procure a Geog0111 dataset. This function will look for the dataset called `dataset_name`, and either provide symbolic links or download the relevant files to a local folder called by default `data`, or with a user-provided name. The other two options are to do with the location of the dataset witin the UCL filesystem (`location`), and the external URL (list `urls`). It is assumed that in either case, `datasest_name` is a valid folder under both `location` and `url`. """ dest_path = Path(destination_folder) if not dest_path.exists(): dest_path.mkdir() output_fname = dest_path.joinpath(dataset_name) if output_fname.exists(): return True done = False fully_qualified_hostname = getfqdn() if fully_qualified_hostname.find("geog.ucl.ac.uk") >= 0: if (verbose): print("Running on UCL's Geography computers") for location in locations: if (verbose): print(f'trying {location}') done = generate_symlinks(dataset_name, location, destination_folder=destination_folder, verbose=verbose) if done: break else: if (verbose): print("Running outside UCL Geography. Will try to download data.\n",\ dataset_name,"\nThis might take a while!") for url in list(urls): if (verbose): print(f'trying {url}') done = download_data(dataset_name, url, verbose=verbose, destination_folder=destination_folder) if done: break if not done: # maybe a modis dataset: try that if its an hdf try: info = dataset_name.split('.') product = info[0] tile = info[2] version = int(info[3]) year = int(info[1][1:5]) doy = int(info[1][5:]) dtype = info[-1] if dtype in ['hdf', 'tif']: for url in modis_urls: try: filename = get_modis_files(doy,year,[tile],base_url=url,\ version=version,\ destination_folder=destination_folder,\ product=product)[0] done = True except: pass except: pass return (done)
#!/usr/bin/env python ''' Python script to pull all MODIS files for LAI product for given years for given tiles into destination_folder ''' from uclgeog.get_modis_files import get_modis_files from datetime import datetime years = [2017, 2016] tiles = ['h17v03', 'h18v03', 'h17v04', 'h18v04'] destination_folder = 'data' for year in years: ndoys = (datetime(year, 12, 31) - datetime(year, 1, 1)).days + 1 for doy in range(1, ndoys + 1, 4): print(year, ndoys, doy, end=' ') filenames = get_modis_files(doy,year,tiles,base_url='https://e4ftl01.cr.usgs.gov/MOTA',\ version=6,\ product='MCD15A3H') print(filenames)