def download_filelist(ftp_texts, file_type=None, outdir=None): """ Reads text file of download links, downloads them. This script reads a text file with urls such as those output from ECHO REVERB and outputs them to an output directory. It will retry failed links 20 times before giving up and outputting a warning to the user. :param ftp_texts: array of txt files ordered from reverb containing ftp links :param file_type: file extension of the desired files, leave blank or False to grab all types. :param outdir: folder where files are to be placed after download :return list failed: list of files which failed to download after the end of the script. """ failed = [] # force inputs to take list format ftp_texts = core.enf_list(ftp_texts) if file_type is not None: file_type = core.enf_list(file_type) for ftptext in ftp_texts: #verify that things exist. core.exists(ftptext) if not outdir: outdir, _ = os.path.split(ftptext) ftp = open(ftptext, 'r') sites = ftp.readlines() print("Attempting to download {0} files!".format(len(sites))) print("Saving all files to {0}".format(outdir)) # perform the first attempt failed = download_urls(sites, outdir, file_type) # for 19 more times, if there are still items in the failed list, try again for i in range(1, 19): if len(failed) > 0: print("retry number {0} to grab {1} failed downloads!".format( i, len(failed))) time.sleep(60) failed = download_urls(failed, file_type, outdir) # once all tries are complete, print a list of files which repeatedly failed if len(failed) > 0: print( 'Files at the following URLs have failed 20 download attempts') print('Manually verify that these files exist on the server:') for i in failed: print(i) else: print('Finished with no errors!') # close the open text files and finish up ftp.close() return failed
def download_filelist(ftptexts, filetypes = False, outdir = False): """ Reads text file of download links, downloads them. This script reads a text file with urls such as those output from ECHO REVERB and outputs them to an output directory. It will retry failed links 20 times before giving up and outputing a warning to the user. Inputs: ftptexts array of txt files ordered from reverb containing ftp links filetype file extension of the desired files, leave blank or False to grab all types. outdir folder where files are to be placed after download Outputs: failed list of files which failed to download after the end of the script. """ # force inputs to take list format ftptexts = core.enf_list(ftptexts) if filetypes: filetypes = core.enf_list(filetypes) for ftptext in ftptexts: #verify that things exist. core.exists(ftptext) if not outdir: outdir,_ = os.path.split(ftptext) ftp = open(ftptext,'r') sites = ftp.readlines() print("Attempting to download {0} files!".format(len(sites))) print("Saving all files to {0}".format(outdir)) # perform the first attempt failed = download_urls(sites, outdir, filetypes) # for 19 more times, if there are still items in the failed list, try again for i in range(1,19): if len(failed)>0: print("retry number {0} to grab {1} failed downloads!".format(i,len(failed))) time.sleep(60) failed = download_urls(failed, filetypes, outdir) # once all tries are complete, print a list of files which repeatedly failed if len(failed)>0: print('Files at the following URLs have failed 20 download attempts') print('Manually verify that these files exist on the server:') for i in failed: print(i) else: print('Finished with no errors!') # close the open text files and finish up ftp.close() return failed
def extract_TRMM_NetCDF(filelist, outdir): """ Function converts NetCDFs to tiffs. Designed to work with TRMM data downloaded from GLOVIS inputs: filelist list of '.nc' files to convert to tiffs. outdir directory to which tif files should be saved returns an output filelist of local filepaths of extracted data. """ # Set up initial parameters. arcpy.env.workspace = outdir filelist = core.enf_list(filelist) output_filelist = [] # convert every file in the list "filelist" for infile in filelist: # use arcpy module to make raster layer from netcdf arcpy.MakeNetCDFRasterLayer_md(infile, "r", "longitude", "latitude", "r", "", "", "BY_VALUE") outname = core.create_outname(outdir, infile, "e", "tif") arcpy.CopyRaster_management("r", outname, "", "", "", "NONE", "NONE", "") output_filelist.append(outname) print("Converted netCDF file " + outname + " to Raster") return output_filelist
def extract_GCMO_NetCDF(netcdf_list, variable, outdir): """ Extracts all time layers from a "Global Climate Model Output" NetCDF layer :param netcdf_list: List of netcdfs from CORDEX climate distribution :param variable: The climate variable of interest (tsmax, tsmin, etc) :param outdir: Output directory to save files. :return output_filelist: returns list of files created by this function """ output_filelist = [] if not os.path.exists(outdir): os.makedirs(outdir) netcdf_list = core.enf_list(netcdf_list) for netcdf in netcdf_list: # get net cdf properties object props = arcpy.NetCDFFileProperties(netcdf) print("finding dimensions") dims = props.getDimensions() for dim in dims: print dim, props.getDimensionSize(dim) # make sure the variable is in this netcdf if variable: if not variable in props.getVariables(): print("Valid variables for this file include {0}".format(props.getVariables())) raise Exception("Variable '{0}' is not in this netcdf!".format(variable)) for dim in dims: if dim == "time": # set other dimensions x_dim = "lon" y_dim = "lat" band_dim = "" valueSelectionMethod = "BY_VALUE" size = props.getDimensionSize(dim) for i in range(size): # sanitize the dimname for invalid characters dimname = props.getDimensionValue(dim,i).replace(" 12:00:00 PM","") dimname = dimname.replace("/","-").replace(" ","_") dim_value = [["time", props.getDimensionValue(dim,i)]] print("extracting '{0}' from '{1}'".format(variable, dim_value)) outname = core.create_outname(outdir, netcdf, dimname, 'tif') output_filelist.append(outname) arcpy.MakeNetCDFRasterLayer_md(netcdf, variable, x_dim, y_dim, "temp", band_dim, dim_value, valueSelectionMethod) arcpy.CopyRaster_management("temp", outname, "", "", "", "NONE", "NONE", "") return output_filelist
def TRMM_NetCDF(filelist, outdir): """ Function converts NetCDFs to tiffs. Designed to work with TRMM data downloaded from GLOVIS :param filelist: list of '.nc' files to convert to tifs. :param outdir: directory to which tif files should be saved :return output_filelist: list of local filepaths of extracted data. """ # Set up initial parameters. arcpy.env.workspace = outdir filelist = core.enf_list(filelist) output_filelist = [] # convert every file in the list "filelist" for infile in filelist: # use arcpy module to make raster layer from netcdf arcpy.MakeNetCDFRasterLayer_md(infile, "r", "longitude", "latitude", "r", "", "", "BY_VALUE") arcpy.CopyRaster_management("r", infile[:-3] + ".tif", "", "", "", "NONE", "NONE", "") output_filelist.append(infile[:-3] + ".tif") print('Converted netCDF file ' + infile + ' to Raster') return output_filelist
def toa_reflectance_8(band_nums, meta_path, outdir = False): """ Converts Landsat 8 bands to Top-of-Atmosphere reflectance. To be performed on raw Landsat 8 level 1 data. See link below for details see here [http://landsat.usgs.gov/Landsat8_Using_Product.php] Inputs: band_nums A list of desired band numbers such as [3,4,5] meta_path The full filepath to the metadata file for those bands outdir Output directory to save converted files. If left False it will save ouput files in the same directory as input files. """ outlist = [] #enforce the list of band numbers and grab metadata from the MTL file band_nums = core.enf_list(band_nums) band_nums = map(str, band_nums) OLI_bands = ['1','2','3','4','5','6','7','8','9'] meta_path = os.path.abspath(meta_path) meta = grab_meta(meta_path) #cycle through each band in the list for calculation, ensuring each is in the list of OLI bands for band_num in band_nums: if band_num in OLI_bands: #scrape data from the given file path and attributes in the MTL file band_path = meta_path.replace("MTL.txt","B{0}.tif".format(band_num)) Qcal = arcpy.Raster(band_path) Mp = getattr(meta,"REFLECTANCE_MULT_BAND_{0}".format(band_num)) # multiplicative scaling factor Ap = getattr(meta,"REFLECTANCE_ADD_BAND_{0}".format(band_num)) # additive rescaling factor SEA = getattr(meta,"SUN_ELEVATION")*(math.pi/180) # sun elevation angle theta_se #get rid of the zero values that show as the black background to avoid skewing values null_raster = arcpy.sa.SetNull(Qcal, Qcal, "VALUE = 0") #calculate top-of-atmosphere reflectance TOA_ref = (((null_raster * Mp) + Ap)/(math.sin(SEA))) #save the data to the automated name if outdir is given or in the parent folder if not if outdir: outdir = os.path.abspath(outdir) outname = core.create_outname(outdir, band_path, "TOA_Ref", "tif") else: folder = os.path.split(meta_path)[0] outname = core.create_outname(folder, band_path, "TOA_Ref", "tif") TOA_ref.save(outname) outlist.append(outname) print("Saved output at {0}".format(outname)) #if listed band is not an OLI sensor band, skip it and print message else: print("Can only perform reflectance conversion on OLI sensor bands") print("Skipping band {0}".format(band_num)) return outlist
def cloud_mask_8(band_nums, BQA_path, outdir=False): """ Removal of cloud-covered pixels in raw Landsat 8 bands using the BQA file included. To be performed on raw Landsat 8 level 1 data. Inputs: band_nums A list of desired band numbers such as [3 4 5] BQA_path The full filepath to the BQA file for the Landsat 8 dataset outdir Output directory to save cloudless band tifs and the cloud mask """ #enforce the input band numbers as a list of strings band_nums = core.enf_list(band_nums) band_nums = map(str, band_nums) #define the range of values in the BQA file to be reclassified as cloud (0) or not cloud (1) outReclass = Reclassify( BQA_path, "Value", RemapRange([[50000, 65000, 0], [28670, 32000, 0], [2, 28669, 1], [32001, 49999, 1], [1, 1, "NoData"]])) #set the name and save the binary cloud mask tiff file Mask_name = BQA_path.replace("_BQA", "") CloudMask_path = core.create_outname(outdir, Mask_name, "Mask", "tif") outReclass.save(CloudMask_path) #for each band listed in band_nums, apply the Con tool to erase cloud pixels and save each band as a new tiff for band_num in band_nums: band_path = BQA_path.replace("BQA.tif", "B{0}.tif".format(band_num)) outname = core.create_outname(outdir, band_path, "NoClds", "tif") outCon = Con(outReclass, band_path, "", "VALUE = 1") outCon.save(outname) return
def fetch_Landsat8(path_row_pairs, start_dto, end_dto, outdir, max_cloud_cover=100, bands=None): """ This function downloads all landsat 8 tiles for the input path_row_pairs and within the bounds of the start_dto and the end_dto, and saves them to the output directory. It uses the amazon web service at [https://aws.amazon.com/public-data-sets/landsat/] :param path_row_pairs: tupled integer values of path,row coordinates of tile. may be a list of several tuples. example: [(1,1),(1,2)] :param start_dto: python datetime object of start date of range :param end_dto: python datetime object of end date of range :param outdir: the folder to save the output landsat files in :param max_cloud_cover: maximum percent cloud cover that is acceptable to download the file. :return output_filelist: A list of tile names downloaded by this function. """ # fetch an updated scene list with custom function. scene_list = fetch_Landsat8_scene_list() path_row_pairs = core.enf_list(path_row_pairs) output_tilenames = [] for path_row_pair in path_row_pairs: #format input strings path, row = path_row_pair path_str = str(path).zfill(3) row_str = str(row).zfill(3) # loop through the scene list # if the date for the given path/row scene is within the date range, download it with landsat_8_scene for row in scene_list: tilename = row[0] datestring = row[1].split(".")[ 0] # removes fractional seconds from datestring date = datetime.datetime.strptime(datestring, "%Y-%m-%d %H:%M:%S") pathrow_id = "LC8{0}{1}".format(path_str, row_str) cloud_cover = float(row[2]) if cloud_cover < max_cloud_cover: if pathrow_id in row[0]: if start_dto <= date <= end_dto: amazon_url = row[-1] fetch_Landsat8_tile(amazon_url, tilename, outdir, bands) output_tilenames.append(os.path.join(outdir, tilename)) print("Finished retrieving landsat 8 data!") return output_tilenames
def fetch_Landsat8_tile(amazon_url, tilename, outdir, bands=None): """ This function makes use of the amazon web service hosted Landsat 8 OLI data. It recieves an amazon web url for a single landsat tile, and downloads the desired files :param amazon_url: url to amazons page hosting these landsat tiles :param tilename: landsat tile name :param outdir: output directory to place landsat data :param bands: list of bands to download when not all are desired, options include any of [1,2,3,4,5,6,7,8,9,10,11,"QA"]. The MTL file is ALWAYS downloaded. :return tilepath: returns a filepath to the new landsat tile folder with .TIFs in it """ if bands is None: bands = map(str, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, "QA"]) else: bands = map(str, (core.enf_list(bands))) # create the scene name from the input parameters and use that to generate the scene's unique url connection = urllib.urlopen(amazon_url) page = connection.read().split("\n") print("Downloading landsat tile {0}".format(tilename)) for line in page: if "<li><a href=" in line: # pull filename from html code filename = line.split('"')[1] # pull out band information band_id = filename.replace(tilename + "_", "").split(".")[0].replace("B", "") good_band = band_id in bands mtl_file = "MTL" in band_id # download desired files. if good_band or mtl_file: link = amazon_url.replace("index.html", filename) savename = os.path.join(outdir, tilename, filename) # try twice if filepath doesn't already exist if not os.path.isfile(savename): try: download_url(link, savename) except: download_url(link, savename) print("\tDownloaded {0}".format(filename)) else: print("\t Found {0}".format(filename)) return os.path.join(outdir, tilename)
def _extract_HDF_datatype(hdf, layer_indexs, outdir=None, datatype=None, force_custom=False, nodata_value=None): """ This function wraps "_extract_HDF_layer_data" and "_gdal_dataset_to_tif" It only works for datatypes listed in the datatype_library.csv :param hdf: a single hdf filepath :param layer_indexs: list of int index values of layers to extract :param outdir: filepath to output directory to place tifs. If left as "None" output geotiffs will be placed right next to input HDF. :param datatype: a dnppy.convert.datatype object created from an entry in the datatype_library.csv :param force_custom: if True, this will force the data to take on the projection and geotransform attributes from the datatype object, even if valid projection and geotransform info can be pulled from the gdal dataset. Should almost never be True. :param nodata_value: the value to set to Nodata :return: list of filepaths to output files """ output_filelist = [] if outdir is None: outdir = os.path.dirname(hdf) data = _extract_HDF_layer_data(hdf, layer_indexs) layer_indexs = core.enf_list(layer_indexs) for layer_index in layer_indexs: dataset = data[layer_index] outpath = core.create_outname(outdir, hdf, str(layer_index), "tif") print("creating dataset at {0}".format(outpath)) _gdal_dataset_to_tif(dataset, outpath, cust_projection=datatype.projectionTXT, cust_geotransform=datatype.geotransform, force_custom=force_custom, nodata_value=nodata_value) output_filelist.append(outpath) return output_filelist
def fetch_Landsat8_tile(amazon_url, tilename, outdir, bands = None): """ This function makes use of the amazon web service hosted Landsat 8 OLI data. It recieves an amazon web url for a single landsat tile, and downloads the desired files :param amazon_url: url to amazons page hosting these landsat tiles :param tilename: landsat tile name :param outdir: output directory to place landsat data :param bands: list of bands to download when not all are desired, options include any of [1,2,3,4,5,6,7,8,9,10,11,"QA"]. The MTL file is ALWAYS downloaded. :return tilepath: returns a filepath to the new landsat tile folder with .TIFs in it """ if bands is None: bands = map(str, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, "QA"]) else: bands = map(str, (core.enf_list(bands))) # create the scene name from the input parameters and use that to generate the scene's unique url connection = urllib.urlopen(amazon_url) page = connection.read().split("\n") print("Downloading landsat tile {0}".format(tilename)) for line in page: if "<li><a href=" in line: # pull filename from html code filename = line.split('"')[1] # pull out band information band_id = filename.replace(tilename + "_","").split(".")[0].replace("B","") good_band = band_id in bands mtl_file = "MTL" in band_id # download desired files. if good_band or mtl_file: link = amazon_url.replace("index.html",filename) savename = os.path.join(outdir, tilename, filename) # try twice if filepath doesn't already exist if not os.path.isfile(savename): try: download_url(link, savename) except: download_url(link, savename) print("\tDownloaded {0}".format(filename)) else: print("\t Found {0}".format(filename)) return os.path.join(outdir, tilename)
def atsat_bright_temp_8(band_nums, meta_path, outdir = False): """ Converts Landsat 8 TIRS bands to at satellite brightnes temperature in Kelvins To be performed on raw Landsat 8 level 1 data. See link below for details see here http://landsat.usgs.gov/Landsat8_Using_Product.php Inputs: band_nums A list of desired band numbers, which should be [10,11] meta_path The full filepath to the metadata file for those bands outdir Output directory to save converted files. If left False it will save ouput files in the same directory as input files. """ band_nums = core.enf_list(band_nums) band_nums = map(str, band_nums) meta = grab_meta(meta_path) for band_num in band_nums: if band_num in ["10","11"]: band_path = meta_path.replace("MTL.txt","B{0}.tif".format(band_num)) Qcal = arcpy.Raster(band_path) # requires first converting to radiance Ml = getattr(meta,"RADIANCE_MULT_BAND_" + band_num) # multiplicative scaling factor Al = getattr(meta,"RADIANCE_ADD_BAND_" + band_num) # additive rescaling factor TOA_rad = (Qcal * Ml) + Al # now convert to at-sattelite brightness temperature K1 = getattr(meta,"K1_CONSTANT_BAND_" + band_num) # thermal conversion constant 1 K2 = getattr(meta,"K2_CONSTANT_BAND_" + band_num) # thermal conversion constant 2 Bright_Temp = K2/(arcpy.sa.Ln((K1/TOA_rad) + 1)) metaname = core.create_outname(outdir, meta_path, "Bright-Temp") shutil.copyfile(meta_path,metaname) outname = core.create_outname(outdir, band_path, "Bright-Temp") Bright_Temp.save(outname) print("Saved output at {0}".format(outname)) del TOA_rad else: print("Can only perform brightness temperature on TIRS sensor bands!") print("Skipping band {0}".format(outname)) return
def fetch_Landsat8(path_row_pairs, start_dto, end_dto, outdir, max_cloud_cover = 100, bands = None): """ This function downloads all landsat 8 tiles for the input path_row_pairs and within the bounds of the start_dto and the end_dto, and saves them to the output directory. It uses the amazon web service at [https://aws.amazon.com/public-data-sets/landsat/] :param path_row_pairs: tupled integer values of path,row coordinates of tile. may be a list of several tuples. example: [(1,1),(1,2)] :param start_dto: python datetime object of start date of range :param end_dto: python datetime object of end date of range :param outdir: the folder to save the output landsat files in :param max_cloud_cover: maximum percent cloud cover that is acceptable to download the file. :return output_filelist: A list of tile names downloaded by this function. """ # fetch an updated scene list with custom function. scene_list = fetch_Landsat8_scene_list() path_row_pairs = core.enf_list(path_row_pairs) output_tilenames = [] for path_row_pair in path_row_pairs: #format input strings path, row = path_row_pair path_str = str(path).zfill(3) row_str = str(row).zfill(3) # loop through the scene list # if the date for the given path/row scene is within the date range, download it with landsat_8_scene for row in scene_list: tilename = row[0] datestring = row[1].split(".")[0] # removes fractional seconds from datestring date = datetime.datetime.strptime(datestring, "%Y-%m-%d %H:%M:%S") pathrow_id = "LC8{0}{1}".format(path_str, row_str) cloud_cover = float(row[2]) if cloud_cover < max_cloud_cover: if pathrow_id in row[0]: if start_dto <= date <= end_dto: amazon_url = row[-1] fetch_Landsat8_tile(amazon_url, tilename, outdir, bands) output_tilenames.append(os.path.join(outdir, tilename)) print("Finished retrieving landsat 8 data!") return output_tilenames
def _extract_HDF_datatype(hdf, layer_indexs, outdir = None, datatype = None, force_custom = False, nodata_value = None): """ This function wraps "_extract_HDF_layer_data" and "_gdal_dataset_to_tif" It only works for datatypes listed in the datatype_library.csv :param hdf: a single hdf filepath :param layer_indexs: list of int index values of layers to extract :param outdir: filepath to output directory to place tifs. If left as "None" output geotiffs will be placed right next to input HDF. :param datatype: a dnppy.convert.datatype object created from an entry in the datatype_library.csv :param force_custom: if True, this will force the data to take on the projection and geotransform attributes from the datatype object, even if valid projection and geotransform info can be pulled from the gdal dataset. Should almost never be True. :param nodata_value: the value to set to Nodata :return: list of filepaths to output files """ output_filelist = [] if outdir is None: outdir = os.path.dirname(hdf) data = _extract_HDF_layer_data(hdf, layer_indexs) layer_indexs = core.enf_list(layer_indexs) for layer_index in layer_indexs: dataset = data[layer_index] outpath = core.create_outname(outdir, hdf, str(layer_index), "tif") print("creating dataset at {0}".format(outpath)) _gdal_dataset_to_tif(dataset, outpath, cust_projection = datatype.projectionTXT, cust_geotransform = datatype.geotransform, force_custom = force_custom, nodata_value = nodata_value) output_filelist.append(outpath) return output_filelist
def fetch_Landsat8_tile(amazon_url, tilename, outdir, bands=None): """ This function makes use of the amazon web service hosted Landsat 8 OLI data. It recieves an amazon web url for a single landsat tile, and downloads the desired files defaults to downlod all bands, but users can call bands = [1,2,3,4,5,6,7,8,9,10,11,"QA"] to control which files are downloaded. The MTL file is ALWAYS downloaded. """ if bands is None: bands = map(str, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, "QA"]) else: bands = map(str, (core.enf_list(bands))) # create the scene name from the input parameters and use that to generate the scene's unique url connection = urllib.urlopen(amazon_url) page = connection.read().split("\n") print("Downloading landsat tile {0}".format(tilename)) for line in page: if "<li><a href=" in line: # pull filename from html code filename = line.split('"')[1] # pull out band information band_id = filename.replace(tilename + "_", "").split(".")[0].replace("B", "") good_band = band_id in bands mtl_file = "MTL" in band_id # download desired files. if good_band or mtl_file: link = amazon_url.replace("index.html", filename) savename = os.path.join(outdir, tilename, filename) download_url(link, savename) print("\tDownloaded {0}".format(filename)) return
def toa_reflectance_8(band_nums, meta_path, outdir = False): """ Converts Landsat 8 bands to Top of atmosphere reflectance. To be performed on raw Landsat 8 level 1 data. See link below for details see here [http://landsat.usgs.gov/Landsat8_Using_Product.php] Inputs: band_nums A list of desired band numbers such as [3,4,5] meta_path The full filepath to the metadata file for those bands outdir Output directory to save converted files. If left False it will save ouput files in the same directory as input files. """ band_nums = core.enf_list(band_nums) band_nums = map(str, band_nums) OLI_bands = ['1','2','3','4','5','6','7','8','9'] meta = grab_meta(meta_path) for band_num in band_nums: if band_num in OLI_bands: band_path = meta_path.replace("MTL.txt","B{0}.tif".format(band_num)) Qcal = arcpy.Raster(band_path) Mp = getattr(meta,"REFLECTANCE_MULT_BAND_" + band_num) # multiplicative scaling factor Ap = getattr(meta,"REFLECTANCE_ADD_BAND_" + band_num) # additive rescaling factor SEA = getattr(meta,"SUN_ELEVATION")*(math.pi/180) # sun elevation angle theta_se TOA_ref = (((Qcal * Mp) + Ap)/(math.sin(SEA))) metaname = core.create_outname(outdir, meta_path, "TOA-Ref", "txt") shutil.copyfile(meta_path,metaname) outname = core.create_outname(outdir, band_path, "TOA-Ref", "tif") TOA_ref.save(outname) print("Saved output at {0}".format(outname)) else: print("Can only perform reflectance conversion on OLI sensor bands!") print("Skipping band {0}".format(band_num)) return
def fetch_Landsat8_tile(amazon_url, tilename, outdir, bands = None): """ This function makes use of the amazon web service hosted Landsat 8 OLI data. It recieves an amazon web url for a single landsat tile, and downloads the desired files defaults to downlod all bands, but users can call bands = [1,2,3,4,5,6,7,8,9,10,11,"QA"] to control which files are downloaded. The MTL file is ALWAYS downloaded. """ if bands is None: bands = map(str,[1,2,3,4,5,6,7,8,9,10,11,"QA"]) else: bands = map(str,(core.enf_list(bands))) # create the scene name from the input parameters and use that to generate the scene's unique url connection = urllib.urlopen(amazon_url) page = connection.read().split("\n") print("Downloading landsat tile {0}".format(tilename)) for line in page: if "<li><a href=" in line: # pull filename from html code filename = line.split('"')[1] # pull out band information band_id = filename.replace(tilename + "_","").split(".")[0].replace("B","") good_band = band_id in bands mtl_file = "MTL" in band_id # download desired files. if good_band or mtl_file: link = amazon_url.replace("index.html",filename) savename = os.path.join(outdir, tilename, filename) download_url(link, savename) print("\tDownloaded {0}".format(filename)) return
def toa_radiance_8(band_nums, meta_path, outdir=False): """ Top of Atmosphere radiance (in Watts/(square meter * steradians * micrometers)) conversion for landsat 8 data To be performed on raw Landsat 8 level 1 data. See link below for details see here http://landsat.usgs.gov/Landsat8_Using_Product.php Inputs: band_nums A list of desired band numbers such as [3 4 5] meta_path The full filepath to the metadata file for those bands outdir Output directory to save converted files. """ band_nums = core.enf_list(band_nums) band_nums = map(str, band_nums) meta = grab_meta(meta_path) for band_num in band_nums: band_path = meta_path.replace("MTL.txt", "B{0}.tif".format(band_num)) Qcal = arcpy.Raster(band_path) Ml = getattr(meta, "RADIANCE_MULT_BAND_" + band_num) # multiplicative scaling factor Al = getattr(meta, "RADIANCE_ADD_BAND_" + band_num) # additive rescaling factor TOA_rad = (Qcal * Ml) + Al metaname = core.create_outname(outdir, meta_path, "TOA-Rad", "txt") shutil.copyfile(meta_path, metaname) outname = core.create_outname(outdir, band_path, "TOA-Rad", "tif") TOA_rad.save(outname) print("Saved toa_radiance at {0}".format(outname)) return
def TRMM_NetCDF(filelist, outdir): """ Function converts NetCDFs to tiffs. Designed to work with TRMM data. inputs: filelist list of '.nc' files to conver to tifs. outdir directory to which tif files should be saved """ # Set up initial parameters. arcpy.env.workspace = outdir filelist = core.enf_list(filelist) # convert every file in the list "filelist" for infile in filelist: # use arcpy module to make raster layer from netcdf arcpy.MakeNetCDFRasterLayer_md(infile, "r", "longitude", "latitude", "r", "", "", "BY_VALUE") arcpy.CopyRaster_management("r", infile[:-3] + ".tif", "", "", "", "NONE", "NONE", "") print('Converted netCDF file ' + infile + ' to Raster') return
def toa_radiance_8(band_nums, meta_path, outdir = False): """ Top of Atmosphere radiance (in Watts/(square meter * steradians * micrometers)) conversion for landsat 8 data To be performed on raw Landsat 8 level 1 data. See link below for details see here http://landsat.usgs.gov/Landsat8_Using_Product.php Inputs: band_nums A list of desired band numbers such as [3 4 5] meta_path The full filepath to the metadata file for those bands outdir Output directory to save converted files. """ band_nums = core.enf_list(band_nums) band_nums = map(str, band_nums) meta = grab_meta(meta_path) for band_num in band_nums: band_path = meta_path.replace("MTL.txt","B{0}.tif".format(band_num)) Qcal = arcpy.Raster(band_path) Ml = getattr(meta,"RADIANCE_MULT_BAND_" + band_num) # multiplicative scaling factor Al = getattr(meta,"RADIANCE_ADD_BAND_" + band_num) # additive rescaling factor TOA_rad = (Qcal * Ml) + Al metaname = core.create_outname(outdir, meta_path, "TOA-Rad", "txt") shutil.copyfile(meta_path,metaname) outname = core.create_outname(outdir, band_path, "TOA-Rad", "tif") TOA_rad.save(outname) print("Saved toa_radiance at {0}".format(outname)) return
def fetch_SRTM(lat_lon_pairs, product, outdir = None, mosaic = None): """ downloads data from the Shuttle Radar Topography Mission (SRTM) [http://e4ftl01.cr.usgs.gov/SRTM/] This data can be used to create DEMS of a variety of resolutions. Inputs: lat_lon_pairs tupled integer values of lat,lon combinations. may be a list of tuples. (N positive, E positive) product short name of product you want. See link below https://lpdaac.usgs.gov/products/measures_products_table outdir local directory to save downloaded files mosaic Set to TRUE to mosaic all downloaded DEM tiles. Returns: tif_list a list of all successfully downloaded tif filepaths for further manipulation Example: lat_lons = [(37,-76), (37,-77)] # Two tiles prod = SRTMGL3 #3 arc second DEM product) download.fetch_SRTM(lat_lons, prod) NOTE: arcmap will open the output hgt files ONLY if they are not renamed. turns out arcmap does some funky things when interpreting these files. """ # build empty return list tif_list = [] # sanitize input list lat_lon_pairs = core.enf_list(lat_lon_pairs) # determine product version if product is "SRTMGL30": print("Download of product SRTMGL30 is supported, but arcmap does not support this filetype") format_string = "{2}{3}{0}{1}.{4}.dem.zip" version = "002" else: format_string = "{0}{1}{2}{3}.{4}.hgt.zip" version = "003" host = "http://e4ftl01.cr.usgs.gov/SRTM" subhost = "{0}/{1}.{2}/2000.02.11/".format(host, product, version) print("Connecting to host at {0}".format(subhost)) for lat_lon_pair in lat_lon_pairs: lat, lon = lat_lon_pair # set North-south, East-West convention. if lat >= 0: NS = "N" else: NS = "S" if lon >= 0: EW = "E" else: EW = "W" if product is "SRTMGL30": if abs(lon) <= 20: lon = 20 elif abs(lon) <=60: lon = 60 elif abs(lon) <= 100: lon = 100 else: lon = 140 if abs(lat) <= 10: lat = 10 elif abs(lat) <=40: lat = 40 else: lat = 90 NS = NS.lower() EW = EW.lower() # build up the filename and file link filename = format_string.format(NS, str(abs(lat)).zfill(2), EW, str(abs(lon)).zfill(3), product) filelink = "{0}/{1}".format(subhost, filename) # decide where to put the file, then download it if outdir is not None: outpath = os.path.join(outdir, filename) else: outpath = filename print("Downloading and extracting {0}".format(filename)) download_url(filelink, outpath) # unzip the file and reassemble descriptive name with zipfile.ZipFile(outpath, "r") as z: itemname = "{0}{1}{2}{3}.hgt".format(NS, str(abs(lat)).zfill(2), EW, str(abs(lon)).zfill(3)) z.extract(itemname, outdir) z.close() # clean up and add this file to output list os.remove(outpath) tif_list.append(os.path.join(outdir,itemname)) if mosaic is True: arcpy.MosaicToNewRaster_management(tif_list, outdir, "SRTM_mosaic.tif", number_of_bands = 1, pixel_type = "32_BIT_SIGNED") print("Finished download and extraction of SRTM data") return tif_list
def extract_from_hdf(file_list, layer_list, layer_names = False, outdir = None): """ Extracts tifs from MODIS HDF files, ensures proper projection. :param file_list: either a list of '.hdf' files from which data should be extracted, or a directory containing '.hdf' files. :param layer_list: list of layer numbers to pull out as individual tifs should be integers such as [0,4] for the 0th and 4th layer respectively. :param layer_names: list of layer names to put more descriptive file suffixes to each layer :param outdir: directory to which tif files should be saved if outdir is left as 'None', files are saved in the same directory as the input file was found. :return output_filelist: returns a list of all files created by this function """ if outdir is not None: if not os.path.exists(outdir): os.makedirs(outdir) # enforce lists for iteration purposes and sanitize inputs file_list = core.enf_filelist(file_list) for filename in file_list: if '.xml' in filename or '.ovr' in filename or not '.hdf' in filename: file_list.remove(filename) layer_list = core.enf_list(layer_list) layer_names = core.enf_list(layer_names) # ignore user input layer_names if they are invalid, but print warnings if layer_names and not len(layer_names) == len(layer_list): Warning('layer_names must be the same length as layer_list!') Warning('Omitting user defined layer_names!') layer_names = False output_filelist = [] # iterate through every file in the input file_list for infile in file_list: # pull the filename and path apart path,name = os.path.split(infile) arcpy.env.workspace = path for i, layer in enumerate(layer_list): # specify the layer names. if layer_names: layername = layer_names[i] else: layername = str(layer).zfill(3) # use the input output directory if the user input one, otherwise build one if outdir: outname = os.path.join(outdir, "{0}_{1}.tif".format(name[:-4], layername)) else: outname = os.path.join(path, "{0}_{1}.tif".format(name[:-4], layername)) # perform the extracting and projection definition try: arcpy.ExtractSubDataset_management(infile, outname, str(layer)) define_projection(outname) output_filelist.append(outname) print("Extracted {0}".format(os.path.basename(outname))) except: print("Failed to extract {0} from {1}".format(os.path.basename(outname), os.path.basename(infile))) return output_filelist
def toa_radiance_8(band_nums, meta_path, outdir=None): """ Top of Atmosphere radiance (in Watts/(square meter x steradians x micrometers)) conversion for landsat 8 data. To be performed on raw Landsat 8 level 1 data. See link below for details: see here http://landsat.usgs.gov/Landsat8_Using_Product.php :param band_nums: A list of desired band numbers such as [3, 4, 5] :param meta_path: The full filepath to the metadata file for those bands :param outdir: Output directory to save converted files. :return output_filelist: List of filepaths created by this function. """ meta_path = os.path.abspath(meta_path) output_filelist = [] #enforce list of band numbers and grab the metadata from the MTL file band_nums = core.enf_list(band_nums) band_nums = map(str, band_nums) meta = landsat_metadata(meta_path) OLI_bands = ['1', '2', '3', '4', '5', '6', '7', '8', '9'] #loop through each band for band_num in band_nums: if band_num in OLI_bands: #create the band name band_path = meta_path.replace("MTL.txt", "B{0}.tif".format(band_num)) Qcal = arcpy.Raster(band_path) null_raster = arcpy.sa.SetNull(Qcal, Qcal, "VALUE = 0") #scrape the attribute data Ml = getattr(meta, "RADIANCE_MULT_BAND_{0}".format( band_num)) # multiplicative scaling factor Al = getattr(meta, "RADIANCE_ADD_BAND_{0}".format( band_num)) # additive rescaling factor #calculate Top-of-Atmosphere radiance TOA_rad = (null_raster * Ml) + Al del null_raster # create the output name and save the TOA radiance tiff if "\\" in meta_path: name = meta_path.split("\\")[-1] elif "//" in meta_path: name = meta_path.split("//")[-1] rad_name = name.replace("_MTL.txt", "_B{0}".format(band_num)) if outdir is not None: outdir = os.path.abspath(outdir) outname = core.create_outname(outdir, rad_name, "TOA_Rad", "tif") else: folder = os.path.split(meta_path)[0] outname = core.create_outname(folder, rad_name, "TOA_Rad", "tif") TOA_rad.save(outname) output_filelist.append(outname) print("Saved toa_radiance at {0}".format(outname)) #if listed band is not a OLI sensor band, skip it and print message else: print( "Can only perform reflectance conversion on OLI sensor bands") print("Skipping band {0}".format(band_num)) return output_filelist
def download_urls(url_list, outdir, file_types=None): """ Downloads a list of files. Retries failed downloads This script downloads a list of files and places it in the output directory. It was built to be nested within "Download_filelist" to allow loops to continuously retry failed files until they are successful or a retry limit is reached. :param url_list: array of urls, probably as read from a text file :param file_types: list of file types to download. Useful for excluding extraneous metadata by only downloading 'hdf' or 'tif' for example. Please note that often times, you actually NEED the metadata. :param outdir: folder where files are to be placed after download :return failed: list of files which failed download """ failed = [] url_list = core.enf_list(url_list) # creates output folder at desired path if it doesn't already exist if not os.path.exists(outdir): os.makedirs(outdir) # establish a wait time that will increase when downloads fail. This helps to reduce # the frequency of REVERB server rejections for requesting too many downloads wait = 0 for site in url_list: download = False url = site.rstrip() sub = url.split("/") leng = len(sub) name = sub[leng - 1] # Determine whether or not to download the file based on filetype. if file_types is not None: for filetype in file_types: if filetype in name[-4:]: download = True else: download = True # attempt download of the file, or skip it. if download: try: # wait for the wait time before attempting writing a file time.sleep(wait) download_url(url, os.path.join(outdir, name)) print("{0} is downloaded {1}".format(name, wait)) # reduce the wait time when downloads succeed. if wait >= 1: wait -= wait # add to the fail count if the download is unsuccessful and wait longer next time. except: print("{0} will be retried! {1}".format(sub[leng - 1], wait)) wait += 5 failed.append(url) print("Finished downloading urls!") return failed
def surface_reflectance_dos_457(band_nums, meta_path, outdir): """ Converts Landsat 4, 5, and 7 band DNs to surface reflectance using dark object subtraction. To be performed on raw Landsat 4,5, and 7 level 1 data. See link below for details see here [http://landsat.usgs.gov/Landsat8_Using_Product.php] Inputs: band_nums A list of desired band numbers such as [3,4,5] meta_path The full filepath to the metadata file for those bands outdir Output directory to save converted files. If left False it will save ouput files in the same directory as input files. """ band_nums = core.enf_list(band_nums) band_nums = map(str, band_nums) TM_ETM_bands = ['1', '2', '3', '4', '5', '7', '8'] #landsat.toa_reflectance_457(band_nums, meta_path, outdir) OutList = [] f = open(meta_path) MText = f.read() metadata = landsat.grab_meta(meta_path) oldMeta = [] newMeta = [] #the presence of a PRODUCT_CREATION_TIME category is used to identify old metadata #if this is not present, the meta data is considered new. #Band6length refers to the length of the Band 6 name string. In the new metadata this string is longer if "PRODUCT_CREATION_TIME" in MText: Meta = oldMeta Band6length = 2 else: Meta = newMeta Band6length = 8 #The tilename is located using the newMeta/oldMeta indixes and the date of capture is recorded if Meta == newMeta: TileName = getattr(metadata, "LANDSAT_SCENE_ID") year = TileName[9:13] jday = TileName[13:16] date = getattr(metadata, "DATE_ACQUIRED") elif Meta == oldMeta: TileName = getattr(metadata, "BAND1_FILE_NAME") year = TileName[13:17] jday = TileName[17:20] date = getattr(metadata, "ACQUISITION_DATE") #the spacecraft from which the imagery was capture is identified #this info determines the solar exoatmospheric irradiance (ESun) for each band spacecraft = getattr(metadata, "SPACECRAFT_ID") if "7" in spacecraft: ESun = (1969.0, 1840.0, 1551.0, 1044.0, 255.700, 0., 82.07, 1368.00) elif "5" in spacecraft: ESun = (1957.0, 1826.0, 1554.0, 1036.0, 215.0, 0., 80.67) elif "4" in spacecraft: ESun = (1957.0, 1825.0, 1557.0, 1033.0, 214.9, 0., 80.72) else: arcpy.AddError("This tool only works for Landsat 4, 5, or 7") raise arcpy.ExecuteError() #determing if year is leap year and setting the Days in year accordingly if float(year) % 4 == 0: DIY = 366. else: DIY = 365. #using the date to determing the distance from the sun theta = 2 * math.pi * float(jday) / DIY dSun2 = (1.00011 + 0.034221 * math.cos(theta) + 0.001280 * math.sin(theta) + 0.000719 * math.cos(2 * theta) + 0.000077 * math.sin(2 * theta)) SZA = 90. - float(getattr(metadata, "SUN_ELEVATION")) for band_num in band_nums: if band_num in TM_ETM_bands: pathname = meta_path.replace("MTL.txt", "B{0}.tif".format(band_num)) Oraster = arcpy.Raster(pathname) null_raster = arcpy.sa.SetNull(Oraster, Oraster, "VALUE = 0") #using the oldMeta/newMeta indixes to pull the min/max for radiance/Digital numbers if Meta == newMeta: LMax = getattr(metadata, "RADIANCE_MAXIMUM_BAND_" + band_num) LMin = getattr(metadata, "RADIANCE_MINIMUM_BAND_" + band_num) QCalMax = getattr(metadata, "QUANTIZE_CAL_MAX_BAND_" + band_num) QCalMin = getattr(metadata, "QUANTIZE_CAL_MIN_BAND_" + band_num) elif Meta == oldMeta: LMax = getattr(metadata, "LMAX_BAND" + band_num) LMin = getattr(metadata, "LMIN_BAND" + band_num) QCalMax = getattr(metadata, "QCALMAX_BAND" + band_num) QCalMin = getattr(metadata, "QCALMIN_BAND" + band_num) Radraster = (((LMax - LMin) / (QCalMax - QCalMin)) * (null_raster - QCalMin)) + LMin Oraster = 0 del null_raster #Calculating temperature for band 6 if present Refraster = (math.pi * Radraster * dSun2) / ( ESun[int(band_num[0]) - 1] * math.cos(SZA * (math.pi / 180))) #Oraster = arcpy.Raster(raster_null) dark_object = arcpy.GetRasterProperties_management( Refraster, "MINIMUM") do_str = str(dark_object) do_flt = float(do_str) #Calculate the minimum value in each band and perform dark object subtraction Surfrefraster = Refraster - do_flt BandPath = "{0}\\{1}_B{2}_SurfRef.tif".format( outdir, TileName, band_num) Surfrefraster.save(BandPath) OutList.append(arcpy.Raster(BandPath)) del Refraster, Radraster, Surfrefraster arcpy.AddMessage( "Surface Reflectance Calculated for Band {0}".format(band_num)) print( "Surface Reflectance Calculated for Band {0}".format(band_num)) f.close() return OutList
def toa_radiance_457(band_nums, meta_path, outdir = None): """ Top of Atmosphere radiance (in Watts/(square meter x steradians x micrometers)) conversion for Landsat 4, 5, and 7 data. To be performed on raw Landsat 4, 5, or 7 level 1 data. :param band_nums: A list of desired band numbers such as [3, 4, 5] :param meta_path: The full filepath to the metadata file for those bands :param outdir: Output directory to save converted files. :return output_filelist: List of filepaths created by this function. """ output_filelist = [] meta_path = os.path.abspath(meta_path) band_nums = core.enf_list(band_nums) band_nums = map(str, band_nums) #metadata format was changed August 29, 2012. This tool can process either the new or old format f = open(meta_path) MText = f.read() metadata = grab_meta(meta_path) #the presence of a PRODUCT_CREATION_TIME category is used to identify old metadata #if this is not present, the meta data is considered new. #Band6length refers to the length of the Band 6 name string. In the new metadata this string is longer if "PRODUCT_CREATION_TIME" in MText: Meta = "oldMeta" Band6length = 2 else: Meta = "newMeta" Band6length = 8 #The tilename is located using the newMeta/oldMeta indixes and the date of capture is recorded if Meta == "newMeta": TileName = getattr(metadata, "LANDSAT_SCENE_ID") year = TileName[9:13] jday = TileName[13:16] date = getattr(metadata, "DATE_ACQUIRED") elif Meta == "oldMeta": TileName = getattr(metadata, "BAND1_FILE_NAME") year = TileName[13:17] jday = TileName[17:20] date = getattr(metadata, "ACQUISITION_DATE") #the spacecraft from which the imagery was capture is identified #this info determines the solar exoatmospheric irradiance (ESun) for each band spacecraft = getattr(metadata, "SPACECRAFT_ID") if "7" in spacecraft: ESun = (1969.0, 1840.0, 1551.0, 1044.0, 255.700, 0., 82.07, 1368.00) TM_ETM_bands = ['1','2','3','4','5','7','8'] elif "5" in spacecraft: ESun = (1957.0, 1826.0, 1554.0, 1036.0, 215.0, 0. ,80.67) TM_ETM_bands = ['1','2','3','4','5','7'] elif "4" in spacecraft: ESun = (1957.0, 1825.0, 1557.0, 1033.0, 214.9, 0. ,80.72) TM_ETM_bands = ['1','2','3','4','5','7'] else: arcpy.AddError("This tool only works for Landsat 4, 5, or 7") raise arcpy.ExecuteError() #Calculating values for each band for band_num in band_nums: if band_num in TM_ETM_bands: print("Processing Band {0}".format(band_num)) pathname = meta_path.replace("MTL.txt", "B{0}.tif".format(band_num)) Oraster = arcpy.Raster(pathname) null_raster = arcpy.sa.SetNull(Oraster, Oraster, "VALUE = 0") #using the oldMeta/newMeta indixes to pull the min/max for radiance/Digital numbers if Meta == "newMeta": LMax = getattr(metadata, "RADIANCE_MAXIMUM_BAND_{0}".format(band_num)) LMin = getattr(metadata, "RADIANCE_MINIMUM_BAND_{0}".format(band_num)) QCalMax = getattr(metadata, "QUANTIZE_CAL_MAX_BAND_{0}".format(band_num)) QCalMin = getattr(metadata, "QUANTIZE_CAL_MIN_BAND_{0}".format(band_num)) elif Meta == "oldMeta": LMax = getattr(metadata, "LMAX_BAND{0}".format(band_num)) LMin = getattr(metadata, "LMIN_BAND{0}".format(band_num)) QCalMax = getattr(metadata, "QCALMAX_BAND{0}".format(band_num)) QCalMin = getattr(metadata, "QCALMIN_BAND{0}".format(band_num)) Radraster = (((LMax - LMin)/(QCalMax-QCalMin)) * (null_raster - QCalMin)) + LMin Oraster = 0 del null_raster band_rad = "{0}_B{1}".format(TileName, band_num) #create the output name and save the TOA radiance tiff if outdir is not None: outdir = os.path.abspath(outdir) outname = core.create_outname(outdir, band_rad, "TOA_Rad", "tif") else: folder = os.path.split(meta_path)[0] outname = core.create_outname(folder, band_rad, "TOA_Rad", "tif") Radraster.save(outname) output_filelist.append(outname) del Radraster print("toa radiance saved for Band {0}".format(band_num)) #if listed band is not a TM/ETM+ sensor band, skip it and print message else: print("Can only perform reflectance conversion on TM/ETM+ sensor bands") print("Skipping band {0}".format(band_num)) f.close() return output_filelist
def fetch_Landsat_WELD(product, tiles, years, outdir): """ Fetch WELD data from the server at [http://e4ftl01.cr.usgs.gov/WELD]. Weld data is corrected and processed Landsat 5 and 7 data that is distributed in the MODIS sinusoidal projection and grid format. Read more about WELD data. https://landsat.usgs.gov/WELD.php http://globalmonitoring.sdstate.edu/projects/weldglobal/ :param product: WELD product to download such as 'USWK','USMO','USYR' :param tiles: list of tiles to grab such as ['h11v12','h11v11'] :param years: list of years to grab such as range(2001,2014) :param outdir: output directory to save downloaded files :return output_filelist: A list of full filepaths to files fetched be this function """ output_filelist = [] # check formats global dates tiles = core.enf_list(tiles) years = core.enf_list(years) years = [str(year) for year in years] # create output directories for tile in tiles: if not os.path.exists(os.path.join(outdir, tile)): os.makedirs(os.path.join(outdir, tile)) print('Connecting to servers!') # Map the contents of the directory site = 'https://e4ftl01.cr.usgs.gov/WELD/WELD' + product + '.001' try: dates = list_http_e4ftl01(site) except: print('Could not connect to site! check inputs!') # find just the folders within the desired year range. good_dates = [] for date in dates: try: y, m, d = date.split(".") if y in years: good_dates.append(date) except: pass print("Found {0} days within year range".format(len(good_dates))) # for all folders within the desired date range, map the subfolder contents. for good_date in good_dates: files = list_http_e4ftl01(site + '/' + good_date) for afile in files: # only list files with desired tilenames and not preview jpgs if not '.jpg' in afile: for tile in tiles: if tile in afile: # assemble the address address = '/'.join([site, good_date, afile]) print("Downloading {0}".format(address)) #download the file. outname = os.path.join(outdir, tile, afile) output_filelist.append(outname) download_url(address, outname) return
def degree_days_accum(rasterlist, critical_values = None, outdir = None): """ Accumulates degree days in a time series rasterlist This function is the logical successor to calc.degree_days. Input a list of rasters containing daily data to be accumulated. Output raster for a given day will be the sum total of the input raster for that day and all preceding days. The last output raster in a years worth of data (image 356) would be the sum of all 365 images. The 25th output raster would be a sum of the first 25 days. Critical value rasters will also be created. Usefull for example: we wish to know on what day of our 365 day sequence every pixel hits a value of 100. Input 100 as a critical value and that output raster will be generated. :param rasterlist: list of files, or directory containing rasters to accumulate :param critical_values: Values at which the user wishes to know WHEN the total accumulation value reaches this point. For every critical value, an output raster will be created. This raster contains integer values denoting the index number of the file at which the value was reached. This input must be a list of ints or floats, not strings. :param outdir: Desired output directory for all output files. :return output_filelist: a list of all files created by this function. """ output_filelist = [] rasterlist = enf_rastlist(rasterlist) if critical_values: critical_values = core.enf_list(critical_values) # critical values of zero are problematic, so replace it with a small value. if 0 in critical_values: critical_values.remove(0) critical_values.append(0.000001) if outdir is not None and not os.path.exists(outdir): os.makedirs(outdir) for i, rast in enumerate(rasterlist): image, meta = to_numpy(rast,"float32") xs, ys = image.shape if i == 0: Sum = numpy.zeros((xs,ys)) Crit = numpy.zeros((len(critical_values),xs,ys)) if image.shape == Sum.shape: # only bother to proceed if at least one pixel is positive if numpy.max(image) >= 0: for x in range(xs): for y in range(ys): if image[x,y] >= 0: Sum[x,y] = Sum[x,y]+image[x,y] if critical_values is not None: for z,critical_value in enumerate(critical_values): if Sum[x,y] >= critical_value and Crit[z,x,y]==0: Crit[z,x,y] = i else: print "Encountered an image of incorrect size! Skipping it!" Sum = Sum.astype('float32') outname = core.create_outname(outdir, rast, "Accum") from_numpy(Sum, meta, outname) output_filelist.append(outname) del image # output critical accumulation rasters using some data from the last raster in previous loop Crit = Crit.astype('int16') crit_meta = meta crit_meta.NoData_Value = 0 head , tail = os.path.split(outname) # place these in the last raster output location for z, critical_value in enumerate(critical_values): outname = os.path.join(head, "Crit_Accum_Index_Val-{0}.tif".format(str(critical_value))) print("Saving {0}".format(outname)) from_numpy(Crit[z,:,:], crit_meta, outname) return output_filelist
def grab_info(filepath, data_type = False, CustGroupings = None): """ Extracts in-filename metadata from common NASA data products This function simply extracts relevant sorting information from a MODIS or Landsat filepath of any type or product and returns object properties relevant to that data. it will be expanded to include additional data products in the future. Inputs: filepath Full or partial filepath to any modis product tile data_type Manually tell the software what the data is. CustGroupings User defined sorting by julian days of specified bin widths. input of 5 for example will group January 1,2,3,4,5 in the first bin and january 6,7,8,9,10 in the second bin, etc. Outputs: info on object containing the attributes (product, year, day, tile) retrieve these values by calling "info.product", "info.year" etc. Attributes by data type: All type,year,j_day,month,day,season,CustGroupings,suffix MODIS product,tile Landsat sensor,satellite,WRSpath,WRSrow,groundstationID,Version,band Attribute descriptions: type NASA data type, for exmaple 'MODIS' and 'Landsat' year four digit year the data was taken j_day julian day 1 to 365 or 366 for leap years month three character month abbreviation day day of the month season 'Winter','Spring','Summer', or 'Autumn' CustGroupings bin number of data according to custom group value. sorted by julian day suffix Any additional trailing information in the filename. used to find details about special product usually a level 3 data product from sensor such as MOD11A1 tile MODIS sinusoidal tile h##v## format sensor Landsat sensor satellite usually 5,7, or 8 for the landsat satellite WRSpath Landsat path designator WRSrow Landsat row designator groundstationID ground station which recieved the data download fromt he satellite Version Version of landsat data product band band of landsat data product, usually 1 through 10 or 11. """ # pull the filename and path apart path, name = os.path.split(filepath) # create an info object class instance class info_object(object):pass info = info_object() # figure out what kind of data these files are. if not data_type: data_type = identify(name) if data_type == 'MODIS': params =['product', 'year', 'j_day', 'tile', 'type', 'version', 'tag', 'suffix'] n = name.split('.') end = n[4] string =[n[0], name[9:13] ,name[13:16], n[2], 'MODIS', n[3], end[:13], end[13:]] elif data_type =='Landsat': params =['sensor', 'satellite', 'WRSpath', 'WRSrow', 'year', 'j_day', 'groundstationID', 'Version', 'band', 'type', 'suffix'] n = name.split('.')[0] string =[n[1], n[2], n[3:6], n[6:9], n[9:13], n[13:16], n[16:19], n[19:21], n[23:].split('_')[0], 'Landsat', '_'.join(n[23:].split('_')[1:])] elif data_type == 'WELD_CONUS' or data_type == 'WELD_AK': params = ['coverage', 'period', 'year', 'tile', 'start_day', 'end_day', 'type'] n = name.split('.') string =[n[0], n[1], n[2] ,n[3], n[4][4:6], n[4][8:11], 'WELD'] # take everything after the first underscore as a suffix if onecore.exists. if '_' in name: params.append('suffix') string.append('_'.join(name.split('_')[1:])) elif data_type == 'ASTER': params = ['product','N','W','type','period'] n = name.split('_') string = [n[0],n[1][1:3],n[1][5:9],n[-1].split('.')[0],'none'] elif data_type == 'TRMM': print '{Grab_Data_Info} no support for TRMM data yet! you could add it!' return(False) elif data_type == 'AMSR_E': print '{Grab_Data_Info} no support for AMSR_E data yet! you could add it!' return False elif data_type == 'AIRS': print '{Grab_Data_Info} no support for AIRS data yet! you could add it!' return False # if data doesnt look like anything! else: print('Data type for file [{0}] could not be identified as any supported type'.format(name)) print('improve this function by adding info for this datatype!') return False # Create atributes and assign parameter names and values for i in range(len(params)): setattr(info,params[i],string[i]) # ................................................................................ # perform additional data gathering only if data has no info.period atribute. Images with # this attribute represent data that is produced from many dates, not just one day. if not hasattr(info,'period'): # fill in date format values and custom grouping and season information based on julian day # many files are named according to julian day. we want the date info for these files. try: tempinfo = datetime.datetime(int(info.year),1,1)+datetime.timedelta(int(int(info.j_day)-1)) info.month = tempinfo.strftime('%b') info.day = tempinfo.day # some files are named according to date. we want the julian day info for these files except: fmt = '%Y.%m.%d' tempinfo = datetime.datetime.strptime('.'.join([info.year,info.month,info.day]),fmt) info.j_day = tempinfo.strftime('%j') # fill in the seasons by checking the value of julian day if int(info.j_day) <=78 or int(info.j_day) >=355: info.season='Winter' elif int(info.j_day) <=171: info.season='Spring' elif int(info.j_day)<=265: info.season='Summer' elif int(info.j_day)<=354: info.season='Autumn' # bin by julian day if integer group width was input if not CustGroupings == None: CustGroupings = core.enf_list(CustGroupings) for grouping in CustGroupings: if isinstance(grouping,int): groupname = 'custom' + str(grouping) setattr(info, groupname, 1+(int(info.j_day)-1)/(grouping)) else: print('invalid custom grouping entered!') print('CustGrouping must be one or more integers in a list') # make sure the filepath input actually leads to a real file, then give user the info if core.exists(filepath): #print('{0} file {1} has attributes:'.format(info.type, name)) #print(vars(info)) return info else: return False
def toa_radiance_457(band_nums, meta_path, outdir = False): """ Top of Atmosphere radiance (in Watts/(square meter * steradians * micrometers)) conversion for Landsat 4, 5, and 7 data. To be performed on raw Landsat 4, 5, or 7 level 1 data. Inputs: band_nums A list of desired band numbers such as [3 4 5] meta_path The full filepath to the metadata file for those bands outdir Output directory to save converted files. """ OutList = [] band_nums = core.enf_list(band_nums) band_nums = map(str, band_nums) TM_ETM_bands = ['1','2','3','4','5','7','8'] #metadata format was changed August 29, 2012. This tool can process either the new or old format f = open(meta_path) MText = f.read() metadata = grab_meta(meta_path) oldMeta = [] newMeta = [] #the presence of a PRODUCT_CREATION_TIME category is used to identify old metadata #if this is not present, the meta data is considered new. #Band6length refers to the length of the Band 6 name string. In the new metadata this string is longer if "PRODUCT_CREATION_TIME" in MText: Meta = oldMeta Band6length = 2 else: Meta = newMeta Band6length = 8 #The tilename is located using the newMeta/oldMeta indixes and the date of capture is recorded if Meta == newMeta: TileName = getattr(metadata, "LANDSAT_SCENE_ID") year = TileName[9:13] jday = TileName[13:16] date = getattr(metadata, "DATE_ACQUIRED") elif Meta == oldMeta: TileName = getattr(metadata, "BAND1_FILE_NAME") year = TileName[13:17] jday = TileName[17:20] date = getattr(metadata, "ACQUISITION_DATE") #the spacecraft from which the imagery was capture is identified #this info determines the solar exoatmospheric irradiance (ESun) for each band spacecraft = getattr(metadata, "SPACECRAFT_ID") if "7" in spacecraft: ESun = (1969.0, 1840.0, 1551.0, 1044.0, 255.700, 0., 82.07, 1368.00) elif "5" in spacecraft: ESun = (1957.0, 1826.0, 1554.0, 1036.0, 215.0, 0. ,80.67) elif "4" in spacecraft: ESun = (1957.0, 1825.0, 1557.0, 1033.0, 214.9, 0. ,80.72) else: arcpy.AddError("This tool only works for Landsat 4, 5, or 7") raise arcpy.ExecuteError() #Calculating values for each band for band_num in band_nums: if band_num in TM_ETM_bands: print("Processing Band {0}".format(band_num)) pathname = meta_path.replace("MTL.txt", "B{0}.tif".format(band_num)) Oraster = arcpy.Raster(pathname) #using the oldMeta/newMeta indixes to pull the min/max for radiance/Digital numbers if Meta == newMeta: LMax = getattr(metadata, "RADIANCE_MAXIMUM_BAND_" + band_num) LMin = getattr(metadata, "RADIANCE_MINIMUM_BAND_" + band_num) QCalMax = getattr(metadata, "QUANTIZE_CAL_MAX_BAND_" + band_num) QCalMin = getattr(metadata, "QUANTIZE_CAL_MIN_BAND_" + band_num) elif Meta == oldMeta: LMax = getattr(metadata, "LMAX_BAND" + band_num) LMin = getattr(metadata, "LMIN_BAND" + band_num) QCalMax = getattr(metadata, "QCALMAX_BAND" + band_num) QCalMin = getattr(metadata, "QCALMIN_BAND" + band_num) Radraster = (((LMax - LMin)/(QCalMax-QCalMin)) * (Oraster - QCalMin)) + LMin Oraster = 0 #Calculating temperature for band 6 if present BandPath = "{0}\\{1}_B{2}_TOA-Rad.tif".format(outdir,TileName,band_num) Radraster.save(BandPath) OutList.append(arcpy.Raster(BandPath)) del Radraster arcpy.AddMessage("toa radiance saved for Band {0}".format(band_num)) print("toa radiance saved for Band {0}".format(band_num)) f.close() return OutList
def toa_radiance_457(band_nums, meta_path, outdir=False): """ Top of Atmosphere radiance (in Watts/(square meter * steradians * micrometers)) conversion for Landsat 4, 5, and 7 data. To be performed on raw Landsat 4, 5, or 7 level 1 data. Inputs: band_nums A list of desired band numbers such as [3 4 5] meta_path The full filepath to the metadata file for those bands outdir Output directory to save converted files. """ OutList = [] band_nums = core.enf_list(band_nums) band_nums = map(str, band_nums) TM_ETM_bands = ['1', '2', '3', '4', '5', '7', '8'] #metadata format was changed August 29, 2012. This tool can process either the new or old format f = open(meta_path) MText = f.read() metadata = grab_meta(meta_path) oldMeta = [] newMeta = [] #the presence of a PRODUCT_CREATION_TIME category is used to identify old metadata #if this is not present, the meta data is considered new. #Band6length refers to the length of the Band 6 name string. In the new metadata this string is longer if "PRODUCT_CREATION_TIME" in MText: Meta = oldMeta Band6length = 2 else: Meta = newMeta Band6length = 8 #The tilename is located using the newMeta/oldMeta indixes and the date of capture is recorded if Meta == newMeta: TileName = getattr(metadata, "LANDSAT_SCENE_ID") year = TileName[9:13] jday = TileName[13:16] date = getattr(metadata, "DATE_ACQUIRED") elif Meta == oldMeta: TileName = getattr(metadata, "BAND1_FILE_NAME") year = TileName[13:17] jday = TileName[17:20] date = getattr(metadata, "ACQUISITION_DATE") #the spacecraft from which the imagery was capture is identified #this info determines the solar exoatmospheric irradiance (ESun) for each band spacecraft = getattr(metadata, "SPACECRAFT_ID") if "7" in spacecraft: ESun = (1969.0, 1840.0, 1551.0, 1044.0, 255.700, 0., 82.07, 1368.00) elif "5" in spacecraft: ESun = (1957.0, 1826.0, 1554.0, 1036.0, 215.0, 0., 80.67) elif "4" in spacecraft: ESun = (1957.0, 1825.0, 1557.0, 1033.0, 214.9, 0., 80.72) else: arcpy.AddError("This tool only works for Landsat 4, 5, or 7") raise arcpy.ExecuteError() #Calculating values for each band for band_num in band_nums: if band_num in TM_ETM_bands: print("Processing Band {0}".format(band_num)) pathname = meta_path.replace("MTL.txt", "B{0}.tif".format(band_num)) Oraster = arcpy.Raster(pathname) #using the oldMeta/newMeta indixes to pull the min/max for radiance/Digital numbers if Meta == newMeta: LMax = getattr(metadata, "RADIANCE_MAXIMUM_BAND_" + band_num) LMin = getattr(metadata, "RADIANCE_MINIMUM_BAND_" + band_num) QCalMax = getattr(metadata, "QUANTIZE_CAL_MAX_BAND_" + band_num) QCalMin = getattr(metadata, "QUANTIZE_CAL_MIN_BAND_" + band_num) elif Meta == oldMeta: LMax = getattr(metadata, "LMAX_BAND" + band_num) LMin = getattr(metadata, "LMIN_BAND" + band_num) QCalMax = getattr(metadata, "QCALMAX_BAND" + band_num) QCalMin = getattr(metadata, "QCALMIN_BAND" + band_num) Radraster = (((LMax - LMin) / (QCalMax - QCalMin)) * (Oraster - QCalMin)) + LMin Oraster = 0 #Calculating temperature for band 6 if present BandPath = "{0}\\{1}_B{2}_TOA-Rad.tif".format(outdir, TileName, band_num) Radraster.save(BandPath) OutList.append(arcpy.Raster(BandPath)) del Radraster arcpy.AddMessage("toa radiance saved for Band {0}".format(band_num)) print("toa radiance saved for Band {0}".format(band_num)) f.close() return OutList
def toa_reflectance_8(band_nums, meta_path, outdir=False): """ Converts Landsat 8 bands to Top-of-Atmosphere reflectance. To be performed on raw Landsat 8 level 1 data. See link below for details see here [http://landsat.usgs.gov/Landsat8_Using_Product.php] Inputs: band_nums A list of desired band numbers such as [3,4,5] meta_path The full filepath to the metadata file for those bands outdir Output directory to save converted files. If left False it will save ouput files in the same directory as input files. """ outlist = [] #enforce the list of band numbers and grab metadata from the MTL file band_nums = core.enf_list(band_nums) band_nums = map(str, band_nums) OLI_bands = ['1', '2', '3', '4', '5', '6', '7', '8', '9'] meta_path = os.path.abspath(meta_path) meta = grab_meta(meta_path) #cycle through each band in the list for calculation, ensuring each is in the list of OLI bands for band_num in band_nums: if band_num in OLI_bands: #scrape data from the given file path and attributes in the MTL file band_path = meta_path.replace("MTL.txt", "B{0}.tif".format(band_num)) Qcal = arcpy.Raster(band_path) Mp = getattr(meta, "REFLECTANCE_MULT_BAND_{0}".format( band_num)) # multiplicative scaling factor Ap = getattr(meta, "REFLECTANCE_ADD_BAND_{0}".format( band_num)) # additive rescaling factor SEA = getattr(meta, "SUN_ELEVATION") * ( math.pi / 180) # sun elevation angle theta_se #get rid of the zero values that show as the black background to avoid skewing values null_raster = arcpy.sa.SetNull(Qcal, Qcal, "VALUE = 0") #calculate top-of-atmosphere reflectance TOA_ref = (((null_raster * Mp) + Ap) / (math.sin(SEA))) #save the data to the automated name if outdir is given or in the parent folder if not if outdir: outdir = os.path.abspath(outdir) outname = core.create_outname(outdir, band_path, "TOA_Ref", "tif") else: folder = os.path.split(meta_path)[0] outname = core.create_outname(folder, band_path, "TOA_Ref", "tif") TOA_ref.save(outname) outlist.append(outname) print("Saved output at {0}".format(outname)) #if listed band is not an OLI sensor band, skip it and print message else: print( "Can only perform reflectance conversion on OLI sensor bands") print("Skipping band {0}".format(band_num)) return outlist
def extract_from_hdf(filelist, layerlist, layernames=False, outdir=None): """ Extracts tifs from MODIS extract_HDF_layer files, ensures proper projection. inputs: filelist list of '.hdf' files from which data should be extracted (or a directory) layerlist list of layer numbers to pull out as individual tifs should be integers such as [0,4] for the 0th and 4th layer respectively. layernames list of layer names to put more descriptive file suffixes to each layer outdir directory to which tif files should be saved if outdir is left as 'False', files are saved in the same directory as the input file was found. """ if outdir is not None: if not os.path.exists(outdir): os.makedirs(outdir) # enforce lists for iteration purposes and sanitize inputs filelist = core.enf_filelist(filelist) for filename in filelist: if '.xml' in filename or not '.hdf' in filename: filelist.remove(filename) layerlist = core.enf_list(layerlist) layernames = core.enf_list(layernames) # ignore user input layernames if they are invalid, but print warnings if layernames and not len(layernames) == len(layerlist): Warning('Layernames must be the same length as layerlist!') Warning('Ommiting user defined layernames!') layernames = False # create empty list to add failed file names into failed = [] # iterate through every file in the input filelist for infile in filelist: # pull the filename and path apart path, name = os.path.split(infile) arcpy.env.workspace = path for i, layer in enumerate(layerlist): # specify the layer names. if layernames: layername = layernames[i] else: layername = str(layer).zfill(3) # use the input output directory if the user input one, otherwise build one if outdir: outname = os.path.join( outdir, "{0}_{1}.tif".format(name[:-4], layername)) else: outname = os.path.join( path, "{0}_{1}.tif".format(name[:-4], layername)) # perform the extracting and projection definition try: # extract the subdataset arcpy.ExtractSubDataset_management(infile, outname, str(layer)) # define the projection as the MODIS Sinusoidal define_projection(outname) print("Extracted {0}".format(os.path.basename(outname))) except: print("Failed to extract {0} from {1}".format( os.path.basename(outname), os.path.basename(infile))) failed.append(infile) print("Finished extracting all hdfs! \n") return failed
def toa_radiance_8(band_nums, meta_path, outdir = None): """ Top of Atmosphere radiance (in Watts/(square meter x steradians x micrometers)) conversion for landsat 8 data. To be performed on raw Landsat 8 level 1 data. See link below for details: see here http://landsat.usgs.gov/Landsat8_Using_Product.php :param band_nums: A list of desired band numbers such as [3, 4, 5] :param meta_path: The full filepath to the metadata file for those bands :param outdir: Output directory to save converted files. :return output_filelist: List of filepaths created by this function. """ meta_path = os.path.abspath(meta_path) output_filelist = [] #enforce list of band numbers and grab the metadata from the MTL file band_nums = core.enf_list(band_nums) band_nums = map(str, band_nums) meta = grab_meta(meta_path) OLI_bands = ['1','2','3','4','5','6','7','8','9'] #loop through each band for band_num in band_nums: if band_num in OLI_bands: #create the band name band_path = meta_path.replace("MTL.txt","B{0}.tif".format(band_num)) Qcal = arcpy.Raster(band_path) null_raster = arcpy.sa.SetNull(Qcal, Qcal, "VALUE = 0") #scrape the attribute data Ml = getattr(meta,"RADIANCE_MULT_BAND_{0}".format(band_num)) # multiplicative scaling factor Al = getattr(meta,"RADIANCE_ADD_BAND_{0}".format(band_num)) # additive rescaling factor #calculate Top-of-Atmosphere radiance TOA_rad = (null_raster * Ml) + Al del null_raster # create the output name and save the TOA radiance tiff if "\\" in meta_path: name = meta_path.split("\\")[-1] elif "//" in meta_path: name = meta_path.split("//")[-1] rad_name = name.replace("_MTL.txt", "_B{0}".format(band_num)) if outdir is not None: outdir = os.path.abspath(outdir) outname = core.create_outname(outdir, rad_name, "TOA_Rad", "tif") else: folder = os.path.split(meta_path)[0] outname = core.create_outname(folder, rad_name, "TOA_Rad", "tif") TOA_rad.save(outname) output_filelist.append(outname) print("Saved toa_radiance at {0}".format(outname)) #if listed band is not a OLI sensor band, skip it and print message else: print("Can only perform reflectance conversion on OLI sensor bands") print("Skipping band {0}".format(band_num)) return output_filelist
def download_urls(url_list, outdir, filetypes = False): """ Downloads a list of files. Retries failed downloads This script downloads a list of files and places it in the output directory. It was built to be nested within "Download_filelist" to allow loops to continuously retry failed files until they are successful or a retry limit is reached. Inputs: url_list array of urls, probably as read from a text file filetypes list of filetypes to download. Useful for excluding extraneous metadata by only downloding 'hdf' or 'tif' for example. Please note that often times, you actually NEED the metadata. outdir folder where files are to be placed after download Output: failed list of files which failed download """ failed = [] url_list = core.enf_list(url_list) # creates output folder at desired path if it doesn't already exist if not os.path.exists(outdir): os.makedirs(outdir) # establish a wait time that will increase when downloads fail. This helps to reduce # the frequency of REVERB server rejections for requesting too many downloads wait = 0 for site in url_list: download = False url = site.rstrip() sub = url.split("/") leng = len(sub) name = sub[leng-1] # Determine whether or not to download the file based on filetype. if filetypes: for filetype in filetypes: if filetype in name[-4:]: download = True else: download = True # attempt download of the file, or skip it. if download: try: # wait for the wait time before attempting writing a file time.sleep(wait) download_url(url, os.path.join(outdir,name)) print("{0} is downloaded {1}".format(name, wait)) # reduce the wait time when downloads succeed. if wait >= 1: wait -= wait # add to the fail count if the download is unsuccessful and wait longer next time. except: print("{0} will be retried! {1}".format(sub[leng-1], wait)) wait += 5 failed.append(url) print("Finished downloading urls!") return failed
def extract_HDF_layers(filelist, layerlist, layernames = None, outdir = None): """ Function extracts tifs from HDFs. Use "Extract_MODIS_HDF" in the modis module for better handling of MODIS data with sinusoidal projections. inputs: filelist list of '.hdf' files from which data should be extracted layerlist list of layer numbers to pull out as individual tifs should be integers such as [0,4] for the 0th and 4th layer respectively. layernames list of layer names to put more descriptive names to each layer outdir directory to which tif files should be saved if outdir is left as 'False', files are saved in the same directory as the input file was found. """ # Set up initial arcpy modules, workspace, and parameters, and sanitize inputs. arcpy.env.overwriteOutput = True # enforce lists for iteration purposes filelist = core.enf_filelist(filelist) layerlist = core.enf_list(layerlist) layernames = core.enf_list(layernames) # ignore user input layernames if they are invalid, but print warnings if layernames and not len(layernames) == len(layerlist): print('layernames must be the same length as layerlist!') print('ommiting user defined layernames!') layernames=False # create empty list to add filenames into produced_files = [] # iterate through every file in the input filelist for infile in filelist: # pull the filename and path apart path,name = os.path.split(infile) arcpy.env.workspace = path for i in range(len(layerlist)): layer=layerlist[i] # specify the layer names. if layernames is not None: layername = layernames[i] else: layername = str(layer).zfill(3) # use the input output directory if the user input one, otherwise build one if outdir is not None: if not os.path.exists(os.path.join(outdir)): os.makedirs(outdir) else: outdir = os.path.dirname(infile) outname = core.create_outname(outdir, infile, layername, ext = "tif") # perform the extracting and projection definition try: # extract the subdataset arcpy.ExtractSubDataset_management(infile, outname, str(layer)) print('Extracted ' + outname) produced_files.append(outname) except: print('Failed to extract '+ outname + ' from ' + infile) return produced_files
def toa_reflectance_457(band_nums, meta_path, outdir=None): """ This function is used to convert Landsat 4, 5, or 7 pixel values from digital numbers to Top-of-Atmosphere Reflectance. To be performed on raw Landsat 4, 5, or 7 data. :param band_nums: A list of desired band numbers such as [3,4,5] :param meta_path: The full filepath to the metadata file for those bands :param outdir: Output directory to save converted files. If left False it will save ouput files in the same directory as input files. :return output_filelist: List of files created by this function """ output_filelist = [] band_nums = core.enf_list(band_nums) band_nums = map(str, band_nums) # metadata format was changed August 29, 2012. This tool can process either the new or old format f = open(meta_path) MText = f.read() meta_path = os.path.abspath(meta_path) metadata = landsat_metadata(meta_path) # the presence of a PRODUCT_CREATION_TIME category is used to identify old metadata # if this is not present, the meta data is considered new. # Band6length refers to the length of the Band 6 name string. In the new metadata this string is longer if "PRODUCT_CREATION_TIME" in MText: Meta = "oldMeta" Band6length = 2 else: Meta = "newMeta" Band6length = 8 # The tilename is located using the newMeta/oldMeta indixes and the date of capture is recorded if Meta == "newMeta": TileName = getattr(metadata, "LANDSAT_SCENE_ID") year = TileName[9:13] jday = TileName[13:16] date = getattr(metadata, "DATE_ACQUIRED") elif Meta == "oldMeta": TileName = getattr(metadata, "BAND1_FILE_NAME") year = TileName[13:17] jday = TileName[17:20] date = getattr(metadata, "ACQUISITION_DATE") # the spacecraft from which the imagery was capture is identified # this info determines the solar exoatmospheric irradiance (ESun) for each band spacecraft = getattr(metadata, "SPACECRAFT_ID") if "7" in spacecraft: ESun = (1969.0, 1840.0, 1551.0, 1044.0, 255.700, 0., 82.07, 1368.00) TM_ETM_bands = ['1', '2', '3', '4', '5', '7', '8'] elif "5" in spacecraft: ESun = (1957.0, 1826.0, 1554.0, 1036.0, 215.0, 0., 80.67) TM_ETM_bands = ['1', '2', '3', '4', '5', '7'] elif "4" in spacecraft: ESun = (1957.0, 1825.0, 1557.0, 1033.0, 214.9, 0., 80.72) TM_ETM_bands = ['1', '2', '3', '4', '5', '7'] else: arcpy.AddError("This tool only works for Landsat 4, 5, or 7") raise arcpy.ExecuteError() # determing if year is leap year and setting the Days in year accordingly if float(year) % 4 == 0: DIY = 366. else: DIY = 365. # using the date to determining the distance from the sun theta = 2 * math.pi * float(jday) / DIY dSun2 = (1.00011 + 0.034221 * math.cos(theta) + 0.001280 * math.sin(theta) + 0.000719 * math.cos(2 * theta) + 0.000077 * math.sin(2 * theta)) SZA = 90. - float(getattr(metadata, "SUN_ELEVATION")) # Calculating values for each band for band_num in band_nums: if band_num in TM_ETM_bands: print("Processing Band {0}".format(band_num)) pathname = meta_path.replace("MTL.txt", "B{0}.tif".format(band_num)) Oraster = arcpy.Raster(pathname) null_raster = arcpy.sa.SetNull(Oraster, Oraster, "VALUE = 0") # using the oldMeta/newMeta indices to pull the min/max for radiance/Digital numbers if Meta == "newMeta": LMax = getattr(metadata, "RADIANCE_MAXIMUM_BAND_{0}".format(band_num)) LMin = getattr(metadata, "RADIANCE_MINIMUM_BAND_{0}".format(band_num)) QCalMax = getattr(metadata, "QUANTIZE_CAL_MAX_BAND_{0}".format(band_num)) QCalMin = getattr(metadata, "QUANTIZE_CAL_MIN_BAND_{0}".format(band_num)) elif Meta == "oldMeta": LMax = getattr(metadata, "LMAX_BAND{0}".format(band_num)) LMin = getattr(metadata, "LMIN_BAND{0}".format(band_num)) QCalMax = getattr(metadata, "QCALMAX_BAND{0}".format(band_num)) QCalMin = getattr(metadata, "QCALMIN_BAND{0}".format(band_num)) Radraster = (((LMax - LMin) / (QCalMax - QCalMin)) * (null_raster - QCalMin)) + LMin Oraster = 0 del null_raster # Calculating temperature for band 6 if present Refraster = (math.pi * Radraster * dSun2) / ( ESun[int(band_num[0]) - 1] * math.cos(SZA * (math.pi / 180))) # construc output names for each band based on whether outdir is set (default is False) if outdir is not None: outdir = os.path.abspath(outdir) BandPath = core.create_outname(outdir, pathname, "TOA_Ref", "tif") else: folder = os.path.split(meta_path)[0] BandPath = core.create_outname(folder, pathname, "TOA_Ref", "tif") Refraster.save(BandPath) output_filelist.append(BandPath) del Refraster, Radraster print("Reflectance Calculated for Band {0}".format(band_num)) # if listed band is not a TM/ETM+ sensor band, skip it and print message else: print( "Can only perform reflectance conversion on TM/ETM+ sensor bands" ) print("Skipping band {0}".format(band_num)) f.close() return output_filelist
def fetch_MODIS(product, version, tiles, outdir, start_dto, end_dto, force_overwrite=False): """ Fetch MODIS Land products from one of two servers. If this function runs and downloads 0 files, check that your inputs are consistent with the naming convention at the appropriate server address. http://e4ftl01.cr.usgs.gov ftp://n5eil01u.ecs.nsidc.org :param product: MODIS product to download such as 'MOD10A1' or 'MYD11A1' :param version: version number, usually '004' or '041' or '005' :param tiles: list of tiles to grab such as ['h11v12','h11v11'] NOTE: for some MODIS products, the h and v are omitted. :param outdir : output directory to save downloaded files :param start_dto: datetime object, the starting date of the range of data to download :param end_dto: datetime object, the ending date of the range of data to download :param force_overwrite: will re-download files even if they already exist :return out_filepaths: a list of filepaths to all files created by this function """ out_filepaths = [] # check formats tiles = core.enf_list(tiles) # create output directories if not os.path.exists(outdir): os.makedirs(outdir) print("Connecting to servers!") # obtain the web address, protocol information, and subdirectory where # this tpe of MODIS data can be found. site, isftp, Dir = _find_modis_product(product, version) if Dir: print("Connected to {0}/{1}".format(site, Dir)) else: print("Connected to {0}".format(site)) # Depending on the type of connection (ftp vs http) populate the file list try: if isftp: dates, _ = list_ftp(site, False, False, Dir) else: dates = list_http_e4ftl01(site) except: raise ValueError("Could not connect to {0}/{1}".format(site, Dir)) # refine contents down to just addresses of valid year and j_day good_dates = [] for date in dates: try: date_dto = datetime.strptime(date, "%Y.%m.%d") if start_dto <= date_dto <= end_dto: good_dates.append(date) except: print("skipping non date folder name {0}".format(date)) print('Found {0} days within range'.format(len(good_dates))) # for all folders within the desired date range, map the subfolder contents. for good_date in good_dates: if isftp: files, _ = list_ftp(site, False, False, Dir + '/' + good_date) else: files = list_http_e4ftl01(site + '/' + good_date) for afile in files: # only list files with desired tile names and not preview jpgs if not '.jpg' in afile: for tile in tiles: if tile in afile: # assemble the address if isftp: address = '/'.join( ['ftp://' + site, Dir, good_date, afile]) else: address = '/'.join([site, good_date, afile]) #download the file outname = os.path.join(outdir, afile) out_filepaths.append(outname) if not os.path.isfile(outname) and not force_overwrite: download_url(address, outname) print('Downloaded {0}'.format(address)) print("Finished retrieving MODIS - {0} data!".format(product)) print("Downloaded {0} files".format(len(out_filepaths))) return out_filepaths
def fetch_MODIS(product, version, tiles, outdir, start_dto, end_dto, force_overwrite = False): """ Fetch MODIS Land products from one of two servers. If this function runs and downloads 0 files, check that your inputs are consistent with the naming convention at the appropriate server address. http://e4ftl01.cr.usgs.gov ftp://n5eil01u.ecs.nsidc.org :param product: MODIS product to download such as 'MOD10A1' or 'MYD11A1' :param version: version number, usually '004' or '041' or '005' :param tiles: list of tiles to grab such as ['h11v12','h11v11'] NOTE: for some MODIS products, the h and v are omitted. :param outdir : output directory to save downloaded files :param start_dto: datetime object, the starting date of the range of data to download :param end_dto: datetime object, the ending date of the range of data to download :param force_overwrite: will re-download files even if they already exist :return out_filepaths: a list of filepaths to all files created by this function """ out_filepaths = [] # check formats tiles = core.enf_list(tiles) # create output directories if not os.path.exists(outdir): os.makedirs(outdir) print("Connecting to servers!") # obtain the web address, protocol information, and subdirectory where # this tpe of MODIS data can be found. site, isftp, Dir = _find_modis_product(product, version) if Dir: print("Connected to {0}/{1}".format(site, Dir)) else: print("Connected to {0}".format(site)) # Depending on the type of connection (ftp vs http) populate the file list try: if isftp: dates,_ = list_ftp(site, False, False, Dir) else: dates = list_http_e4ftl01(site) except: raise ValueError("Could not connect to {0}/{1}".format(site,Dir)) # refine contents down to just addresses of valid year and j_day good_dates = [] for date in dates: try: date_dto = datetime.strptime(date, "%Y.%m.%d") if start_dto <= date_dto <= end_dto: good_dates.append(date) except: print("skipping non date folder name {0}".format(date)) print('Found {0} days within range'.format(len(good_dates))) # for all folders within the desired date range, map the subfolder contents. for good_date in good_dates: if isftp: files,_ = list_ftp(site, False, False, Dir + '/' + good_date) else: files = list_http_e4ftl01(site + '/' + good_date) for afile in files: # only list files with desired tile names and not preview jpgs if not '.jpg' in afile: for tile in tiles: if tile in afile: # assemble the address if isftp: address='/'.join(['ftp://'+site, Dir, good_date, afile]) else: address='/'.join([site, good_date, afile]) #download the file outname = os.path.join(outdir, afile) out_filepaths.append(outname) if not os.path.isfile(outname) and not force_overwrite: download_url(address, outname) print('Downloaded {0}'.format(address)) print("Finished retrieving MODIS - {0} data!".format(product)) print("Downloaded {0} files".format(len(out_filepaths))) return out_filepaths
def Landsat_WELD(product, tiles, years, outdir): """ Fetch WELD data from the server at [http://e4ftl01.cr.usgs.gov/WELD] Weld data is corrected and processed Landsat 5 and 7 data that is distributed in the MODIS sinusoidal projection and grid format. Read more about WELD data. https://landsat.usgs.gov/WELD.php http://globalmonitoring.sdstate.edu/projects/weldglobal/ Inputs: product WELD product to download such as 'USWK','USMO','USYR' tiles list of tiles to grab such as ['h11v12','h11v11'] years list of years to grab such as range(2001,2014) outdir output directory to save downloaded files """ # check formats global dates tiles = core.enf_list(tiles) years = core.enf_list(years) years = [str(year) for year in years] # create output directories for tile in tiles: if not os.path.exists(os.path.join(outdir,tile)): os.makedirs(os.path.join(outdir,tile)) print '{Fetch_Landsat_WELD} Connecting to servers!' # Map the contents of the directory site= 'http://e4ftl01.cr.usgs.gov/WELD/WELD'+product+'.001' try: dates = list_http(site) except: print '{Fetch_Landsat_WELD} Could not connect to site! check inputs!' # find just the folders within the desired year range. good_dates=[] for date in dates: try: y, m, d = date.split(".") if y in years: good_dates.append(date) except: pass print 'Found ' + str(len(good_dates)) + ' days within year range' # for all folders within the desired date range, map the subfolder contents. for good_date in good_dates: files = list_http(site+'/'+good_date) for afile in files: # only list files with desired tilenames and not preview jpgs if not '.jpg' in afile: for tile in tiles: if tile in afile: # assemble the address address = '/'.join([site,good_date,afile]) print '{Fetch_Landsat_WELD} Downloading' + address #download the file. outname = os.path.join(outdir,tile,afile) download_url(address, outname) return
def degree_days_accum(rasterlist, critical_values=None, outdir=None): """ Accumulates degree days in a time series rasterlist This function is the logical successor to calc.degree_days. Input a list of rasters containing daily data to be accumulated. Output raster for a given day will be the sum total of the input raster for that day and all preceding days. The last output raster in a years worth of data (image 356) would be the sum of all 365 images. The 25th output raster would be a sum of the first 25 days. Critical value rasters will also be created. Usefull for example: we wish to know on what day of our 365 day sequence every pixel hits a value of 100. Input 100 as a critical value and that output raster will be generated. :param rasterlist: list of files, or directory containing rasters to accumulate :param critical_values: Values at which the user wishes to know WHEN the total accumulation value reaches this point. For every critical value, an output raster will be created. This raster contains integer values denoting the index number of the file at which the value was reached. This input must be a list of ints or floats, not strings. :param outdir: Desired output directory for all output files. :return output_filelist: a list of all files created by this function. """ output_filelist = [] rasterlist = enf_rastlist(rasterlist) if critical_values: critical_values = core.enf_list(critical_values) # critical values of zero are problematic, so replace it with a small value. if 0 in critical_values: critical_values.remove(0) critical_values.append(0.000001) if outdir is not None and not os.path.exists(outdir): os.makedirs(outdir) for i, rast in enumerate(rasterlist): image, meta = to_numpy(rast, "float32") xs, ys = image.shape if i == 0: Sum = numpy.zeros((xs, ys)) Crit = numpy.zeros((len(critical_values), xs, ys)) if image.shape == Sum.shape: # only bother to proceed if at least one pixel is positive if numpy.max(image) >= 0: for x in range(xs): for y in range(ys): if image[x, y] >= 0: Sum[x, y] = Sum[x, y] + image[x, y] if critical_values is not None: for z, critical_value in enumerate( critical_values): if Sum[x, y] >= critical_value and Crit[z, x, y] == 0: Crit[z, x, y] = i else: print "Encountered an image of incorrect size! Skipping it!" Sum = Sum.astype('float32') outname = core.create_outname(outdir, rast, "Accum") from_numpy(Sum, meta, outname) output_filelist.append(outname) del image # output critical accumulation rasters using some data from the last raster in previous loop Crit = Crit.astype('int16') crit_meta = meta crit_meta.NoData_Value = 0 head, tail = os.path.split( outname) # place these in the last raster output location for z, critical_value in enumerate(critical_values): outname = os.path.join( head, "Crit_Accum_Index_Val-{0}.tif".format(str(critical_value))) print("Saving {0}".format(outname)) from_numpy(Crit[z, :, :], crit_meta, outname) return output_filelist
def toa_radiance_457(band_nums, meta_path, outdir=None): """ Top of Atmosphere radiance (in Watts/(square meter x steradians x micrometers)) conversion for Landsat 4, 5, and 7 data. To be performed on raw Landsat 4, 5, or 7 level 1 data. :param band_nums: A list of desired band numbers such as [3, 4, 5] :param meta_path: The full filepath to the metadata file for those bands :param outdir: Output directory to save converted files. :return output_filelist: List of filepaths created by this function. """ output_filelist = [] meta_path = os.path.abspath(meta_path) band_nums = core.enf_list(band_nums) band_nums = map(str, band_nums) #metadata format was changed August 29, 2012. This tool can process either the new or old format f = open(meta_path) MText = f.read() metadata = landsat_metadata(meta_path) #the presence of a PRODUCT_CREATION_TIME category is used to identify old metadata #if this is not present, the meta data is considered new. #Band6length refers to the length of the Band 6 name string. In the new metadata this string is longer if "PRODUCT_CREATION_TIME" in MText: Meta = "oldMeta" Band6length = 2 else: Meta = "newMeta" Band6length = 8 #The tilename is located using the newMeta/oldMeta indixes and the date of capture is recorded if Meta == "newMeta": TileName = getattr(metadata, "LANDSAT_SCENE_ID") year = TileName[9:13] jday = TileName[13:16] date = getattr(metadata, "DATE_ACQUIRED") elif Meta == "oldMeta": TileName = getattr(metadata, "BAND1_FILE_NAME") year = TileName[13:17] jday = TileName[17:20] date = getattr(metadata, "ACQUISITION_DATE") #the spacecraft from which the imagery was capture is identified #this info determines the solar exoatmospheric irradiance (ESun) for each band spacecraft = getattr(metadata, "SPACECRAFT_ID") if "7" in spacecraft: ESun = (1969.0, 1840.0, 1551.0, 1044.0, 255.700, 0., 82.07, 1368.00) TM_ETM_bands = ['1', '2', '3', '4', '5', '7', '8'] elif "5" in spacecraft: ESun = (1957.0, 1826.0, 1554.0, 1036.0, 215.0, 0., 80.67) TM_ETM_bands = ['1', '2', '3', '4', '5', '7'] elif "4" in spacecraft: ESun = (1957.0, 1825.0, 1557.0, 1033.0, 214.9, 0., 80.72) TM_ETM_bands = ['1', '2', '3', '4', '5', '7'] else: arcpy.AddError("This tool only works for Landsat 4, 5, or 7") raise arcpy.ExecuteError() #Calculating values for each band for band_num in band_nums: if band_num in TM_ETM_bands: print("Processing Band {0}".format(band_num)) pathname = meta_path.replace("MTL.txt", "B{0}.tif".format(band_num)) Oraster = arcpy.Raster(pathname) null_raster = arcpy.sa.SetNull(Oraster, Oraster, "VALUE = 0") #using the oldMeta/newMeta indixes to pull the min/max for radiance/Digital numbers if Meta == "newMeta": LMax = getattr(metadata, "RADIANCE_MAXIMUM_BAND_{0}".format(band_num)) LMin = getattr(metadata, "RADIANCE_MINIMUM_BAND_{0}".format(band_num)) QCalMax = getattr(metadata, "QUANTIZE_CAL_MAX_BAND_{0}".format(band_num)) QCalMin = getattr(metadata, "QUANTIZE_CAL_MIN_BAND_{0}".format(band_num)) elif Meta == "oldMeta": LMax = getattr(metadata, "LMAX_BAND{0}".format(band_num)) LMin = getattr(metadata, "LMIN_BAND{0}".format(band_num)) QCalMax = getattr(metadata, "QCALMAX_BAND{0}".format(band_num)) QCalMin = getattr(metadata, "QCALMIN_BAND{0}".format(band_num)) Radraster = (((LMax - LMin) / (QCalMax - QCalMin)) * (null_raster - QCalMin)) + LMin Oraster = 0 del null_raster band_rad = "{0}_B{1}".format(TileName, band_num) #create the output name and save the TOA radiance tiff if outdir is not None: outdir = os.path.abspath(outdir) outname = core.create_outname(outdir, band_rad, "TOA_Rad", "tif") else: folder = os.path.split(meta_path)[0] outname = core.create_outname(folder, band_rad, "TOA_Rad", "tif") Radraster.save(outname) output_filelist.append(outname) del Radraster print("toa radiance saved for Band {0}".format(band_num)) #if listed band is not a TM/ETM+ sensor band, skip it and print message else: print( "Can only perform reflectance conversion on TM/ETM+ sensor bands" ) print("Skipping band {0}".format(band_num)) f.close() return output_filelist
def GCMO_NetCDF(netcdf_list, variable, outdir): """ Extracts all time layers from a "Global Climate Model Output" NetCDF layer Inputs: netcdf_list list of netcdfs from CORDEX climate distribution varaible the climate variable of interest (tsmax, tsmin, etc) outdir output directory to save files. """ if not os.path.exists(outdir): os.makedirs(outdir) netcdf_list = core.enf_list(netcdf_list) for netcdf in netcdf_list: # get net cdf properties object props = arcpy.NetCDFFileProperties(netcdf) print("finding dimensions") dims = props.getDimensions() for dim in dims: print dim, props.getDimensionSize(dim) # make sure the variable is in this netcdf if variable: if not variable in props.getVariables(): print("Valid variables for this file include {0}".format( props.getVariables())) raise Exception( "Variable '{0}' is not in this netcdf!".format(variable)) for dim in dims: if dim == "time": # set other dimensions x_dim = "lon" y_dim = "lat" band_dim = "" valueSelectionMethod = "BY_VALUE" size = props.getDimensionSize(dim) for i in range(size): # sanitize the dimname for invalid characters dimname = props.getDimensionValue(dim, i).replace( " 12:00:00 PM", "") dimname = dimname.replace("/", "-").replace(" ", "_") dim_value = [["time", props.getDimensionValue(dim, i)]] print("extracting '{0}' from '{1}'".format( variable, dim_value)) outname = core.create_outname(outdir, netcdf, dimname, 'tif') arcpy.MakeNetCDFRasterLayer_md(netcdf, variable, x_dim, y_dim, "temp", band_dim, dim_value, valueSelectionMethod) arcpy.CopyRaster_management("temp", outname, "", "", "", "NONE", "NONE", "") return
def toa_reflectance_457(band_nums, meta_path, outdir = False): """ This function is used to convert Landsat 4,5, or 7 pixel values from digital numbers to Radiance, Reflectance, or Temperature (if using Band 6) Inputs: band_nums A list of desired band numbers such as [3,4,5] meta_path The full filepath to the metadata file for those bands outdir Output directory to save converted files. If left False it will save ouput files in the same directory as input files. """ OutList = [] band_nums = core.enf_list(band_nums) band_nums = map(str, band_nums) TM_ETM_bands = ['1','2','3','4','5','7','8'] #metadata format was changed August 29, 2012. This tool can process either the new or old format f = open(meta_path) MText = f.read() metadata = grab_meta(meta_path) oldMeta = [] newMeta = [] #the presence of a PRODUCT_CREATION_TIME category is used to identify old metadata #if this is not present, the meta data is considered new. #Band6length refers to the length of the Band 6 name string. In the new metadata this string is longer if "PRODUCT_CREATION_TIME" in MText: Meta = oldMeta Band6length = 2 else: Meta = newMeta Band6length = 8 #The tilename is located using the newMeta/oldMeta indixes and the date of capture is recorded if Meta == newMeta: TileName = getattr(metadata, "LANDSAT_SCENE_ID") year = TileName[9:13] jday = TileName[13:16] date = getattr(metadata, "DATE_ACQUIRED") elif Meta == oldMeta: TileName = getattr(metadata, "BAND1_FILE_NAME") year = TileName[13:17] jday = TileName[17:20] date = getattr(metadata, "ACQUISITION_DATE") #the spacecraft from which the imagery was capture is identified #this info determines the solar exoatmospheric irradiance (ESun) for each band spacecraft = getattr(metadata, "SPACECRAFT_ID") if "7" in spacecraft: ESun = (1969.0, 1840.0, 1551.0, 1044.0, 255.700, 0., 82.07, 1368.00) elif "5" in spacecraft: ESun = (1957.0, 1826.0, 1554.0, 1036.0, 215.0, 0. ,80.67) elif "4" in spacecraft: ESun = (1957.0, 1825.0, 1557.0, 1033.0, 214.9, 0. ,80.72) else: arcpy.AddError("This tool only works for Landsat 4, 5, or 7") raise arcpy.ExecuteError() #determing if year is leap year and setting the Days in year accordingly if float(year) % 4 == 0: DIY = 366. else:DIY=365. #using the date to determing the distance from the sun theta = 2 * math.pi * float(jday)/DIY dSun2 = (1.00011 + 0.034221 * math.cos(theta) + 0.001280 * math.sin(theta) + 0.000719 * math.cos(2*theta)+ 0.000077 * math.sin(2 * theta)) SZA = 90. - float(getattr(metadata, "SUN_ELEVATION")) #Calculating values for each band for band_num in band_nums: if band_num in TM_ETM_bands: print("Processing Band {0}".format(band_num)) pathname = meta_path.replace("MTL.txt", "B{0}.tif".format(band_num)) Oraster = arcpy.Raster(pathname) #using the oldMeta/newMeta indixes to pull the min/max for radiance/Digital numbers if Meta == newMeta: LMax = getattr(metadata, "RADIANCE_MAXIMUM_BAND_" + band_num) LMin = getattr(metadata, "RADIANCE_MINIMUM_BAND_" + band_num) QCalMax = getattr(metadata, "QUANTIZE_CAL_MAX_BAND_" + band_num) QCalMin = getattr(metadata, "QUANTIZE_CAL_MIN_BAND_" + band_num) elif Meta == oldMeta: LMax = getattr(metadata, "LMAX_BAND" + band_num) LMin = getattr(metadata, "LMIN_BAND" + band_num) QCalMax = getattr(metadata, "QCALMAX_BAND" + band_num) QCalMin = getattr(metadata, "QCALMIN_BAND" + band_num) Radraster = (((LMax - LMin)/(QCalMax-QCalMin)) * (Oraster - QCalMin)) + LMin Oraster = 0 #Calculating temperature for band 6 if present Refraster = (math.pi * Radraster * dSun2) / (ESun[int(band_num[0])-1] * math.cos(SZA*(math.pi/180))) BandPath = "{0}\\{1}_B{2}_TOA-Ref.tif".format(outdir,TileName,band_num) Refraster.save(BandPath) OutList.append(arcpy.Raster(BandPath)) del Refraster,Radraster arcpy.AddMessage("Reflectance Calculated for Band {0}".format(band_num)) print("Reflectance Calculated for Band {0}".format(band_num)) f.close() return OutList
def toa_reflectance_457(band_nums, meta_path, outdir = None): """ This function is used to convert Landsat 4, 5, or 7 pixel values from digital numbers to Top-of-Atmosphere Reflectance. To be performed on raw Landsat 4, 5, or 7 data. :param band_nums: A list of desired band numbers such as [3,4,5] :param meta_path: The full filepath to the metadata file for those bands :param outdir: Output directory to save converted files. If left False it will save ouput files in the same directory as input files. :return output_filelist: List of files created by this function """ output_filelist = [] band_nums = core.enf_list(band_nums) band_nums = map(str, band_nums) # metadata format was changed August 29, 2012. This tool can process either the new or old format f = open(meta_path) MText = f.read() meta_path = os.path.abspath(meta_path) metadata = grab_meta(meta_path) # the presence of a PRODUCT_CREATION_TIME category is used to identify old metadata # if this is not present, the meta data is considered new. # Band6length refers to the length of the Band 6 name string. In the new metadata this string is longer if "PRODUCT_CREATION_TIME" in MText: Meta = "oldMeta" Band6length = 2 else: Meta = "newMeta" Band6length = 8 # The tilename is located using the newMeta/oldMeta indixes and the date of capture is recorded if Meta == "newMeta": TileName = getattr(metadata, "LANDSAT_SCENE_ID") year = TileName[9:13] jday = TileName[13:16] date = getattr(metadata, "DATE_ACQUIRED") elif Meta == "oldMeta": TileName = getattr(metadata, "BAND1_FILE_NAME") year = TileName[13:17] jday = TileName[17:20] date = getattr(metadata, "ACQUISITION_DATE") # the spacecraft from which the imagery was capture is identified # this info determines the solar exoatmospheric irradiance (ESun) for each band spacecraft = getattr(metadata, "SPACECRAFT_ID") if "7" in spacecraft: ESun = (1969.0, 1840.0, 1551.0, 1044.0, 255.700, 0., 82.07, 1368.00) TM_ETM_bands = ['1','2','3','4','5','7','8'] elif "5" in spacecraft: ESun = (1957.0, 1826.0, 1554.0, 1036.0, 215.0, 0. ,80.67) TM_ETM_bands = ['1','2','3','4','5','7'] elif "4" in spacecraft: ESun = (1957.0, 1825.0, 1557.0, 1033.0, 214.9, 0. ,80.72) TM_ETM_bands = ['1','2','3','4','5','7'] else: arcpy.AddError("This tool only works for Landsat 4, 5, or 7") raise arcpy.ExecuteError() # determing if year is leap year and setting the Days in year accordingly if float(year) % 4 == 0: DIY = 366. else: DIY=365. # using the date to determining the distance from the sun theta = 2 * math.pi * float(jday)/DIY dSun2 = (1.00011 + 0.034221 * math.cos(theta) + 0.001280 * math.sin(theta) + 0.000719 * math.cos(2*theta)+ 0.000077 * math.sin(2 * theta)) SZA = 90. - float(getattr(metadata, "SUN_ELEVATION")) # Calculating values for each band for band_num in band_nums: if band_num in TM_ETM_bands: print("Processing Band {0}".format(band_num)) pathname = meta_path.replace("MTL.txt", "B{0}.tif".format(band_num)) Oraster = arcpy.Raster(pathname) null_raster = arcpy.sa.SetNull(Oraster, Oraster, "VALUE = 0") # using the oldMeta/newMeta indices to pull the min/max for radiance/Digital numbers if Meta == "newMeta": LMax = getattr(metadata, "RADIANCE_MAXIMUM_BAND_{0}".format(band_num)) LMin = getattr(metadata, "RADIANCE_MINIMUM_BAND_{0}".format(band_num)) QCalMax = getattr(metadata, "QUANTIZE_CAL_MAX_BAND_{0}".format(band_num)) QCalMin = getattr(metadata, "QUANTIZE_CAL_MIN_BAND_{0}".format(band_num)) elif Meta == "oldMeta": LMax = getattr(metadata, "LMAX_BAND{0}".format(band_num)) LMin = getattr(metadata, "LMIN_BAND{0}".format(band_num)) QCalMax = getattr(metadata, "QCALMAX_BAND{0}".format(band_num)) QCalMin = getattr(metadata, "QCALMIN_BAND{0}".format(band_num)) Radraster = (((LMax - LMin)/(QCalMax-QCalMin)) * (null_raster - QCalMin)) + LMin Oraster = 0 del null_raster # Calculating temperature for band 6 if present Refraster = (math.pi * Radraster * dSun2) / (ESun[int(band_num[0])-1] * math.cos(SZA*(math.pi/180))) # construc output names for each band based on whether outdir is set (default is False) if outdir is not None: outdir = os.path.abspath(outdir) BandPath = core.create_outname(outdir, pathname, "TOA_Ref", "tif") else: folder = os.path.split(meta_path)[0] BandPath = core.create_outname(folder, pathname, "TOA_Ref", "tif") Refraster.save(BandPath) output_filelist.append(BandPath) del Refraster, Radraster print("Reflectance Calculated for Band {0}".format(band_num)) # if listed band is not a TM/ETM+ sensor band, skip it and print message else: print("Can only perform reflectance conversion on TM/ETM+ sensor bands") print("Skipping band {0}".format(band_num)) f.close() return output_filelist
def fetch_SRTM(lat_lon_pairs, product, outdir = None, mosaic = None): """ downloads data from the Shuttle Radar Topography Mission (SRTM) [http://e4ftl01.cr.usgs.gov/SRTM/] This data can be used to create DEMS of a variety of resolutions. Inputs: lat_lon_pairs tupled integer values of lat,lon combinations. may be a list of tuples. (N positive, E positive) product short name of product you want. See link below https://lpdaac.usgs.gov/products/measures_products_table outdir local directory to save downloaded files mosaic Set to TRUE to mosaic all downloaded DEM tiles. Returns: tif_list a list of all successfully downloaded tif filepaths for further manipulation Example: lat_lons = [(37,-76), (37,-77)] # Two tiles prod = "SRTMGL3" #3 arc second DEM product) download.fetch_SRTM(lat_lons, prod) NOTE: arcmap will open the output hgt files ONLY if they are not renamed. turns out arcmap does some funky things when interpreting these files. """ # build empty return list tif_list = [] # sanitize input list lat_lon_pairs = core.enf_list(lat_lon_pairs) # determine product version if product is "SRTMGL30": print("Download of product SRTMGL30 is supported, but arcmap does not support this filetype") format_string = "{2}{3}{0}{1}.{4}.dem.zip" version = "002" else: format_string = "{0}{1}{2}{3}.{4}.hgt.zip" version = "003" host = "http://e4ftl01.cr.usgs.gov/SRTM" subhost = "{0}/{1}.{2}/2000.02.11/".format(host, product, version) print("Connecting to host at {0}".format(subhost)) for lat_lon_pair in lat_lon_pairs: lat, lon = lat_lon_pair # set North-south, East-West convention. if lat >= 0: NS = "N" else: NS = "S" if lon >= 0: EW = "E" else: EW = "W" if product is "SRTMGL30": if abs(lon) <= 20: lon = 20 elif abs(lon) <=60: lon = 60 elif abs(lon) <= 100: lon = 100 else: lon = 140 if abs(lat) <= 10: lat = 10 elif abs(lat) <=40: lat = 40 else: lat = 90 NS = NS.lower() EW = EW.lower() # build up the filename and file link filename = format_string.format(NS, str(abs(lat)).zfill(2), EW, str(abs(lon)).zfill(3), product) filelink = "{0}/{1}".format(subhost, filename) # decide where to put the file, then download it if outdir is not None: outpath = os.path.join(outdir, filename) else: outpath = filename print("Downloading and extracting {0}".format(filename)) download_url(filelink, outpath) # unzip the file and reassemble descriptive name with zipfile.ZipFile(outpath, "r") as z: itemname = "{0}{1}{2}{3}.hgt".format(NS, str(abs(lat)).zfill(2), EW, str(abs(lon)).zfill(3)) z.extract(itemname, outdir) z.close() # clean up and add this file to output list os.remove(outpath) tif_list.append(os.path.join(outdir,itemname)) if mosaic is True: arcpy.MosaicToNewRaster_management(tif_list, outdir, "SRTM_mosaic.tif", number_of_bands = 1, pixel_type = "32_BIT_SIGNED") print("Finished download and extraction of SRTM data") return tif_list
def fetch_MODIS(product, version, tiles, outdir, years, j_days = False, force_overwrite = False): """ Fetch MODIS Land products from one of two servers. http://e4ftl01.cr.usgs.gov ftp://n5eil01u.ecs.nsidc.org Inputs: product MODIS product to download such as 'MOD10A1' or 'MYD11A1' version version number, usually '004' or '041' or '005' tiles list of tiles to grab such as ['h11v12','h11v11'] outdir output directory to save downloaded files years list of years to grab such as range(2001,2014) j_days list of days to grab such as range(31:60). Defaults to all days in year force_overwrite will re-download files even if they already exist """ def Find_MODIS_Product(product, version): """ Subfunction to determine server properties for MODIS data product. returns http/ftp handles the two current servers where aqua/terra MODIS data can be downloaded are site1='http://e4ftl01.cr.usgs.gov' site2='n5eil01u.ecs.nsidc.org' Inputs: product modis product such as 'MOD10A1' versions modis version, usually '005', '004', or '041' Outputs: site server address where data can be found ftp ftp handle for open ftp session Dir subdirectory of server to further search for files of input product. """ sat_designation = product[0:3] prod_ID = product[3:] site1 = 'http://e4ftl01.cr.usgs.gov/' site2 = 'n5eil01u.ecs.nsidc.org' isftp = False Dir = False # refine the address of the desired data product if '10' in prod_ID: isftp = True site = site2 if sat_designation == 'MOD': if isftp: Dir = 'MOST/' + product + '.' + version else: site = site1+'MOLT/' + product + '.' + version elif sat_designation == 'MYD': if isftp: Dir = 'DP1/MOSA/' + product + '.' + version else: site = site1+'MOLA/' + product+'.' + version elif sat_designation == 'MCD': site = site1+'MOTA/' + product + '.' + version else: print('No such MODIS product is available for download with this script!') site = "None" return site, isftp, Dir # check formats tiles = core.enf_list(tiles) years = core.enf_list(years) years = [str(year) for year in years] if isinstance(j_days, list): js = [str(j_day).zfill(3) for j_day in j_days] elif isinstance(j_days, int) and j_days != False: js = [str(j_days)] else: js = [str(x).zfill(3) for x in range(367)] # do a quick input tile check for 6 characters. for tile in tiles: if not len(tile) == 6: print("Warning! your tiles appear to be invalid!") print("Warning! make sure they are in format 'h##v##") # create output directories if not os.path.exists(outdir): os.makedirs(outdir) print("Connecting to servers!") # obtain the web address, protocol information, and subdirectory where # this tpe of MODIS data can be found. site, isftp, Dir = Find_MODIS_Product(product, version) if Dir: print("Connected to {0}/{1}".format(site, Dir)) else: print("Connected to {0}".format(site)) # Depending on the type of connection (ftp vs http) populate the file list try: if isftp: dates,_ = list_ftp(site, False, False, Dir) else: dates = list_http(site) except: raise ValueError("Could not connect to {0}/{1}".format(site,Dir)) # refine contents down to just addresses of valid year and j_day good_dates=[] for date in dates: try: dto = datetime.strptime(date, "%Y.%m.%d") j_day = dto.strftime("%j") year = dto.strftime("%Y") if year in years: good_dates.append(date) if j_days: if j_day not in js: good_dates.remove(date) except ValueError: print("skipping non date folder name {0}".format(date)) print('Found {0} days within range'.format(len(good_dates))) # for all folders within the desired date range, map the subfolder contents. for good_date in good_dates: if isftp: files,_ = list_ftp(site, False, False, Dir + '/' + good_date) else: files = list_http(site + '/' + good_date) for afile in files: # only list files with desired tile names and not preview jpgs if not '.jpg' in afile: for tile in tiles: if tile in afile: # assemble the address if isftp: address='/'.join(['ftp://'+site, Dir, good_date, afile]) else: address='/'.join([site, good_date, afile]) #download the file outname = os.path.join(outdir, afile) if not os.path.isfile(outname) and not force_overwrite: download_url(address, outname) print('Downloaded {0}'.format(address)) print('Finished retrieving MODIS - {0} data!'.format(product)) return
def extract_from_hdf(filelist, layerlist, layernames = False, outdir = None): """ Extracts tifs from MODIS extract_HDF_layer files, ensures proper projection. inputs: filelist list of '.hdf' files from which data should be extracted (or a directory) layerlist list of layer numbers to pull out as individual tifs should be integers such as [0,4] for the 0th and 4th layer respectively. layernames list of layer names to put more descriptive file suffixes to each layer outdir directory to which tif files should be saved if outdir is left as 'False', files are saved in the same directory as the input file was found. """ if outdir is not None: if not os.path.exists(outdir): os.makedirs(outdir) # enforce lists for iteration purposes and sanitize inputs filelist = core.enf_filelist(filelist) for filename in filelist: if '.xml' in filename or not '.hdf' in filename: filelist.remove(filename) layerlist = core.enf_list(layerlist) layernames = core.enf_list(layernames) # ignore user input layernames if they are invalid, but print warnings if layernames and not len(layernames) == len(layerlist): Warning('Layernames must be the same length as layerlist!') Warning('Ommiting user defined layernames!') layernames = False # create empty list to add failed file names into failed = [] # iterate through every file in the input filelist for infile in filelist: # pull the filename and path apart path,name = os.path.split(infile) arcpy.env.workspace = path for i,layer in enumerate(layerlist): # specify the layer names. if layernames: layername = layernames[i] else: layername = str(layer).zfill(3) # use the input output directory if the user input one, otherwise build one if outdir: outname = os.path.join(outdir, "{0}_{1}.tif".format(name[:-4], layername)) else: outname = os.path.join(path, "{0}_{1}.tif".format(name[:-4], layername)) # perform the extracting and projection definition try: # extract the subdataset arcpy.ExtractSubDataset_management(infile, outname, str(layer)) # define the projection as the MODIS Sinusoidal define_projection(outname) print("Extracted {0}".format(os.path.basename(outname))) except: print("Failed to extract {0} from {1}".format(os.path.basename(outname), os.path.basename(infile))) failed.append(infile) print("Finished extracting all hdfs! \n") return failed
def surface_reflectance_dos_457(band_nums, meta_path, outdir): """ Converts Landsat 4, 5, and 7 band DNs to surface reflectance using dark object subtraction. To be performed on raw Landsat 4,5, and 7 level 1 data. See link below for details see here [http://landsat.usgs.gov/Landsat8_Using_Product.php] Inputs: band_nums A list of desired band numbers such as [3,4,5] meta_path The full filepath to the metadata file for those bands outdir Output directory to save converted files. If left False it will save ouput files in the same directory as input files. """ band_nums = core.enf_list(band_nums) band_nums = map(str, band_nums) TM_ETM_bands = ['1','2','3','4','5','7','8'] #landsat.toa_reflectance_457(band_nums, meta_path, outdir) OutList = [] f = open(meta_path) MText = f.read() metadata = landsat.grab_meta(meta_path) oldMeta = [] newMeta = [] #the presence of a PRODUCT_CREATION_TIME category is used to identify old metadata #if this is not present, the meta data is considered new. #Band6length refers to the length of the Band 6 name string. In the new metadata this string is longer if "PRODUCT_CREATION_TIME" in MText: Meta = oldMeta Band6length = 2 else: Meta = newMeta Band6length = 8 #The tilename is located using the newMeta/oldMeta indixes and the date of capture is recorded if Meta == newMeta: TileName = getattr(metadata, "LANDSAT_SCENE_ID") year = TileName[9:13] jday = TileName[13:16] date = getattr(metadata, "DATE_ACQUIRED") elif Meta == oldMeta: TileName = getattr(metadata, "BAND1_FILE_NAME") year = TileName[13:17] jday = TileName[17:20] date = getattr(metadata, "ACQUISITION_DATE") #the spacecraft from which the imagery was capture is identified #this info determines the solar exoatmospheric irradiance (ESun) for each band spacecraft = getattr(metadata, "SPACECRAFT_ID") if "7" in spacecraft: ESun = (1969.0, 1840.0, 1551.0, 1044.0, 255.700, 0., 82.07, 1368.00) elif "5" in spacecraft: ESun = (1957.0, 1826.0, 1554.0, 1036.0, 215.0, 0. ,80.67) elif "4" in spacecraft: ESun = (1957.0, 1825.0, 1557.0, 1033.0, 214.9, 0. ,80.72) else: arcpy.AddError("This tool only works for Landsat 4, 5, or 7") raise arcpy.ExecuteError() #determing if year is leap year and setting the Days in year accordingly if float(year) % 4 == 0: DIY = 366. else:DIY=365. #using the date to determing the distance from the sun theta = 2 * math.pi * float(jday)/DIY dSun2 = (1.00011 + 0.034221 * math.cos(theta) + 0.001280 * math.sin(theta) + 0.000719 * math.cos(2*theta)+ 0.000077 * math.sin(2 * theta)) SZA = 90. - float(getattr(metadata, "SUN_ELEVATION")) for band_num in band_nums: if band_num in TM_ETM_bands: pathname = meta_path.replace("MTL.txt", "B{0}.tif".format(band_num)) Oraster = arcpy.Raster(pathname) null_raster = arcpy.sa.SetNull(Oraster, Oraster, "VALUE = 0") #using the oldMeta/newMeta indixes to pull the min/max for radiance/Digital numbers if Meta == newMeta: LMax = getattr(metadata, "RADIANCE_MAXIMUM_BAND_" + band_num) LMin = getattr(metadata, "RADIANCE_MINIMUM_BAND_" + band_num) QCalMax = getattr(metadata, "QUANTIZE_CAL_MAX_BAND_" + band_num) QCalMin = getattr(metadata, "QUANTIZE_CAL_MIN_BAND_" + band_num) elif Meta == oldMeta: LMax = getattr(metadata, "LMAX_BAND" + band_num) LMin = getattr(metadata, "LMIN_BAND" + band_num) QCalMax = getattr(metadata, "QCALMAX_BAND" + band_num) QCalMin = getattr(metadata, "QCALMIN_BAND" + band_num) Radraster = (((LMax - LMin)/(QCalMax-QCalMin)) * (null_raster - QCalMin)) + LMin Oraster = 0 del null_raster #Calculating temperature for band 6 if present Refraster = (math.pi * Radraster * dSun2) / (ESun[int(band_num[0]) - 1] * math.cos(SZA * (math.pi/180))) #Oraster = arcpy.Raster(raster_null) dark_object = arcpy.GetRasterProperties_management(Refraster, "MINIMUM") do_str = str(dark_object) do_flt = float(do_str) #Calculate the minimum value in each band and perform dark object subtraction Surfrefraster = Refraster - do_flt BandPath = "{0}\\{1}_B{2}_SurfRef.tif".format(outdir, TileName, band_num) Surfrefraster.save(BandPath) OutList.append(arcpy.Raster(BandPath)) del Refraster, Radraster, Surfrefraster arcpy.AddMessage("Surface Reflectance Calculated for Band {0}".format(band_num)) print("Surface Reflectance Calculated for Band {0}".format(band_num)) f.close() return OutList