def get_ntl_file_urls(district_gdf: gpd.geodataframe.GeoDataFrame, products: str, startTime: str, endTime: str, collection: int, district_json_file: str): (w, s, e, n) = ode.district_extents(district_gdf) # Create a ModapsClient object a = modapsclient.ModapsClient() a.headers["Authorization"] = "Bearer " + os.environ.get("LADS_TOKEN") collections = a.getCollections('VNP46A2') fileIDs = a.searchForFiles(products=products, startTime=startTime, endTime=endTime, north=n, south=s, east=e, west=w, coordsOrTiles='coords', dayNightBoth='N', collection=collection) file_url_list = [] file_name_list = [] for id in fileIDs: file_url = a.getFileUrls(id)[0] file_url_list.append(file_url) file_name_start_pos = file_url.rfind("/") + 1 file_name = file_url[file_name_start_pos:] file_name_list.append(file_name) import json #file_name_list_json = json.dumps(file_name_list) district_json = {"district_id": int(district_gdf.index[0]), "start_time": startTime, "end_time": endTime, "file_list": file_name_list} # write json to file with open(district_json_file + "-" + startTime + "-" + endTime + ".json", 'w', encoding='utf-8') as district_json_dumped: json.dump(json.dumps(district_json), district_json_dumped) return file_url_list
def get_LAADS_filepaths(product, start_time, end_time, collection, bbox=None, connection_type="https"): #-------------------------------------------------------------------------. # Check bbox if (bbox is None): bbox = check_bbox(bbox) coordsOrTiles = "global" else: coordsOrTiles = "coords" #-------------------------------------------------------------------------. # Process start_time and end_time # TODO: enhance modaps to accept also hour and minute # start_time_str = start_time.strftime('%Y-%m-%d %H:%M:%S') # end_time_str = end_time.strftime('%Y-%m-%d %H:%M:%S') start_time_str = start_time.strftime('%Y-%m-%d') # %H:%M:%S') end_time_str = end_time.strftime('%Y-%m-%d') # %H:%M:%S') #-------------------------------------------------------------------------. # Retrieve fileIDs proxy = m.ModapsClient() fileIDs = proxy.searchForFiles(products=product, startTime=start_time_str, endTime=end_time_str, west=bbox[0], south=bbox[1], east=bbox[2], north=bbox[3], coordsOrTiles=coordsOrTiles, collection=collection) #-------------------------------------------------------------------------. if ((len(fileIDs) == 1) and (fileIDs[0] == 'No results')): return #-------------------------------------------------------------------------. # Check file is online for direct download # TODO: Log which fail is not online ! fileIDs_formatted = ",".join(fileIDs) l_properties = proxy.getFileProperties(fileIDs_formatted) idx_online = [ properties['online'] == "true" for properties in l_properties ] fileIDs = list(np.array(fileIDs)[idx_online]) #-------------------------------------------------------------------------. # Retrieve file urls fileIDs_formatted = ",".join(fileIDs) filepaths = sorted(proxy.getFileUrls(fileIDs_formatted)) #-------------------------------------------------------------------------. # Change to opendap if specified if (connection_type == "opendap"): filepaths = convert_to_OPeNDAP_filepaths(filepaths) #-------------------------------------------------------------------------. # Return file paths return (filepaths)
def download_MOD44B(): product='MOD44B' print("Downloading -> ", product) years= np.unique(np.array([startdate_obj.year,enddate_obj.year])) for year in years: try: ''' The MOD44B data product layers include percent tree cover, percent non-tree cover, percent non-vegetated, cloud cover, and quality indicators. The start date of the annual period for this product begins with day of year (DOY) 65 (March 5). ''' temp = datetime.strptime( str(year) +'-01-01', '%Y-%m-%d') temp= temp + timedelta(days=64) print(temp) doy = int(datetime(temp.year,temp.month,temp.day).timetuple().tm_yday) print(doy) path_= os.path.join( path,product, str(temp.year)) if not os.path.exists(path_): print(path_) os.makedirs(path_) a = modapsclient.ModapsClient() items=a.searchForFiles(products=product, startTime=temp, endTime=temp, north=north,south=south, west=west,east=east, collection=collection) print("Products count = > ",len(items),items[0]) for p in items: if(p.strip() =='No results'.strip()): continue url=a.getFileUrls(p)[0] print('=====>',p,url) # cmd=('wget --user hiik324 --password Ecology123 {0} --header "Authorization: Bearer C88B2F44-881A-11E9-B4DB-D7883D88392C" -P {1} '.format( url, path_)) cmd=('wget --user hiik324 --password Ecology123 {0} -P {1} '.format( url, path_)) os.system(cmd) except: print("Unexpected error:", sys.exc_info()[0]) continue
def download_modis_products(): products = [ "MCD12C1", "MCD15A3H", "MCD43D59", "MCD43D60", "MCD43D61", "MCD43D62", "MCD43D63", "MOD15A2H", "MOD44B" ] for product in products: print("Downloading -> ", product) try: for day in range(int((enddate_obj - startdate_obj).days)): temp = startdate_obj + timedelta(days=day) doy = int( datetime(temp.year, temp.month, temp.day).timetuple().tm_yday) if (product == 'MOD44B'): path_ = os.path.join(path, product, str(temp.year)) else: path_ = os.path.join(path, product, str(temp.year), str(doy)) if not os.path.exists(path_): print(path_) os.makedirs(path_) a = modapsclient.ModapsClient() items = a.searchForFiles(products=product, startTime=temp, endTime=temp, north=north, south=south, west=west, east=east, collection=collection) print("Products count = > ", len(items)) # for p in items: # url=a.getFileUrls(p)[0] # print(p,url) # # cmd=('wget --user hiik324 --password Ecology123 {0} --header "Authorization: Bearer C88B2F44-881A-11E9-B4DB-D7883D88392C" -P {1} '.format( url, path_)) # cmd=('wget --user hiik324 --password Ecology123 {0} -P {1} '.format( url, path_)) # os.system(cmd) except: print("Error") print("Unexpected error:", sys.exc_info()[0])
def download_Aqua_Terra_L2(): products=["MYD04_L2","MYD06_L2","MYD07_L2","MYD11_L2","MOD04_L2","MOD06_L2","MOD07_L2","MOD11_L2"] for product in products: print("Downloading ->", product) try: for day in range (int((enddate_obj - startdate_obj).days)): temp= startdate_obj + timedelta(days=day) doy = int(datetime(temp.year,temp.month,temp.day).timetuple().tm_yday) if(product[:3]=='MYD'): path_= os.path.join( path,"Aqua_L2", str(temp.year),str(doy),product) elif (product[:3]=='MOD'): path_= os.path.join( path,"Terra_L2", str(temp.year),str(doy),product) if not os.path.exists(path_): print(path_) os.makedirs(path_) a = modapsclient.ModapsClient() if(product=='MYD011_L2' or product =="MOD11_L2"): collection="6" else: collection="61" items=a.searchForFiles(products=product, startTime=startdate, endTime=enddate, north=north,south=south, west=west,east=east, collection=collection) print("Products count = > ",len(items)) # download_items(a,items,path_) except Exception as e: print("Error: ",e) print("Unexpected error:", sys.exc_info()[0]) continue
def get_products(satellite, instrument): modaps_inst = _modaps_sat_acronym(satellite, instrument) # Retrieve products proxy = m.ModapsClient() products = sorted(proxy.listProductsByInstrument(modaps_inst)) return products
def get_collection_most_recent(product): proxy = m.ModapsClient() collections = proxy.getCollections(product) collections = [int(v) for v in collections.keys()] most_recent_collection = str(max(collections)) return most_recent_collection
def get_collections(product): proxy = m.ModapsClient() collections = proxy.getCollections(product) return collections
def find_modis_data(location, date=datetime.utcnow(), ndays=1, products=['MODATML2', 'MYDATML2'], collection='61', dbg=False): """Finds near-realtime MODIS products from the Aqua/Terra satellites for a specific location <location>, starting at a specifc ([date]; defaults to 'now') for a number of days ([ndays]; defaults to 1). By default the MODIS Joint Atmosphere products, MODATML2/MYDATML2 in Collection 61: https://ladsweb.modaps.eosdis.nasa.gov/missions-and-measurements/products/l2-joint-atmosphere/MYDATML2/ are searched but this can be overridden by passing in [products] and [collection]. A dictionary with the name of HDF file products as keys is returned; the values for each HDF file is a dictionary of the following form: url: URL link to retrieve the HDF file, checksum: CRC checksum of the file (`cksum` can be used to generate the checksum and size, size: size of the HDF file in bytes, name: name of the HDF file, """ import modapsclient as m import logging logging.basicConfig(level=logging.WARNING) a = m.ModapsClient() start_date = date.date() end_date = start_date + timedelta(days=ndays) west_val, south_val, east_val, north_val = make_bounding_box(location) dl_files = {} for product in products: if dbg: print(product) collections = a.getCollections(product) if collection not in collections: print("Did not find specified collection (%s) in %s".format( collection, collections)) return None files = a.searchForFiles(product, start_date, end_date, north=north_val, west=west_val, east=east_val, south=south_val, collection=collection) if dbg: print(files) for hdf_file in files: url = a.getFileUrls(hdf_file) props = a.getFileProperties(hdf_file) if len(props) == 1: props = props[0] else: print("Unexpected number of file properties found") continue if props.get('online', 'false') == 'true': dl_files[hdf_file] = { 'url': url, 'checksum': props.get('checksum', ''), 'size': props.get('fileSizeBytes', ''), 'name': props.get('fileName', '') } else: print("File id %d (%s) is not online".format( hdf_file, props.get('fileName', 'UNKNOWN'))) return dl_files
def realmodaps(): a = modapsclient.ModapsClient() return a
def mockmodaps(): a = modapsclient.ModapsClient() a._makeurl = fakeurl a._rawresponse = mockedresponse return a
def __init__(self, config_name="ModisAPI"): self._config = load_config(Modis.CONFIG_PATH, config_name) self._defaults = self._config['defaults'] self._modis_tiles = self._load_modis_tiles() self._api_key = get_api_key(self._config) self._m = m.ModapsClient()