def get_landsat_scenes(wlon, nlat, elon, slat, startdate, enddate, cloud,
formal_name: str):
"""List landsat scenes from USGS."""
credentials = get_credentials()['landsat']
api = API(credentials['username'], credentials['password'])
landsat_folder_id = EE_FOLDER.get(formal_name)
if landsat_folder_id is None:
raise ValueError(
'Invalid Landsat product name. Expected one of {}'.format(
EE_FOLDER.keys()))
# Request
scenes_result = api.search(dataset=formal_name,
bbox=(slat, wlon, nlat, elon),
start_date=startdate,
end_date=enddate,
max_cloud_cover=cloud or 100,
max_results=50000)
scenes_output = {}
for scene in scenes_result:
if scene['displayId'].endswith('RT'):
logging.warning('Skipping Real Time {}'.format(scene['displayId']))
continue
copy_scene = dict()
copy_scene['sceneid'] = scene['displayId']
copy_scene['scene_id'] = scene['entityId']
copy_scene['cloud'] = int(scene['cloudCover'])
copy_scene['date'] = scene['acquisitionDate']
xmin, ymin, xmax, ymax = scene['sceneBounds'].split(',')
copy_scene['wlon'] = float(xmin)
copy_scene['slat'] = float(ymin)
copy_scene['elon'] = float(xmax)
copy_scene['nlat'] = float(ymax)
copy_scene['link'] = EE_DOWNLOAD_URL.format(folder=landsat_folder_id,
sid=scene['entityId'])
pathrow = scene['displayId'].split('_')[2]
copy_scene['path'] = pathrow[:3]
copy_scene['row'] = pathrow[3:]
scenes_output[scene['displayId']] = copy_scene
return scenes_output
def __init__(self, username, password, archive, backup_archive):
"""Connect API.
Raise ProcessorFailedError on failure
:param str username: username
:param str password: password
:param str archive: not used by Landsat implementation
:param str backup_archive: not used by Landsat implementation
"""
from landsatxplore.api import API as LandsatAPI
from landsatxplore.exceptions import EarthExplorerError
try:
self.api = LandsatAPI(
username, password
)
except EarthExplorerError as e:
raise ProcessorFailedError(
self,
"Unable to connect API: {}".format(e),
set_status=False
)
except json.JSONDecodeError as e:
raise ProcessorFailedError(
self,
"Landsat server is down. It raised a JSON exception: "
"{}".format(e),
set_status=False
)
class EarthExplorer(object):
"""Access Earth Explorer portal."""
def __init__(self, username, password):
"""Access Earth Explorer portal."""
self.session = requests.session()
self.login(username, password)
self.api = API(username, password)
def logged_in(self):
"""Check if the log-in has been successfull. Search for
the log-out URL in the portal html body.
"""
rsp = self.session.get(EE_URL)
return EE_LOGOUT_URL in rsp.text
def login(self, username, password):
"""Login to Earth Explorer."""
rsp = self.session.get(EE_LOGIN_URL)
csrf, ncform = _get_tokens(rsp.text)
payload = {
'username': username,
'password': password,
'csrf_token': csrf,
'__ncforminfo': ncform
}
rsp = self.session.post(EE_LOGIN_URL,
data=payload,
allow_redirects=False)
if not self.logged_in():
raise EarthExplorerError('EE: login failed.')
def logout(self):
"""Log out from Earth Explorer."""
self.session.get(EE_LOGOUT_URL)
def _download(self, url, output_dir, file_size, chunk_size=1024):
"""Download remote file given its URL."""
with tqdm(total=file_size, unit_scale=True, unit='B') as pbar:
with self.session.get(url, stream=True, allow_redirects=True) as r:
local_filename = r.headers['Content-Disposition'].split(
'=')[-1]
local_filename = os.path.join(output_dir, local_filename)
with open(local_filename, 'wb') as f:
for chunk in r.iter_content(chunk_size=chunk_size):
if chunk:
f.write(chunk)
pbar.update(chunk_size)
return local_filename
def download(self, scene_id, output_dir):
"""Download a Landsat scene given its identifier and an output
directory.
"""
dataset = guess_dataset(scene_id)
if is_product_id(scene_id):
scene_id = self.api.lookup(dataset, [scene_id], inverse=True)[0]
url = EE_DOWNLOAD_URL.format(folder=EE_FOLDER[dataset], sid=scene_id)
filename = self._download(url, output_dir, file_size=SIZES[dataset])
return filename
def get_landsat_scenes(wlon, nlat, elon, slat, startdate, enddate, cloud,
limit):
"""List landsat scenes from USGS."""
credentials = get_credentials()['landsat']
api = API(credentials['username'], credentials['password'])
# Request
scenes_result = api.search(dataset='LANDSAT_8_C1',
bbox=(slat, wlon, nlat, elon),
start_date=startdate,
end_date=enddate,
max_cloud_cover=cloud or 100,
max_results=10000)
scenes_output = {}
for scene in scenes_result:
if scene['displayId'].endswith('RT'):
logging.warning('Skipping Real Time {}'.format(scene['displayId']))
continue
copy_scene = dict()
copy_scene['sceneid'] = scene['displayId']
copy_scene['scene_id'] = scene['entityId']
copy_scene['cloud'] = int(scene['cloudCover'])
copy_scene['date'] = scene['acquisitionDate']
xmin, ymin, xmax, ymax = scene['sceneBounds'].split(',')
copy_scene['wlon'] = float(xmin)
copy_scene['slat'] = float(ymin)
copy_scene['elon'] = float(xmax)
copy_scene['nlat'] = float(ymax)
copy_scene[
'link'] = 'https://earthexplorer.usgs.gov/download/12864/{}/STANDARD/EE'.format(
scene['entityId'])
pathrow = scene['displayId'].split('_')[2]
copy_scene['path'] = pathrow[:3]
copy_scene['row'] = pathrow[3:]
scenes_output[scene['displayId']] = copy_scene
return scenes_output
def search(
username, password, dataset, location, bbox, clouds, start, end, output, limit
):
"""Search for scenes."""
api = API(username, password)
where = {"dataset": dataset}
if location:
latitude, longitude = location
where.update(latitude=latitude, longitude=longitude)
if bbox:
where.update(bbox=bbox)
if clouds:
where.update(max_cloud_cover=clouds)
if start:
where.update(start_date=start)
if end:
where.update(end_date=end)
if limit:
where.update(max_results=limit)
results = api.search(**where)
api.logout()
if not results:
return
if output == "entity_id":
for scene in results:
click.echo(scene["entity_id"])
if output == "display_id":
for scene in results:
click.echo(scene["display_id"])
if output == "json":
dump = json.dumps(results, indent=True)
click.echo(dump)
if output == "csv":
with StringIO("tmp.csv") as f:
w = csv.DictWriter(f, results[0].keys())
w.writeheader()
w.writerows(results)
click.echo(f.getvalue())
def downloader(sat_choice=None,
block=2027,
dates=None,
indice_requested=[False, False, False, True, False],
test=False,
root=str):
api = API(username, password)
if test == True:
sat_choice = satellite_selector()
dates = get_dates(sat_choice=sat_choice, test=False)
indice_requested = indices_requested()
request = str(random.randint(5, 101))
try:
try:
os.rmdir(f'./Images/Request.{request}')
except:
os.mkdir(f'./Images/Request.{request}')
os.mkdir(f"./Images/Request.{request}/Start_date")
os.mkdir(f"./Images/Request.{request}/End_date")
except:
pass
co_ord = csv_crawler(block=block, clipper=True, root=root)
# print(co_ord)
for (id, date) in enumerate(dates):
scenes = scene_finder(api, sat_choice, date, coordinates=co_ord)
print(scenes)
# scene_list, scene_data = download_selector(scenes, test=True, sat_choice=sat_choice)
scene_list, scene_data = download_selector(scenes,
sat_choice=sat_choice)
scene_downloader(scene_list,
id=id,
request=request,
sat_choice=sat_choice)
api.logout()
return sat_choice, indice_requested
class QCConnectLandsat:
def __init__(self, username, password, archive, backup_archive):
"""Connect API.
Raise ProcessorFailedError on failure
:param str username: username
:param str password: password
:param str archive: not used by Landsat implementation
:param str backup_archive: not used by Landsat implementation
"""
from landsatxplore.api import API as LandsatAPI
from landsatxplore.exceptions import EarthExplorerError
try:
self.api = LandsatAPI(
username, password
)
except EarthExplorerError as e:
raise ProcessorFailedError(
self,
"Unable to connect API: {}".format(e),
set_status=False
)
except json.JSONDecodeError as e:
raise ProcessorFailedError(
self,
"Landsat server is down. It raised a JSON exception: "
"{}".format(e),
set_status=False
)
def __del__(self):
if not hasattr(self, "api"):
return
from landsatxplore.exceptions import EarthExplorerError
try:
self.api.logout()
except EarthExplorerError as e:
Logger.error("Landsat server is down. {}".format(e))
def query(self, footprint, kwargs):
"""Query API.
:return: result
"""
from landsatxplore.exceptions import EarthExplorerError
kwargs['bbox'] = wkt2bbox(footprint, switch_axis=True)
kwargs['max_results'] = 500
del kwargs['producttype'] # used only for testing
try:
items = self.api.search(**kwargs)
except EarthExplorerError as e:
raise ProcessorFailedError(
self,
"Landsat server is down. "
"{}".format(e),
set_status=False
)
dict_items = {}
for item in items:
selected = False
if self.filter_by_tiles:
for tile in self.filter_by_tiles:
if str(tile) in item['entityId']:
selected = True
break
else:
selected = True
if selected:
dict_items[item['entityId']] = item
# used tests only
dict_items[item['entityId']]['producttype'] = item['displayId'].split('_')[1]
dict_items[item['entityId']]['beginposition'] = \
datetime.strptime(item['startTime'], '%Y-%m-%d')
else:
Logger.info("IP {} skipped by tile filter".format(item['entityId']))
return dict_items
def __init__(self, username, password):
"""Access Earth Explorer portal."""
self.session = requests.session()
self.login(username, password)
self.api = API(username, password)
from landsatxplore.api import API
# Initialize a new API instance and get an access key
api = API(username, password)
# Perform a request. Results are returned in a dictionnary
# creamos nuestra clase agenda
class Agenda:
# iniciamos nuestra clase
def __init__(self):
# crearemos una lista donde guardaremos los datos de nuestra agenda
self.contactos = []
# menu del programa
def menu(self):
print()
menu = [['Agenda Personal'], ['1. Añadir Contacto', "anadir"],
['2. Lista de contactos'], ['3. Buscar contacto'],
['4. Editar contacto'], ['5. Cerrar agenda']]
for x in range(6):
print(menu[x][0])
opcion = int(input("Introduzca la opción deseada: "))
if opcion == 1:
self.anadir()
elif opcion == 2:
self.lista()
elif opcion == 3:
longitude = -103.85
end_date = dt.date.today()
start_date = dt.date(2021, 2, 1)
dataset = "landsat_8_c1"
# https://pypi.org/project/landsatxplore/
# Landsat 7 ETM+ Collection 1 Level 1 landsat_etm_c1
# Landsat 7 ETM+ Collection 2 Level 1 landsat_etm_c2_l1
# Landsat 7 ETM+ Collection 2 Level 2 landsat_etm_c2_l2
# Landsat 8 Collection 1 Level 1 landsat_8_c1
# Landsat 8 Collection 2 Level 1 landsat_ot_c2_l1
# Landsat 8 Collection 2 Level 2 landsat_ot_c2_l2
# Initialize a new API instance and search for Landsat TM scenes
print("Searching from " + str(start_date) + " to " + str(end_date))
password = keyring.get_password("EarthExplorer", username)
api = API(username, password)
scenes = api.search(dataset=dataset,
latitude=latitude,
longitude=longitude,
start_date=str(start_date),
end_date=(str(end_date)),
max_cloud_cover=10)
print(f"{len(scenes)} scenes found:")
for scene in scenes:
print(scene['acquisition_date'])
# Initialize nvdi_array
nvdi_array = np.zeros((len(scenes), YSize, XSize))
nvdi_array = nvdi_array.astype('float32')
print(nvdi_array.shape)
class EarthExplorer(object):
"""Access Earth Explorer portal."""
def __init__(self, username, password):
"""Access Earth Explorer portal."""
self.session = requests.Session()
self.login(username, password)
self.api = API(username, password)
def logged_in(self):
"""Check if the log-in has been successfull based on session cookies."""
eros_sso = self.session.cookies.get("EROS_SSO_production_secure")
return bool(eros_sso)
def login(self, username, password):
"""Login to Earth Explorer."""
rsp = self.session.get(EE_LOGIN_URL)
csrf, ncform = _get_tokens(rsp.text)
payload = {
"username": username,
"password": password,
"csrf": csrf,
"__ncforminfo": ncform,
}
rsp = self.session.post(EE_LOGIN_URL,
data=payload,
allow_redirects=True)
if not self.logged_in():
raise EarthExplorerError("EE: login failed.")
def logout(self):
"""Log out from Earth Explorer."""
self.session.get(EE_LOGOUT_URL)
def _download(self, url, output_dir, timeout, chunk_size=1024, skip=False):
"""Download remote file given its URL."""
# Check availability of the requested product
# EarthExplorer should respond with JSON
with self.session.get(url,
allow_redirects=False,
stream=True,
timeout=timeout) as r:
r.raise_for_status()
error_msg = r.json().get("errorMessage")
if error_msg:
raise EarthExplorerError(error_msg)
download_url = r.json().get("url")
try:
with self.session.get(download_url,
stream=True,
allow_redirects=True,
timeout=timeout) as r:
file_size = int(r.headers.get("Content-Length"))
with tqdm(total=file_size,
unit_scale=True,
unit="B",
unit_divisor=1024) as pbar:
local_filename = r.headers["Content-Disposition"].split(
"=")[-1]
local_filename = local_filename.replace('"', "")
local_filename = os.path.join(output_dir, local_filename)
if skip:
return local_filename
with open(local_filename, "wb") as f:
for chunk in r.iter_content(chunk_size=chunk_size):
if chunk:
f.write(chunk)
pbar.update(chunk_size)
except requests.exceptions.Timeout:
raise EarthExplorerError(
"Connection timeout after {} seconds.".format(timeout))
return local_filename
def download(self,
identifier,
output_dir,
dataset=None,
timeout=300,
skip=False):
"""Download a Landsat scene.
Parameters
----------
identifier : str
Scene Entity ID or Display ID.
output_dir : str
Output directory. Automatically created if it does not exist.
dataset : str, optional
Dataset name. If not provided, automatically guessed from scene id.
timeout : int, optional
Connection timeout in seconds.
skip : bool, optional
Skip download, only returns the remote filename.
Returns
-------
filename : str
Path to downloaded file.
"""
os.makedirs(output_dir, exist_ok=True)
if not dataset:
dataset = guess_dataset(identifier)
if is_display_id(identifier):
entity_id = self.api.get_entity_id(identifier, dataset)
else:
entity_id = identifier
url = EE_DOWNLOAD_URL.format(data_product_id=DATA_PRODUCTS[dataset],
entity_id=entity_id)
filename = self._download(url, output_dir, timeout=timeout, skip=skip)
return filename
def get_earth_explorer_api():
credentials = get_credentials()['landsat']
return API(credentials['username'], credentials['password'])
class EarthExplorer(object):
"""Access Earth Explorer portal."""
def __init__(self, username, password):
"""Access Earth Explorer portal."""
self.session = requests.session()
self.login(username, password)
self.api = API(username, password)
def logged_in(self):
"""Check if the log-in has been successfull based on session cookies."""
eros_sso = self.session.cookies.get("EROS_SSO_production_secure")
return bool(eros_sso)
def login(self, username, password):
"""Login to Earth Explorer."""
rsp = self.session.get(EE_LOGIN_URL)
csrf, ncform = _get_tokens(rsp.text)
payload = {
'username': username,
'password': password,
'csrf': csrf,
'__ncforminfo': ncform
}
rsp = self.session.post(EE_LOGIN_URL,
data=payload,
allow_redirects=True)
if not self.logged_in():
raise EarthExplorerError('EE: login failed.')
def logout(self):
"""Log out from Earth Explorer."""
self.session.get(EE_LOGOUT_URL)
def _download(self, url, output_dir, chunk_size=1024):
"""Download remote file given its URL."""
with self.session.get(url, stream=True, allow_redirects=True) as r:
file_size = int(r.headers.get("Content-Length"))
with tqdm(total=file_size,
unit_scale=True,
unit='B',
unit_divisor=1024) as pbar:
local_filename = r.headers['Content-Disposition'].split(
'=')[-1]
local_filename = local_filename.replace("\"", "")
local_filename = os.path.join(output_dir, local_filename)
with open(local_filename, 'wb') as f:
for chunk in r.iter_content(chunk_size=chunk_size):
if chunk:
f.write(chunk)
pbar.update(chunk_size)
return local_filename
def download(self, scene_id, output_dir):
"""Download a Landsat scene given its identifier and an output
directory.
"""
os.makedirs(output_dir, exist_ok=True)
dataset = guess_dataset(scene_id)
if is_product_id(scene_id):
scene_id = self.api.lookup(dataset, [scene_id], inverse=True)[0]
url = EE_DOWNLOAD_URL.format(dataset_id=DATASETS[dataset],
scene_id=scene_id)
filename = self._download(url, output_dir)
return filename
latitude, longitude = sys.argv[1], sys.argv[2]
start_date = sys.argv[3]
end_date = None
months=None
max_cloud_cover=None
path = "/mnt/c/Users/avaro/Desktop/LST/" + site_id + "/"
user = "******"
pw = "pepsiav123pepsiav123"
api = api(user,pw)
# scenes = api.search( dataset, latitude=latitude, longitude=longitude, bbox=None, start_date=start_date, end_date=end_date, max_cloud_cover=max_cloud_cover, months=months, max_results=2)
scenes = api.search( dataset, latitude=latitude, longitude=longitude,max_cloud_cover=max_cloud_cover, months=months)
print('{} scenes found.'.format(len(scenes)))
api.logout()
#init downloader
downloader = ee(user, pw)
# entity id of latest
print(scenes[0]['entityId'])
# download & persist
downloader.download(scenes[0]['entityId'],path)
#close session
downloader.logout()
