class LandsatWrapper:
def __init__(self):
logger.info("connect to landsat API")
# connection to API for search queries
self.api = landsatxplore.api.API(config.usgsUser, config.usgsPW)
# connection to EarthExplorer for download requests
self.earth_explorer = EarthExplorer(config.usgsUser, config.usgsPW)
logger.info("landsat API connected")
def __del__(self):
# logout from API and EarthExplorer
self.api.logout()
self.earth_explorer.logout()
def get_landsat_products(self,
lat,
lon,
date_from,
date_to,
platform,
max_cloud_coverage=100,
limit=0):
# datasets: [LANDSAT_TM_C1|LANDSAT_ETM_C1|LANDSAT_8_C1]
# format for dates: 'YYYY-MM-DD'
logger.info("start landsat query")
if int(limit) > 0:
scenes = self.api.search(dataset=platform,
latitude=lat,
longitude=lon,
start_date=date_from,
end_date=date_to,
max_cloud_cover=max_cloud_coverage,
max_results=limit)
else:
scenes = self.api.search(dataset=platform,
latitude=lat,
longitude=lon,
start_date=date_from,
end_date=date_to,
max_cloud_cover=maxCloudCoverage)
return scenes
def get_product_data(self, platform, product_id):
scenes = self.api.metadata(platform, [product_id])
if len(scenes) > 0:
return scenes[0]
else:
return None
def download_landsat_product(self, scene_id):
self.earth_explorer.download(scene_id=scene_id,
output_dir=config.bigTilesDir)
def __init__(self):
logger.info("connect to landsat API")
# connection to API for search queries
self.api = landsatxplore.api.API(config.usgsUser, config.usgsPW)
# connection to EarthExplorer for download requests
self.earthExplorer = EarthExplorer(config.usgsUser, config.usgsPW)
logger.info("landsat API connected")
def download(username, password, output, scenes):
"""Download one or several Landsat scenes."""
ee = EarthExplorer(username, password)
output_dir = os.path.abspath(output)
for scene in scenes:
if not ee.logged_in():
ee = EarthExplorer(username, password)
ee.download(scene, output_dir)
ee.logout()
def download(self, scene_id: str, *args, **kwargs):
"""Download Landsat product from USGS."""
self._api()
destination = kwargs.get('output')
earth_explorer = EarthExplorer(self.kwargs['username'],
self.kwargs['password'])
try:
file_name = earth_explorer.download(scene_id, destination)
return file_name
except EarthExplorerError as e:
raise DownloadError(str(e))
def downloadEartExplorer(displayId, entityId, badFiles):
if chkFileExist(displayId):
print(' All files: {} '.format(displayId), time.strftime("%d.%m.%Y %H:%M:%S"))
return 1
print('Logging ee in: ', time.strftime("%d.%m.%Y %H:%M:%S"))
ee = EarthExplorer(usr, psw)
print(' -> ee: ',ee)
print('Download ', time.strftime("%d.%m.%Y %H:%M:%S"), displayId, entityId)
try:
ee.download(scene_id=entityId, output_dir=image_path + 'tempE/')
except IOError as e:
print(" -> I/O error({0}): {1}".format(e.errno, e.strerror))
return -1
except:
print(" -> Unexpected error:", sys.exc_info()[0])
return -2
print('UnTarGz ', time.strftime("%d.%m.%Y %H:%M:%S"))
if (file_untar(image_path + 'tempE/', displayId + '.tar.gz') < 0):
print(' -> unTar error: ', image_path + 'tempE/', image_id + '.tar.gz')
return -3
print('Move Img ', time.strftime("%d.%m.%Y %H:%M:%S"))
filelist = [ f for f in os.listdir(image_path + 'tempE/')]
for f in filelist:
os.replace(image_path + 'tempE/' + f, image_path + f)
#if os.path.isfile(image_path + image_id + '_B10.TIF') == False or os.path.isfile(image_path + image_id + '_B11.TIF') == False:
# print(' -> no file in directory error: ', image_id, ' -- ', os.path.isfile(image_path + image_id + '_B10.TIF'), ' -- ', os.path.isfile(image_path + image_id + '_B11.TIF'))
# if (badFiles['badFile'] == image_id).any():
# badFiles = badFiles.append({'badFile' : image_id}, ignore_index=True)
# print(' -> Bad file was added to the list !!!!!!!!!:', image_id)
# badFiles.to_csv(file_path + '1_DataExtract/badFiles.txt', index = False, header=True, sep = ';')
# return -4
print('Loging ee out: ', time.strftime("%d.%m.%Y %H:%M:%S"))
ee.logout()
print('Finnished at: ', time.strftime("%d.%m.%Y %H:%M:%S"))
return 1
def landsat8_download(scenes, output_dir='./data'):
username = '******'
password = '******'
ee = EarthExplorer(username, password)
try:
for idx in range(len(scenes)):
ee.download(scene_id=scenes[idx]['entityId'],
output_dir=output_dir) # download with entityId
except KeyError:
ee.download(scene_id=scenes['entityId'], output_dir=output_dir)
except:
print('Please input a valid Landsat8 scene entityId')
ee.logout()
def download(username, password, dataset, output, timeout, skip, scenes):
"""Download one or several scenes."""
ee = EarthExplorer(username, password)
output_dir = os.path.abspath(output)
if dataset and dataset not in DATASETS:
raise LandsatxploreError(f"`{dataset}` is not a supported dataset.")
for scene in scenes:
if not ee.logged_in():
ee = EarthExplorer(username, password)
fname = ee.download(
scene, output_dir, dataset=dataset, timeout=timeout, skip=skip
)
if skip:
click.echo(fname)
ee.logout()
def download_merge_landsat():
'''
Check if all IDs have been downloaded and merged and do so if necessary.
Downloads landsat scenes to from landsat[n]/download/[Scene ID].tar
Outputs merged 3-band tiffs to landsat[n]/[roi]/merge-sr/[Scene ID]_SR_B[Band order].tif
'''
for ID in bases_list: # e.g. LT05_L2SP_070014_19850831_20200918_02_T1
## Switch params based on Landsat number
if 'LT05' in ID:
ls_dir_use = ls_dirs[0] # e.g. /mnt/gcs/landsat5
merge_band_order = ls_bands[0] # Use bands 423 for NGR
elif 'LC08' in ID:
ls_dir_use = ls_dirs[1]
merge_band_order = ls_bands[1] # Use bands 534 for NGR
else:
RuntimeError('EDK: unrecognized Landsat number')
## From here on, parse paths in this function, not in environment22.py, becasue they are specific to landsat number
ls_download_dir_use = os.path.join(ls_dir_use, 'download')
ls_tar_pth = os.path.join(
ls_download_dir_use, ID + '.tar'
) # e.g. /mnt/gcs/landsat5/download/LT05_L2SP_070014_19850831_20200918_02_T1.tar
if not os.path.exists(ls_tar_pth):
print(f'\nFile not present. Downloading: {ls_tar_pth}')
from landsatxplore.earthexplorer import EarthExplorer
ee = EarthExplorer('*****@*****.**', 'mcmurdo2030')
ee.download(ID, output_dir=ls_download_dir_use)
ee.logout()
## Create 3-band images using gdal (yes I know I'm calling a python function from command line from subprocess...) (and make COG)
# print('\n-----------------------------------\nCreating 3-band images')
region = file_inputs['bases'][ID][
'region'] # list(file_inputs['bases'][ID][0].keys())[0].replace('-roi','') # take first item in list to use as roi name # e.g. fairbanks
merge_sr_uint16_dir = os.path.join(
ls_dir_use, region,
'merge-sr') # e.g. /mnt/gcs/landsat5/fairbanks/merge-sr
print(region)
## GDAL MERGE
merge_sr_uint16_pth = f'{merge_sr_uint16_dir}/{ID}_SR_B{merge_band_order[0]}{merge_band_order[1]}{merge_band_order[2]}.tif' # AKA # three_band_full_pth in "rescale-landsat.py" # e.g. /mnt/gcs/landsat5/fairbanks/merge-sr/LT05_L2SP_070014_19850831_20200918_02_T1_SR_B423.tif
if not os.path.exists(merge_sr_uint16_pth):
os.makedirs(merge_sr_uint16_dir, exist_ok=True)
print(f'\nMerging into 3-band image: {merge_sr_uint16_pth}')
cmd = f'gdal_merge.py -co TILED=YES -co COMPRESS=LZW -seperate /vsitar/{ls_tar_pth}/{ID}_SR_B{merge_band_order[0]}.TIF /vsitar/{ls_tar_pth}/{ID}_SR_B{merge_band_order[1]}.TIF /vsitar/{ls_tar_pth}/{ID}_SR_B{merge_band_order[2]}.TIF -o {merge_sr_uint16_pth}'
check_print_output(cmd)
## COG dir # don't need for now
merge_sr_uint16_COG_dir = os.path.join(merge_sr_uint16_dir, 'COG')
merge_sr_uint16_COG_pth = os.path.join(
os.path.dirname(merge_sr_uint16_pth), 'COG',
os.path.basename(merge_sr_uint16_pth))
if not os.path.exists(merge_sr_uint16_COG_pth):
os.makedirs(os.path.dirname(merge_sr_uint16_COG_pth),
exist_ok=True)
print(f'\nMaking COG: {merge_sr_uint16_COG_pth}')
cmd = f'gdal_translate -of COG {merge_sr_uint16_pth} {merge_sr_uint16_COG_pth}'
check_print_output(cmd)
pass
class QCConnectLandsatDownload(QCConnectLandsat):
def __init__(self, username, password, archive, backup_archive):
from landsatxplore.earthexplorer import EarthExplorer
self.api = EarthExplorer(
username, password
)
def download_file(self, uuid, output_dir):
from landsatxplore.util import guess_dataset
from landsatxplore.earthexplorer import EE_DOWNLOAD_URL, DATASETS
self.api.download(
uuid, output_dir
)
# pseudo-checksum control
dataset = guess_dataset(uuid)
url = EE_DOWNLOAD_URL.format(dataset_id=DATASETS[dataset], scene_id=uuid)
with self.api.session.get(url, stream=True, allow_redirects=True) as r:
expected_filesize = int(r.headers['Content-Length'])
return expected_filesize
def downloading(self):
print('youpiiiii')
print(
self.dlg.tableResult.item(self.dlg.tableResult.currentRow(),
0).text())
var_id_product = self.dlg.tableResult.item(
self.dlg.tableResult.currentRow(), 0).text()
ee = EarthExplorer('Boutaina', 'boutaina123456789')
ee.download(scene_id=var_id_product, output_dir='/SatDownloaded/Sat')
ee.logout()
def __download_scene(self, scene_id, output_dir):
try:
ee = EarthExplorer(self.username, self.password)
ee.download(scene_id, output_dir)
ee.logout()
return scene_id, True
except Exception:
scene_file = f'{output_dir}/{scene_id}.tar.gz'
if os.path.exists(scene_file):
os.remove(scene_file)
return scene_id, False
def landsat8_download_entityId(entityId, output_dir='./data'):
"""
entityId: list type
"""
username = '******'
password = '******'
ee = EarthExplorer(username, password)
for idx in range(len(entityId)):
ee.download(scene_id=entityId[idx],
output_dir=output_dir) # download with entityId
ee.logout()
def main():
api = earthexpl_api_handler()
api.login(username, pswd)
api.scene_search(ll, ur, start_date, end_date)
print("Searching result")
latest_id = api.get_latest_displayID()
for scene in latest_id:
print(scene)
print("Try to dowload latest scence")
ee = EarthExplorer(username, pswd)
ee.download(scene_id=latest_id[0]['displayId'], output_dir=output_dir)
ee.logout()
def download_task(self,gis,ts):
lock(None,self,"\n"+gis+"\n"+ts)
coordinates = wkt.loads(gis)
coordinates=coordinates['coordinates'][0]
proj_in = pyproj.Proj("epsg:3857")
proj_out = pyproj.Proj("epsg:4326")
tuples=[]
for coord in coordinates:
#POTREBBE ESSERE NECESSARIO CONVERTIRE IN METRI LE COORDINATE PRIMA DI FARE TRANSFORM, DA ANALIZZARE PIU' ACCURATAMENTE
lon, lat = pyproj.transform(proj_in,proj_out,coord[0],coord[1])
tuples.append((lon,lat))
coordinates=tuples
print(coordinates)
gis = Polygon(coordinates)
stop_date = datetime.datetime.strptime(ts.split('T')[0], '%d/%m/%Y')
start_date = calculate_start_date(stop_date)
result,products_df_sorted,scenes_df_sorted,n_products,n_scenes=check(gis,start_date,stop_date)
path = os.getcwd()+"/"+constants.data_dir
if result==True:
release(None)
return res_ok("already downloaded",False,True)
num_record_sentinel, campi=products_df_sorted.shape
api = SentinelAPI(sentineluser, sentinelpwd, 'https://scihub.copernicus.eu/dhus')
for i in range (0,num_record_sentinel):
index=products_df_sorted['uuid'][i]
api.download(index, path, checksum=True)
num_record_landsat, campi=scenes_df_sorted.shape
for i in range (0,num_record_landsat):
index=scenes_df_sorted['entityId'][i]
ee = EarthExplorer(landsatuser, landsatpwd)
ee.download(scene_id=index, output_dir=path)
ee.logout()
release(None)
return res_ok("ok",False,True)
scenes = api.search(dataset='LANDSAT_8_C1',
latitude=row['lat'],
longitude=row['lon'],
start_date='2016-01-01',
end_date='2019-11-01',
max_cloud_cover=10)
print('{} scenes found.'.format(len(scenes)))
length = len(scenes)
place = row['place']
place_dir = os.path.join("landsat", f'{place}')
path = os.path.join(f'{os.getcwd()}', f'{place_dir}')
print(path)
if not os.path.exists(path):
os.makedirs(path)
while (length > 1):
ee = EarthExplorer(username, password)
ee.download(scene_id=scenes[length - 1]['displayId'],
output_dir=f'{path}')
obj = s3_resource.Object(
bucket_name="sabudh-displacement-data",
key=(f"landsat/{place}/{scenes[length-1]['displayId']}.tar.gz"))
obj.upload_file(f"{place_dir}/{scenes[length-1]['displayId']}.tar.gz")
os.remove(f"{place_dir}/{scenes[length-1]['displayId']}.tar.gz")
length = length - 1
for scene in combined_scenes:
time = scene['acquisitionDate']
if time[-5:] != "02-29":
scenedate = datetime.strptime(
time, '%Y-%m-%d').date().replace(year=today.year)
if datemin < scenedate < datemax:
selected_scenes.append(scene)
print('{} scenes found.'.format(len(selected_scenes)))
api.logout()
# Start downloading
ee = EarthExplorer("remoteSensingErsl", "ErslRemoteSensing00")
# Creating the "images" directory
images_dir = os.path.join(os.getcwd(), 'images')
if not os.path.isdir(images_dir):
os.mkdir(images_dir)
# Downloading scenes
print("Downloading selected scenes")
for scene in tqdm(selected_scenes):
year = scene['acquisitionDate'][:4]
scene_id = scene['entityId']
display_id = scene['displayId']
pathdir = os.path.join(images_dir, year)
if not os.path.isdir(pathdir):
# Iterate through list of scenes
for i in range(len(scenes)):
scene = scenes[i]
landsat_product_id = scene['landsat_product_id']
print("Processing " + landsat_product_id)
landsat_dir = os.path.join(output_dir, landsat_product_id)
zip_path = os.path.join(landsat_dir, landsat_product_id + ".tar.gz")
# Download from EarthExplorer
if os.path.exists(landsat_dir):
print("Folder exists for landsat_product_id. Skipping download.")
else:
print("Logging into EarthExplorer...")
password = keyring.get_password("EarthExplorer", username)
ee = EarthExplorer(username, password)
print("Downloading " + landsat_product_id + ". Imagery aquired ",
scene['acquisition_date'])
ee.download(landsat_product_id, output_dir=landsat_dir)
ee.logout()
# Extract tar.gz file and crop each band
sample_cropped = os.path.join(landsat_dir,
landsat_product_id + "_B1_cropped.TIF")
if os.path.exists(sample_cropped):
print("Cropped tif exists. Skipping extract and crop.")
else:
print("Extracting compressed file " + zip_path)
tar = tarfile.open(zip_path, "r:gz")
tar.extractall(landsat_dir)
tar.close()
import landsatxplore.api
from landsatxplore.earthexplorer import EarthExplorer
api = landsatxplore.api.API('******', '*******')
ee = EarthExplorer('******', '******')
# Request
scenes = api.search(dataset='LANDSAT_8_C1',
latitude=37.01,
longitude=-5.93,
start_date=m,
end_date=t,
max_cloud_cover=100)
print('{} scenes found.'.format(len(scenes)))
for scene in scenes:
sc = scene['displayId']
if scene['displayId'].split('_')[2] == '202034':
print('downloading:', sc)
ee.download(scene_id=sc, output_dir='/home/last/Descargas')
ee.logout()
api.logout()
import os
import arrow
import landsatxplore.api
from landsatxplore.earthexplorer import EarthExplorer
api = landsatxplore.api.API(os.environ.get('usgs_user'), os.environ.get('usgs_password'))
ee = EarthExplorer(os.environ.get('usgs_user'), os.environ.get('usgs_password'))
today = arrow.now().format('YYYY-MM-DD')
month = arrow.now().shift(days=-45).format('YYYY-MM-DD')
scenes_oli = api.search(
dataset='LANDSAT_8_C1',
latitude=37.47944444,
longitude=-6.25861111,
start_date=month,
end_date=today,
max_cloud_cover=100,
max_results=1000)
scenes_etm = api.search(
dataset='LANDSAT_ETM_C1',
latitude=37.47944444,
longitude=-6.25861111,
start_date=month,
end_date=today,
max_cloud_cover=100,
max_results=1000)
scenes_tm = api.search(
dataset='LANDSAT_TM_C1',
def ee():
return EarthExplorer(os.getenv("LANDSATXPLORE_USERNAME"),
os.getenv("LANDSATXPLORE_PASSWORD"))
def __init__(self, username, password, archive, backup_archive):
from landsatxplore.earthexplorer import EarthExplorer
self.api = EarthExplorer(
username, password
)
def main():
user = password = None
if options["settings"] == "-":
# stdin
import getpass
user = raw_input(_("Insert username: "******"Insert password: "******"settings"], "r") as fd:
lines = list(filter(None,
(line.rstrip()
for line in fd))) # non-blank lines only
if len(lines) < 2:
gs.fatal(_("Invalid settings file"))
user = lines[0].strip()
password = lines[1].strip()
except IOError as e:
gs.fatal(_("Unable to open settings file: {}").format(e))
landsat_api = landsatxplore.api.API(user, password)
if user is None or password is None:
gs.fatal(_("No user or password given"))
start_date = options["start"]
delta_days = timedelta(60)
if not options["start"]:
start_date = date.today() - delta_days
start_date = start_date.strftime("%Y-%m-%d")
end_date = options["end"]
if not options["end"]:
end_date = date.today().strftime("%Y-%m-%d")
# outdir = ""
if options["output"]:
outdir = options["output"]
if os.path.isdir(outdir):
if not os.access(outdir, os.W_OK):
gs.fatal(
_("Output directory <{}> is not writable").format(outdir))
else:
gs.fatal(
_("Output directory <{}> is not a directory").format(outdir))
else:
outdir = "/tmp"
# Download by ID
if options["id"]:
ids = options["id"].split(",")
ee = EarthExplorer(user, password)
for i in ids:
try:
ee.download(scene_id=i, output_dir=outdir)
except OSError:
gs.fatal(_("Scene ID <{}> not valid or not found").format(i))
ee.logout()
else:
bb = get_bb(options["map"])
# List available scenes
scenes = landsat_api.search(dataset=options["dataset"],
bbox=bb,
start_date=start_date,
end_date=end_date,
max_cloud_cover=options["clouds"],
max_results=50)
if options["tier"]:
scenes = list(
filter(lambda s: options["tier"] in s["displayId"], scenes))
# Output number of scenes found
gs.message(_("{} scenes found.".format(len(scenes))))
sort_vars = options["sort"].split(",")
reverse = False
if options["order"] == "desc":
reverse = True
# Sort scenes
sorted_scenes = sorted(scenes,
key=lambda i:
(i[sort_vars[0]], i[sort_vars[1]]),
reverse=reverse)
landsat_api.logout()
if flags["l"]:
# Output sorted list of scenes found
# print('ID', 'DisplayID', 'Date', 'Clouds')
for scene in sorted_scenes:
print(
scene["entityId"],
scene["displayId"],
scene["acquisitionDate"],
scene["cloudCover"],
)
gs.message(
_("To download all scenes found, re-run the previous "
"command without -l flag. Note that if no output "
"option is provided, files will be downloaded in /tmp"))
else:
ee = EarthExplorer(user, password)
for scene in sorted_scenes:
gs.message(
_("Downloading scene <{}> ...").format(scene["entityId"]))
ee.download(scene_id=scene["entityId"], output_dir=outdir)
ee.logout()
def test_ee_login_error():
with pytest.raises(EarthExplorerError):
EarthExplorer("bad_username", "bad_password")
#
## Perform a request. Results are returned in a dictionnary
#response = api.request('<request_code>', parameter1=value1, parameter2=value2)
#
## Log out
#api.logout()
#import landsatxplore.api
# Initialize a new API instance and get an access key
api = landsatxplore.api.API('username', 'password')
# Request
scenes = api.search(dataset='LANDSAT_ETM_C1',
latitude=19.53,
longitude=-1.53,
start_date='1995-01-01',
end_date='1997-01-01',
max_cloud_cover=10)
print('{} scenes found.'.format(len(scenes)))
for scene in scenes:
print(scene['acquisitionDate'])
api.logout()
ee = EarthExplorer('username', 'password')
ee.download(scene_id='LT51960471995178MPS00', output_dir='./data')
ee.logout()
def download(self, output, sleep=False, maxretry=False,
datasource='ESA_COAH'):
if self._products_df_sorted is None:
return
create_dir(output)
gs.message(_("Downloading data into <{}>...").format(output))
if datasource == 'USGS_EE':
from landsatxplore.earthexplorer import EarthExplorer
from landsatxplore.errors import EarthExplorerError
from zipfile import ZipFile
ee_login = False
while ee_login is False:
# avoid login conflict in possible parallel execution
try:
ee = EarthExplorer(self._user, self._password)
ee_login = True
except EarthExplorerError as e:
time.sleep(1)
for idx in range(len(self._products_df_sorted['entity_id'])):
scene = self._products_df_sorted['entity_id'][idx]
identifier = self._products_df_sorted['display_id'][idx]
zip_file = os.path.join(output, '{}.zip'.format(identifier))
gs.message(_("Downloading {}...").format(identifier))
try:
ee.download(identifier=identifier, output_dir=output, timeout=600)
except EarthExplorerError as e:
gs.fatal(_(e))
ee.logout()
# extract .zip to get "usual" .SAFE
with ZipFile(zip_file, 'r') as zip:
safe_name = zip.namelist()[0].split('/')[0]
outpath = os.path.join(output, safe_name)
zip.extractall(path=output)
gs.message(_("Downloaded to <{}>").format(outpath))
try:
os.remove(zip_file)
except Exception as e:
gs.warning(_("Unable to remove {0}:{1}").format(
zip_file, e))
elif datasource == "ESA_COAH":
for idx in range(len(self._products_df_sorted['uuid'])):
gs.message('{} -> {}.SAFE'.format(
self._products_df_sorted['uuid'][idx],
os.path.join(output, self._products_df_sorted['identifier'][idx])
))
# download
out = self._api.download(self._products_df_sorted['uuid'][idx],
output)
if sleep:
x = 1
online = out['Online']
while not online:
# sleep is in minutes so multiply by 60
time.sleep(int(sleep) * 60)
out = self._api.download(self._products_df_sorted['uuid'][idx],
output)
x += 1
if x > maxretry:
online = True
elif datasource == 'GCS':
for scene_id in self._products_df_sorted['identifier']:
gs.message(_("Downloading {}...").format(scene_id))
dl_code = download_gcs(scene_id, output)
if dl_code == 0:
gs.message(_("Downloaded to {}").format(
os.path.join(output, '{}.SAFE'.format(scene_id))))
else:
# remove incomplete file
del_folder = os.path.join(output,
'{}.SAFE'.format(scene_id))
try:
shutil.rmtree(del_folder)
except Exception as e:
gs.warning(_("Unable to removed unfinished "
"download {}".format(del_folder)))
def download(self, output, sleep=False, maxretry=False):
if self._products_df_sorted is None:
return
if not os.path.exists(output):
os.makedirs(output)
gs.message(_('Downloading data into <{}>...').format(output))
if self._apiname == 'USGS_EE':
from landsatxplore.earthexplorer import EarthExplorer
#from landsatxplore.exceptions import EarthExplorerError
from landsatxplore.errors import EarthExplorerError
from zipfile import ZipFile
ee_login = False
while ee_login is False:
# avoid login conflict in possible parallel execution
try:
ee = EarthExplorer(self._user, self._password)
ee_login = True
except EarthExplorerError as e:
time.sleep(1)
for idx in range(len(self._products_df_sorted['entity_id'])):
scene = self._products_df_sorted['entity_id'][idx]
identifier = self._products_df_sorted['display_id'][idx]
zip_file = os.path.join(output, '{}.zip'.format(identifier))
gs.message('Downloading {}...'.format(identifier))
try:
ee.download(identifier=identifier,
output_dir=output,
timeout=600)
except EarthExplorerError as e:
gs.fatal(_(e))
ee.logout()
# extract .zip to get "usual" .SAFE
with ZipFile(zip_file, 'r') as zip:
safe_name = zip.namelist()[0].split('/')[0]
outpath = os.path.join(output, safe_name)
zip.extractall(path=output)
gs.message(_('Downloaded to <{}>').format(outpath))
try:
os.remove(zip_file)
except Exception as e:
gs.warning(
_('Unable to remove {0}:{1}').format(zip_file, e))
else:
for idx in range(len(self._products_df_sorted['uuid'])):
gs.message('{} -> {}.SAFE'.format(
self._products_df_sorted['uuid'][idx],
os.path.join(output,
self._products_df_sorted['identifier'][idx])))
# download
out = self._api.download(self._products_df_sorted['uuid'][idx],
output)
if sleep:
x = 1
online = out['Online']
while not online:
# sleep is in minutes so multiply by 60
time.sleep(int(sleep) * 60)
out = self._api.download(
self._products_df_sorted['uuid'][idx], output)
x += 1
if x > maxretry:
online = True
