def get_quicklook(self,
filename=None,
base_dir=None,
progress_callback=None):
"""Download the quick look image of a given EOProduct from its provider if it
exists.
:param filename: (optional) the name to give to the downloaded quicklook.
:type filename: str
:param base_dir: (optional) the absolute path of the directory where to store
the quicklooks in the filesystem. If it is not given, it
defaults to the `quicklooks` directory under this EO product
downloader's ``outputs_prefix`` config param
:type base_dir: str
:param progress_callback: (optional) A method or a callable object
which takes a current size and a maximum
size as inputs and handle progress bar
creation and update to give the user a
feedback on the download progress
:type progress_callback: :class:`~eodag.utils.ProgressCallback` or None
:returns: The absolute path of the downloaded quicklook
:rtype: str
.. versionchanged::
1.0
* Added the ``base_dir`` optional parameter to choose where to download
the retrieved quicklook
"""
def format_quicklook_address():
"""If the quicklook address is a Python format string, resolve the
formatting with the properties of the product."""
fstrmatch = re.match(r".*{.+}*.*", self.properties["quicklook"])
if fstrmatch:
self.properties["quicklook"].format({
prop_key: prop_val
for prop_key, prop_val in self.properties.items()
if prop_key != "quicklook"
})
if progress_callback is None:
progress_callback = ProgressCallback()
if self.properties["quicklook"] is None:
logger.warning(
"Missing information to retrieve quicklook for EO product: %s",
self.properties["id"],
)
return ""
format_quicklook_address()
if base_dir is not None:
quicklooks_base_dir = os.path.abspath(os.path.realpath(base_dir))
else:
quicklooks_base_dir = os.path.join(
self.downloader.config.outputs_prefix, "quicklooks")
if not os.path.isdir(quicklooks_base_dir):
os.makedirs(quicklooks_base_dir)
quicklook_file = os.path.join(
quicklooks_base_dir,
filename if filename is not None else self.properties["id"],
)
if not os.path.isfile(quicklook_file):
# VERY SPECIAL CASE (introduced by the onda provider): first check if
# it is a HTTP URL. If not, we assume it is a byte string, in which case
# we just write the content into the quicklook_file and return it.
if not (self.properties["quicklook"].startswith("http")
or self.properties["quicklook"].startswith("https")):
with open(quicklook_file, "wb") as fd:
fd.write(self.properties["quicklook"])
return quicklook_file
auth = (self.downloader_auth.authenticate()
if self.downloader_auth is not None else None)
with requests.get(self.properties["quicklook"],
stream=True,
auth=auth) as stream:
try:
stream.raise_for_status()
except HTTPError as e:
import traceback as tb
logger.error("Error while getting resource :\n%s",
tb.format_exc())
return str(e)
else:
stream_size = int(stream.headers.get("content-length", 0))
with open(quicklook_file, "wb") as fhandle:
for chunk in stream.iter_content(chunk_size=64 * 1024):
if chunk:
fhandle.write(chunk)
progress_callback(len(chunk), stream_size)
logger.info("Download recorded in %s", quicklook_file)
return quicklook_file
def download(self,
progress_callback=None,
wait=DEFAULT_DOWNLOAD_WAIT,
timeout=DEFAULT_DOWNLOAD_TIMEOUT,
**kwargs):
"""Download the EO product using the provided download plugin and the
authenticator if necessary.
The actual download of the product occurs only at the first call of this
method. A side effect of this method is that it changes the ``location``
attribute of an EOProduct, from its remote address to the local address.
:param progress_callback: (optional) A method or a callable object
which takes a current size and a maximum
size as inputs and handle progress bar
creation and update to give the user a
feedback on the download progress
:type progress_callback: :class:`~eodag.utils.ProgressCallback` or None
:param wait: (optional) If download fails, wait time in minutes between
two download tries (default=2')
:type wait: int
:param timeout: (optional) If download fails, maximum time in minutes
before stop retrying to download (default=20')
:type timeout: int
:param dict kwargs: `outputs_prefix` (str), `extract` (bool) and
`dl_url_params` (dict) can be provided as additional kwargs
and will override any other values defined in a configuration
file or with environment variables.
:returns: The absolute path to the downloaded product on the local filesystem
:rtype: str
:raises: :class:`~eodag.utils.exceptions.PluginImplementationError`
:raises: :class:`RuntimeError`
"""
if progress_callback is None:
progress_callback = ProgressCallback()
if self.downloader is None:
raise RuntimeError(
"EO product is unable to download itself due to lacking of a "
"download plugin")
auth = (self.downloader_auth.authenticate()
if self.downloader_auth is not None else self.downloader_auth)
fs_location = self.downloader.download(
self,
auth=auth,
progress_callback=progress_callback,
wait=wait,
timeout=timeout,
**kwargs)
if fs_location is None:
raise DownloadError(
"Invalid file location returned by download process: '{}'".
format(fs_location))
self.location = "file://{}".format(fs_location)
logger.debug(
"Product location updated from '%s' to '%s'",
self.remote_location,
self.location,
)
logger.info(
"Remote location of the product is still available through its "
"'remote_location' property: %s",
self.remote_location,
)
return self.location
def downloadProducts(descDict):
products, _ = dag.search(**descDict)
path = dag.download(products[0], progress_callback=ProgressCallback())
def download_all(
self,
products,
auth=None,
progress_callback=None,
wait=DEFAULT_DOWNLOAD_WAIT,
timeout=DEFAULT_DOWNLOAD_TIMEOUT,
):
"""
A sequential download_all implementation
using download method for every products
"""
paths = []
# initiate retry loop
start_time = datetime.now()
stop_time = datetime.now() + timedelta(minutes=timeout)
nb_products = len(products)
retry_count = 0
# another output for notbooks
nb_info = NotebookWidgets()
for product in products:
product.next_try = start_time
with tqdm(total=len(products),
unit="product",
desc="Downloading products") as bar:
while "Loop until all products are download or timeout is reached":
# try downloading each product before retry
for idx, product in enumerate(products):
if datetime.now() >= product.next_try:
products[idx].next_try += timedelta(minutes=wait)
try:
if progress_callback is None:
progress_callback = ProgressCallback()
if product.downloader is None:
raise RuntimeError(
"EO product is unable to download itself due to lacking of a "
"download plugin")
auth = (product.downloader_auth.authenticate()
if product.downloader_auth is not None else
product.downloader_auth)
# resolve remote location if needed with downloader configuration
product.remote_location = product.remote_location % vars(
product.downloader.config)
paths.append(
self.download(
product,
auth=auth,
progress_callback=progress_callback,
wait=wait,
timeout=-1,
))
# product downloaded, to not retry it
products.remove(product)
bar.update(1)
# reset stop time for next product
stop_time = datetime.now() + timedelta(
minutes=timeout)
except NotAvailableError as e:
logger.info(e)
continue
except RuntimeError:
import traceback as tb
logger.error(
"A problem occurred during download of product: %s. "
"Skipping it",
product,
)
logger.debug("\n%s", tb.format_exc())
stop_time = datetime.now()
except Exception:
import traceback as tb
logger.warning(
"A problem occurred during download of product: %s. "
"Skipping it",
product,
)
logger.debug("\n%s", tb.format_exc())
if (len(products) > 0 and datetime.now() < products[0].next_try
and datetime.now() < stop_time):
wait_seconds = (products[0].next_try -
datetime.now()).seconds
retry_count += 1
info_message = (
"[Retry #%s, %s/%s D/L] Waiting %ss until next download try (retry every %s' for %s')"
% (
retry_count,
(nb_products - len(products)),
nb_products,
wait_seconds,
wait,
timeout,
))
logger.info(info_message)
nb_info.display_html(info_message)
sleep(wait_seconds + 1)
elif len(products) > 0 and datetime.now() >= stop_time:
logger.warning(
"%s products could not be downloaded: %s",
len(products),
[prod.properties["title"] for prod in products],
)
break
elif len(products) == 0:
break
return paths
def download(
self,
progress_callback=None,
wait=DEFAULT_DOWNLOAD_WAIT,
timeout=DEFAULT_DOWNLOAD_TIMEOUT,
):
"""Download the EO product using the provided download plugin and the
authenticator if necessary.
The actual download of the product occurs only at the first call of this
method. A side effect of this method is that it changes the ``location``
attribute of an EOProduct, from its remote address to the local address.
:param progress_callback: (optional) A method or a callable object
which takes a current size and a maximum
size as inputs and handle progress bar
creation and update to give the user a
feedback on the download progress
:type progress_callback: :class:`~eodag.utils.ProgressCallback` or None
:returns: The absolute path to the downloaded product on the local filesystem
:rtype: str or unicode
:raises: :class:`~eodag.utils.exceptions.PluginImplementationError`
:raises: :class:`RuntimeError`
"""
if progress_callback is None:
progress_callback = ProgressCallback()
if self.downloader is None:
raise RuntimeError(
"EO product is unable to download itself due to lacking of a "
"download plugin"
)
auth = (
self.downloader_auth.authenticate()
if self.downloader_auth is not None
else self.downloader_auth
)
fs_location = self.downloader.download(
self,
auth=auth,
progress_callback=progress_callback,
wait=wait,
timeout=timeout,
)
if fs_location is None:
raise DownloadError(
"Invalid file location returned by download process: '{}'".format(
fs_location
)
)
self.location = "file://{}".format(fs_location)
logger.debug(
"Product location updated from '%s' to '%s'",
self.remote_location,
self.location,
)
logger.info(
"Remote location of the product is still available through its "
"'remote_location' property: %s",
self.remote_location,
)
return self.location