def get_quicklook(self, filename=None, base_dir=None, progress_callback=None): """Download the quick look image of a given EOProduct from its provider if it exists. :param filename: (optional) the name to give to the downloaded quicklook. :type filename: str :param base_dir: (optional) the absolute path of the directory where to store the quicklooks in the filesystem. If it is not given, it defaults to the `quicklooks` directory under this EO product downloader's ``outputs_prefix`` config param :type base_dir: str :param progress_callback: (optional) A method or a callable object which takes a current size and a maximum size as inputs and handle progress bar creation and update to give the user a feedback on the download progress :type progress_callback: :class:`~eodag.utils.ProgressCallback` or None :returns: The absolute path of the downloaded quicklook :rtype: str .. versionchanged:: 1.0 * Added the ``base_dir`` optional parameter to choose where to download the retrieved quicklook """ def format_quicklook_address(): """If the quicklook address is a Python format string, resolve the formatting with the properties of the product.""" fstrmatch = re.match(r".*{.+}*.*", self.properties["quicklook"]) if fstrmatch: self.properties["quicklook"].format({ prop_key: prop_val for prop_key, prop_val in self.properties.items() if prop_key != "quicklook" }) if progress_callback is None: progress_callback = ProgressCallback() if self.properties["quicklook"] is None: logger.warning( "Missing information to retrieve quicklook for EO product: %s", self.properties["id"], ) return "" format_quicklook_address() if base_dir is not None: quicklooks_base_dir = os.path.abspath(os.path.realpath(base_dir)) else: quicklooks_base_dir = os.path.join( self.downloader.config.outputs_prefix, "quicklooks") if not os.path.isdir(quicklooks_base_dir): os.makedirs(quicklooks_base_dir) quicklook_file = os.path.join( quicklooks_base_dir, filename if filename is not None else self.properties["id"], ) if not os.path.isfile(quicklook_file): # VERY SPECIAL CASE (introduced by the onda provider): first check if # it is a HTTP URL. If not, we assume it is a byte string, in which case # we just write the content into the quicklook_file and return it. if not (self.properties["quicklook"].startswith("http") or self.properties["quicklook"].startswith("https")): with open(quicklook_file, "wb") as fd: fd.write(self.properties["quicklook"]) return quicklook_file auth = (self.downloader_auth.authenticate() if self.downloader_auth is not None else None) with requests.get(self.properties["quicklook"], stream=True, auth=auth) as stream: try: stream.raise_for_status() except HTTPError as e: import traceback as tb logger.error("Error while getting resource :\n%s", tb.format_exc()) return str(e) else: stream_size = int(stream.headers.get("content-length", 0)) with open(quicklook_file, "wb") as fhandle: for chunk in stream.iter_content(chunk_size=64 * 1024): if chunk: fhandle.write(chunk) progress_callback(len(chunk), stream_size) logger.info("Download recorded in %s", quicklook_file) return quicklook_file
def download(self, progress_callback=None, wait=DEFAULT_DOWNLOAD_WAIT, timeout=DEFAULT_DOWNLOAD_TIMEOUT, **kwargs): """Download the EO product using the provided download plugin and the authenticator if necessary. The actual download of the product occurs only at the first call of this method. A side effect of this method is that it changes the ``location`` attribute of an EOProduct, from its remote address to the local address. :param progress_callback: (optional) A method or a callable object which takes a current size and a maximum size as inputs and handle progress bar creation and update to give the user a feedback on the download progress :type progress_callback: :class:`~eodag.utils.ProgressCallback` or None :param wait: (optional) If download fails, wait time in minutes between two download tries (default=2') :type wait: int :param timeout: (optional) If download fails, maximum time in minutes before stop retrying to download (default=20') :type timeout: int :param dict kwargs: `outputs_prefix` (str), `extract` (bool) and `dl_url_params` (dict) can be provided as additional kwargs and will override any other values defined in a configuration file or with environment variables. :returns: The absolute path to the downloaded product on the local filesystem :rtype: str :raises: :class:`~eodag.utils.exceptions.PluginImplementationError` :raises: :class:`RuntimeError` """ if progress_callback is None: progress_callback = ProgressCallback() if self.downloader is None: raise RuntimeError( "EO product is unable to download itself due to lacking of a " "download plugin") auth = (self.downloader_auth.authenticate() if self.downloader_auth is not None else self.downloader_auth) fs_location = self.downloader.download( self, auth=auth, progress_callback=progress_callback, wait=wait, timeout=timeout, **kwargs) if fs_location is None: raise DownloadError( "Invalid file location returned by download process: '{}'". format(fs_location)) self.location = "file://{}".format(fs_location) logger.debug( "Product location updated from '%s' to '%s'", self.remote_location, self.location, ) logger.info( "Remote location of the product is still available through its " "'remote_location' property: %s", self.remote_location, ) return self.location
def downloadProducts(descDict): products, _ = dag.search(**descDict) path = dag.download(products[0], progress_callback=ProgressCallback())
def download_all( self, products, auth=None, progress_callback=None, wait=DEFAULT_DOWNLOAD_WAIT, timeout=DEFAULT_DOWNLOAD_TIMEOUT, ): """ A sequential download_all implementation using download method for every products """ paths = [] # initiate retry loop start_time = datetime.now() stop_time = datetime.now() + timedelta(minutes=timeout) nb_products = len(products) retry_count = 0 # another output for notbooks nb_info = NotebookWidgets() for product in products: product.next_try = start_time with tqdm(total=len(products), unit="product", desc="Downloading products") as bar: while "Loop until all products are download or timeout is reached": # try downloading each product before retry for idx, product in enumerate(products): if datetime.now() >= product.next_try: products[idx].next_try += timedelta(minutes=wait) try: if progress_callback is None: progress_callback = ProgressCallback() if product.downloader is None: raise RuntimeError( "EO product is unable to download itself due to lacking of a " "download plugin") auth = (product.downloader_auth.authenticate() if product.downloader_auth is not None else product.downloader_auth) # resolve remote location if needed with downloader configuration product.remote_location = product.remote_location % vars( product.downloader.config) paths.append( self.download( product, auth=auth, progress_callback=progress_callback, wait=wait, timeout=-1, )) # product downloaded, to not retry it products.remove(product) bar.update(1) # reset stop time for next product stop_time = datetime.now() + timedelta( minutes=timeout) except NotAvailableError as e: logger.info(e) continue except RuntimeError: import traceback as tb logger.error( "A problem occurred during download of product: %s. " "Skipping it", product, ) logger.debug("\n%s", tb.format_exc()) stop_time = datetime.now() except Exception: import traceback as tb logger.warning( "A problem occurred during download of product: %s. " "Skipping it", product, ) logger.debug("\n%s", tb.format_exc()) if (len(products) > 0 and datetime.now() < products[0].next_try and datetime.now() < stop_time): wait_seconds = (products[0].next_try - datetime.now()).seconds retry_count += 1 info_message = ( "[Retry #%s, %s/%s D/L] Waiting %ss until next download try (retry every %s' for %s')" % ( retry_count, (nb_products - len(products)), nb_products, wait_seconds, wait, timeout, )) logger.info(info_message) nb_info.display_html(info_message) sleep(wait_seconds + 1) elif len(products) > 0 and datetime.now() >= stop_time: logger.warning( "%s products could not be downloaded: %s", len(products), [prod.properties["title"] for prod in products], ) break elif len(products) == 0: break return paths
def download( self, progress_callback=None, wait=DEFAULT_DOWNLOAD_WAIT, timeout=DEFAULT_DOWNLOAD_TIMEOUT, ): """Download the EO product using the provided download plugin and the authenticator if necessary. The actual download of the product occurs only at the first call of this method. A side effect of this method is that it changes the ``location`` attribute of an EOProduct, from its remote address to the local address. :param progress_callback: (optional) A method or a callable object which takes a current size and a maximum size as inputs and handle progress bar creation and update to give the user a feedback on the download progress :type progress_callback: :class:`~eodag.utils.ProgressCallback` or None :returns: The absolute path to the downloaded product on the local filesystem :rtype: str or unicode :raises: :class:`~eodag.utils.exceptions.PluginImplementationError` :raises: :class:`RuntimeError` """ if progress_callback is None: progress_callback = ProgressCallback() if self.downloader is None: raise RuntimeError( "EO product is unable to download itself due to lacking of a " "download plugin" ) auth = ( self.downloader_auth.authenticate() if self.downloader_auth is not None else self.downloader_auth ) fs_location = self.downloader.download( self, auth=auth, progress_callback=progress_callback, wait=wait, timeout=timeout, ) if fs_location is None: raise DownloadError( "Invalid file location returned by download process: '{}'".format( fs_location ) ) self.location = "file://{}".format(fs_location) logger.debug( "Product location updated from '%s' to '%s'", self.remote_location, self.location, ) logger.info( "Remote location of the product is still available through its " "'remote_location' property: %s", self.remote_location, ) return self.location