def get_local_filename(self, url): """ Returns the local file name under which to save a remote image (at URL url). By default this is Util.get_local_name(url), but subclasses can override. :param url: the URL of the image :return: the full local path, under the current target_folder """ return Util.get_local_name(url)
def get_local_filename(self, url): """ Returns the local file path where to save a remote image (at URL url) :param url: the URL of the image :return: the full local path, under the current target_folder """ if self.target_folder is None: raise Exception( 'update_download_folder was not called before downloading') return os.path.join(self.target_folder, Util.get_local_name(url))
def test_get_local_name(self): self.assertEqual("img.jpg", Util.get_local_name("http://example.com/a/img?a=b")) self.assertEqual("img.jpg", Util.get_local_name("http://example.com/a/img.jpg?a=b")) self.assertEqual("img.jpg", Util.get_local_name("http://example.com/a/img.jpg#x")) self.assertEqual("img.jpg", Util.get_local_name("http://example.com/a/img.jpg?a=b#x")) self.assertEqual("im g.jpg", Util.get_local_name("http://example.com/a/im%20g.jpg?a=b#x")) self.assertEqual("im_g.jpg", Util.get_local_name("http://example.com/a/im%22g.jpg?a=b#x")) self.assertTrue(len(Util.get_local_name("a" * 1000 + ".jpg")) < 255)
def fetch( url, to_folder, origin_url=None, source_type=None, source_location=None, source_name=None, extra_metadata=None, progress_reporter=lambda a, b: None, verbose=True, ): reported = verbose try: logger.info(lambda: "Trying to fetch URL %s to %s " % (url, to_folder)) if verbose: progress_reporter(_("Fetching"), url) if url.startswith("javascript:"): if verbose: progress_reporter(_("Not an image"), url) return None if url.find("://") < 0: url = "file://" + url r = Util.request(url, stream=True) if not "content-type" in r.headers: logger.info(lambda: "Unknown content-type for url " + url) if verbose: progress_reporter(_("Not an image"), url) return None ct = r.headers["content-type"] if not ct.startswith("image/"): logger.info(lambda: "Unsupported content-type for url " + url + ": " + ct) if verbose: progress_reporter(_("Not an image"), url) return None local_name = Util.get_local_name(r.url) if "content-disposition" in r.headers: cd = r.headers["content-disposition"] cd_name = ImageFetcher.extract_filename_from_content_disposition( cd) if cd_name: local_name = cd_name filename = os.path.join(to_folder, local_name) if os.path.exists(filename): m = Util.read_metadata(filename) if m and m.get("imageURL") == url: logger.info( lambda: "Local file already exists (%s)" % filename) return filename else: logger.info( lambda: "File with same name already exists, but from different imageURL; renaming new download" ) filename = Util.find_unique_name(filename) logger.info(lambda: "Fetching to " + filename) if not reported: reported = True progress_reporter(_("Fetching"), url) local_filepath_partial = filename + ".partial" with open(local_filepath_partial, "wb") as f: Util.request_write_to(r, f) try: img = Image.open(local_filepath_partial) except Exception: progress_reporter(_("Not an image"), url) Util.safe_unlink(local_filepath_partial) return None if img.size[0] < 400 or img.size[1] < 400: # too small - delete and do not use progress_reporter(_("Image too small, ignoring it"), url) Util.safe_unlink(local_filepath_partial) return None metadata = { "sourceType": source_type or "fetched", "sourceName": source_name or "Fetched", "sourceURL": origin_url or url, "imageURL": url, } if source_location: metadata["sourceLocation"] = source_location metadata.update(extra_metadata or {}) Util.write_metadata(local_filepath_partial, metadata) os.rename(local_filepath_partial, filename) logger.info(lambda: "Fetched %s to %s." % (url, filename)) return filename except Exception as e: # pylint: disable=no-member logger.exception(lambda: "Fetch failed for URL " + url) if reported: if isinstance( e, HTTPError) and e.response.status_code in (403, 404): progress_reporter( _("Sorry, got %s error...") % str(e.response.status_code), _("This means the link is no longer valid"), ) else: progress_reporter( _("Fetch failed for some reason"), _("To get more information, please run Variety from terminal with -v option and retry the action" ), ) return None
def is_in_favorites(self, url): return self.get_variety() and os.path.exists( os.path.join(self.get_variety().options.favorites_folder, Util.get_local_name(url)))
def get_local_filename(self, url): return os.path.join(self.target_folder, Util.get_local_name(url))
def is_in_favorites(self, url): return self.parent and os.path.exists(os.path.join(self.parent.options.favorites_folder, Util.get_local_name(url)))
def fetch(url, to_folder, origin_url=None, source_type=None, source_location=None, source_name=None, extra_metadata={}, progress_reporter=lambda a, b: None, verbose=True): reported = verbose try: logger.info(lambda: "Trying to fetch URL %s to %s " % (url, to_folder)) if verbose: progress_reporter(_("Fetching"), url) if url.startswith('javascript:'): if verbose: progress_reporter(_("Not an image"), url) return None if url.find('://') < 0: url = "file://" + url r = Util.request(url, stream=True) if not "content-type" in r.headers: logger.info(lambda: "Unknown content-type for url " + url) if verbose: progress_reporter(_("Not an image"), url) return None ct = r.headers["content-type"] if not ct.startswith("image/"): logger.info(lambda: "Unsupported content-type for url " + url + ": " + ct) if verbose: progress_reporter(_("Not an image"), url) return None local_name = Util.get_local_name(r.url) if "content-disposition" in r.headers: cd = r.headers["content-disposition"] cd_name = ImageFetcher.extract_filename_from_content_disposition(cd) if cd_name: local_name = cd_name filename = os.path.join(to_folder, local_name) if os.path.exists(filename): m = Util.read_metadata(filename) if m and m.get("imageURL") == url: logger.info(lambda: "Local file already exists (%s)" % filename) return filename else: logger.info(lambda: "File with same name already exists, but from different imageURL; renaming new download") filename = Util.find_unique_name(filename) local_name = os.path.basename(filename) logger.info(lambda: "Fetching to " + filename) if not reported: reported = True progress_reporter(_("Fetching"), url) with open(filename, 'wb') as f: Util.request_write_to(r, f) try: img = Image.open(filename) except Exception: progress_reporter(_("Not an image"), url) os.unlink(filename) return None if img.size[0] < 400 or img.size[1] < 400: # too small - delete and do not use progress_reporter(_("Image too small, ignoring it"), url) os.unlink(filename) return None metadata = {"sourceType": source_type or 'fetched', "sourceName": source_name or "Fetched", "sourceURL": origin_url or url, "imageURL": url} if source_location: metadata["sourceLocation"] = source_location metadata.update(extra_metadata) Util.write_metadata(filename, metadata) logger.info(lambda: "Fetched %s to %s." % (url, filename)) return filename except Exception, e: logger.exception(lambda: "Fetch failed for URL " + url) if reported: if isinstance(e, HTTPError) and e.response.status_code in (403, 404): progress_reporter( _("Sorry, got %s error...") % str(e.response.status_code), _("This means the link is no longer valid")) else: progress_reporter( _("Fetch failed for some reason"), _("To get more information, please run Variety from terminal with -v option and retry the action")) return None