def get_local_filename(self, url):
"""
Returns the local file name under which to save a remote image (at URL url).
By default this is Util.get_local_name(url), but subclasses can override.
:param url: the URL of the image
:return: the full local path, under the current target_folder
"""
return Util.get_local_name(url)
def get_local_filename(self, url):
"""
Returns the local file path where to save a remote image (at URL url)
:param url: the URL of the image
:return: the full local path, under the current target_folder
"""
if self.target_folder is None:
raise Exception(
'update_download_folder was not called before downloading')
return os.path.join(self.target_folder, Util.get_local_name(url))
def test_get_local_name(self):
self.assertEqual("img.jpg", Util.get_local_name("http://example.com/a/img?a=b"))
self.assertEqual("img.jpg", Util.get_local_name("http://example.com/a/img.jpg?a=b"))
self.assertEqual("img.jpg", Util.get_local_name("http://example.com/a/img.jpg#x"))
self.assertEqual("img.jpg", Util.get_local_name("http://example.com/a/img.jpg?a=b#x"))
self.assertEqual("im g.jpg", Util.get_local_name("http://example.com/a/im%20g.jpg?a=b#x"))
self.assertEqual("im_g.jpg", Util.get_local_name("http://example.com/a/im%22g.jpg?a=b#x"))
self.assertTrue(len(Util.get_local_name("a" * 1000 + ".jpg")) < 255)
def test_get_local_name(self):
self.assertEqual("img.jpg", Util.get_local_name("http://example.com/a/img?a=b"))
self.assertEqual("img.jpg", Util.get_local_name("http://example.com/a/img.jpg?a=b"))
self.assertEqual("img.jpg", Util.get_local_name("http://example.com/a/img.jpg#x"))
self.assertEqual("img.jpg", Util.get_local_name("http://example.com/a/img.jpg?a=b#x"))
self.assertEqual("im g.jpg", Util.get_local_name("http://example.com/a/im%20g.jpg?a=b#x"))
self.assertEqual("im_g.jpg", Util.get_local_name("http://example.com/a/im%22g.jpg?a=b#x"))
self.assertTrue(len(Util.get_local_name("a" * 1000 + ".jpg")) < 255)
def fetch(
url,
to_folder,
origin_url=None,
source_type=None,
source_location=None,
source_name=None,
extra_metadata=None,
progress_reporter=lambda a, b: None,
verbose=True,
):
reported = verbose
try:
logger.info(lambda: "Trying to fetch URL %s to %s " %
(url, to_folder))
if verbose:
progress_reporter(_("Fetching"), url)
if url.startswith("javascript:"):
if verbose:
progress_reporter(_("Not an image"), url)
return None
if url.find("://") < 0:
url = "file://" + url
r = Util.request(url, stream=True)
if not "content-type" in r.headers:
logger.info(lambda: "Unknown content-type for url " + url)
if verbose:
progress_reporter(_("Not an image"), url)
return None
ct = r.headers["content-type"]
if not ct.startswith("image/"):
logger.info(lambda: "Unsupported content-type for url " + url +
": " + ct)
if verbose:
progress_reporter(_("Not an image"), url)
return None
local_name = Util.get_local_name(r.url)
if "content-disposition" in r.headers:
cd = r.headers["content-disposition"]
cd_name = ImageFetcher.extract_filename_from_content_disposition(
cd)
if cd_name:
local_name = cd_name
filename = os.path.join(to_folder, local_name)
if os.path.exists(filename):
m = Util.read_metadata(filename)
if m and m.get("imageURL") == url:
logger.info(
lambda: "Local file already exists (%s)" % filename)
return filename
else:
logger.info(
lambda:
"File with same name already exists, but from different imageURL; renaming new download"
)
filename = Util.find_unique_name(filename)
logger.info(lambda: "Fetching to " + filename)
if not reported:
reported = True
progress_reporter(_("Fetching"), url)
local_filepath_partial = filename + ".partial"
with open(local_filepath_partial, "wb") as f:
Util.request_write_to(r, f)
try:
img = Image.open(local_filepath_partial)
except Exception:
progress_reporter(_("Not an image"), url)
Util.safe_unlink(local_filepath_partial)
return None
if img.size[0] < 400 or img.size[1] < 400:
# too small - delete and do not use
progress_reporter(_("Image too small, ignoring it"), url)
Util.safe_unlink(local_filepath_partial)
return None
metadata = {
"sourceType": source_type or "fetched",
"sourceName": source_name or "Fetched",
"sourceURL": origin_url or url,
"imageURL": url,
}
if source_location:
metadata["sourceLocation"] = source_location
metadata.update(extra_metadata or {})
Util.write_metadata(local_filepath_partial, metadata)
os.rename(local_filepath_partial, filename)
logger.info(lambda: "Fetched %s to %s." % (url, filename))
return filename
except Exception as e:
# pylint: disable=no-member
logger.exception(lambda: "Fetch failed for URL " + url)
if reported:
if isinstance(
e, HTTPError) and e.response.status_code in (403, 404):
progress_reporter(
_("Sorry, got %s error...") %
str(e.response.status_code),
_("This means the link is no longer valid"),
)
else:
progress_reporter(
_("Fetch failed for some reason"),
_("To get more information, please run Variety from terminal with -v option and retry the action"
),
)
return None
def is_in_favorites(self, url):
return self.get_variety() and os.path.exists(
os.path.join(self.get_variety().options.favorites_folder,
Util.get_local_name(url)))
def get_local_filename(self, url):
return os.path.join(self.target_folder, Util.get_local_name(url))
def is_in_favorites(self, url):
return self.parent and os.path.exists(os.path.join(self.parent.options.favorites_folder, Util.get_local_name(url)))
def get_local_filename(self, url):
return os.path.join(self.target_folder, Util.get_local_name(url))
def fetch(url, to_folder, origin_url=None,
source_type=None, source_location=None, source_name=None,
extra_metadata={},
progress_reporter=lambda a, b: None, verbose=True):
reported = verbose
try:
logger.info(lambda: "Trying to fetch URL %s to %s " % (url, to_folder))
if verbose:
progress_reporter(_("Fetching"), url)
if url.startswith('javascript:'):
if verbose:
progress_reporter(_("Not an image"), url)
return None
if url.find('://') < 0:
url = "file://" + url
r = Util.request(url, stream=True)
if not "content-type" in r.headers:
logger.info(lambda: "Unknown content-type for url " + url)
if verbose:
progress_reporter(_("Not an image"), url)
return None
ct = r.headers["content-type"]
if not ct.startswith("image/"):
logger.info(lambda: "Unsupported content-type for url " + url + ": " + ct)
if verbose:
progress_reporter(_("Not an image"), url)
return None
local_name = Util.get_local_name(r.url)
if "content-disposition" in r.headers:
cd = r.headers["content-disposition"]
cd_name = ImageFetcher.extract_filename_from_content_disposition(cd)
if cd_name:
local_name = cd_name
filename = os.path.join(to_folder, local_name)
if os.path.exists(filename):
m = Util.read_metadata(filename)
if m and m.get("imageURL") == url:
logger.info(lambda: "Local file already exists (%s)" % filename)
return filename
else:
logger.info(lambda:
"File with same name already exists, but from different imageURL; renaming new download")
filename = Util.find_unique_name(filename)
local_name = os.path.basename(filename)
logger.info(lambda: "Fetching to " + filename)
if not reported:
reported = True
progress_reporter(_("Fetching"), url)
with open(filename, 'wb') as f:
Util.request_write_to(r, f)
try:
img = Image.open(filename)
except Exception:
progress_reporter(_("Not an image"), url)
os.unlink(filename)
return None
if img.size[0] < 400 or img.size[1] < 400:
# too small - delete and do not use
progress_reporter(_("Image too small, ignoring it"), url)
os.unlink(filename)
return None
metadata = {"sourceType": source_type or 'fetched',
"sourceName": source_name or "Fetched",
"sourceURL": origin_url or url,
"imageURL": url}
if source_location:
metadata["sourceLocation"] = source_location
metadata.update(extra_metadata)
Util.write_metadata(filename, metadata)
logger.info(lambda: "Fetched %s to %s." % (url, filename))
return filename
except Exception, e:
logger.exception(lambda: "Fetch failed for URL " + url)
if reported:
if isinstance(e, HTTPError) and e.response.status_code in (403, 404):
progress_reporter(
_("Sorry, got %s error...") % str(e.response.status_code),
_("This means the link is no longer valid"))
else:
progress_reporter(
_("Fetch failed for some reason"),
_("To get more information, please run Variety from terminal with -v option and retry the action"))
return None
