def save_locally(self, origin_url, image_url,
source_type=None, source_location=None, source_name=None,
force_download=False, extra_metadata={}, local_filename=None):
if not source_type:
source_type = self.source_type
if not source_name:
source_name = self.name
if not source_location:
source_location = self.location
if not force_download and self.parent and origin_url in self.parent.banned:
logger.info(lambda: "URL " + origin_url + " is banned, skip downloading")
return None
try:
os.makedirs(self.target_folder)
except Exception:
pass
if origin_url.startswith('//'):
origin_url = 'https:' + origin_url
if image_url.startswith('//'):
image_url = origin_url.split('//')[0] + image_url
if not local_filename:
local_filename = self.get_local_filename(image_url)
logger.info(lambda: "Origin URL: " + origin_url)
logger.info(lambda: "Image URL: " + image_url)
logger.info(lambda: "Local name: " + local_filename)
if not force_download and os.path.exists(local_filename):
logger.info(lambda: "File already exists, skip downloading")
return None
if self.parent and self.parent.options.safe_mode:
sfw_rating = Smart.get_sfw_rating(origin_url)
if sfw_rating is not None and sfw_rating < 100:
logger.info(lambda: "Skipping non-safe download %s. Is the source %s:%s "
"suitable for Safe mode?" % (origin_url, source_type, self.location))
return None
if self.parent and self.parent.options.safe_mode and 'keywords' in extra_metadata:
blacklisted = set(k.lower() for k in extra_metadata['keywords']) & Smart.get_safe_mode_keyword_blacklist()
if len(blacklisted) > 0:
logger.info(lambda: "Skipping non-safe download %s due to blacklisted keywords (%s). "
"Is the source %s:%s suitable for Safe mode?" %
(origin_url, str(blacklisted), source_type, self.location))
return None
try:
r = Util.request(image_url, stream=True)
with open(local_filename, 'wb') as f:
Util.request_write_to(r, f)
except Exception, e:
logger.info(lambda: "Download failed from image URL: %s (source location: %s) " % (image_url, self.location))
raise e
def save_locally(self,
origin_url,
image_url,
source_type=None,
source_location=None,
source_name=None,
force_download=False,
extra_metadata={},
local_filename=None):
if not source_type:
source_type = self.source_type
if not source_name:
source_name = self.name
if not source_location:
source_location = self.location
if not force_download and self.parent and origin_url in self.parent.banned:
logger.info(
lambda: "URL " + origin_url + " is banned, skip downloading")
return None
try:
os.makedirs(self.target_folder)
except Exception:
pass
if origin_url.startswith('//'):
origin_url = 'https:' + origin_url
if image_url.startswith('//'):
image_url = origin_url.split('//')[0] + image_url
if not local_filename:
local_filename = self.get_local_filename(image_url)
logger.info(lambda: "Origin URL: " + origin_url)
logger.info(lambda: "Image URL: " + image_url)
logger.info(lambda: "Local name: " + local_filename)
if not force_download and os.path.exists(local_filename):
logger.info(lambda: "File already exists, skip downloading")
return None
if self.parent and self.parent.options.safe_mode:
sfw_rating = Smart.get_sfw_rating(origin_url)
if sfw_rating is not None and sfw_rating < 100:
logger.info(
lambda:
"Skipping non-safe download %s. Is the source %s:%s "
"suitable for Safe mode?" %
(origin_url, source_type, self.location))
return None
if self.parent and self.parent.options.safe_mode and 'keywords' in extra_metadata:
blacklisted = set(k.lower() for k in extra_metadata['keywords']
) & Smart.get_safe_mode_keyword_blacklist()
if len(blacklisted) > 0:
logger.info(
lambda:
"Skipping non-safe download %s due to blacklisted keywords (%s). "
"Is the source %s:%s suitable for Safe mode?" %
(origin_url, str(blacklisted), source_type, self.location))
return None
try:
r = Util.request(image_url, stream=True)
with open(local_filename, 'wb') as f:
Util.request_write_to(r, f)
except Exception, e:
logger.info(
lambda:
"Download failed from image URL: %s (source location: %s) " %
(image_url, self.location))
raise e
