def __init__(self):
self.reddit = praw.Reddit("reddit-wall")
self.imgur = ImgurDownloader(CONFIG["Imgur"]["client_id"],
CONFIG["Imgur"]["client_secret"])
self.subreddits = []
self.multireddits = []
self.allow_nsfw = CONFIG["Downloads"].getboolean("AllowNSFW")
self.output_dir = CONFIG["Downloads"]["OutputDirectory"]
def download_from_imgur(link, post):
MAX_ALBUM_LEN = 20 # only download full albums if >20 total images
try:
downloader = ImgurDownloader(link)
if(downloader.num_images() <= MAX_ALBUM_LEN):
# Group images by redditor name
album_name = post.author.name
downloader.save_images("./saved/images/{}".format(album_name))
except:
print("Error saving Imgur image: {}".format(link))
def download_one(link_tuple):
link_type, link, name, id = link_tuple
msg = "Downloading {} of type {}".format(link, link_type)
print('=' * 50, msg, sep='\n')
new_post_id = name + '_' + id
if link_type == 'imgur_album':
try:
ImgurDownloader(link,
DOWNLOAD_DIR).save_images(post_id=new_post_id)
return True
except (KeyboardInterrupt, SystemExit):
raise
except ImgurException as e:
print("Failed to download image(s) from album at: {}".format(link),
e.msg)
except:
print(
"Failed to download image(s) from album at: {} for unknown reasons"
.format(link))
if link_type == 'image':
try:
file_path = os.path.join(DOWNLOAD_DIR, new_post_id + '.jpg')
urllib.request.urlretrieve(link, file_path)
return True
except ConnectionResetError:
print("Failed to download image at {} due to connection reset".
format(link))
except urllib.error.URLError as e:
print("Failed to download image at: {}".format(link), e.reason)
except:
print("Failed to download image at: {} for unknown reasons".format(
link))
return False
def download_one(link_tuple):
link_type, post_id, link = link_tuple
download_dir = os.path.join(OUTPUT_DIR, post_id)
if not os.path.exists(download_dir):
# A comment must have a top level submission
return False
msg = "Downloading {} of type {} with post id {}".format(
link, link_type, post_id)
print('=' * 50, msg, sep='\n')
new_post_id = post_id + '_' + str(get_rand_num())
if link_type == 'imgur_album':
try:
ImgurDownloader(link,
download_dir).save_images(post_id=new_post_id)
return True
except (KeyboardInterrupt, SystemExit):
raise
except ImgurException as e:
print("Failed to download image(s) from album at: {}".format(link),
e.msg)
except:
print(
"Failed to download image(s) from album at: {} for unknown reasons"
.format(link))
elif link_type == 'image':
try:
urllib.request.urlretrieve(
link, os.path.join(download_dir, new_post_id + '.jpg'))
return True
except ConnectionResetError:
print("Failed to download image at {} due to connection reset".
format(link))
except urllib.error.URLError as e:
print("Failed to download image at: {}".format(link), e.reason)
except:
print("Failed to download image at: {} for unknown reasons".format(
link))
return False
def download_one(link_tuple):
link_type, post_id, link = link_tuple
if link == "http://i.imgur.com/ET4VfDd.jpg":
return False
download_dir = os.path.join(OUTPUT_DIR, post_id)
make_dir(download_dir)
msg = "Downloading {} of type {} with post id {}".format(
link, link_type, post_id)
print('=' * 50, msg, sep='\n')
if link_type == 'imgur_album':
try:
ImgurDownloader(link, download_dir).save_images(post_id=post_id)
return True
except (KeyboardInterrupt, SystemExit):
raise
except ImgurException as e:
print("Failed to download image(s) from album at: {}".format(link),
e.msg)
except Exception as e:
print(
"Failed to download image(s) from album at: {} for unknown reasons"
.format(link))
if link_type == 'image':
try:
urllib.request.urlretrieve(
link, os.path.join(download_dir, post_id + '.jpg'))
return True
except ConnectionResetError:
print("Failed to download image at {} due to connection reset".
format(link))
except urllib.error.URLError as e:
print("Failed to download image at: {}".format(link), e.reason)
except:
print("Failed to download image at: {} for unknown reasons".format(
link))
return False
def imgurDownload(url, user):
vids = [ "gif", "gifv", "mp4" ]
try:
downloader = ImgurDownloader(url)
if downloader.num_images() == 1:
verifyCreateDir(user, "images")
if any(x in url for x in vids) or any(x in str(downloader.list_extensions()) for x in vids):
dltype = "videos"
else:
dltype = "images"
if downloader.imageIDs[0][0] == "":
filename = url.split(r'/')[-1]
else:
if "gif" in downloader.imageIDs[0][1]:
filename = downloader.imageIDs[0][0] + ".mp4"
else:
filename = downloader.imageIDs[0][0] + downloader.imageIDs[0][1]
url = "https://i.imgur.com/" + filename
downloadFile(url, filename, user, dltype)
elif downloader.num_images() == 0:
logging.info("IMGUR_EMPTY_ALBUM: {}".format(url))
return
else:
if not os.path.exists(user + "/albums/" + downloader.get_album_key()):
try:
downloader.save_images(user + "/albums/" + downloader.get_album_key())
printDownload(user, "albums", downloader.get_album_key() + "/all.html")
allhtml = open(user + "/albums/" + downloader.get_album_key() + "/all.html", "w")
allhtml.write("<style>\n")
allhtml.write(".fixed-ratio-resize { /* basic responsive img class=\"fixed-ratio-resize\" */\n")
allhtml.write(" max-width: 100%;\n")
allhtml.write(" height: auto;\n")
allhtml.write(" width: auto\9; /* IE8 */\n")
allhtml.write("}\n")
allhtml.write("</style>\n")
allhtml.write("\n")
allhtml.write("<body text=\"#ffffff\" bgcolor=\"#000000\">")
for filename in sorted(os.listdir(user + "/albums/" + downloader.get_album_key())):
if "image" in subprocess.run(['file','--brief','--mime-type', user + "/albums/" + downloader.get_album_key() + "/" + filename], stdout=subprocess.PIPE).stdout.decode("utf-8"):
allhtml.write("<a href=" + filename + "><img class=\"fixed-ratio-resize\" src=" + filename + "></a><br>" + filename + "<hr>\n")
allhtml.write("</body>")
allhtml.close()
logging.info("IMGUR_ALBUM_DOWNLOAD: {}".format(url))
except Exception as e:
logging.info("IMGUR_ALBUM_EXCEPTION: {}".format(e))
pass
else:
logging.info("ALBUM_EXISTS: {}".format(user + "/albums/" + downloader.get_album_key()))
except Exception as e:
logging.info("IMGUR_EXCEPTION: {}".format(e))
pass
class RedditHandler:
"""The reddit handler.
Attributes
----------
reddit : praw.reddit
A connection to the reddit api.
imgur : ImgurDownloader
Handler for downloading images from Imgur.
subreddits : list
List of reddit.subreddit objects that will be parsed.
multireddits : list
List of reddit.multireddit objects that will be parsed.
allow_nsfw : bool
Whether to parse subreddits or submissions marked NSFW.
output_dir : string
The directory where images will be saved.
"""
#FIXME: Check if praw.ini exists and allow user to specify creds at runtime.
def __init__(self):
self.reddit = praw.Reddit("reddit-wall")
self.imgur = ImgurDownloader(CONFIG["Imgur"]["client_id"],
CONFIG["Imgur"]["client_secret"])
self.subreddits = []
self.multireddits = []
self.allow_nsfw = CONFIG["Downloads"].getboolean("AllowNSFW")
self.output_dir = CONFIG["Downloads"]["OutputDirectory"]
def add_subreddit(self, name):
"""Add a subreddit for the handler to manage.
Parameters
----------
name : string
Name of the subreddit to add.
"""
subreddit = self.reddit.subreddit(name)
if not self.allow_nsfw and subreddit.over18:
return
self.subreddits.append(subreddit)
def add_multireddit(self, user, name):
"""Add a multireddit for the handler to manage.
Parameters
----------
user : string
Username of the owner of the multireddit.
name : string
Name of the multireddit.
"""
multireddit = self.reddit.multireddit(user, name)
if not self.allow_nsfw and multireddit.over_18:
return
self.multireddits.append(multireddit)
def combined_subreddits(self):
"""Generate a string representing all subreddits being handled.
On reddit, subs can be combined by adding '+' between the names.
This effectively acts as one large subreddit.
Returns
-------
combo_str : string
String listing all subreddits being handled.
"""
subs = [sub.display_name.lower() for sub in self.subreddits]
for multireddit in self.multireddits:
for subreddit in multireddit.subreddits:
subs.append(subreddit.display_name.lower())
subs = set(subs)
combo_str = '+'.join(subs)
return combo_str
def valid_submission(self, submission):
"""Check if the given submission has images that can be downloaded.
Parameters
----------
submission : praw.models.submission
The submission to check.
Returns
-------
valid : bool
Whether the submission has downloadable images.
"""
image_types = (".png", ".jpg", ".jpeg", ".gif")
is_self = submission.is_self
is_image = submission.url.endswith(image_types)
is_album = "imgur.com/a/" in submission.url
is_gallery = "imgur.com/gallery/" in submission.url
if not self.allow_nsfw and submission.over_18:
is_age_appropriate = False
else:
is_age_appropriate = True
return (is_self or is_image or is_album
or is_gallery) and is_age_appropriate
def download_submission_image(self, submission):
"""Download the image associated with the given reddit submission.
Parameters
----------
submission : praw.models.submission
The submission to download an image from.
"""
if not self.valid_submission(submission):
return False
# FIXME: Check file permissions when attempting to create directory
output_dirpath = pathlib.Path(self.output_dir).expanduser()
if str(output_dirpath) == '.':
output_dirpath = pathlib.Path.cwd() # Get the absolute path
if not output_dirpath.exists():
output_dirpath.mkdir(parents=True)
if "imgur.com/a/" in submission.url:
return self.imgur.download_album(submission.url, output_dirpath)
if "imgur.com/gallery/" in submission.url:
return self.imgur.download_gallery(submission.url, output_dirpath)
filename = submission.url[submission.url.rfind('/'):]
output_filepath = str(output_dirpath) + filename
with open(output_filepath, "wb") as handle:
response = requests.get(submission.url, stream=True)
if not response.ok:
print(response)
return False
for block in response.iter_content(1024):
if not block:
break
handle.write(block)
return True