def __init__(self): self.reddit = praw.Reddit("reddit-wall") self.imgur = ImgurDownloader(CONFIG["Imgur"]["client_id"], CONFIG["Imgur"]["client_secret"]) self.subreddits = [] self.multireddits = [] self.allow_nsfw = CONFIG["Downloads"].getboolean("AllowNSFW") self.output_dir = CONFIG["Downloads"]["OutputDirectory"]
def download_from_imgur(link, post): MAX_ALBUM_LEN = 20 # only download full albums if >20 total images try: downloader = ImgurDownloader(link) if(downloader.num_images() <= MAX_ALBUM_LEN): # Group images by redditor name album_name = post.author.name downloader.save_images("./saved/images/{}".format(album_name)) except: print("Error saving Imgur image: {}".format(link))
def download_one(link_tuple): link_type, link, name, id = link_tuple msg = "Downloading {} of type {}".format(link, link_type) print('=' * 50, msg, sep='\n') new_post_id = name + '_' + id if link_type == 'imgur_album': try: ImgurDownloader(link, DOWNLOAD_DIR).save_images(post_id=new_post_id) return True except (KeyboardInterrupt, SystemExit): raise except ImgurException as e: print("Failed to download image(s) from album at: {}".format(link), e.msg) except: print( "Failed to download image(s) from album at: {} for unknown reasons" .format(link)) if link_type == 'image': try: file_path = os.path.join(DOWNLOAD_DIR, new_post_id + '.jpg') urllib.request.urlretrieve(link, file_path) return True except ConnectionResetError: print("Failed to download image at {} due to connection reset". format(link)) except urllib.error.URLError as e: print("Failed to download image at: {}".format(link), e.reason) except: print("Failed to download image at: {} for unknown reasons".format( link)) return False
def download_one(link_tuple): link_type, post_id, link = link_tuple download_dir = os.path.join(OUTPUT_DIR, post_id) if not os.path.exists(download_dir): # A comment must have a top level submission return False msg = "Downloading {} of type {} with post id {}".format( link, link_type, post_id) print('=' * 50, msg, sep='\n') new_post_id = post_id + '_' + str(get_rand_num()) if link_type == 'imgur_album': try: ImgurDownloader(link, download_dir).save_images(post_id=new_post_id) return True except (KeyboardInterrupt, SystemExit): raise except ImgurException as e: print("Failed to download image(s) from album at: {}".format(link), e.msg) except: print( "Failed to download image(s) from album at: {} for unknown reasons" .format(link)) elif link_type == 'image': try: urllib.request.urlretrieve( link, os.path.join(download_dir, new_post_id + '.jpg')) return True except ConnectionResetError: print("Failed to download image at {} due to connection reset". format(link)) except urllib.error.URLError as e: print("Failed to download image at: {}".format(link), e.reason) except: print("Failed to download image at: {} for unknown reasons".format( link)) return False
def download_one(link_tuple): link_type, post_id, link = link_tuple if link == "http://i.imgur.com/ET4VfDd.jpg": return False download_dir = os.path.join(OUTPUT_DIR, post_id) make_dir(download_dir) msg = "Downloading {} of type {} with post id {}".format( link, link_type, post_id) print('=' * 50, msg, sep='\n') if link_type == 'imgur_album': try: ImgurDownloader(link, download_dir).save_images(post_id=post_id) return True except (KeyboardInterrupt, SystemExit): raise except ImgurException as e: print("Failed to download image(s) from album at: {}".format(link), e.msg) except Exception as e: print( "Failed to download image(s) from album at: {} for unknown reasons" .format(link)) if link_type == 'image': try: urllib.request.urlretrieve( link, os.path.join(download_dir, post_id + '.jpg')) return True except ConnectionResetError: print("Failed to download image at {} due to connection reset". format(link)) except urllib.error.URLError as e: print("Failed to download image at: {}".format(link), e.reason) except: print("Failed to download image at: {} for unknown reasons".format( link)) return False
def imgurDownload(url, user): vids = [ "gif", "gifv", "mp4" ] try: downloader = ImgurDownloader(url) if downloader.num_images() == 1: verifyCreateDir(user, "images") if any(x in url for x in vids) or any(x in str(downloader.list_extensions()) for x in vids): dltype = "videos" else: dltype = "images" if downloader.imageIDs[0][0] == "": filename = url.split(r'/')[-1] else: if "gif" in downloader.imageIDs[0][1]: filename = downloader.imageIDs[0][0] + ".mp4" else: filename = downloader.imageIDs[0][0] + downloader.imageIDs[0][1] url = "https://i.imgur.com/" + filename downloadFile(url, filename, user, dltype) elif downloader.num_images() == 0: logging.info("IMGUR_EMPTY_ALBUM: {}".format(url)) return else: if not os.path.exists(user + "/albums/" + downloader.get_album_key()): try: downloader.save_images(user + "/albums/" + downloader.get_album_key()) printDownload(user, "albums", downloader.get_album_key() + "/all.html") allhtml = open(user + "/albums/" + downloader.get_album_key() + "/all.html", "w") allhtml.write("<style>\n") allhtml.write(".fixed-ratio-resize { /* basic responsive img class=\"fixed-ratio-resize\" */\n") allhtml.write(" max-width: 100%;\n") allhtml.write(" height: auto;\n") allhtml.write(" width: auto\9; /* IE8 */\n") allhtml.write("}\n") allhtml.write("</style>\n") allhtml.write("\n") allhtml.write("<body text=\"#ffffff\" bgcolor=\"#000000\">") for filename in sorted(os.listdir(user + "/albums/" + downloader.get_album_key())): if "image" in subprocess.run(['file','--brief','--mime-type', user + "/albums/" + downloader.get_album_key() + "/" + filename], stdout=subprocess.PIPE).stdout.decode("utf-8"): allhtml.write("<a href=" + filename + "><img class=\"fixed-ratio-resize\" src=" + filename + "></a><br>" + filename + "<hr>\n") allhtml.write("</body>") allhtml.close() logging.info("IMGUR_ALBUM_DOWNLOAD: {}".format(url)) except Exception as e: logging.info("IMGUR_ALBUM_EXCEPTION: {}".format(e)) pass else: logging.info("ALBUM_EXISTS: {}".format(user + "/albums/" + downloader.get_album_key())) except Exception as e: logging.info("IMGUR_EXCEPTION: {}".format(e)) pass
class RedditHandler: """The reddit handler. Attributes ---------- reddit : praw.reddit A connection to the reddit api. imgur : ImgurDownloader Handler for downloading images from Imgur. subreddits : list List of reddit.subreddit objects that will be parsed. multireddits : list List of reddit.multireddit objects that will be parsed. allow_nsfw : bool Whether to parse subreddits or submissions marked NSFW. output_dir : string The directory where images will be saved. """ #FIXME: Check if praw.ini exists and allow user to specify creds at runtime. def __init__(self): self.reddit = praw.Reddit("reddit-wall") self.imgur = ImgurDownloader(CONFIG["Imgur"]["client_id"], CONFIG["Imgur"]["client_secret"]) self.subreddits = [] self.multireddits = [] self.allow_nsfw = CONFIG["Downloads"].getboolean("AllowNSFW") self.output_dir = CONFIG["Downloads"]["OutputDirectory"] def add_subreddit(self, name): """Add a subreddit for the handler to manage. Parameters ---------- name : string Name of the subreddit to add. """ subreddit = self.reddit.subreddit(name) if not self.allow_nsfw and subreddit.over18: return self.subreddits.append(subreddit) def add_multireddit(self, user, name): """Add a multireddit for the handler to manage. Parameters ---------- user : string Username of the owner of the multireddit. name : string Name of the multireddit. """ multireddit = self.reddit.multireddit(user, name) if not self.allow_nsfw and multireddit.over_18: return self.multireddits.append(multireddit) def combined_subreddits(self): """Generate a string representing all subreddits being handled. On reddit, subs can be combined by adding '+' between the names. This effectively acts as one large subreddit. Returns ------- combo_str : string String listing all subreddits being handled. """ subs = [sub.display_name.lower() for sub in self.subreddits] for multireddit in self.multireddits: for subreddit in multireddit.subreddits: subs.append(subreddit.display_name.lower()) subs = set(subs) combo_str = '+'.join(subs) return combo_str def valid_submission(self, submission): """Check if the given submission has images that can be downloaded. Parameters ---------- submission : praw.models.submission The submission to check. Returns ------- valid : bool Whether the submission has downloadable images. """ image_types = (".png", ".jpg", ".jpeg", ".gif") is_self = submission.is_self is_image = submission.url.endswith(image_types) is_album = "imgur.com/a/" in submission.url is_gallery = "imgur.com/gallery/" in submission.url if not self.allow_nsfw and submission.over_18: is_age_appropriate = False else: is_age_appropriate = True return (is_self or is_image or is_album or is_gallery) and is_age_appropriate def download_submission_image(self, submission): """Download the image associated with the given reddit submission. Parameters ---------- submission : praw.models.submission The submission to download an image from. """ if not self.valid_submission(submission): return False # FIXME: Check file permissions when attempting to create directory output_dirpath = pathlib.Path(self.output_dir).expanduser() if str(output_dirpath) == '.': output_dirpath = pathlib.Path.cwd() # Get the absolute path if not output_dirpath.exists(): output_dirpath.mkdir(parents=True) if "imgur.com/a/" in submission.url: return self.imgur.download_album(submission.url, output_dirpath) if "imgur.com/gallery/" in submission.url: return self.imgur.download_gallery(submission.url, output_dirpath) filename = submission.url[submission.url.rfind('/'):] output_filepath = str(output_dirpath) + filename with open(output_filepath, "wb") as handle: response = requests.get(submission.url, stream=True) if not response.ok: print(response) return False for block in response.iter_content(1024): if not block: break handle.write(block) return True