def extractDetails(posts, SINGLE_POST=False): """Check posts and decide if it can be downloaded. If so, create a dictionary with post details and append them to a list. Write all of posts to file. Return the list """ postList = [] postCount = 1 allPosts = {} print("\nGETTING POSTS") postsFile = createLogFile("POSTS") if SINGLE_POST: submission = posts postCount += 1 try: details = { 'POSTID': submission.id, 'TITLE': submission.title, 'REDDITOR': str(submission.author), 'TYPE': None, 'CONTENTURL': submission.url, 'SUBREDDIT': submission.subreddit.display_name, 'UPVOTES': submission.score, 'FLAIR': submission.link_flair_text, 'DATE': str( time.strftime("%Y-%m-%d_%H-%M", time.localtime(submission.created_utc))) } except AttributeError: pass if not any(domain in submission.domain for domain in GLOBAL.arguments.skip_domain): result = matchWithDownloader(submission) if result is not None: details = {**details, **result} postList.append(details) postsFile.add({postCount: details}) else: try: for submission in posts: if postCount % 100 == 0: sys.stdout.write("• ") sys.stdout.flush() if postCount % 1000 == 0: sys.stdout.write("\n" + " " * 14) sys.stdout.flush() try: details = { 'POSTID': submission.id, 'TITLE': submission.title, 'REDDITOR': str(submission.author), 'TYPE': None, 'CONTENTURL': submission.url, 'SUBREDDIT': submission.subreddit.display_name, 'UPVOTES': submission.score, 'FLAIR': submission.link_flair_text, 'DATE': str( time.strftime( "%Y-%m-%d_%H-%M", time.localtime(submission.created_utc))) } except AttributeError: continue if details['POSTID'] in GLOBAL.downloadedPosts(): continue if not any(domain in submission.domain for domain in GLOBAL.arguments.skip_domain): result = matchWithDownloader(submission) if result is not None: details = {**details, **result} postList.append(details) allPosts[postCount] = details postCount += 1 except KeyboardInterrupt: print("\nKeyboardInterrupt", noPrint=True) postsFile.add(allPosts) if not len(postList) == 0: print() return postList else: raise NoMatchingSubmissionFound("No matching submission was found")
def download(submissions): """Analyze list of submissions and call the right function to download each one, catch errors, update the log files """ downloadedCount = 0 duplicates = 0 FAILED_FILE = createLogFile("FAILED") if GLOBAL.arguments.unsave: reddit = Reddit(GLOBAL.config['credentials']['reddit']).begin() subsLenght = len(submissions) for i in range(len(submissions)): print(f"\n({i+1}/{subsLenght})",end=" — ") print(submissions[i]['POSTID'], f"r/{submissions[i]['SUBREDDIT']}", f"u/{submissions[i]['REDDITOR']}", submissions[i]['FLAIR'] if submissions[i]['FLAIR'] else "", sep=" — ", end="") print(f" – {submissions[i]['TYPE'].upper()}",end="",noPrint=True) directory = GLOBAL.directory / GLOBAL.config["folderpath"].format(**submissions[i]) details = { **submissions[i], **{ "TITLE": nameCorrector( submissions[i]['TITLE'], reference = str(directory) + GLOBAL.config['filename'].format(**submissions[i]) + ".ext" ) } } filename = GLOBAL.config['filename'].format(**details) if isPostExists(details,directory): print() print(directory) print(filename) print("It already exists") duplicates += 1 continue if any(domain in submissions[i]['CONTENTURL'] for domain in GLOBAL.arguments.skip): print() print(submissions[i]['CONTENTURL']) print("Domain found in skip domains, skipping post...") continue try: downloadPost(details,directory) GLOBAL.downloadedPosts.add(details['POSTID']) try: if GLOBAL.arguments.unsave: reddit.submission(id=details['POSTID']).unsave() except InsufficientScope: reddit = Reddit().begin() reddit.submission(id=details['POSTID']).unsave() downloadedCount += 1 except FileAlreadyExistsError: print("It already exists") GLOBAL.downloadedPosts.add(details['POSTID']) duplicates += 1 except ImgurLoginError: print( "Imgur login failed. \nQuitting the program "\ "as unexpected errors might occur." ) sys.exit() except ImgurLimitError as exception: FAILED_FILE.add({int(i+1):[ "{class_name}: {info}".format( class_name=exception.__class__.__name__,info=str(exception) ), details ]}) except NotADownloadableLinkError as exception: print( "{class_name}: {info}".format( class_name=exception.__class__.__name__,info=str(exception) ) ) FAILED_FILE.add({int(i+1):[ "{class_name}: {info}".format( class_name=exception.__class__.__name__,info=str(exception) ), submissions[i] ]}) except TypeInSkip: print() print(submissions[i]['CONTENTURL']) print("Skipping post...") except DomainInSkip: print() print(submissions[i]['CONTENTURL']) print("Skipping post...") except NoSuitablePost: print("No match found, skipping...") except FailedToDownload: print("Failed to download the posts, skipping...") except AlbumNotDownloadedCompletely: print("Album did not downloaded completely.") FAILED_FILE.add({int(i+1):[ "{class_name}: {info}".format( class_name=exc.__class__.__name__,info=str(exc) ), submissions[i] ]}) except Exception as exc: print( "{class_name}: {info}\nSee CONSOLE_LOG.txt for more information".format( class_name=exc.__class__.__name__,info=str(exc) ) ) logging.error(sys.exc_info()[0].__name__, exc_info=full_exc_info(sys.exc_info())) print(GLOBAL.log_stream.getvalue(),noPrint=True) FAILED_FILE.add({int(i+1):[ "{class_name}: {info}".format( class_name=exc.__class__.__name__,info=str(exc) ), submissions[i] ]}) if duplicates: print(f"\nThere {'were' if duplicates > 1 else 'was'} " \ f"{duplicates} duplicate{'s' if duplicates > 1 else ''}") if downloadedCount == 0: print("Nothing is downloaded :(") else: print(f"Total of {downloadedCount} " \ f"link{'s' if downloadedCount > 1 else ''} downloaded!")
def download(submissions): """Analyze list of submissions and call the right function to download each one, catch errors, update the log files """ subsLenght = len(submissions) global lastRequestTime lastRequestTime = 0 downloadedCount = subsLenght duplicates = 0 FAILED_FILE = createLogFile("FAILED") for i in range(subsLenght): print( f"\n({i+1}/{subsLenght}) – {submissions[i]['postId']} – r/{submissions[i]['postSubreddit']}", end="") print(f" – {submissions[i]['postType'].upper()}", end="", noPrint=True) if isPostExists(submissions[i]): print(f"\n" \ f"{submissions[i]['postSubmitter']}_" f"{nameCorrector(submissions[i]['postTitle'])}") print("It already exists") duplicates += 1 downloadedCount -= 1 continue try: downloadPost(submissions[i]) except FileAlreadyExistsError: print("It already exists") duplicates += 1 downloadedCount -= 1 except ImgurLoginError: print( "Imgur login failed. \nQuitting the program "\ "as unexpected errors might occur." ) sys.exit() except ImgurLimitError as exception: FAILED_FILE.add({ int(i + 1): [ "{class_name}: {info}".format( class_name=exception.__class__.__name__, info=str(exception)), submissions[i] ] }) downloadedCount -= 1 except NotADownloadableLinkError as exception: print("{class_name}: {info}".format( class_name=exception.__class__.__name__, info=str(exception))) FAILED_FILE.add({ int(i + 1): [ "{class_name}: {info}".format( class_name=exception.__class__.__name__, info=str(exception)), submissions[i] ] }) downloadedCount -= 1 except NoSuitablePost: print("No match found, skipping...") downloadedCount -= 1 except Exception as exception: # raise exception print("{class_name}: {info}".format( class_name=exception.__class__.__name__, info=str(exception))) FAILED_FILE.add({ int(i + 1): [ "{class_name}: {info}".format( class_name=exception.__class__.__name__, info=str(exception)), submissions[i] ] }) downloadedCount -= 1 if duplicates: print(f"\nThere {'were' if duplicates > 1 else 'was'} " \ f"{duplicates} duplicate{'s' if duplicates > 1 else ''}") if downloadedCount == 0: print("Nothing downloaded :(") else: print(f"Total of {downloadedCount} " \ f"link{'s' if downloadedCount > 1 else ''} downloaded!")
def redditSearcher(posts,SINGLE_POST=False): """Check posts and decide if it can be downloaded. If so, create a dictionary with post details and append them to a list. Write all of posts to file. Return the list """ subList = [] global subCount subCount = 0 global orderCount orderCount = 0 global gfycatCount gfycatCount = 0 global redgifsCount redgifsCount = 0 global imgurCount imgurCount = 0 global eromeCount eromeCount = 0 global gifDeliveryNetworkCount gifDeliveryNetworkCount = 0 global directCount directCount = 0 global selfCount selfCount = 0 allPosts = {} print("\nGETTING POSTS") if GLOBAL.arguments.verbose: print("\n") postsFile = createLogFile("POSTS") if SINGLE_POST: submission = posts subCount += 1 try: details = {'postId':submission.id, 'postTitle':submission.title, 'postSubmitter':str(submission.author), 'postType':None, 'postURL':submission.url, 'postSubreddit':submission.subreddit.display_name} except AttributeError: pass result = checkIfMatching(submission) if result is not None: details = result orderCount += 1 if GLOBAL.arguments.verbose: printSubmission(submission,subCount,orderCount) subList.append(details) postsFile.add({subCount:[details]}) else: try: for submission in posts: subCount += 1 if subCount % 100 == 0 and not GLOBAL.arguments.verbose: sys.stdout.write("• ") sys.stdout.flush() if subCount % 1000 == 0: sys.stdout.write("\n"+" "*14) sys.stdout.flush() try: details = {'postId':submission.id, 'postTitle':submission.title, 'postSubmitter':str(submission.author), 'postType':None, 'postURL':submission.url, 'postSubreddit':submission.subreddit.display_name} except AttributeError: continue result = checkIfMatching(submission) if result is not None: details = result orderCount += 1 if GLOBAL.arguments.verbose: printSubmission(submission,subCount,orderCount) subList.append(details) allPosts[subCount] = [details] except KeyboardInterrupt: print("\nKeyboardInterrupt",noPrint=True) postsFile.add(allPosts) if not len(subList) == 0: if GLOBAL.arguments.NoDownload or GLOBAL.arguments.verbose: print( f"\n\nTotal of {len(subList)} submissions found!" ) print( f"{gfycatCount} GFYCATs, {imgurCount} IMGURs, " \ f"{eromeCount} EROMEs, {directCount} DIRECTs " \ f"and {selfCount} SELF POSTS",noPrint=True ) else: print() return subList else: raise NoMatchingSubmissionFound("No matching submission was found")
def extractDetails(posts, SINGLE_POST=False): """Check posts and decide if it can be downloaded. If so, create a dictionary with post details and append them to a list. Write all of posts to file. Return the list """ postList = [] postCount = 1 allPosts = {} print("\nGETTING POSTS") postsFile = createLogFile("POSTS") if SINGLE_POST: submission = posts postCount += 1 # All options offered by PRAW are author, clicked, comments, created_utc, distinguished, edited, id, # is_original_content, is_self, link_flair_template_id, link_flair_text, locked, name, num_comments, over_18, # permalink, poll_data, score, selftext, spoiler, stickied, subreddit, title, upvote_ratio, url try: details = { 'POSTID': submission.id, 'TITLE': submission.title, 'REDDITOR': str(submission.author), 'TYPE': None, 'CONTENTURL': submission.url, 'SUBREDDIT': submission.subreddit.display_name, 'UPVOTES': submission.score, 'FLAIR': submission.link_flair_text, 'OVER18': submission.over_18, 'DATE': str( time.strftime("%Y-%m-%d_%H-%M", time.localtime(submission.created_utc))) } except AttributeError: pass if not any(domain in submission.domain for domain in GLOBAL.arguments.skip_domain): result = matchWithDownloader(submission) if result is not None: details = {**details, **result} postList.append(details) postsFile.add({postCount: details}) else: try: for submission in posts: if postCount % 100 == 0: sys.stdout.write("• ") sys.stdout.flush() if postCount % 1000 == 0: sys.stdout.write("\n" + " " * 14) sys.stdout.flush() try: details = { 'POSTID': submission.id, 'TITLE': submission.title, 'REDDITOR': str(submission.author), 'TYPE': None, 'CONTENTURL': submission.url, 'SUBREDDIT': submission.subreddit.display_name, 'UPVOTES': submission.score, 'FLAIR': submission.link_flair_text, 'OVER18': submission.over_18, 'DATE': str( time.strftime( "%Y-%m-%d_%H-%M", time.localtime(submission.created_utc))) } except AttributeError: continue # Check if the grabbed POSTID is already in the downloadedPosts from Prior runs, if enabled if details['POSTID'] in GLOBAL.downloadedPosts(): continue # Check to make sure the post is not in the skip_domain list if not any(domain in submission.domain for domain in GLOBAL.arguments.skip_domain): result = matchWithDownloader(submission) if result is not None: details = {**details, **result} postList.append(details) allPosts[postCount] = details postCount += 1 except KeyboardInterrupt: print("\nKeyboardInterrupt", noPrint=True) postsFile.add(allPosts) if not len(postList) == 0: print() return postList else: raise NoMatchingSubmissionFound("No matching submission was found")