def download(submissions): """Analyze list of submissions and call the right function to download each one, catch errors, update the log files """ subsLenght = len(submissions) lastRequestTime = 0 downloadedCount = subsLenght duplicates = 0 BACKUP = {} FAILED_FILE = createLogFile("FAILED") for i in range(subsLenght): print("\n({}/{})".format(i + 1, subsLenght)) print("https://reddit.com/r/{subreddit}/comments/{id}".format( subreddit=submissions[i]['postSubreddit'], id=submissions[i]['postId'])) if postExists(submissions[i]): result = False print("It already exists") duplicates += 1 downloadedCount -= 1 continue directory = GLOBAL.directory / submissions[i]['postSubreddit'] if submissions[i]['postType'] == 'imgur': print("IMGUR", end="") while int(time.time() - lastRequestTime) <= 2: pass credit = Imgur.get_credits() IMGUR_RESET_TIME = credit['UserReset'] - time.time() USER_RESET = ("after " \ + str(int(IMGUR_RESET_TIME/60)) \ + " Minutes " \ + str(int(IMGUR_RESET_TIME%60)) \ + " Seconds") print(" => Client: {} - User: {} - Reset {}".format( credit['ClientRemaining'], credit['UserRemaining'], USER_RESET)) if not (credit['UserRemaining'] == 0 or \ credit['ClientRemaining'] == 0): """This block of code is needed """ while int(time.time() - lastRequestTime) <= 2: pass lastRequestTime = time.time() try: Imgur(directory, submissions[i]) except FileAlreadyExistsError: print("It already exists") duplicates += 1 downloadedCount -= 1 except ImgurLoginError: print( "Imgur login failed. Quitting the program "\ "as unexpected errors might occur." ) quit() except Exception as exception: print(exception) FAILED_FILE.add( {int(i + 1): [str(exception), submissions[i]]}) downloadedCount -= 1 else: if credit['UserRemaining'] == 0: KEYWORD = "user" elif credit['ClientRemaining'] == 0: KEYWORD = "client" print('{} LIMIT EXCEEDED\n'.format(KEYWORD.upper())) FAILED_FILE.add({ int(i + 1): [ '{} LIMIT EXCEEDED\n'.format(KEYWORD.upper()), submissions[i] ] }) downloadedCount -= 1 elif submissions[i]['postType'] == 'gfycat': print("GFYCAT") try: Gfycat(directory, submissions[i]) except FileAlreadyExistsError: print("It already exists") duplicates += 1 downloadedCount -= 1 except NotADownloadableLinkError as exception: print("Could not read the page source") FAILED_FILE.add({int(i + 1): [str(exception), submissions[i]]}) downloadedCount -= 1 except Exception as exception: print(exception) FAILED_FILE.add({int(i + 1): [str(exception), submissions[i]]}) downloadedCount -= 1 elif submissions[i]['postType'] == 'direct': print("DIRECT") try: Direct(directory, submissions[i]) except FileAlreadyExistsError: print("It already exists") downloadedCount -= 1 duplicates += 1 except Exception as exception: print(exception) FAILED_FILE.add({int(i + 1): [str(exception), submissions[i]]}) downloadedCount -= 1 else: print("No match found, skipping...") downloadedCount -= 1 if duplicates: print("\n There was {} duplicates".format(duplicates)) if downloadedCount == 0: print(" Nothing downloaded :(") else: print(" Total of {} links downloaded!".format(downloadedCount))
def redditSearcher(posts, SINGLE_POST=False): """Check posts and decide if it can be downloaded. If so, create a dictionary with post details and append them to a list. Write all of posts to file. Return the list """ subList = [] global subCount subCount = 0 global orderCount orderCount = 0 global gfycatCount gfycatCount = 0 global imgurCount imgurCount = 0 global directCount directCount = 0 postsFile = createLogFile("POSTS") if SINGLE_POST: submission = posts subCount += 1 try: details = { 'postId': submission.id, 'postTitle': submission.title, 'postSubmitter': str(submission.author), 'postType': None, 'postURL': submission.url, 'postSubreddit': submission.subreddit.display_name } except AttributeError: pass postsFile.add({subCount: [details]}) details = checkIfMatching(submission) if details is not None: if not details["postType"] == "self": orderCount += 1 printSubmission(submission, subCount, orderCount) subList.append(details) else: postsFile.add({subCount: [details]}) else: for submission in posts: subCount += 1 try: details = { 'postId': submission.id, 'postTitle': submission.title, 'postSubmitter': str(submission.author), 'postType': None, 'postURL': submission.url, 'postSubreddit': submission.subreddit.display_name } except AttributeError: continue postsFile.add({subCount: [details]}) details = checkIfMatching(submission) if details is not None: if not details["postType"] == "self": orderCount += 1 printSubmission(submission, subCount, orderCount) subList.append(details) else: postsFile.add({subCount: [details]}) if not len(subList) == 0: print( "\nTotal of {} submissions found!\n"\ "{} GFYCATs, {} IMGURs and {} DIRECTs\n" .format(len(subList),gfycatCount,imgurCount,directCount) ) return subList else: raise NoMatchingSubmissionFound
def redditSearcher(posts, SINGLE_POST=False): """Check posts and decide if it can be downloaded. If so, create a dictionary with post details and append them to a list. Write all of posts to file. Return the list """ subList = [] global subCount subCount = 0 global orderCount orderCount = 0 global gfycatCount gfycatCount = 0 global imgurCount imgurCount = 0 global eromeCount eromeCount = 0 global directCount directCount = 0 global selfCount selfCount = 0 allPosts = {} print("\nGETTING POSTS") if GLOBAL.arguments.verbose: print("\n") postsFile = createLogFile("POSTS") if SINGLE_POST: submission = posts subCount += 1 try: details = { 'postId': submission.id, 'postTitle': submission.title, 'postSubmitter': str(submission.author), 'postType': None, 'postURL': submission.url, 'postSubreddit': submission.subreddit.display_name } except AttributeError: pass result = checkIfMatching(submission) if result is not None: details = result orderCount += 1 if GLOBAL.arguments.verbose: printSubmission(submission, subCount, orderCount) subList.append(details) postsFile.add({subCount: [details]}) else: try: for submission in posts: subCount += 1 if subCount % 100 == 0 and not GLOBAL.arguments.verbose: sys.stdout.write("• ") sys.stdout.flush() if subCount % 1000 == 0: sys.stdout.write("\n" + " " * 14) sys.stdout.flush() try: details = { 'postId': submission.id, 'postTitle': submission.title, 'postSubmitter': str(submission.author), 'postType': None, 'postURL': submission.url, 'postSubreddit': submission.subreddit.display_name } except AttributeError: continue result = checkIfMatching(submission) if result is not None: details = result orderCount += 1 if GLOBAL.arguments.verbose: printSubmission(submission, subCount, orderCount) subList.append(details) allPosts[subCount] = [details] except KeyboardInterrupt: print("\nKeyboardInterrupt", noPrint=True) postsFile.add(allPosts) if not len(subList) == 0: if GLOBAL.arguments.NoDownload or GLOBAL.arguments.verbose: print(f"\n\nTotal of {len(subList)} submissions found!") print( f"{gfycatCount} GFYCATs, {imgurCount} IMGURs, " \ f"{eromeCount} EROMEs, {directCount} DIRECTs " \ f"and {selfCount} SELF POSTS",noPrint=True ) else: print() return subList else: raise NoMatchingSubmissionFound("No matching submission was found")
def download(submissions): """Analyze list of submissions and call the right function to download each one, catch errors, update the log files """ subsLenght = len(submissions) global lastRequestTime lastRequestTime = 0 downloadedCount = subsLenght duplicates = 0 FAILED_FILE = createLogFile("FAILED") for i in range(subsLenght): print(f"\n({i+1}/{subsLenght}) – r/{submissions[i]['postSubreddit']}", end="") print(f" – {submissions[i]['postType'].upper()}", end="", noPrint=True) if isPostExists(submissions[i]): print(f"\n" \ f"{submissions[i]['postSubmitter']}_" f"{nameCorrector(submissions[i]['postTitle'])}") print("It already exists") duplicates += 1 downloadedCount -= 1 continue try: downloadPost(submissions[i]) except FileAlreadyExistsError: print("It already exists") duplicates += 1 downloadedCount -= 1 except ImgurLoginError: print( "Imgur login failed. \nQuitting the program "\ "as unexpected errors might occur." ) sys.exit() except ImgurLimitError as exception: FAILED_FILE.add({ int(i + 1): [ "{class_name}: {info}".format( class_name=exception.__class__.__name__, info=str(exception)), submissions[i] ] }) downloadedCount -= 1 except NotADownloadableLinkError as exception: print("{class_name}: {info}".format( class_name=exception.__class__.__name__, info=str(exception))) FAILED_FILE.add({ int(i + 1): [ "{class_name}: {info}".format( class_name=exception.__class__.__name__, info=str(exception)), submissions[i] ] }) downloadedCount -= 1 except NoSuitablePost: print("No match found, skipping...") downloadedCount -= 1 except Exception as exception: # raise exception print("{class_name}: {info}".format( class_name=exception.__class__.__name__, info=str(exception))) FAILED_FILE.add({ int(i + 1): [ "{class_name}: {info}".format( class_name=exception.__class__.__name__, info=str(exception)), submissions[i] ] }) downloadedCount -= 1 if duplicates: print(f"\nThere {'were' if duplicates > 1 else 'was'} " \ f"{duplicates} duplicate{'s' if duplicates > 1 else ''}") if downloadedCount == 0: print("Nothing downloaded :(") else: print(f"Total of {downloadedCount} " \ f"link{'s' if downloadedCount > 1 else ''} downloaded!")
def redditSearcher(posts): """Check posts and decide if it can be downloaded. If so, create a dictionary with post details and append them to a list. Write all of posts to file. Return the list """ subList = [] subCount = 0 orderCount = 0 gfycatCount = 0 imgurCount = 0 directCount = 0 found = False postsFile = createLogFile("POSTS") for submission in posts: subCount += 1 found = False try: details = { 'postId': submission.id, 'postTitle': submission.title, 'postSubmitter': str(submission.author), 'postType': None, 'postURL': submission.url, 'postSubreddit': submission.subreddit.display_name } except AttributeError: continue if ('gfycat' in submission.domain) or \ ('imgur' in submission.domain): found = True if 'gfycat' in submission.domain: details['postType'] = 'gfycat' gfycatCount += 1 elif 'imgur' in submission.domain: details['postType'] = 'imgur' imgurCount += 1 orderCount += 1 printSubmission(submission, subCount, orderCount) elif isDirectLink(submission.url) is True: found = True orderCount += 1 directCount += 1 details['postType'] = 'direct' printSubmission(submission, subCount, orderCount) if found: subList.append(details) postsFile.add({subCount: [details]}) print( "\nTotal of {} submissions found!\n{} GFYCATs, {} IMGURs and {} DIRECTs\n" .format(len(subList), gfycatCount, imgurCount, directCount)) return subList