def download(submissions):
"""Analyze list of submissions and call the right function
to download each one, catch errors, update the log files
"""
subsLenght = len(submissions)
lastRequestTime = 0
downloadedCount = subsLenght
duplicates = 0
BACKUP = {}
FAILED_FILE = createLogFile("FAILED")
for i in range(subsLenght):
print("\n({}/{})".format(i + 1, subsLenght))
print("https://reddit.com/r/{subreddit}/comments/{id}".format(
subreddit=submissions[i]['postSubreddit'],
id=submissions[i]['postId']))
if postExists(submissions[i]):
result = False
print("It already exists")
duplicates += 1
downloadedCount -= 1
continue
directory = GLOBAL.directory / submissions[i]['postSubreddit']
if submissions[i]['postType'] == 'imgur':
print("IMGUR", end="")
while int(time.time() - lastRequestTime) <= 2:
pass
credit = Imgur.get_credits()
IMGUR_RESET_TIME = credit['UserReset'] - time.time()
USER_RESET = ("after " \
+ str(int(IMGUR_RESET_TIME/60)) \
+ " Minutes " \
+ str(int(IMGUR_RESET_TIME%60)) \
+ " Seconds")
print(" => Client: {} - User: {} - Reset {}".format(
credit['ClientRemaining'], credit['UserRemaining'],
USER_RESET))
if not (credit['UserRemaining'] == 0 or \
credit['ClientRemaining'] == 0):
"""This block of code is needed
"""
while int(time.time() - lastRequestTime) <= 2:
pass
lastRequestTime = time.time()
try:
Imgur(directory, submissions[i])
except FileAlreadyExistsError:
print("It already exists")
duplicates += 1
downloadedCount -= 1
except ImgurLoginError:
print(
"Imgur login failed. Quitting the program "\
"as unexpected errors might occur."
)
quit()
except Exception as exception:
print(exception)
FAILED_FILE.add(
{int(i + 1): [str(exception), submissions[i]]})
downloadedCount -= 1
else:
if credit['UserRemaining'] == 0:
KEYWORD = "user"
elif credit['ClientRemaining'] == 0:
KEYWORD = "client"
print('{} LIMIT EXCEEDED\n'.format(KEYWORD.upper()))
FAILED_FILE.add({
int(i + 1): [
'{} LIMIT EXCEEDED\n'.format(KEYWORD.upper()),
submissions[i]
]
})
downloadedCount -= 1
elif submissions[i]['postType'] == 'gfycat':
print("GFYCAT")
try:
Gfycat(directory, submissions[i])
except FileAlreadyExistsError:
print("It already exists")
duplicates += 1
downloadedCount -= 1
except NotADownloadableLinkError as exception:
print("Could not read the page source")
FAILED_FILE.add({int(i + 1): [str(exception), submissions[i]]})
downloadedCount -= 1
except Exception as exception:
print(exception)
FAILED_FILE.add({int(i + 1): [str(exception), submissions[i]]})
downloadedCount -= 1
elif submissions[i]['postType'] == 'direct':
print("DIRECT")
try:
Direct(directory, submissions[i])
except FileAlreadyExistsError:
print("It already exists")
downloadedCount -= 1
duplicates += 1
except Exception as exception:
print(exception)
FAILED_FILE.add({int(i + 1): [str(exception), submissions[i]]})
downloadedCount -= 1
else:
print("No match found, skipping...")
downloadedCount -= 1
if duplicates:
print("\n There was {} duplicates".format(duplicates))
if downloadedCount == 0:
print(" Nothing downloaded :(")
else:
print(" Total of {} links downloaded!".format(downloadedCount))
def redditSearcher(posts, SINGLE_POST=False):
"""Check posts and decide if it can be downloaded.
If so, create a dictionary with post details and append them to a list.
Write all of posts to file. Return the list
"""
subList = []
global subCount
subCount = 0
global orderCount
orderCount = 0
global gfycatCount
gfycatCount = 0
global imgurCount
imgurCount = 0
global directCount
directCount = 0
postsFile = createLogFile("POSTS")
if SINGLE_POST:
submission = posts
subCount += 1
try:
details = {
'postId': submission.id,
'postTitle': submission.title,
'postSubmitter': str(submission.author),
'postType': None,
'postURL': submission.url,
'postSubreddit': submission.subreddit.display_name
}
except AttributeError:
pass
postsFile.add({subCount: [details]})
details = checkIfMatching(submission)
if details is not None:
if not details["postType"] == "self":
orderCount += 1
printSubmission(submission, subCount, orderCount)
subList.append(details)
else:
postsFile.add({subCount: [details]})
else:
for submission in posts:
subCount += 1
try:
details = {
'postId': submission.id,
'postTitle': submission.title,
'postSubmitter': str(submission.author),
'postType': None,
'postURL': submission.url,
'postSubreddit': submission.subreddit.display_name
}
except AttributeError:
continue
postsFile.add({subCount: [details]})
details = checkIfMatching(submission)
if details is not None:
if not details["postType"] == "self":
orderCount += 1
printSubmission(submission, subCount, orderCount)
subList.append(details)
else:
postsFile.add({subCount: [details]})
if not len(subList) == 0:
print(
"\nTotal of {} submissions found!\n"\
"{} GFYCATs, {} IMGURs and {} DIRECTs\n"
.format(len(subList),gfycatCount,imgurCount,directCount)
)
return subList
else:
raise NoMatchingSubmissionFound
def redditSearcher(posts, SINGLE_POST=False):
"""Check posts and decide if it can be downloaded.
If so, create a dictionary with post details and append them to a list.
Write all of posts to file. Return the list
"""
subList = []
global subCount
subCount = 0
global orderCount
orderCount = 0
global gfycatCount
gfycatCount = 0
global imgurCount
imgurCount = 0
global eromeCount
eromeCount = 0
global directCount
directCount = 0
global selfCount
selfCount = 0
allPosts = {}
print("\nGETTING POSTS")
if GLOBAL.arguments.verbose: print("\n")
postsFile = createLogFile("POSTS")
if SINGLE_POST:
submission = posts
subCount += 1
try:
details = {
'postId': submission.id,
'postTitle': submission.title,
'postSubmitter': str(submission.author),
'postType': None,
'postURL': submission.url,
'postSubreddit': submission.subreddit.display_name
}
except AttributeError:
pass
result = checkIfMatching(submission)
if result is not None:
details = result
orderCount += 1
if GLOBAL.arguments.verbose:
printSubmission(submission, subCount, orderCount)
subList.append(details)
postsFile.add({subCount: [details]})
else:
try:
for submission in posts:
subCount += 1
if subCount % 100 == 0 and not GLOBAL.arguments.verbose:
sys.stdout.write("• ")
sys.stdout.flush()
if subCount % 1000 == 0:
sys.stdout.write("\n" + " " * 14)
sys.stdout.flush()
try:
details = {
'postId': submission.id,
'postTitle': submission.title,
'postSubmitter': str(submission.author),
'postType': None,
'postURL': submission.url,
'postSubreddit': submission.subreddit.display_name
}
except AttributeError:
continue
result = checkIfMatching(submission)
if result is not None:
details = result
orderCount += 1
if GLOBAL.arguments.verbose:
printSubmission(submission, subCount, orderCount)
subList.append(details)
allPosts[subCount] = [details]
except KeyboardInterrupt:
print("\nKeyboardInterrupt", noPrint=True)
postsFile.add(allPosts)
if not len(subList) == 0:
if GLOBAL.arguments.NoDownload or GLOBAL.arguments.verbose:
print(f"\n\nTotal of {len(subList)} submissions found!")
print(
f"{gfycatCount} GFYCATs, {imgurCount} IMGURs, " \
f"{eromeCount} EROMEs, {directCount} DIRECTs " \
f"and {selfCount} SELF POSTS",noPrint=True
)
else:
print()
return subList
else:
raise NoMatchingSubmissionFound("No matching submission was found")
def download(submissions):
"""Analyze list of submissions and call the right function
to download each one, catch errors, update the log files
"""
subsLenght = len(submissions)
global lastRequestTime
lastRequestTime = 0
downloadedCount = subsLenght
duplicates = 0
FAILED_FILE = createLogFile("FAILED")
for i in range(subsLenght):
print(f"\n({i+1}/{subsLenght}) – r/{submissions[i]['postSubreddit']}",
end="")
print(f" – {submissions[i]['postType'].upper()}", end="", noPrint=True)
if isPostExists(submissions[i]):
print(f"\n" \
f"{submissions[i]['postSubmitter']}_"
f"{nameCorrector(submissions[i]['postTitle'])}")
print("It already exists")
duplicates += 1
downloadedCount -= 1
continue
try:
downloadPost(submissions[i])
except FileAlreadyExistsError:
print("It already exists")
duplicates += 1
downloadedCount -= 1
except ImgurLoginError:
print(
"Imgur login failed. \nQuitting the program "\
"as unexpected errors might occur."
)
sys.exit()
except ImgurLimitError as exception:
FAILED_FILE.add({
int(i + 1): [
"{class_name}: {info}".format(
class_name=exception.__class__.__name__,
info=str(exception)), submissions[i]
]
})
downloadedCount -= 1
except NotADownloadableLinkError as exception:
print("{class_name}: {info}".format(
class_name=exception.__class__.__name__, info=str(exception)))
FAILED_FILE.add({
int(i + 1): [
"{class_name}: {info}".format(
class_name=exception.__class__.__name__,
info=str(exception)), submissions[i]
]
})
downloadedCount -= 1
except NoSuitablePost:
print("No match found, skipping...")
downloadedCount -= 1
except Exception as exception:
# raise exception
print("{class_name}: {info}".format(
class_name=exception.__class__.__name__, info=str(exception)))
FAILED_FILE.add({
int(i + 1): [
"{class_name}: {info}".format(
class_name=exception.__class__.__name__,
info=str(exception)), submissions[i]
]
})
downloadedCount -= 1
if duplicates:
print(f"\nThere {'were' if duplicates > 1 else 'was'} " \
f"{duplicates} duplicate{'s' if duplicates > 1 else ''}")
if downloadedCount == 0:
print("Nothing downloaded :(")
else:
print(f"Total of {downloadedCount} " \
f"link{'s' if downloadedCount > 1 else ''} downloaded!")
def redditSearcher(posts):
"""Check posts and decide if it can be downloaded.
If so, create a dictionary with post details and append them to a list.
Write all of posts to file. Return the list
"""
subList = []
subCount = 0
orderCount = 0
gfycatCount = 0
imgurCount = 0
directCount = 0
found = False
postsFile = createLogFile("POSTS")
for submission in posts:
subCount += 1
found = False
try:
details = {
'postId': submission.id,
'postTitle': submission.title,
'postSubmitter': str(submission.author),
'postType': None,
'postURL': submission.url,
'postSubreddit': submission.subreddit.display_name
}
except AttributeError:
continue
if ('gfycat' in submission.domain) or \
('imgur' in submission.domain):
found = True
if 'gfycat' in submission.domain:
details['postType'] = 'gfycat'
gfycatCount += 1
elif 'imgur' in submission.domain:
details['postType'] = 'imgur'
imgurCount += 1
orderCount += 1
printSubmission(submission, subCount, orderCount)
elif isDirectLink(submission.url) is True:
found = True
orderCount += 1
directCount += 1
details['postType'] = 'direct'
printSubmission(submission, subCount, orderCount)
if found:
subList.append(details)
postsFile.add({subCount: [details]})
print(
"\nTotal of {} submissions found!\n{} GFYCATs, {} IMGURs and {} DIRECTs\n"
.format(len(subList), gfycatCount, imgurCount, directCount))
return subList
