def extractDetails(posts, SINGLE_POST=False):
"""Check posts and decide if it can be downloaded.
If so, create a dictionary with post details and append them to a list.
Write all of posts to file. Return the list
"""
postList = []
postCount = 1
allPosts = {}
print("\nGETTING POSTS")
postsFile = createLogFile("POSTS")
if SINGLE_POST:
submission = posts
postCount += 1
try:
details = {
'POSTID':
submission.id,
'TITLE':
submission.title,
'REDDITOR':
str(submission.author),
'TYPE':
None,
'CONTENTURL':
submission.url,
'SUBREDDIT':
submission.subreddit.display_name,
'UPVOTES':
submission.score,
'FLAIR':
submission.link_flair_text,
'DATE':
str(
time.strftime("%Y-%m-%d_%H-%M",
time.localtime(submission.created_utc)))
}
except AttributeError:
pass
if not any(domain in submission.domain
for domain in GLOBAL.arguments.skip_domain):
result = matchWithDownloader(submission)
if result is not None:
details = {**details, **result}
postList.append(details)
postsFile.add({postCount: details})
else:
try:
for submission in posts:
if postCount % 100 == 0:
sys.stdout.write("• ")
sys.stdout.flush()
if postCount % 1000 == 0:
sys.stdout.write("\n" + " " * 14)
sys.stdout.flush()
try:
details = {
'POSTID':
submission.id,
'TITLE':
submission.title,
'REDDITOR':
str(submission.author),
'TYPE':
None,
'CONTENTURL':
submission.url,
'SUBREDDIT':
submission.subreddit.display_name,
'UPVOTES':
submission.score,
'FLAIR':
submission.link_flair_text,
'DATE':
str(
time.strftime(
"%Y-%m-%d_%H-%M",
time.localtime(submission.created_utc)))
}
except AttributeError:
continue
if details['POSTID'] in GLOBAL.downloadedPosts(): continue
if not any(domain in submission.domain
for domain in GLOBAL.arguments.skip_domain):
result = matchWithDownloader(submission)
if result is not None:
details = {**details, **result}
postList.append(details)
allPosts[postCount] = details
postCount += 1
except KeyboardInterrupt:
print("\nKeyboardInterrupt", noPrint=True)
postsFile.add(allPosts)
if not len(postList) == 0:
print()
return postList
else:
raise NoMatchingSubmissionFound("No matching submission was found")
def download(submissions):
"""Analyze list of submissions and call the right function
to download each one, catch errors, update the log files
"""
downloadedCount = 0
duplicates = 0
FAILED_FILE = createLogFile("FAILED")
if GLOBAL.arguments.unsave:
reddit = Reddit(GLOBAL.config['credentials']['reddit']).begin()
subsLenght = len(submissions)
for i in range(len(submissions)):
print(f"\n({i+1}/{subsLenght})",end=" — ")
print(submissions[i]['POSTID'],
f"r/{submissions[i]['SUBREDDIT']}",
f"u/{submissions[i]['REDDITOR']}",
submissions[i]['FLAIR'] if submissions[i]['FLAIR'] else "",
sep=" — ",
end="")
print(f" – {submissions[i]['TYPE'].upper()}",end="",noPrint=True)
directory = GLOBAL.directory / GLOBAL.config["folderpath"].format(**submissions[i])
details = {
**submissions[i],
**{
"TITLE": nameCorrector(
submissions[i]['TITLE'],
reference = str(directory)
+ GLOBAL.config['filename'].format(**submissions[i])
+ ".ext"
)
}
}
filename = GLOBAL.config['filename'].format(**details)
if isPostExists(details,directory):
print()
print(directory)
print(filename)
print("It already exists")
duplicates += 1
continue
if any(domain in submissions[i]['CONTENTURL'] for domain in GLOBAL.arguments.skip):
print()
print(submissions[i]['CONTENTURL'])
print("Domain found in skip domains, skipping post...")
continue
try:
downloadPost(details,directory)
GLOBAL.downloadedPosts.add(details['POSTID'])
try:
if GLOBAL.arguments.unsave:
reddit.submission(id=details['POSTID']).unsave()
except InsufficientScope:
reddit = Reddit().begin()
reddit.submission(id=details['POSTID']).unsave()
downloadedCount += 1
except FileAlreadyExistsError:
print("It already exists")
GLOBAL.downloadedPosts.add(details['POSTID'])
duplicates += 1
except ImgurLoginError:
print(
"Imgur login failed. \nQuitting the program "\
"as unexpected errors might occur."
)
sys.exit()
except ImgurLimitError as exception:
FAILED_FILE.add({int(i+1):[
"{class_name}: {info}".format(
class_name=exception.__class__.__name__,info=str(exception)
),
details
]})
except NotADownloadableLinkError as exception:
print(
"{class_name}: {info}".format(
class_name=exception.__class__.__name__,info=str(exception)
)
)
FAILED_FILE.add({int(i+1):[
"{class_name}: {info}".format(
class_name=exception.__class__.__name__,info=str(exception)
),
submissions[i]
]})
except TypeInSkip:
print()
print(submissions[i]['CONTENTURL'])
print("Skipping post...")
except DomainInSkip:
print()
print(submissions[i]['CONTENTURL'])
print("Skipping post...")
except NoSuitablePost:
print("No match found, skipping...")
except FailedToDownload:
print("Failed to download the posts, skipping...")
except AlbumNotDownloadedCompletely:
print("Album did not downloaded completely.")
FAILED_FILE.add({int(i+1):[
"{class_name}: {info}".format(
class_name=exc.__class__.__name__,info=str(exc)
),
submissions[i]
]})
except Exception as exc:
print(
"{class_name}: {info}\nSee CONSOLE_LOG.txt for more information".format(
class_name=exc.__class__.__name__,info=str(exc)
)
)
logging.error(sys.exc_info()[0].__name__,
exc_info=full_exc_info(sys.exc_info()))
print(GLOBAL.log_stream.getvalue(),noPrint=True)
FAILED_FILE.add({int(i+1):[
"{class_name}: {info}".format(
class_name=exc.__class__.__name__,info=str(exc)
),
submissions[i]
]})
if duplicates:
print(f"\nThere {'were' if duplicates > 1 else 'was'} " \
f"{duplicates} duplicate{'s' if duplicates > 1 else ''}")
if downloadedCount == 0:
print("Nothing is downloaded :(")
else:
print(f"Total of {downloadedCount} " \
f"link{'s' if downloadedCount > 1 else ''} downloaded!")
def download(submissions):
"""Analyze list of submissions and call the right function
to download each one, catch errors, update the log files
"""
subsLenght = len(submissions)
global lastRequestTime
lastRequestTime = 0
downloadedCount = subsLenght
duplicates = 0
FAILED_FILE = createLogFile("FAILED")
for i in range(subsLenght):
print(
f"\n({i+1}/{subsLenght}) – {submissions[i]['postId']} – r/{submissions[i]['postSubreddit']}",
end="")
print(f" – {submissions[i]['postType'].upper()}", end="", noPrint=True)
if isPostExists(submissions[i]):
print(f"\n" \
f"{submissions[i]['postSubmitter']}_"
f"{nameCorrector(submissions[i]['postTitle'])}")
print("It already exists")
duplicates += 1
downloadedCount -= 1
continue
try:
downloadPost(submissions[i])
except FileAlreadyExistsError:
print("It already exists")
duplicates += 1
downloadedCount -= 1
except ImgurLoginError:
print(
"Imgur login failed. \nQuitting the program "\
"as unexpected errors might occur."
)
sys.exit()
except ImgurLimitError as exception:
FAILED_FILE.add({
int(i + 1): [
"{class_name}: {info}".format(
class_name=exception.__class__.__name__,
info=str(exception)), submissions[i]
]
})
downloadedCount -= 1
except NotADownloadableLinkError as exception:
print("{class_name}: {info}".format(
class_name=exception.__class__.__name__, info=str(exception)))
FAILED_FILE.add({
int(i + 1): [
"{class_name}: {info}".format(
class_name=exception.__class__.__name__,
info=str(exception)), submissions[i]
]
})
downloadedCount -= 1
except NoSuitablePost:
print("No match found, skipping...")
downloadedCount -= 1
except Exception as exception:
# raise exception
print("{class_name}: {info}".format(
class_name=exception.__class__.__name__, info=str(exception)))
FAILED_FILE.add({
int(i + 1): [
"{class_name}: {info}".format(
class_name=exception.__class__.__name__,
info=str(exception)), submissions[i]
]
})
downloadedCount -= 1
if duplicates:
print(f"\nThere {'were' if duplicates > 1 else 'was'} " \
f"{duplicates} duplicate{'s' if duplicates > 1 else ''}")
if downloadedCount == 0:
print("Nothing downloaded :(")
else:
print(f"Total of {downloadedCount} " \
f"link{'s' if downloadedCount > 1 else ''} downloaded!")
def redditSearcher(posts,SINGLE_POST=False):
"""Check posts and decide if it can be downloaded.
If so, create a dictionary with post details and append them to a list.
Write all of posts to file. Return the list
"""
subList = []
global subCount
subCount = 0
global orderCount
orderCount = 0
global gfycatCount
gfycatCount = 0
global redgifsCount
redgifsCount = 0
global imgurCount
imgurCount = 0
global eromeCount
eromeCount = 0
global gifDeliveryNetworkCount
gifDeliveryNetworkCount = 0
global directCount
directCount = 0
global selfCount
selfCount = 0
allPosts = {}
print("\nGETTING POSTS")
if GLOBAL.arguments.verbose: print("\n")
postsFile = createLogFile("POSTS")
if SINGLE_POST:
submission = posts
subCount += 1
try:
details = {'postId':submission.id,
'postTitle':submission.title,
'postSubmitter':str(submission.author),
'postType':None,
'postURL':submission.url,
'postSubreddit':submission.subreddit.display_name}
except AttributeError:
pass
result = checkIfMatching(submission)
if result is not None:
details = result
orderCount += 1
if GLOBAL.arguments.verbose:
printSubmission(submission,subCount,orderCount)
subList.append(details)
postsFile.add({subCount:[details]})
else:
try:
for submission in posts:
subCount += 1
if subCount % 100 == 0 and not GLOBAL.arguments.verbose:
sys.stdout.write("• ")
sys.stdout.flush()
if subCount % 1000 == 0:
sys.stdout.write("\n"+" "*14)
sys.stdout.flush()
try:
details = {'postId':submission.id,
'postTitle':submission.title,
'postSubmitter':str(submission.author),
'postType':None,
'postURL':submission.url,
'postSubreddit':submission.subreddit.display_name}
except AttributeError:
continue
result = checkIfMatching(submission)
if result is not None:
details = result
orderCount += 1
if GLOBAL.arguments.verbose:
printSubmission(submission,subCount,orderCount)
subList.append(details)
allPosts[subCount] = [details]
except KeyboardInterrupt:
print("\nKeyboardInterrupt",noPrint=True)
postsFile.add(allPosts)
if not len(subList) == 0:
if GLOBAL.arguments.NoDownload or GLOBAL.arguments.verbose:
print(
f"\n\nTotal of {len(subList)} submissions found!"
)
print(
f"{gfycatCount} GFYCATs, {imgurCount} IMGURs, " \
f"{eromeCount} EROMEs, {directCount} DIRECTs " \
f"and {selfCount} SELF POSTS",noPrint=True
)
else:
print()
return subList
else:
raise NoMatchingSubmissionFound("No matching submission was found")
def extractDetails(posts, SINGLE_POST=False):
"""Check posts and decide if it can be downloaded.
If so, create a dictionary with post details and append them to a list.
Write all of posts to file. Return the list
"""
postList = []
postCount = 1
allPosts = {}
print("\nGETTING POSTS")
postsFile = createLogFile("POSTS")
if SINGLE_POST:
submission = posts
postCount += 1
# All options offered by PRAW are author, clicked, comments, created_utc, distinguished, edited, id,
# is_original_content, is_self, link_flair_template_id, link_flair_text, locked, name, num_comments, over_18,
# permalink, poll_data, score, selftext, spoiler, stickied, subreddit, title, upvote_ratio, url
try:
details = {
'POSTID':
submission.id,
'TITLE':
submission.title,
'REDDITOR':
str(submission.author),
'TYPE':
None,
'CONTENTURL':
submission.url,
'SUBREDDIT':
submission.subreddit.display_name,
'UPVOTES':
submission.score,
'FLAIR':
submission.link_flair_text,
'OVER18':
submission.over_18,
'DATE':
str(
time.strftime("%Y-%m-%d_%H-%M",
time.localtime(submission.created_utc)))
}
except AttributeError:
pass
if not any(domain in submission.domain
for domain in GLOBAL.arguments.skip_domain):
result = matchWithDownloader(submission)
if result is not None:
details = {**details, **result}
postList.append(details)
postsFile.add({postCount: details})
else:
try:
for submission in posts:
if postCount % 100 == 0:
sys.stdout.write("• ")
sys.stdout.flush()
if postCount % 1000 == 0:
sys.stdout.write("\n" + " " * 14)
sys.stdout.flush()
try:
details = {
'POSTID':
submission.id,
'TITLE':
submission.title,
'REDDITOR':
str(submission.author),
'TYPE':
None,
'CONTENTURL':
submission.url,
'SUBREDDIT':
submission.subreddit.display_name,
'UPVOTES':
submission.score,
'FLAIR':
submission.link_flair_text,
'OVER18':
submission.over_18,
'DATE':
str(
time.strftime(
"%Y-%m-%d_%H-%M",
time.localtime(submission.created_utc)))
}
except AttributeError:
continue
# Check if the grabbed POSTID is already in the downloadedPosts from Prior runs, if enabled
if details['POSTID'] in GLOBAL.downloadedPosts(): continue
# Check to make sure the post is not in the skip_domain list
if not any(domain in submission.domain
for domain in GLOBAL.arguments.skip_domain):
result = matchWithDownloader(submission)
if result is not None:
details = {**details, **result}
postList.append(details)
allPosts[postCount] = details
postCount += 1
except KeyboardInterrupt:
print("\nKeyboardInterrupt", noPrint=True)
postsFile.add(allPosts)
if not len(postList) == 0:
print()
return postList
else:
raise NoMatchingSubmissionFound("No matching submission was found")
