def _praw_apply_filter(praw_object, order_by='new', limit=None, time='all'):
""" Accepts a Praw object (subreddit/multireddit/user posts/etc) and applies filters to it. Returns a Generator. """
if order_by == 'best':
print(
'Sorting submissions by "best" is no longer supported. Use "top" instead.'
)
order_by = 'top'
time = 'day'
order = [o for o in post_orders() if o[0] == order_by]
assert len(order) > 0 # The order must be a valid value.
assert time in time_filters()
if limit < 1:
limit = None
order = order[0]
try:
if not order[1]:
gen = getattr(praw_object, order[0])(limit=limit)
else:
gen = getattr(praw_object, order[0])(limit=limit, time_filter=time)
for g in gen:
yield RedditElement(g)
except TypeError as e:
stringutil.error('Invalid Praw order configuration! [%s]' % order_by)
print(order)
print(e)
def load_sources(self): # !cover
importlib.reload(custom_sources)
custom_sources.load_userlist()
sources = []
settings_sources = settings.get_sources()
if self.sources is None:
for s in settings_sources:
print('Loaded Source: ', s.get_alias())
sources.append(s)
else:
for so in self.sources:
regexp = re.compile(str(so), re.IGNORECASE)
for s in settings_sources:
if regexp.search(str(s.get_alias())):
print('Matched Source: ', s.get_alias())
sources.append(s)
break
if len(sources) == 0:
if len(settings_sources) == 0:
su.error('No sources were found from the settings file.')
else:
su.error(
'No sources were found from the settings file matching the supplied patterns.'
)
sys.exit(20)
return sources
def _submission(self, post):
""" Handle a Submission. """
# out("[Post](%s): %s" % (post.subreddit.display_name, post.title) )
self.type = 'Submission'
self.id = str(post.fullname)
self.title = str(post.title)
self.subreddit = str(post.subreddit.display_name)
if post.author is None:
self.author = 'Deleted'
else:
self.author = str(post.author.name)
self.over_18 = post.over_18
self.num_comments = post.num_comments
self.score = post.score
self.body = post.selftext
if post.selftext.strip() != '':
# This post probably doesn't have a URL, and has selftext instead.
for url in stringutil.html_elements(post.selftext_html, 'a',
'href'):
self.add_url(url)
if getattr(post, 'is_gallery', False) and getattr(
post, 'media_metadata', False):
for k, img in post.media_metadata.items():
try:
self.add_url(img['s']['u'])
except:
stringutil.error('Unable to parse URL from reddit album.')
elif post.url is not None and post.url.strip() != '':
self.add_url(post.url)
def user_liked_saved(username,
scan_upvoted=True,
scan_saved=True,
scan_sub=None):
""" Gets all the upvoted/saved comments and/or submissions for the given User. Allows filtering by Subreddit. """
params = {'sr': scan_sub} if scan_sub else None
try:
if _user.name.lower() == username.lower():
redditor = _user
else:
redditor = _reddit.redditor(username)
if scan_saved:
for saved in redditor.saved(limit=None, params=params):
re = RedditElement(saved)
yield re
if scan_upvoted:
for upvoted in redditor.upvoted(limit=None, params=params):
re = RedditElement(upvoted)
yield re
except prawcore.exceptions.NotFound:
stringutil.error(
'Cannot locate comments or submissions for nonexistent user: %s' %
username)
except prawcore.Forbidden:
stringutil.error(
'Cannot load Upvoted/Saved Posts from the User "%s", because they are private!'
% username)
def _ps_submission(self, post):
""" Handle a PushShift Submission. """
self.type = 'Submission'
self.id = post.id
if 't3_' not in self.id:
self.id = 't3_%s' % self.id
self.title = post.title
self.subreddit = post.subreddit
if getattr(post, 'author', None) is None or post.author == '[deleted]':
self.author = 'Deleted'
else:
self.author = str(post.author)
self.over_18 = post.over_18
self.num_comments = post.num_comments
self.score = post.score
self.body = html.unescape(str(
post.selftext)) if 'selftext' in post and post.selftext else ''
if getattr(post, 'is_gallery', False) and getattr(
post, 'media_metadata', False):
for k, img in post.media_metadata.items():
try:
self.add_url(html.unescape(img['s']['u']))
except:
stringutil.error(
'Unable to parse URL from praw reddit album')
elif post.url is not None and post.url.strip(
) != '' and 'reddit.com' not in post.url:
self.add_url(post.url)
if self.body.strip():
for u in re.findall(r'\[.+?\]\s*?\((.+?)\)', self.body):
self.add_url(u)
def _convert_imported_limit(self, val):
""" Overrides default to convert user-supplied string dates to timestamps. """
if not stringutil.is_numeric(val):
stringutil.error(
"Error in UTC Filter: Expects time in a numeric UTC timestamp."
) # !cover
return val
def user_posts(username,
find_submissions,
find_comments,
find_limit=None,
deep_find_submissions=False,
deep_find_comments=False): #vy
""" Generator for all the posts made by the given Redditor. """
try:
limit = find_limit if (find_limit is not None
and find_limit > 0) else None
if find_comments or deep_find_comments:
for c in _reddit.redditor(username).comments.new(limit=limit):
yield RedditElement(c)
if deep_find_comments:
for c in _reddit.redditor(username).comments.top(limit=limit):
yield RedditElement(c)
if find_submissions or deep_find_submissions:
for c in _reddit.redditor(username).submissions.new(limit=limit):
yield RedditElement(c)
if deep_find_submissions:
for c in _reddit.redditor(username).submissions.top(limit=limit):
yield RedditElement(c)
except prawcore.exceptions.NotFound:
stringutil.error(
'Cannot locate comments or submissions for nonexistent user: %s' %
username)
with open('user.nonexistent.log', 'a+', newline='\n') as f:
f.write(username + '\n')
except prawcore.exceptions.Forbidden:
stringutil.error(
'Cannot locate posts from a suspended user account: %s' % username)
with open('user.suspended.log', 'a+', newline='\n') as f:
f.write(username + '\n')
def _safe_commit(self):
""" Commit and catch any exceptions, usually raised if the session is not dirty. """
try:
self._session.commit(
) # After commiting, the URL generated IDs will be filled in.
except Exception as e:
stringutil.error("RedditLoader: Error persisting session: %s" % e)
pass
def user_posts(username, find_submissions, find_comments):
""" Generator for all the posts made by the given Redditor. """
try:
if find_comments:
for c in _reddit.redditor(username).comments.new():
yield RedditElement(c)
if find_submissions:
for c in _reddit.redditor(username).submissions.new():
yield RedditElement(c)
except prawcore.exceptions.NotFound:
stringutil.error('Cannot locate comments or submissions for nonexistent user: %s' % username)
def handle(handler_task, progress_obj):
"""
Pass the given HandlerTask into the handler list, and try to find one that can download the given file.
"""
for h in sorted_list():
try:
progress_obj.set_handler(h.tag)
result = h.handle(task=handler_task, progress=progress_obj)
if result:
return result
except Exception as ex: # There are too many possible exceptions between all handlers to catch properly.
stringutil.error('Handler Exception [%s] :: {%s} :: %s' % (h.tag, handler_task.url, ex))
# We don't *really* care if a Handler fails, since there are plenty of reasons a download could.
traceback.print_exc()
pass
return HandlerResponse(success=False, handler=None, failure_reason="No Handlers could process this URL.")
def run():
logging.basicConfig(level=logging.WARN, format='%(levelname)-5.5s [%(name)s] %(message)s', datefmt='%H:%M:%S')
su.print_color('green', "\r\n" +
'====================================\r\n' +
(' Reddit Media Downloader %s\r\n' % meta.current_version) +
'====================================\r\n' +
' (By ShadowMoose @ Github)\r\n')
if args.version:
sys.exit(0)
if args.run_tests:
error_count = tests.runner.run_tests(test_subdir=args.run_tests)
sys.exit(error_count)
if args.list_settings:
print('All valid overridable settings:')
for _s in settings.get_all():
if _s.public:
print("%s.%s" % (_s.category, _s.name))
print('\tDescription: %s' % _s.description)
if not _s.opts:
print('\tValid value: \n\t\tAny %s' % _s.type)
else:
print('\tValid values:')
for o in _s.opts:
print('\t\t"%s": %s' % o)
print()
sys.exit()
settings_file = args.settings or fs.find_file('settings.json')
_loaded = settings.load(settings_file)
for ua in unknown_args:
if '=' not in ua or '/comments/' in ua:
if '/comments/' in ua:
direct_sources.append(DirectURLSource(url=ua))
continue
elif 'r/' or 'u/' in ua:
direct_sources.append(DirectInputSource(txt=ua, args={'limit': args.limit}))
continue
else:
su.error("ERROR: Unkown argument: %s" % ua)
sys.exit(1)
k = ua.split('=')[0].strip('- ')
v = ua.split('=', 2)[1].strip()
try:
settings.put(k, v, save_after=False)
except KeyError:
print('Unknown setting: %s' % k)
sys.exit(50)
if args.source:
matched_sources = set()
for s in args.source:
for stt in settings.get_sources():
if re.match(s, stt.get_alias()):
matched_sources.add(stt)
direct_sources.extend(matched_sources)
first_time_auth = False
if not _loaded and not direct_sources: # First-time configuration.
su.error('Could not find an existing settings file. A new one will be generated!')
if not console.confirm('Would you like to start the WebUI to help set things up?', True):
su.print_color('red', "If you don't open the webUI now, you'll need to edit the settings file yourself.")
if console.confirm("Are you sure you'd like to edit settings without the UI (if 'yes', these prompts will not show again)?"):
settings.put('interface.start_server', False, save_after=True) # Creates a save.
print('A settings file has been created for you, at "%s". Please customize it.' % settings_file)
first_time_auth = True
else:
print('Please re-run RMD to configure again.')
sys.exit(1)
else:
mode = console.prompt_list('How would you like to open the UI?',
settings.get('interface.browser', full_obj=True).opts)
settings.put('interface.browser', mode, save_after=False)
settings.put('interface.start_server', True)
if args.authorize or first_time_auth: # In-console oAuth authentication flow
from static import praw_wrapper
from urllib.parse import urlparse, parse_qs
url = praw_wrapper.get_reddit_token_url()
su.print_color('green', '\nTo manually authorize your account, visit the below URL.')
su.print_color('yellow', 'Once there, authorize RMD, then copy the URL it redirects you to.')
su.print_color('yellow', 'NOTE: The redirect page will likely not load, and that is ok.')
su.print_color('cyan', '\n%s\n' % url)
token_url = console.col_input('Paste the URL you are redirected to here: ')
if token_url.strip():
qs = parse_qs(urlparse(token_url).query)
if 'state' not in qs or 'code' not in qs:
su.error('The url provided was not a valid reddit redirect. Please make sure you copied it right!')
elif qs['state'][0].strip() != settings.get('auth.oauth_key').strip():
su.error('Invalid reddit redirect state. Please restart and try again.')
else:
code = qs['code'][0]
su.print_color('green', 'Got code. Authorizing account...')
refresh = praw_wrapper.get_refresh_token(code)
if refresh:
settings.put('auth.refresh_token', refresh)
usr = praw_wrapper.get_current_username()
su.print_color('cyan', 'Authorized to view account: %s' % usr)
su.print_color('green', 'Saved authorization token! Please restart RMD to begin downloading!')
else:
su.error('Failed to gain an account access token from Reddit with that code. Please try again.')
sys.exit(0)
if not ffmpeg_download.install_local():
print("RMD was unable to locate (or download) a working FFmpeg binary.")
print("For downloading and post-processing, this is a required tool.")
print("Please Install FFmpeg manually, or download it from here: https://rmd.page.link/ffmpeg")
sys.exit(15)
# Initialize Database
sql.init_from_settings()
print('Using manifest file [%s].' % sql.get_file_location())
if direct_sources:
settings.disable_saving()
settings.put('processing.retry_failed', False)
for s in settings.get_sources():
settings.remove_source(s, save_after=False)
for d in direct_sources:
settings.add_source(d, prevent_duplicate=False, save_after=False)
if settings.get('interface.start_server') and not direct_sources:
print("Starting WebUI...")
ui = WebUI()
else:
ui = TerminalUI()
ui.display()
def run():
su.print_color(
'green', "\r\n" + '====================================\r\n' +
(' Reddit Media Downloader %s\r\n' % meta.current_version) +
'====================================\r\n' +
' (By ShadowMoose @ Github)\r\n')
if args.version:
sys.exit(0)
if args.run_tests:
error_count = tests.runner.run_tests(test_subdir=args.run_tests)
sys.exit(error_count)
if args.list_settings:
print('All valid overridable settings:')
for _s in settings.get_all():
if _s.public:
print("%s.%s" % (_s.category, _s.name))
print('\tDescription: %s' % _s.description)
if not _s.opts:
print('\tValid value: \n\t\tAny %s' % _s.type)
else:
print('\tValid values:')
for o in _s.opts:
print('\t\t"%s": %s' % o)
print()
sys.exit()
settings_file = args.settings or fs.find_file('settings.json')
_loaded = settings.load(settings_file)
for ua in unknown_args:
if '=' not in ua:
if 'r/' or 'u/' in ua:
direct_sources.append(
DirectInputSource(txt=ua, args={'limit': args.limit}))
continue
else:
su.error("ERROR: Unkown argument: %s" % ua)
sys.exit(1)
k = ua.split('=')[0].strip('- ')
v = ua.split('=', 2)[1].strip()
try:
settings.put(k, v, save_after=False)
except KeyError:
print('Unknown setting: %s' % k)
sys.exit(50)
if args.source:
matched_sources = set()
for s in args.source:
for stt in settings.get_sources():
if re.match(s, stt.get_alias()):
matched_sources.add(stt)
direct_sources.extend(matched_sources)
if not ffmpeg_download.install_local():
print(
"RMD was unable to locate (or download) a working FFmpeg binary.")
print("For downloading and post-processing, this is a required tool.")
print(
"Please Install FFmpeg manually, or download it from here: https://rmd.page.link/ffmpeg"
)
sys.exit(15)
if not _loaded and not direct_sources:
# First-time configuration.
su.error(
'Could not find an existing settings file. A new one will be generated!'
)
if not console.confirm(
'Would you like to start the WebUI to help set things up?',
True):
su.print_color(
'red',
"If you don't open the webUI now, you'll need to edit the settings file yourself."
)
if console.confirm(
"Are you sure you'd like to edit settings without the UI (if 'yes', these prompts will not show again)?"
):
settings.put('interface.start_server',
False) # Creates a save.
print(
'A settings file has been created for you, at "%s". Please customize it.'
% settings_file)
else:
print('Please re-run RMD to configure again.')
sys.exit(1)
else:
mode = console.prompt_list(
'How would you like to open the UI?',
settings.get('interface.browser', full_obj=True).opts)
settings.put('interface.browser', mode, save_after=False)
settings.put('interface.start_server', True)
# Initialize Database
sql.init_from_settings()
if direct_sources:
settings.disable_saving()
for s in settings.get_sources():
settings.remove_source(s, save_after=False)
for d in direct_sources:
settings.add_source(d, prevent_duplicate=False, save_after=False)
ui = None
if settings.get('interface.start_server') and not direct_sources:
print("Starting WebUI...")
ui = WebUI()
else:
ui = TerminalUI()
ui.display()
if _s.public:
print("%s.%s" % (_s.category, _s.name))
print('\tDescription: %s' % _s.description)
if not _s.opts:
print('\tValid value: \n\t\tAny %s' % _s.type)
else:
print('\tValid values:')
for o in _s.opts:
print('\t\t"%s": %s' % o)
print()
sys.exit()
_loaded = settings.load(args.settings)
for ua in unknown_args:
if '=' not in ua:
su.error("ERROR: Unkown argument: %s" % ua)
sys.exit(1)
k = ua.split('=')[0].strip('- ')
v = ua.split('=', 2)[1].strip()
try:
settings.put(k, v, save_after=False)
except KeyError:
print('Unknown setting: %s' % k)
sys.exit(50)
if not ffmpeg_download.install_local():
print(
"RMD was unable to locate (or download) a working FFmpeg binary.")
print("For downloading and post-processing, this is a required tool.")
print(
"Please Install FFmpeg manually, or download it from here: https://rmd.page.link/ffmpeg"
def test_error(self):
""" Error coloring should print """
su.error('test error')
