def test_pattern_loader(self):
""" All pattern tags should work """
settings.put(
'output.file_name_pattern',
'[type]-[reddit_id]-[title]-[author]-[subreddit]-[source_alias]-[created_utc]-[created_date]-[created_time]'
)
tp = self.sess.query(sql.Post).filter(sql.Post.title == 'test').first()
file = ng.choose_file_name(tp.urls[0], tp, sql.session(), album_size=1)
self.assertRegex(
file,
r'Submission-t3_b1rycu-test-testuser-aww-newsource-1552739416-2019-..-..-..\...\...',
msg='Failed to convert basic Test post!')
def api_save_settings(settings_obj):
print('WebUI wants to change settings:', settings_obj)
# noinspection PyBroadException
try:
for k, v in settings_obj.items():
settings.put(k, v, save_after=False)
settings.save()
except Exception:
import traceback
traceback.print_exc()
return False
return True
def _authorize_rmd_token():
state = eel.btl.request.query.state
print('New refresh code request: ', state, eel.btl.request.query.code)
if state.strip() == settings.get('auth.oauth_key').strip():
code = eel.btl.request.query.code
print('Saving new reddit code.')
refresh = praw_wrapper.get_refresh_token(code)
if refresh:
settings.put('auth.refresh_token', refresh)
praw_wrapper.init()
praw_wrapper.login()
return 'Saved authorization token! Close this page to continue.'
return 'Cannot save the new auth key, something went wrong.<br><a href="../index.html">Back</a>'
def setUp(self):
global download_ran, session, thread
if not download_ran:
download_ran = True
self.wui = WebUI('test_version')
self.db_path = join(settings.get('output.base_dir'), 'manifest.sqlite')
self.url = 'http://%s:%s/index.html#' % (settings.get('interface.host'), settings.get('interface.port'))
settings.load(self.settings_file)
settings.put('interface.start_server', True)
sql.init_from_settings()
session = sql.session()
thread = Thread(target=self.wui.display)
thread.setDaemon(True)
thread.start()
self.assertTrue(self.wui.waitFor(10), msg='WebUI Failed to start!')
def load_userlist():
#existing_users = [x[0] for x in _session.query(Post.author).distinct()]
newest_utc = dict(
_session.query(Post.author,
func.max(Post.created_utc)).group_by(Post.author))
source_dicts = []
if os.path.exists('userlist'):
lnum = 0
with open('userlist', 'r') as f:
for u in f:
lnum += 1
if lnum <= -1 or lnum > 10000:
continue
u = u.split('#')[0].strip()
if not u:
continue
if u in newest_utc:
check_utc = newest_utc[u]
check_last = 50
check_utc = min(check_utc,
datetime(2021, 3, 31).timestamp())
source_dicts.append(
user_source(u,
alias='p%03d %s' % (lnum, u),
check_last=check_last,
check_utc=check_utc,
ps=True))
source_dicts.append(
user_source(u,
alias='r%03d %s' % (lnum, u),
check_last=check_last,
check_utc=check_utc,
ps=False))
elif u:
print(lnum, '(no posts found)', u)
source_dicts.append(
user_source(u,
alias='p%03d %s (new)' % (lnum, u),
deep=True,
ps=True))
source_dicts.append(
user_source(u,
alias='r%03d %s (new)' % (lnum, u),
deep=True,
ps=False))
print("\n%d sources added from userlist" % len(source_dicts))
settings.put('sources', source_dicts, save_after=False)
def start(self):
os.makedirs(self.new_save_base, exist_ok=True)
if not settings.load(self.settings_file):
raise Exception("You must provide a valid settings.json file!")
settings.put('output.base_dir', self.new_save_base)
sql.init_from_settings()
self.session = sql.session()
print("Scanning legacy posts...")
self.scan()
print("Found %s elements total." % len(self.posts.keys()))
self.process_posts()
self.session.commit()
print("Processed:", len(self.posts), "Posts.")
print("Failed to convert %s Posts." % len(self.failures))
outfile = os.path.join(self.new_save_base, 'failed_conversion_posts.json')
with open(outfile, 'w') as o:
o.write(json.dumps(self.failures, indent=4, sort_keys=True, separators=(',', ': ')))
print("Saved a list of failed posts to", outfile, ', be sure to check these before deleting anything!')
def setUp(self):
settings.put('imgur.client_id', os.environ['RMD_IMGUR_ID'])
settings.put('imgur.client_secret', os.environ['RMD_IMGUR_SECRET'])
def run():
logging.basicConfig(level=logging.WARN, format='%(levelname)-5.5s [%(name)s] %(message)s', datefmt='%H:%M:%S')
su.print_color('green', "\r\n" +
'====================================\r\n' +
(' Reddit Media Downloader %s\r\n' % meta.current_version) +
'====================================\r\n' +
' (By ShadowMoose @ Github)\r\n')
if args.version:
sys.exit(0)
if args.run_tests:
error_count = tests.runner.run_tests(test_subdir=args.run_tests)
sys.exit(error_count)
if args.list_settings:
print('All valid overridable settings:')
for _s in settings.get_all():
if _s.public:
print("%s.%s" % (_s.category, _s.name))
print('\tDescription: %s' % _s.description)
if not _s.opts:
print('\tValid value: \n\t\tAny %s' % _s.type)
else:
print('\tValid values:')
for o in _s.opts:
print('\t\t"%s": %s' % o)
print()
sys.exit()
settings_file = args.settings or fs.find_file('settings.json')
_loaded = settings.load(settings_file)
for ua in unknown_args:
if '=' not in ua or '/comments/' in ua:
if '/comments/' in ua:
direct_sources.append(DirectURLSource(url=ua))
continue
elif 'r/' or 'u/' in ua:
direct_sources.append(DirectInputSource(txt=ua, args={'limit': args.limit}))
continue
else:
su.error("ERROR: Unkown argument: %s" % ua)
sys.exit(1)
k = ua.split('=')[0].strip('- ')
v = ua.split('=', 2)[1].strip()
try:
settings.put(k, v, save_after=False)
except KeyError:
print('Unknown setting: %s' % k)
sys.exit(50)
if args.source:
matched_sources = set()
for s in args.source:
for stt in settings.get_sources():
if re.match(s, stt.get_alias()):
matched_sources.add(stt)
direct_sources.extend(matched_sources)
first_time_auth = False
if not _loaded and not direct_sources: # First-time configuration.
su.error('Could not find an existing settings file. A new one will be generated!')
if not console.confirm('Would you like to start the WebUI to help set things up?', True):
su.print_color('red', "If you don't open the webUI now, you'll need to edit the settings file yourself.")
if console.confirm("Are you sure you'd like to edit settings without the UI (if 'yes', these prompts will not show again)?"):
settings.put('interface.start_server', False, save_after=True) # Creates a save.
print('A settings file has been created for you, at "%s". Please customize it.' % settings_file)
first_time_auth = True
else:
print('Please re-run RMD to configure again.')
sys.exit(1)
else:
mode = console.prompt_list('How would you like to open the UI?',
settings.get('interface.browser', full_obj=True).opts)
settings.put('interface.browser', mode, save_after=False)
settings.put('interface.start_server', True)
if args.authorize or first_time_auth: # In-console oAuth authentication flow
from static import praw_wrapper
from urllib.parse import urlparse, parse_qs
url = praw_wrapper.get_reddit_token_url()
su.print_color('green', '\nTo manually authorize your account, visit the below URL.')
su.print_color('yellow', 'Once there, authorize RMD, then copy the URL it redirects you to.')
su.print_color('yellow', 'NOTE: The redirect page will likely not load, and that is ok.')
su.print_color('cyan', '\n%s\n' % url)
token_url = console.col_input('Paste the URL you are redirected to here: ')
if token_url.strip():
qs = parse_qs(urlparse(token_url).query)
if 'state' not in qs or 'code' not in qs:
su.error('The url provided was not a valid reddit redirect. Please make sure you copied it right!')
elif qs['state'][0].strip() != settings.get('auth.oauth_key').strip():
su.error('Invalid reddit redirect state. Please restart and try again.')
else:
code = qs['code'][0]
su.print_color('green', 'Got code. Authorizing account...')
refresh = praw_wrapper.get_refresh_token(code)
if refresh:
settings.put('auth.refresh_token', refresh)
usr = praw_wrapper.get_current_username()
su.print_color('cyan', 'Authorized to view account: %s' % usr)
su.print_color('green', 'Saved authorization token! Please restart RMD to begin downloading!')
else:
su.error('Failed to gain an account access token from Reddit with that code. Please try again.')
sys.exit(0)
if not ffmpeg_download.install_local():
print("RMD was unable to locate (or download) a working FFmpeg binary.")
print("For downloading and post-processing, this is a required tool.")
print("Please Install FFmpeg manually, or download it from here: https://rmd.page.link/ffmpeg")
sys.exit(15)
# Initialize Database
sql.init_from_settings()
print('Using manifest file [%s].' % sql.get_file_location())
if direct_sources:
settings.disable_saving()
settings.put('processing.retry_failed', False)
for s in settings.get_sources():
settings.remove_source(s, save_after=False)
for d in direct_sources:
settings.add_source(d, prevent_duplicate=False, save_after=False)
if settings.get('interface.start_server') and not direct_sources:
print("Starting WebUI...")
ui = WebUI()
else:
ui = TerminalUI()
ui.display()
def run():
su.print_color(
'green', "\r\n" + '====================================\r\n' +
(' Reddit Media Downloader %s\r\n' % meta.current_version) +
'====================================\r\n' +
' (By ShadowMoose @ Github)\r\n')
if args.version:
sys.exit(0)
if args.run_tests:
error_count = tests.runner.run_tests(test_subdir=args.run_tests)
sys.exit(error_count)
if args.list_settings:
print('All valid overridable settings:')
for _s in settings.get_all():
if _s.public:
print("%s.%s" % (_s.category, _s.name))
print('\tDescription: %s' % _s.description)
if not _s.opts:
print('\tValid value: \n\t\tAny %s' % _s.type)
else:
print('\tValid values:')
for o in _s.opts:
print('\t\t"%s": %s' % o)
print()
sys.exit()
settings_file = args.settings or fs.find_file('settings.json')
_loaded = settings.load(settings_file)
for ua in unknown_args:
if '=' not in ua:
if 'r/' or 'u/' in ua:
direct_sources.append(
DirectInputSource(txt=ua, args={'limit': args.limit}))
continue
else:
su.error("ERROR: Unkown argument: %s" % ua)
sys.exit(1)
k = ua.split('=')[0].strip('- ')
v = ua.split('=', 2)[1].strip()
try:
settings.put(k, v, save_after=False)
except KeyError:
print('Unknown setting: %s' % k)
sys.exit(50)
if args.source:
matched_sources = set()
for s in args.source:
for stt in settings.get_sources():
if re.match(s, stt.get_alias()):
matched_sources.add(stt)
direct_sources.extend(matched_sources)
if not ffmpeg_download.install_local():
print(
"RMD was unable to locate (or download) a working FFmpeg binary.")
print("For downloading and post-processing, this is a required tool.")
print(
"Please Install FFmpeg manually, or download it from here: https://rmd.page.link/ffmpeg"
)
sys.exit(15)
if not _loaded and not direct_sources:
# First-time configuration.
su.error(
'Could not find an existing settings file. A new one will be generated!'
)
if not console.confirm(
'Would you like to start the WebUI to help set things up?',
True):
su.print_color(
'red',
"If you don't open the webUI now, you'll need to edit the settings file yourself."
)
if console.confirm(
"Are you sure you'd like to edit settings without the UI (if 'yes', these prompts will not show again)?"
):
settings.put('interface.start_server',
False) # Creates a save.
print(
'A settings file has been created for you, at "%s". Please customize it.'
% settings_file)
else:
print('Please re-run RMD to configure again.')
sys.exit(1)
else:
mode = console.prompt_list(
'How would you like to open the UI?',
settings.get('interface.browser', full_obj=True).opts)
settings.put('interface.browser', mode, save_after=False)
settings.put('interface.start_server', True)
# Initialize Database
sql.init_from_settings()
if direct_sources:
settings.disable_saving()
for s in settings.get_sources():
settings.remove_source(s, save_after=False)
for d in direct_sources:
settings.add_source(d, prevent_duplicate=False, save_after=False)
ui = None
if settings.get('interface.start_server') and not direct_sources:
print("Starting WebUI...")
ui = WebUI()
else:
ui = TerminalUI()
ui.display()
else:
print('\tValid values:')
for o in _s.opts:
print('\t\t"%s": %s' % o)
print()
sys.exit()
_loaded = settings.load(args.settings)
for ua in unknown_args:
if '=' not in ua:
su.error("ERROR: Unkown argument: %s" % ua)
sys.exit(1)
k = ua.split('=')[0].strip('- ')
v = ua.split('=', 2)[1].strip()
try:
settings.put(k, v, save_after=False)
except KeyError:
print('Unknown setting: %s' % k)
sys.exit(50)
if not ffmpeg_download.install_local():
print(
"RMD was unable to locate (or download) a working FFmpeg binary.")
print("For downloading and post-processing, this is a required tool.")
print(
"Please Install FFmpeg manually, or download it from here: https://rmd.page.link/ffmpeg"
)
sys.exit(15)
if not _loaded:
# First-time configuration.
def setUp(self):
importlib.reload(settings)
importlib.reload(sql)
settings.load(self.settings_file)
settings.put('output.base_dir', self.dir)
sql.init_from_settings()
