def setUp(self):
importlib.reload(settings)
importlib.reload(sql)
importlib.reload(ng)
settings.load(self.settings_file)
sql.init_from_settings()
self.sess = sql.session()
def setUp(self):
global download_ran, session
if not download_ran:
download_ran = True
settings.load(self.settings_file)
tui = TerminalUI()
tui.display()
# self.db_path = join(settings.get('output.base_dir'), 'manifest.sqlite')
self.db_path = join(settings.get('output.manifest_for_sqlite_dir'), 'manifest.sqlite') # This is part of the change to save manifest.sqlite to a different directory than the downloads
sql.init_from_settings()
session = sql.session()
def setUp(self):
global download_ran, session
if not download_ran:
download_ran = True
settings.load(self.settings_file)
tui = TerminalUI()
tui.display()
self.db_path = join(settings.get('output.base_dir'),
'manifest.sqlite')
sql.init_from_settings()
session = sql.session()
def setUp(self):
global download_ran, session
if not download_ran:
download_ran = True
importlib.reload(settings)
importlib.reload(sql)
settings.load(self.settings_file)
tui = TerminalUI()
tui.display()
sql.init_from_settings()
self.db_path = sql.get_file_location()
session = sql.session()
def setUp(self):
global download_ran, session, thread
if not download_ran:
download_ran = True
self.wui = WebUI('test_version')
self.db_path = join(settings.get('output.base_dir'), 'manifest.sqlite')
self.url = 'http://%s:%s/index.html#' % (settings.get('interface.host'), settings.get('interface.port'))
settings.load(self.settings_file)
settings.put('interface.start_server', True)
sql.init_from_settings()
session = sql.session()
thread = Thread(target=self.wui.display)
thread.setDaemon(True)
thread.start()
self.assertTrue(self.wui.waitFor(10), msg='WebUI Failed to start!')
def start(self):
os.makedirs(self.new_save_base, exist_ok=True)
if not settings.load(self.settings_file):
raise Exception("You must provide a valid settings.json file!")
settings.put('output.base_dir', self.new_save_base)
sql.init_from_settings()
self.session = sql.session()
print("Scanning legacy posts...")
self.scan()
print("Found %s elements total." % len(self.posts.keys()))
self.process_posts()
self.session.commit()
print("Processed:", len(self.posts), "Posts.")
print("Failed to convert %s Posts." % len(self.failures))
outfile = os.path.join(self.new_save_base, 'failed_conversion_posts.json')
with open(outfile, 'w') as o:
o.write(json.dumps(self.failures, indent=4, sort_keys=True, separators=(',', ': ')))
print("Saved a list of failed posts to", outfile, ', be sure to check these before deleting anything!')
def run():
logging.basicConfig(level=logging.WARN, format='%(levelname)-5.5s [%(name)s] %(message)s', datefmt='%H:%M:%S')
su.print_color('green', "\r\n" +
'====================================\r\n' +
(' Reddit Media Downloader %s\r\n' % meta.current_version) +
'====================================\r\n' +
' (By ShadowMoose @ Github)\r\n')
if args.version:
sys.exit(0)
if args.run_tests:
error_count = tests.runner.run_tests(test_subdir=args.run_tests)
sys.exit(error_count)
if args.list_settings:
print('All valid overridable settings:')
for _s in settings.get_all():
if _s.public:
print("%s.%s" % (_s.category, _s.name))
print('\tDescription: %s' % _s.description)
if not _s.opts:
print('\tValid value: \n\t\tAny %s' % _s.type)
else:
print('\tValid values:')
for o in _s.opts:
print('\t\t"%s": %s' % o)
print()
sys.exit()
settings_file = args.settings or fs.find_file('settings.json')
_loaded = settings.load(settings_file)
for ua in unknown_args:
if '=' not in ua or '/comments/' in ua:
if '/comments/' in ua:
direct_sources.append(DirectURLSource(url=ua))
continue
elif 'r/' or 'u/' in ua:
direct_sources.append(DirectInputSource(txt=ua, args={'limit': args.limit}))
continue
else:
su.error("ERROR: Unkown argument: %s" % ua)
sys.exit(1)
k = ua.split('=')[0].strip('- ')
v = ua.split('=', 2)[1].strip()
try:
settings.put(k, v, save_after=False)
except KeyError:
print('Unknown setting: %s' % k)
sys.exit(50)
if args.source:
matched_sources = set()
for s in args.source:
for stt in settings.get_sources():
if re.match(s, stt.get_alias()):
matched_sources.add(stt)
direct_sources.extend(matched_sources)
first_time_auth = False
if not _loaded and not direct_sources: # First-time configuration.
su.error('Could not find an existing settings file. A new one will be generated!')
if not console.confirm('Would you like to start the WebUI to help set things up?', True):
su.print_color('red', "If you don't open the webUI now, you'll need to edit the settings file yourself.")
if console.confirm("Are you sure you'd like to edit settings without the UI (if 'yes', these prompts will not show again)?"):
settings.put('interface.start_server', False, save_after=True) # Creates a save.
print('A settings file has been created for you, at "%s". Please customize it.' % settings_file)
first_time_auth = True
else:
print('Please re-run RMD to configure again.')
sys.exit(1)
else:
mode = console.prompt_list('How would you like to open the UI?',
settings.get('interface.browser', full_obj=True).opts)
settings.put('interface.browser', mode, save_after=False)
settings.put('interface.start_server', True)
if args.authorize or first_time_auth: # In-console oAuth authentication flow
from static import praw_wrapper
from urllib.parse import urlparse, parse_qs
url = praw_wrapper.get_reddit_token_url()
su.print_color('green', '\nTo manually authorize your account, visit the below URL.')
su.print_color('yellow', 'Once there, authorize RMD, then copy the URL it redirects you to.')
su.print_color('yellow', 'NOTE: The redirect page will likely not load, and that is ok.')
su.print_color('cyan', '\n%s\n' % url)
token_url = console.col_input('Paste the URL you are redirected to here: ')
if token_url.strip():
qs = parse_qs(urlparse(token_url).query)
if 'state' not in qs or 'code' not in qs:
su.error('The url provided was not a valid reddit redirect. Please make sure you copied it right!')
elif qs['state'][0].strip() != settings.get('auth.oauth_key').strip():
su.error('Invalid reddit redirect state. Please restart and try again.')
else:
code = qs['code'][0]
su.print_color('green', 'Got code. Authorizing account...')
refresh = praw_wrapper.get_refresh_token(code)
if refresh:
settings.put('auth.refresh_token', refresh)
usr = praw_wrapper.get_current_username()
su.print_color('cyan', 'Authorized to view account: %s' % usr)
su.print_color('green', 'Saved authorization token! Please restart RMD to begin downloading!')
else:
su.error('Failed to gain an account access token from Reddit with that code. Please try again.')
sys.exit(0)
if not ffmpeg_download.install_local():
print("RMD was unable to locate (or download) a working FFmpeg binary.")
print("For downloading and post-processing, this is a required tool.")
print("Please Install FFmpeg manually, or download it from here: https://rmd.page.link/ffmpeg")
sys.exit(15)
# Initialize Database
sql.init_from_settings()
print('Using manifest file [%s].' % sql.get_file_location())
if direct_sources:
settings.disable_saving()
settings.put('processing.retry_failed', False)
for s in settings.get_sources():
settings.remove_source(s, save_after=False)
for d in direct_sources:
settings.add_source(d, prevent_duplicate=False, save_after=False)
if settings.get('interface.start_server') and not direct_sources:
print("Starting WebUI...")
ui = WebUI()
else:
ui = TerminalUI()
ui.display()
def run():
su.print_color(
'green', "\r\n" + '====================================\r\n' +
(' Reddit Media Downloader %s\r\n' % meta.current_version) +
'====================================\r\n' +
' (By ShadowMoose @ Github)\r\n')
if args.version:
sys.exit(0)
if args.run_tests:
error_count = tests.runner.run_tests(test_subdir=args.run_tests)
sys.exit(error_count)
if args.list_settings:
print('All valid overridable settings:')
for _s in settings.get_all():
if _s.public:
print("%s.%s" % (_s.category, _s.name))
print('\tDescription: %s' % _s.description)
if not _s.opts:
print('\tValid value: \n\t\tAny %s' % _s.type)
else:
print('\tValid values:')
for o in _s.opts:
print('\t\t"%s": %s' % o)
print()
sys.exit()
settings_file = args.settings or fs.find_file('settings.json')
_loaded = settings.load(settings_file)
for ua in unknown_args:
if '=' not in ua:
if 'r/' or 'u/' in ua:
direct_sources.append(
DirectInputSource(txt=ua, args={'limit': args.limit}))
continue
else:
su.error("ERROR: Unkown argument: %s" % ua)
sys.exit(1)
k = ua.split('=')[0].strip('- ')
v = ua.split('=', 2)[1].strip()
try:
settings.put(k, v, save_after=False)
except KeyError:
print('Unknown setting: %s' % k)
sys.exit(50)
if args.source:
matched_sources = set()
for s in args.source:
for stt in settings.get_sources():
if re.match(s, stt.get_alias()):
matched_sources.add(stt)
direct_sources.extend(matched_sources)
if not ffmpeg_download.install_local():
print(
"RMD was unable to locate (or download) a working FFmpeg binary.")
print("For downloading and post-processing, this is a required tool.")
print(
"Please Install FFmpeg manually, or download it from here: https://rmd.page.link/ffmpeg"
)
sys.exit(15)
if not _loaded and not direct_sources:
# First-time configuration.
su.error(
'Could not find an existing settings file. A new one will be generated!'
)
if not console.confirm(
'Would you like to start the WebUI to help set things up?',
True):
su.print_color(
'red',
"If you don't open the webUI now, you'll need to edit the settings file yourself."
)
if console.confirm(
"Are you sure you'd like to edit settings without the UI (if 'yes', these prompts will not show again)?"
):
settings.put('interface.start_server',
False) # Creates a save.
print(
'A settings file has been created for you, at "%s". Please customize it.'
% settings_file)
else:
print('Please re-run RMD to configure again.')
sys.exit(1)
else:
mode = console.prompt_list(
'How would you like to open the UI?',
settings.get('interface.browser', full_obj=True).opts)
settings.put('interface.browser', mode, save_after=False)
settings.put('interface.start_server', True)
# Initialize Database
sql.init_from_settings()
if direct_sources:
settings.disable_saving()
for s in settings.get_sources():
settings.remove_source(s, save_after=False)
for d in direct_sources:
settings.add_source(d, prevent_duplicate=False, save_after=False)
ui = None
if settings.get('interface.start_server') and not direct_sources:
print("Starting WebUI...")
ui = WebUI()
else:
ui = TerminalUI()
ui.display()
if args.list_settings:
print('All valid overridable settings:')
for _s in settings.get_all():
if _s.public:
print("%s.%s" % (_s.category, _s.name))
print('\tDescription: %s' % _s.description)
if not _s.opts:
print('\tValid value: \n\t\tAny %s' % _s.type)
else:
print('\tValid values:')
for o in _s.opts:
print('\t\t"%s": %s' % o)
print()
sys.exit()
_loaded = settings.load(args.settings)
for ua in unknown_args:
if '=' not in ua:
su.error("ERROR: Unkown argument: %s" % ua)
sys.exit(1)
k = ua.split('=')[0].strip('- ')
v = ua.split('=', 2)[1].strip()
try:
settings.put(k, v, save_after=False)
except KeyError:
print('Unknown setting: %s' % k)
sys.exit(50)
if not ffmpeg_download.install_local():
print(
"RMD was unable to locate (or download) a working FFmpeg binary.")
def setUp(self):
settings.load(self.settings_file)
"Custom source importer"
import os
import static.filesystem as fs
import static.settings as settings
import sources
from datetime import datetime
import sql
from sql import Post, File, URL, Hash
from sqlalchemy.sql.expression import func
#datetime.now().strftime("%c %X")
settings_file = fs.find_file('settings.json')
_loaded = settings.load(settings_file)
_session = sql.session()
url_patt = r'^(?:(?!(youtu\.be|youtube\.com|amazon\.c|twitter\.c|instagram\.com)).)*$'
def strf_utc(sec):
return datetime.fromtimestamp(sec).strftime("%y-%m-%d %H:%M:%S")
def user_source(name,
alias=None,
ps=False,
limit=None,
deep=False,
check_last=None,
check_utc=0):
u = name.replace('/u/', '').replace('u/', '').strip('/')
out = {
def setUp(self):
importlib.reload(settings)
importlib.reload(sql)
settings.load(self.settings_file)
settings.put('output.base_dir', self.dir)
sql.init_from_settings()
def setUp(self):
settings.load(self.settings_file)
dfs = sources.DirectFileSource(file=os.path.join(
self.dir, 'export.csv'),
slow_fallback=True)
settings.add_source(dfs, prevent_duplicate=True, save_after=False)
def setUp(self):
importlib.reload(sql)
self.assertFalse(sql._engine, msg="Failed to clear SQL session.")
settings.load(self.settings_file)
sql.init_from_settings()
self.ps = sql.PostSearcher(sql.session())
def setUp(self):
importlib.reload(settings)
importlib.reload(sql)
settings.load(self.settings_file)
from alembic import command
from static import console
from static import settings
import static.filesystem as fs
import shutil
from datetime import datetime
def make_migration():
conn = sql.session()
alembic_cfg, script_, context = sql.get_alembic_ctx(conn)
message = console.string('Enter a message for the migration')
if not message:
print('Skipping migration.')
return
res = command.revision(message=message,
autogenerate=True,
config=alembic_cfg)
print('Generated Migration:', res)
print('Finished.')
if __name__ == '__main__':
settings.load(fs.find_file('settings.json'))
sql.init_from_settings()
# noinspection PyProtectedMember
pth = sql._db_path
bkup = ('%s-bkup-%s.sqlite' % (pth, str(datetime.now()).replace(':', '.')))
shutil.copy2(pth, bkup)
make_migration()
