def load(self):
""" Threaded loading of elements. """
settings.from_json(self.settings)
sql.init_from_settings()
self._session = sql.session()
self.progress.set_scanning(True)
retry_failed = settings.get('processing.retry_failed')
# Query for all unhandled URLs, and submit them before scanning for new Posts.
unfinished = self._session\
.query(sql.URL)\
.filter((sql.URL.processed == False) | (retry_failed and sql.URL.failed == True))\
.all()
self._push_url_list(unfinished)
self._scan_sources()
self.progress.set_scanning(False)
# Wait for any remaining ACKS to come in, before closing the writing pipe.
# ...Until the Downloaders have confirmed completion of everything, more album URLS may come in.
while len(self._open_ack) > 0 and not self._stop_event.is_set():
self._handle_acks(timeout=0.5)
print("Finished loading.")
sql.close()
def setUp(self):
importlib.reload(settings)
importlib.reload(sql)
importlib.reload(ng)
settings.load(self.settings_file)
sql.init_from_settings()
self.sess = sql.session()
def test_relative_create(self):
""" The database should build from relative paths """
importlib.reload(sql)
settings.put('output.manifest', './test.sqlite', save_after=False)
sql.init_from_settings()
self.assertTrue(isfile(sql.get_file_location()),
"Failed to create sqlite file.")
def load(self):
""" Threaded loading of elements. """
settings.from_json(self.settings)
sql.init_from_settings()
self._session = sql.session()
t_start = datetime.now() #vy
print("Started loading.") #vy
self.progress.set_scanning(True)
retry_failed = settings.get('processing.retry_failed')
# Query for all unhandled URLs, and submit them before scanning for new Posts.
unfinished = self._session\
.query(sql.URL)\
.filter((sql.URL.processed == False) | \
(retry_failed and sql.URL.failed and \
sql.not_(sql.URL.failure_reason.contains('404'))))\
.all()
print("Loading %s unfinished urls" % len(unfinished))
self._push_url_list(unfinished)
self._scan_sources()
self.progress.set_scanning(False)
# Wait for any remaining ACKS to come in, before closing the writing pipe.
# ...Until the Downloaders have confirmed completion of everything, more album URLS may come in.
while len(self._open_ack) > 0 and not self._stop_event.is_set():
self._handle_acks(timeout=1.0, clear=True)
print("Finished loading.") #vy
print("Elapsed time: %s" % str(datetime.now() - t_start)) #vy
sql.close()
def test_absolute_create(self):
""" The database should build from absolute paths """
importlib.reload(sql)
settings.put('output.manifest',
abspath(join(self.dir, 'test-manifest.sqlite')),
save_after=False)
sql.init_from_settings()
self.assertTrue(isfile(sql.get_file_location()),
"Failed to create sqlite file.")
def setUp(self):
global download_ran, session
if not download_ran:
download_ran = True
settings.load(self.settings_file)
tui = TerminalUI()
tui.display()
self.db_path = join(settings.get('output.base_dir'),
'manifest.sqlite')
sql.init_from_settings()
session = sql.session()
def setUp(self):
global download_ran, session
if not download_ran:
download_ran = True
settings.load(self.settings_file)
tui = TerminalUI()
tui.display()
# self.db_path = join(settings.get('output.base_dir'), 'manifest.sqlite')
self.db_path = join(settings.get('output.manifest_for_sqlite_dir'), 'manifest.sqlite') # This is part of the change to save manifest.sqlite to a different directory than the downloads
sql.init_from_settings()
session = sql.session()
def setUp(self):
global download_ran, session
if not download_ran:
download_ran = True
importlib.reload(settings)
importlib.reload(sql)
settings.load(self.settings_file)
tui = TerminalUI()
tui.display()
sql.init_from_settings()
self.db_path = sql.get_file_location()
session = sql.session()
def setUp(self):
global download_ran, session, thread
if not download_ran:
download_ran = True
self.wui = WebUI('test_version')
self.db_path = join(settings.get('output.base_dir'), 'manifest.sqlite')
self.url = 'http://%s:%s/index.html#' % (settings.get('interface.host'), settings.get('interface.port'))
settings.load(self.settings_file)
settings.put('interface.start_server', True)
sql.init_from_settings()
session = sql.session()
thread = Thread(target=self.wui.display)
thread.setDaemon(True)
thread.start()
self.assertTrue(self.wui.waitFor(10), msg='WebUI Failed to start!')
def run(self):
""" Threaded loading of elements. """
settings.from_json(self._settings)
sql.init_from_settings()
self._session = sql.session()
self.progress.clear(status="Starting up...")
self.progress.set_running(True)
while not self._stop_event.is_set():
self._dedupe()
self.progress.set_status("Waiting for new files...")
self._stop_event.wait(2)
self._dedupe() # Run one final pass after downloading stops.
self.progress.set_running(False)
sql.close()
self.progress.clear("Finished.")
def start(self):
os.makedirs(self.new_save_base, exist_ok=True)
if not settings.load(self.settings_file):
raise Exception("You must provide a valid settings.json file!")
settings.put('output.base_dir', self.new_save_base)
sql.init_from_settings()
self.session = sql.session()
print("Scanning legacy posts...")
self.scan()
print("Found %s elements total." % len(self.posts.keys()))
self.process_posts()
self.session.commit()
print("Processed:", len(self.posts), "Posts.")
print("Failed to convert %s Posts." % len(self.failures))
outfile = os.path.join(self.new_save_base, 'failed_conversion_posts.json')
with open(outfile, 'w') as o:
o.write(json.dumps(self.failures, indent=4, sort_keys=True, separators=(',', ': ')))
print("Saved a list of failed posts to", outfile, ', be sure to check these before deleting anything!')
def __init__(self, source_patterns=None):
super().__init__()
sql.init_from_settings() # Make sure the database is built & migrated before starting threads.
sql.close()
self.daemon = False
self.sources = source_patterns
self.sources = self.load_sources()
self.db_lock = RLock()
# initialize Loader
self.loader = RedditLoader(sources=self.sources, settings_json=settings.to_json(), db_lock=self.db_lock)
self.deduplicator = Deduplicator(
settings_json=settings.to_json(),
stop_event=self.loader.get_stop_event(),
db_lock=self.db_lock
)
self._downloaders = self._create_downloaders()
self._all_processes = [self.loader, *self._downloaders]
if settings.get('processing.deduplicate_files'):
self._all_processes.append(self.deduplicator)
def run(self):
""" Threaded loading of elements. """
settings.from_json(self._settings)
sql.init_from_settings()
try:
self._session = sql.session()
self.progress.clear(status="Starting up...")
self.progress.set_running(True)
while not self._stop_event.is_set():
self._dedupe()
self.progress.set_status("Ready for new files...")
self._stop_event.wait(2)
self._dedupe() # Run one final pass after downloading stops.
self.progress.clear(status="Finished.", running=False)
except Exception as ex:
print('Deduplication Process Error:', ex)
self.progress.set_error(ex)
self.progress.set_running(False)
traceback.print_exc()
finally:
sql.close()
def run(self):
""" Threaded loading of elements. """
settings.from_json(self._settings)
sql.init_from_settings()
print("Starting up...", debug=True)
try:
self._session = sql.session()
self.progress.clear(status="Starting up...")
self.progress.set_running(True)
self.dedup_ignore_ids = set()
self.prune_counter = 0
self.special_hashes = self._session.query(Hash).filter(
Hash.id < 0).all()
while not self._stop_event.is_set():
#print("_stop_event is %s"%self._stop_event.is_set(), debug=True)
completed = self._dedupe()
if completed:
self.progress.set_status(
"Completed %s files. Ready for new files..." %
completed)
self._stop_event.wait(1)
else:
self._stop_event.wait(10)
print("_stop_event is %s" % self._stop_event.is_set(), debug=True)
self._dedupe() # Run one final pass after downloading stops.
self.progress.clear(status="Finished.", running=False)
except Exception as ex:
print('Deduplication Process Error:', ex)
self.progress.set_error(ex)
self.progress.set_running(False)
traceback.print_exc()
finally:
print("Finished process, _stop_event is %s" %
self._stop_event.is_set(),
debug=True)
sql.close()
"If you don't open the webUI now, you'll need to edit the settings file yourself."
)
if console.confirm(
"Are you sure you'd like to edit settings without the UI (if 'yes', these prompts will not show again)?"
):
settings.put('interface.start_server',
False) # Creates a save.
print(
'A settings file has been created for you, at "%s". Please customize it.'
% args.settings)
else:
print('Please re-run RMD to configure again.')
sys.exit(1)
else:
mode = console.prompt_list(
'How would you like to open the UI?',
settings.get('interface.browser', full_obj=True).opts)
settings.put('interface.browser', mode, save_after=False)
settings.put('interface.start_server', True)
# Initialize Database
sql.init_from_settings()
ui = None
if settings.get('interface.start_server'):
print("Starting WebUI...")
ui = WebUI(__version__)
else:
ui = TerminalUI(__version__)
ui.display()
def setUp(self):
importlib.reload(settings)
importlib.reload(sql)
settings.load(self.settings_file)
settings.put('output.base_dir', self.dir)
sql.init_from_settings()
def run():
su.print_color(
'green', "\r\n" + '====================================\r\n' +
(' Reddit Media Downloader %s\r\n' % meta.current_version) +
'====================================\r\n' +
' (By ShadowMoose @ Github)\r\n')
if args.version:
sys.exit(0)
if args.run_tests:
error_count = tests.runner.run_tests(test_subdir=args.run_tests)
sys.exit(error_count)
if args.list_settings:
print('All valid overridable settings:')
for _s in settings.get_all():
if _s.public:
print("%s.%s" % (_s.category, _s.name))
print('\tDescription: %s' % _s.description)
if not _s.opts:
print('\tValid value: \n\t\tAny %s' % _s.type)
else:
print('\tValid values:')
for o in _s.opts:
print('\t\t"%s": %s' % o)
print()
sys.exit()
settings_file = args.settings or fs.find_file('settings.json')
_loaded = settings.load(settings_file)
for ua in unknown_args:
if '=' not in ua:
if 'r/' or 'u/' in ua:
direct_sources.append(
DirectInputSource(txt=ua, args={'limit': args.limit}))
continue
else:
su.error("ERROR: Unkown argument: %s" % ua)
sys.exit(1)
k = ua.split('=')[0].strip('- ')
v = ua.split('=', 2)[1].strip()
try:
settings.put(k, v, save_after=False)
except KeyError:
print('Unknown setting: %s' % k)
sys.exit(50)
if args.source:
matched_sources = set()
for s in args.source:
for stt in settings.get_sources():
if re.match(s, stt.get_alias()):
matched_sources.add(stt)
direct_sources.extend(matched_sources)
if not ffmpeg_download.install_local():
print(
"RMD was unable to locate (or download) a working FFmpeg binary.")
print("For downloading and post-processing, this is a required tool.")
print(
"Please Install FFmpeg manually, or download it from here: https://rmd.page.link/ffmpeg"
)
sys.exit(15)
if not _loaded and not direct_sources:
# First-time configuration.
su.error(
'Could not find an existing settings file. A new one will be generated!'
)
if not console.confirm(
'Would you like to start the WebUI to help set things up?',
True):
su.print_color(
'red',
"If you don't open the webUI now, you'll need to edit the settings file yourself."
)
if console.confirm(
"Are you sure you'd like to edit settings without the UI (if 'yes', these prompts will not show again)?"
):
settings.put('interface.start_server',
False) # Creates a save.
print(
'A settings file has been created for you, at "%s". Please customize it.'
% settings_file)
else:
print('Please re-run RMD to configure again.')
sys.exit(1)
else:
mode = console.prompt_list(
'How would you like to open the UI?',
settings.get('interface.browser', full_obj=True).opts)
settings.put('interface.browser', mode, save_after=False)
settings.put('interface.start_server', True)
# Initialize Database
sql.init_from_settings()
if direct_sources:
settings.disable_saving()
for s in settings.get_sources():
settings.remove_source(s, save_after=False)
for d in direct_sources:
settings.add_source(d, prevent_duplicate=False, save_after=False)
ui = None
if settings.get('interface.start_server') and not direct_sources:
print("Starting WebUI...")
ui = WebUI()
else:
ui = TerminalUI()
ui.display()
def test_init_db(self):
""" The database should build """
sql.init_from_settings()
self.assertTrue(
isfile(join(settings.get("output.base_dir"), "manifest.sqlite")),
"Failed to create sqlite file.")
def run(self):
""" Threaded loading of elements. """
settings.from_json(self._settings)
sql.init_from_settings()
self._session = sql.session()
self.progress.clear(status="Starting up...", running=True)
failed = False
for nxt_id in self._reader:
try:
url = self._session.query(
sql.URL).filter(sql.URL.id == nxt_id).first()
if not url:
raise Exception("Unknown URL ID provided: (%s}" % nxt_id)
file = url.file
path = SanitizedRelFile(base=settings.get("output.base_dir"),
file_path=str(file.path))
self.progress.set_file(path.relative())
self.progress.set_status("Attempting to Handle URL...")
self.progress.set_running(True)
task = handlers.HandlerTask(url=url.address, file_obj=path)
resp = handlers.handle(task, self.progress)
is_album_parent = False
with self._db_lock:
if resp.album_urls:
if url.album_id:
resp.album_urls = [
] # Ignore nested Albums to avoid recursion.
else:
url.album_id = str(uuid.uuid4())
is_album_parent = True
else:
resp.album_urls = []
url.failed = not resp.success
url.failure_reason = resp.failure_reason
url.last_handler = resp.handler
url.album_is_parent = is_album_parent
if resp.rel_file:
file.downloaded = True
file.path = resp.rel_file.relative()
file.hash = None
utime(resp.rel_file.absolute(), times=(time(), time()))
self._session.commit()
# Once *all* processing is completed on this URL, the Downloader needs to ACK it.
# If any additional Album URLS were located, they should be sent before the ACK.
self._ack_queue.put(
AckPacket(url_id=nxt_id, extra_urls=resp.album_urls))
self.progress.clear(status="Waiting for URL...")
except Exception as ex:
failed = str(ex)
self._ack_queue.put(AckPacket(url_id=nxt_id, extra_urls=[]))
print(ex)
traceback.print_exc()
self.progress.set_error("Exited with error: {%s}" % failed)
break
sql.close()
self.progress.clear(
"Finished." if not failed else "Exited with error: %s" % failed,
running=False)
def test_init_db(self):
""" The database should build """
sql.init_from_settings()
# self.assertTrue(isfile(join(settings.get("output.base_dir"), "manifest.sqlite")), "Failed to create sqlite file.")
self.assertTrue(isfile(join(settings.get("output.manifest_for_sqlite_dir"), "manifest.sqlite")), "Failed to create sqlite file.") # This is part of the change to save manifest.sqlite to a different directory than the downloads
def run():
logging.basicConfig(level=logging.WARN, format='%(levelname)-5.5s [%(name)s] %(message)s', datefmt='%H:%M:%S')
su.print_color('green', "\r\n" +
'====================================\r\n' +
(' Reddit Media Downloader %s\r\n' % meta.current_version) +
'====================================\r\n' +
' (By ShadowMoose @ Github)\r\n')
if args.version:
sys.exit(0)
if args.run_tests:
error_count = tests.runner.run_tests(test_subdir=args.run_tests)
sys.exit(error_count)
if args.list_settings:
print('All valid overridable settings:')
for _s in settings.get_all():
if _s.public:
print("%s.%s" % (_s.category, _s.name))
print('\tDescription: %s' % _s.description)
if not _s.opts:
print('\tValid value: \n\t\tAny %s' % _s.type)
else:
print('\tValid values:')
for o in _s.opts:
print('\t\t"%s": %s' % o)
print()
sys.exit()
settings_file = args.settings or fs.find_file('settings.json')
_loaded = settings.load(settings_file)
for ua in unknown_args:
if '=' not in ua or '/comments/' in ua:
if '/comments/' in ua:
direct_sources.append(DirectURLSource(url=ua))
continue
elif 'r/' or 'u/' in ua:
direct_sources.append(DirectInputSource(txt=ua, args={'limit': args.limit}))
continue
else:
su.error("ERROR: Unkown argument: %s" % ua)
sys.exit(1)
k = ua.split('=')[0].strip('- ')
v = ua.split('=', 2)[1].strip()
try:
settings.put(k, v, save_after=False)
except KeyError:
print('Unknown setting: %s' % k)
sys.exit(50)
if args.source:
matched_sources = set()
for s in args.source:
for stt in settings.get_sources():
if re.match(s, stt.get_alias()):
matched_sources.add(stt)
direct_sources.extend(matched_sources)
first_time_auth = False
if not _loaded and not direct_sources: # First-time configuration.
su.error('Could not find an existing settings file. A new one will be generated!')
if not console.confirm('Would you like to start the WebUI to help set things up?', True):
su.print_color('red', "If you don't open the webUI now, you'll need to edit the settings file yourself.")
if console.confirm("Are you sure you'd like to edit settings without the UI (if 'yes', these prompts will not show again)?"):
settings.put('interface.start_server', False, save_after=True) # Creates a save.
print('A settings file has been created for you, at "%s". Please customize it.' % settings_file)
first_time_auth = True
else:
print('Please re-run RMD to configure again.')
sys.exit(1)
else:
mode = console.prompt_list('How would you like to open the UI?',
settings.get('interface.browser', full_obj=True).opts)
settings.put('interface.browser', mode, save_after=False)
settings.put('interface.start_server', True)
if args.authorize or first_time_auth: # In-console oAuth authentication flow
from static import praw_wrapper
from urllib.parse import urlparse, parse_qs
url = praw_wrapper.get_reddit_token_url()
su.print_color('green', '\nTo manually authorize your account, visit the below URL.')
su.print_color('yellow', 'Once there, authorize RMD, then copy the URL it redirects you to.')
su.print_color('yellow', 'NOTE: The redirect page will likely not load, and that is ok.')
su.print_color('cyan', '\n%s\n' % url)
token_url = console.col_input('Paste the URL you are redirected to here: ')
if token_url.strip():
qs = parse_qs(urlparse(token_url).query)
if 'state' not in qs or 'code' not in qs:
su.error('The url provided was not a valid reddit redirect. Please make sure you copied it right!')
elif qs['state'][0].strip() != settings.get('auth.oauth_key').strip():
su.error('Invalid reddit redirect state. Please restart and try again.')
else:
code = qs['code'][0]
su.print_color('green', 'Got code. Authorizing account...')
refresh = praw_wrapper.get_refresh_token(code)
if refresh:
settings.put('auth.refresh_token', refresh)
usr = praw_wrapper.get_current_username()
su.print_color('cyan', 'Authorized to view account: %s' % usr)
su.print_color('green', 'Saved authorization token! Please restart RMD to begin downloading!')
else:
su.error('Failed to gain an account access token from Reddit with that code. Please try again.')
sys.exit(0)
if not ffmpeg_download.install_local():
print("RMD was unable to locate (or download) a working FFmpeg binary.")
print("For downloading and post-processing, this is a required tool.")
print("Please Install FFmpeg manually, or download it from here: https://rmd.page.link/ffmpeg")
sys.exit(15)
# Initialize Database
sql.init_from_settings()
print('Using manifest file [%s].' % sql.get_file_location())
if direct_sources:
settings.disable_saving()
settings.put('processing.retry_failed', False)
for s in settings.get_sources():
settings.remove_source(s, save_after=False)
for d in direct_sources:
settings.add_source(d, prevent_duplicate=False, save_after=False)
if settings.get('interface.start_server') and not direct_sources:
print("Starting WebUI...")
ui = WebUI()
else:
ui = TerminalUI()
ui.display()
def setUp(self):
importlib.reload(sql)
self.assertFalse(sql._engine, msg="Failed to clear SQL session.")
settings.load(self.settings_file)
sql.init_from_settings()
self.ps = sql.PostSearcher(sql.session())
