def start(web_dir):
global _web_dir, started, _session
_web_dir = os.path.abspath(web_dir)
_session = sql.session()
if not settings.get('interface.start_server'):
print('WebUI is disabled by settings.')
return False
browser = settings.get('interface.browser').lower().strip()
browser = None if (browser == 'off') else browser
options = {
'app_mode': '-app' in browser,
'mode': browser.replace('-app', ''),
'host': settings.get('interface.host'),
'port': settings.get('interface.port'),
'block': False,
'close_callback': _websocket_close,
'all_interfaces': False,
# 'app': btl.default_app()
}
eel.init(web_dir)
eel.start('index.html', **options)
print('Started WebUI!')
if browser:
print('Awaiting connection from browser...')
else:
print('Browser auto-opening is disabled! Please open a browser to http://%s:%s/index.html !' %
(options['host'], options['port']))
started = True
return True
def load(self):
""" Threaded loading of elements. """
settings.from_json(self.settings)
sql.init_from_settings()
self._session = sql.session()
t_start = datetime.now() #vy
print("Started loading.") #vy
self.progress.set_scanning(True)
retry_failed = settings.get('processing.retry_failed')
# Query for all unhandled URLs, and submit them before scanning for new Posts.
unfinished = self._session\
.query(sql.URL)\
.filter((sql.URL.processed == False) | \
(retry_failed and sql.URL.failed and \
sql.not_(sql.URL.failure_reason.contains('404'))))\
.all()
print("Loading %s unfinished urls" % len(unfinished))
self._push_url_list(unfinished)
self._scan_sources()
self.progress.set_scanning(False)
# Wait for any remaining ACKS to come in, before closing the writing pipe.
# ...Until the Downloaders have confirmed completion of everything, more album URLS may come in.
while len(self._open_ack) > 0 and not self._stop_event.is_set():
self._handle_acks(timeout=1.0, clear=True)
print("Finished loading.") #vy
print("Elapsed time: %s" % str(datetime.now() - t_start)) #vy
sql.close()
def load(self):
""" Threaded loading of elements. """
settings.from_json(self.settings)
sql.init_from_settings()
self._session = sql.session()
self.progress.set_scanning(True)
retry_failed = settings.get('processing.retry_failed')
# Query for all unhandled URLs, and submit them before scanning for new Posts.
unfinished = self._session\
.query(sql.URL)\
.filter((sql.URL.processed == False) | (retry_failed and sql.URL.failed == True))\
.all()
self._push_url_list(unfinished)
self._scan_sources()
self.progress.set_scanning(False)
# Wait for any remaining ACKS to come in, before closing the writing pipe.
# ...Until the Downloaders have confirmed completion of everything, more album URLS may come in.
while len(self._open_ack) > 0 and not self._stop_event.is_set():
self._handle_acks(timeout=0.5)
print("Finished loading.")
sql.close()
def start(web_dir, file_dir, rmd_version):
global _file_dir, _web_dir, _rmd_version, started, _session
_file_dir = os.path.abspath(file_dir)
_web_dir = os.path.abspath(web_dir)
_rmd_version = rmd_version
_session = sql.session()
if not settings.get('interface.start_server'):
print('WebUI is disabled by settings.')
return False
browser = settings.get('interface.browser').lower().strip()
browser = None if (browser == 'off') else browser
options = {
'mode': browser,
'host': settings.get('interface.host'),
'port': settings.get('interface.port'),
'chromeFlags': []
}
eel.init(web_dir)
eel.start('index.html', options=options, block=False, callback=_websocket_close)
# interface.port
print('Started WebUI!')
if browser:
print('Awaiting connection from browser...')
else:
print('Browser auto-opening is disabled! Please open a browser to http://%s:%s/index.html !' %
(options['host'], options['port']))
started = True
return True
def test_existing_file_name(self):
""" Generating an incremented file name should work """
tp = self.sess.query(
sql.Post).filter(sql.Post.reddit_id == 't3_ahal9v').first()
file = ng.choose_file_name(tp.urls[0], tp, sql.session(), album_size=1)
self.assertTrue(file.endswith(' - 2'),
msg='Failed to increment duplicate post!')
def test_choose_file_name(self):
""" Generating a new file name should work """
tp = self.sess.query(sql.Post).filter(sql.Post.title == 'test').first()
file = ng.choose_file_name(tp.urls[0], tp, sql.session(), album_size=1)
self.assertEqual('aww/test - (testuser)',
file,
msg='Failed to convert basic Test post!')
def setUp(self):
importlib.reload(settings)
importlib.reload(sql)
importlib.reload(ng)
settings.load(self.settings_file)
sql.init_from_settings()
self.sess = sql.session()
def test_pattern_fail_load(self):
""" Invalid patterns should fail """
settings.put('output.file_name_pattern',
'[type]-[id]-[title]-[author-[subreddit]-[source_alias]')
tp = self.sess.query(sql.Post).filter(sql.Post.title == 'test').first()
with self.assertRaises(Exception,
msg='Failed to catch broken pattern!'):
ng.choose_file_name(tp.urls[0], tp, sql.session(), album_size=1)
def test_album_filename(self):
""" Generating new & incremented album names should work """
tp = self.sess.query(sql.Post).join(
sql.URL).filter(sql.Post.reddit_id == 't3_98crc8').first()
file = ng.choose_file_name(tp.urls[0],
tp,
sql.session(),
album_size=1000)
self.assertEqual('aww/album - (testuser2)/0001',
file,
msg='Failed to generate new Album foldername!')
np = self.sess.query(sql.Post).join(
sql.URL).filter(sql.Post.reddit_id == 't3_awyf90').first()
file = ng.choose_file_name(np.urls[0], np, sql.session(), album_size=1)
self.assertEqual('aww/album - (testuser2) - 2/1',
file,
msg='Failed to create separate album folder!')
def setUp(self):
global download_ran, session
if not download_ran:
download_ran = True
settings.load(self.settings_file)
tui = TerminalUI()
tui.display()
self.db_path = join(settings.get('output.base_dir'),
'manifest.sqlite')
sql.init_from_settings()
session = sql.session()
def setUp(self):
global download_ran, session
if not download_ran:
download_ran = True
settings.load(self.settings_file)
tui = TerminalUI()
tui.display()
# self.db_path = join(settings.get('output.base_dir'), 'manifest.sqlite')
self.db_path = join(settings.get('output.manifest_for_sqlite_dir'), 'manifest.sqlite') # This is part of the change to save manifest.sqlite to a different directory than the downloads
sql.init_from_settings()
session = sql.session()
def make_migration():
conn = sql.session()
alembic_cfg, script_, context = sql.get_alembic_ctx(conn)
message = console.string('Enter a message for the migration')
if not message:
print('Skipping migration.')
return
res = command.revision(message=message,
autogenerate=True,
config=alembic_cfg)
print('Generated Migration:', res)
print('Finished.')
def test_pattern_loader(self):
""" All pattern tags should work """
settings.put(
'output.file_name_pattern',
'[type]-[reddit_id]-[title]-[author]-[subreddit]-[source_alias]-[created_utc]-[created_date]-[created_time]'
)
tp = self.sess.query(sql.Post).filter(sql.Post.title == 'test').first()
file = ng.choose_file_name(tp.urls[0], tp, sql.session(), album_size=1)
self.assertRegex(
file,
r'Submission-t3_b1rycu-test-testuser-aww-newsource-1552739416-2019-..-..-..\...\...',
msg='Failed to convert basic Test post!')
def setUp(self):
global download_ran, session
if not download_ran:
download_ran = True
importlib.reload(settings)
importlib.reload(sql)
settings.load(self.settings_file)
tui = TerminalUI()
tui.display()
sql.init_from_settings()
self.db_path = sql.get_file_location()
session = sql.session()
def setUp(self):
global download_ran, session, thread
if not download_ran:
download_ran = True
self.wui = WebUI('test_version')
self.db_path = join(settings.get('output.base_dir'), 'manifest.sqlite')
self.url = 'http://%s:%s/index.html#' % (settings.get('interface.host'), settings.get('interface.port'))
settings.load(self.settings_file)
settings.put('interface.start_server', True)
sql.init_from_settings()
session = sql.session()
thread = Thread(target=self.wui.display)
thread.setDaemon(True)
thread.start()
self.assertTrue(self.wui.waitFor(10), msg='WebUI Failed to start!')
def test_full_encode(self):
""" The entire nested relation should serialize """
p = sql.session().query(sql.Post).join(sql.URL).join(sql.File).first()
self.assertTrue(p, msg='Failed to find a test Post.')
for u in p.urls:
self.assertTrue(u.file, msg="URL is missing a file! %s" %
u) # These are lazy loaded, so check them all.
ser = sql.encode_safe(p)
self.assertTrue(
ser, msg='Failed to properly encode full stack Post into Object!')
self.assertGreater(len(ser['urls']),
0,
msg='Lost Post URLs in encode!')
for u in ser['urls']:
self.assertIn('file', u, msg='Lost file in URL encode! %s' % u)
def run(self):
""" Threaded loading of elements. """
settings.from_json(self._settings)
sql.init_from_settings()
self._session = sql.session()
self.progress.clear(status="Starting up...")
self.progress.set_running(True)
while not self._stop_event.is_set():
self._dedupe()
self.progress.set_status("Waiting for new files...")
self._stop_event.wait(2)
self._dedupe() # Run one final pass after downloading stops.
self.progress.set_running(False)
sql.close()
self.progress.clear("Finished.")
def add_test(inf):
sess = sql.session()
lst = sess.query(sql.URL).all()
st = time.time()
sent = []
for l in lst:
in_queue.put_nowait(l.id)
inf['sent'] += 1
sent.append(l.id)
while time.time() - st < 30 and sent:
try:
rd = ack_queue.get(block=True, timeout=.5)
inf['ack'] += 1
sent.remove(rd.url_id)
except queue.Empty:
pass
sess.close()
stop_event.set()
def start(self):
os.makedirs(self.new_save_base, exist_ok=True)
if not settings.load(self.settings_file):
raise Exception("You must provide a valid settings.json file!")
settings.put('output.base_dir', self.new_save_base)
sql.init_from_settings()
self.session = sql.session()
print("Scanning legacy posts...")
self.scan()
print("Found %s elements total." % len(self.posts.keys()))
self.process_posts()
self.session.commit()
print("Processed:", len(self.posts), "Posts.")
print("Failed to convert %s Posts." % len(self.failures))
outfile = os.path.join(self.new_save_base, 'failed_conversion_posts.json')
with open(outfile, 'w') as o:
o.write(json.dumps(self.failures, indent=4, sort_keys=True, separators=(',', ': ')))
print("Saved a list of failed posts to", outfile, ', be sure to check these before deleting anything!')
def run(self):
""" Threaded loading of elements. """
settings.from_json(self._settings)
sql.init_from_settings()
try:
self._session = sql.session()
self.progress.clear(status="Starting up...")
self.progress.set_running(True)
while not self._stop_event.is_set():
self._dedupe()
self.progress.set_status("Ready for new files...")
self._stop_event.wait(2)
self._dedupe() # Run one final pass after downloading stops.
self.progress.clear(status="Finished.", running=False)
except Exception as ex:
print('Deduplication Process Error:', ex)
self.progress.set_error(ex)
self.progress.set_running(False)
traceback.print_exc()
finally:
sql.close()
def run(self):
""" Threaded loading of elements. """
settings.from_json(self._settings)
sql.init_from_settings()
print("Starting up...", debug=True)
try:
self._session = sql.session()
self.progress.clear(status="Starting up...")
self.progress.set_running(True)
self.dedup_ignore_ids = set()
self.prune_counter = 0
self.special_hashes = self._session.query(Hash).filter(
Hash.id < 0).all()
while not self._stop_event.is_set():
#print("_stop_event is %s"%self._stop_event.is_set(), debug=True)
completed = self._dedupe()
if completed:
self.progress.set_status(
"Completed %s files. Ready for new files..." %
completed)
self._stop_event.wait(1)
else:
self._stop_event.wait(10)
print("_stop_event is %s" % self._stop_event.is_set(), debug=True)
self._dedupe() # Run one final pass after downloading stops.
self.progress.clear(status="Finished.", running=False)
except Exception as ex:
print('Deduplication Process Error:', ex)
self.progress.set_error(ex)
self.progress.set_running(False)
traceback.print_exc()
finally:
print("Finished process, _stop_event is %s" %
self._stop_event.is_set(),
debug=True)
sql.close()
def __init__(self, bot: Bot):
self.bot = bot
self.sql: Session = sql.session()
self.handler = BotMessageHandler(self.sql)
self.settings = settings.SettingWrapper(self.__class__.__name__)
print("Registered Cog:", self.__class__.__name__)
"Custom source importer"
import os
import static.filesystem as fs
import static.settings as settings
import sources
from datetime import datetime
import sql
from sql import Post, File, URL, Hash
from sqlalchemy.sql.expression import func
#datetime.now().strftime("%c %X")
settings_file = fs.find_file('settings.json')
_loaded = settings.load(settings_file)
_session = sql.session()
url_patt = r'^(?:(?!(youtu\.be|youtube\.com|amazon\.c|twitter\.c|instagram\.com)).)*$'
def strf_utc(sec):
return datetime.fromtimestamp(sec).strftime("%y-%m-%d %H:%M:%S")
def user_source(name,
alias=None,
ps=False,
limit=None,
deep=False,
check_last=None,
check_utc=0):
u = name.replace('/u/', '').replace('u/', '').strip('/')
out = {
def run(self):
""" Threaded loading of elements. """
settings.from_json(self._settings)
sql.init_from_settings()
self._session = sql.session()
self.progress.clear(status="Starting up...", running=True)
failed = False
for nxt_id in self._reader:
try:
url = self._session.query(
sql.URL).filter(sql.URL.id == nxt_id).first()
if not url:
raise Exception("Unknown URL ID provided: (%s}" % nxt_id)
file = url.file
path = SanitizedRelFile(base=settings.get("output.base_dir"),
file_path=str(file.path))
self.progress.set_file(path.relative())
self.progress.set_status("Attempting to Handle URL...")
self.progress.set_running(True)
task = handlers.HandlerTask(url=url.address, file_obj=path)
resp = handlers.handle(task, self.progress)
is_album_parent = False
with self._db_lock:
if resp.album_urls:
if url.album_id:
resp.album_urls = [
] # Ignore nested Albums to avoid recursion.
else:
url.album_id = str(uuid.uuid4())
is_album_parent = True
else:
resp.album_urls = []
url.failed = not resp.success
url.failure_reason = resp.failure_reason
url.last_handler = resp.handler
url.album_is_parent = is_album_parent
if resp.rel_file:
file.downloaded = True
file.path = resp.rel_file.relative()
file.hash = None
utime(resp.rel_file.absolute(), times=(time(), time()))
self._session.commit()
# Once *all* processing is completed on this URL, the Downloader needs to ACK it.
# If any additional Album URLS were located, they should be sent before the ACK.
self._ack_queue.put(
AckPacket(url_id=nxt_id, extra_urls=resp.album_urls))
self.progress.clear(status="Waiting for URL...")
except Exception as ex:
failed = str(ex)
self._ack_queue.put(AckPacket(url_id=nxt_id, extra_urls=[]))
print(ex)
traceback.print_exc()
self.progress.set_error("Exited with error: {%s}" % failed)
break
sql.close()
self.progress.clear(
"Finished." if not failed else "Exited with error: %s" % failed,
running=False)
def setUp(self):
importlib.reload(sql)
self.assertFalse(sql._engine, msg="Failed to clear SQL session.")
settings.load(self.settings_file)
sql.init_from_settings()
self.ps = sql.PostSearcher(sql.session())
