def run_gallery_dl(url: str,
subscription_mode: bool,
ignore_anchor: bool,
metadata_only: bool,
log_file: str,
console_output_file: str,
unsupported_urls_file: str,
overwrite_existing: bool,
filter_: Optional[str] = None,
chapter_filter: Optional[str] = None,
abort_after: Optional[int] = None,
test_mode: bool = False,
old_log_file: Optional[str] = None,
old_unsupported_urls_file: Optional[str] = None,
max_file_count: Optional[int] = None) -> str:
"""
Downloads a URL with gallery-dl using the current hydownloader environment.
"""
run_args = [str(db.get_conf('gallery-dl.executable'))]
run_args += ['--ignore-config']
run_args += ['--verbose']
if os.path.isfile(db.get_rootpath() + "/gallery-dl-config.json"):
run_args += ["-c", db.get_rootpath() + "/gallery-dl-config.json"]
if os.path.isfile(db.get_rootpath() + "/gallery-dl-user-config.json"):
run_args += ["-c", db.get_rootpath() + "/gallery-dl-user-config.json"]
run_args += ['--cookies', db.get_rootpath() + '/cookies.txt']
if not test_mode:
run_args += ['--dest', db.get_rootpath() + '/data/gallery-dl']
else:
run_args += ['--dest', db.get_rootpath() + '/test/data/gallery-dl']
run_args += ['--write-metadata']
if metadata_only:
run_args += ['--no-download']
if old_log_file: append_file_contents(log_file, old_log_file)
run_args += ['--write-log', log_file]
if old_unsupported_urls_file:
append_file_contents(unsupported_urls_file, old_unsupported_urls_file)
run_args += ['--write-unsupported', unsupported_urls_file]
if overwrite_existing:
run_args += ['--no-skip']
if not ignore_anchor:
if not test_mode:
if override := str(db.get_conf("gallery-dl.archive-override")):
run_args += ["--download-archive", override]
else:
run_args += [
"--download-archive",
db.get_rootpath() + '/anchor.db'
]
else:
run_args += [
"--download-archive",
db.get_rootpath() + '/test/anchor.db'
]
def api_worker(path: str, debug: bool) -> None:
global _srv
if db.get_conf('daemon.ssl') and os.path.isfile(path+"/server.pem"):
log.info("hydownloader", "Starting daemon (with SSL)...")
_srv = SSLWSGIRefServer(path+"/server.pem", host=db.get_conf('daemon.host'), port=db.get_conf('daemon.port'))
bottle.run(server=_srv, debug=debug)
else:
if db.get_conf('daemon.ssl'):
log.warning("hydownloader", "SSL enabled in config, but no server.pem file found in the db folder, continuing without SSL...")
log.info("hydownloader", "Starting daemon...")
_srv = SSLWSGIRefServer("", host=db.get_conf('daemon.host'), port=db.get_conf('daemon.port'))
bottle.run(server=_srv, debug=debug)
def start(path : str, debug : bool) -> None:
log.init(path, debug)
db.init(path)
process_additional_data()
subs_thread = threading.Thread(target=subscription_worker, name='Subscription worker', daemon=True)
subs_thread.start()
url_thread = threading.Thread(target=url_queue_worker, name='Single URL queue worker', daemon=True)
url_thread.start()
if db.get_conf('daemon.ssl') and os.path.isfile(path+"/server.pem"):
log.info("hydownloader", "Starting daemon (with SSL)...")
srv = SSLWSGIRefServer(path+"/server.pem", host=db.get_conf('daemon.host'), port=db.get_conf('daemon.port'))
bottle.run(server=srv, debug=debug)
else:
if db.get_conf('daemon.ssl'):
log.warning("hydownloader", "SSL enabled in config, but no server.pem file found in the db folder, continuing without SSL...")
log.info("hydownloader", "Starting daemon...")
srv = SSLWSGIRefServer("", host=db.get_conf('daemon.host'), port=db.get_conf('daemon.port'))
bottle.run(server=srv, debug=debug)
def run_gallery_dl_with_custom_args(
args: list[str],
capture_output: bool = False) -> subprocess.CompletedProcess:
"""
This function runs gallery-dl with the given arguments in the current hydownloader environment.
Some arguments beyond the ones passed in by the caller will be added (these are needed to make
gallery-dl use the current hydownloader environment and conventions).
"""
run_args = [str(db.get_conf('gallery-dl.executable'))]
run_args += ['--ignore-config']
run_args += ['--verbose']
if os.path.isfile(db.get_rootpath() + "/gallery-dl-config.json"):
run_args += ["-c", db.get_rootpath() + "/gallery-dl-config.json"]
if os.path.isfile(db.get_rootpath() + "/gallery-dl-user-config.json"):
run_args += ["-c", db.get_rootpath() + "/gallery-dl-user-config.json"]
run_args += args
result = subprocess.run(run_args,
capture_output=capture_output,
text=capture_output,
check=False)
return result
def check_access() -> None:
if not db.get_conf("daemon.access_key") == bottle.request.headers.get("HyDownloader-Access-Key"):
bottle.abort(403)
def run_gallery_dl(url: str,
subscription_mode: bool,
ignore_anchor: bool,
metadata_only: bool,
log_file: str,
console_output_file: str,
unsupported_urls_file: str,
overwrite_existing: bool,
filter_: Optional[str] = None,
chapter_filter: Optional[str] = None,
abort_after: Optional[int] = None,
test_mode: bool = False,
old_log_file: Optional[str] = None,
old_unsupported_urls_file: Optional[str] = None,
max_file_count: Optional[int] = None,
process_id: Optional[str] = None) -> str:
"""
Downloads a URL with gallery-dl using the current hydownloader environment.
"""
global _process_map
run_args = [str(db.get_conf('gallery-dl.executable'))]
run_args += ['--ignore-config']
run_args += ['--verbose']
if os.path.isfile(db.get_rootpath() + "/gallery-dl-config.json"):
run_args += ["-c", db.get_rootpath() + "/gallery-dl-config.json"]
if os.path.isfile(db.get_rootpath() + "/gallery-dl-user-config.json"):
run_args += ["-c", db.get_rootpath() + "/gallery-dl-user-config.json"]
run_args += ['--cookies', db.get_rootpath() + '/cookies.txt']
if not test_mode:
run_args += ['--dest', db.get_datapath() + '/gallery-dl']
else:
run_args += ['--dest', db.get_rootpath() + '/test/data/gallery-dl']
run_args += ['--write-metadata']
if metadata_only:
run_args += ['--no-download']
if old_log_file: append_file_contents(log_file, old_log_file)
run_args += ['-o', f'output.logfile.path={json.dumps(log_file)}']
run_args += [
'-o',
'output.logfile.format="[{name}][{levelname}][{asctime}] {message}"'
]
db.add_log_file_to_parse_queue(log_file,
process_id if process_id else 'unknown')
if old_unsupported_urls_file:
append_file_contents(unsupported_urls_file, old_unsupported_urls_file)
run_args += ['--write-unsupported', unsupported_urls_file]
if overwrite_existing:
run_args += ['--no-skip']
if not ignore_anchor:
if not test_mode:
if override := str(db.get_conf("gallery-dl.archive-override")):
run_args += ["--download-archive", override]
else:
run_args += [
"--download-archive",
db.get_rootpath() + '/anchor.db'
]
else:
run_args += [
"--download-archive",
db.get_rootpath() + '/test/anchor.db'
]