def mass_add_subscriptions(path: str, file_: str, downloader: str, additional_data: Optional[str], paused: bool, filter_: Optional[str], abort_after: int, max_files_initial: Optional[int], max_files_regular: Optional[int]) -> None:
log.init(path, True)
db.init(path)
for line in open(file_, 'r'):
line = line.strip()
if line:
db.add_or_update_subscriptions([{
'keywords': line,
'downloader': downloader,
'time_created': time.time(),
'additional_data': additional_data,
'filter': filter_,
'max_files_initial': max_files_initial,
'max_files_regular': max_files_regular,
'abort_after': abort_after,
'paused': paused
}])
log.info("hydownloader-tools", f"Added subscription {line} with downloader {downloader}")
def mass_add_subscriptions(path: str, file_: str, downloader: str,
additional_data: Optional[str], paused: bool,
filter_: Optional[str], abort_after: int,
max_files_initial: Optional[int],
max_files_regular: Optional[int],
check_interval: int, random_check_interval: int,
encode_keywords: bool) -> None:
log.init(path, True)
db.init(path)
for line in open(file_, 'r', encoding='utf-8-sig'):
line = line.strip()
if encode_keywords:
line = line.replace(' ', '+')
line = urllib.parse.quote(line, safe='/+').lower()
if line:
new_sub = {
'keywords':
line,
'downloader':
downloader,
'time_created':
time.time(),
'additional_data':
additional_data,
'filter':
filter_,
'paused':
paused,
'check_interval':
check_interval + random.randint(0, random_check_interval)
}
if max_files_initial is not None:
new_sub['max_files_initial'] = max_files_initial
if max_files_regular is not None:
new_sub['max_files_regular'] = max_files_regular
if abort_after is not None:
new_sub['abort_after'] = abort_after
db.add_or_update_subscriptions([new_sub])
log.info(
"hydownloader-tools",
f"Added subscription {line} with downloader {downloader}")
def subscription_worker() -> None:
global _sub_worker_ended_flag
try:
log.info("hydownloader", "Starting subscription worker thread...")
with _worker_lock:
_sub_worker_ended_flag = False
while True:
time.sleep(2)
with _worker_lock:
if _end_threads_flag:
break
subs_due = db.get_due_subscriptions()
if not subs_due:
with _worker_lock:
if _sub_worker_paused_flag:
set_subscription_worker_status("paused")
else:
set_subscription_worker_status("nothing to do: checked for due subscriptions, found none")
sub = subs_due[0] if subs_due else None
while sub:
with _worker_lock:
if _end_threads_flag:
break
if _sub_worker_paused_flag:
set_subscription_worker_status("paused")
break
initial_check = sub['last_check'] is None
url = urls.subscription_data_to_url(sub['downloader'], sub['keywords'])
check_started_time = time.time()
status_msg = f"checking subscription: {sub['id']} (downloader: {sub['downloader']}, keywords: {sub['keywords']})"
set_subscription_worker_status(status_msg)
log.info(f"subscription-{sub['id']}", status_msg.capitalize())
if initial_check:
log.info(f"subscription-{sub['id']}", "This is the first check for this subscription")
result = gallery_dl_utils.run_gallery_dl(
url=url,
ignore_anchor=False,
metadata_only=False,
log_file=db.get_rootpath()+f"/logs/subscription-{sub['id']}-gallery-dl-latest.txt",
old_log_file=db.get_rootpath()+f"/logs/subscription-{sub['id']}-gallery-dl-old.txt",
console_output_file=db.get_rootpath()+f"/temp/subscription-{sub['id']}-gallery-dl-output.txt",
unsupported_urls_file=db.get_rootpath()+f"/logs/subscription-{sub['id']}-unsupported-urls-gallery-dl-latest.txt",
old_unsupported_urls_file=db.get_rootpath()+f"/logs/subscription-{sub['id']}-unsupported-urls-gallery-dl-old.txt",
overwrite_existing=False,
filter_=sub['filter'],
chapter_filter=None,
subscription_mode=True,
abort_after=sub['abort_after'],
max_file_count = sub['max_files_initial'] if initial_check else sub['max_files_regular']
)
if result:
log.warning(f"subscription-{sub['id']}", "Error: "+result)
else:
sub['last_successful_check'] = check_started_time
sub['last_check'] = check_started_time
new_files, skipped_files = process_additional_data(subscription_id = sub['id'])
check_ended_time = time.time()
db.add_subscription_check(sub['id'], new_files=new_files, already_seen_files=skipped_files, time_started=check_started_time, time_finished=check_ended_time, status=result)
db.add_or_update_subscriptions([sub])
status_msg = f"finished checking subscription: {sub['id']} (downloader: {sub['downloader']}, keywords: {sub['keywords']}), new files: {new_files}, skipped: {skipped_files}"
set_subscription_worker_status(status_msg)
log.info(f"subscription-{sub['id']}", status_msg.capitalize())
subs_due = db.get_due_subscriptions()
sub = subs_due[0] if subs_due else None
with _worker_lock:
if _end_threads_flag:
break
with _worker_lock:
if _end_threads_flag:
log.info("hydownloader", "Stopping subscription worker thread")
_sub_worker_ended_flag = True
except Exception as e:
log.fatal("hydownloader", "Uncaught exception in subscription worker thread", e)
shutdown()
def route_add_or_update_subscriptions() -> dict:
check_access()
return {'status': db.add_or_update_subscriptions(bottle.request.json)}