def check_results_of_post_url(data: dict, sitename: str) -> bool:
"""
Downloads a URL with gallery-dl, then checks if the
downloaded filenames, file content and anchor entries match what was provided by the caller.
"""
url = data['url']
filenames = data['filenames']
anchors = data['anchors']
log.info("hydownloader-test", f'Testing downloading of posts for site {sitename}')
log_file = db.get_rootpath()+f"/logs/test-site-{sitename}-gallery-dl.txt"
result_txt = gallery_dl_utils.run_gallery_dl(
url=url,
ignore_anchor=False,
metadata_only=False,
log_file=log_file,
console_output_file=db.get_rootpath()+f"/test/test-site-{sitename}-gallery-dl-output.txt",
unsupported_urls_file=db.get_rootpath()+f"/test/test-site-{sitename}-unsupported-urls-gallery-dl.txt",
overwrite_existing=False,
subscription_mode=False,
test_mode = True
)
result = True
if result_txt:
log.error("hydownloader-test", f"Error returned for {sitename} download: {result_txt}")
result = False
else:
log.info("hydownloader-test", f"Return code for {sitename} download OK")
for fname in filenames:
abs_fname = db.get_rootpath()+"/test/data/gallery-dl/"+fname
if not os.path.isfile(abs_fname):
log.error("hydownloader-test", f"Missing expected file: {fname}")
result = False
else:
log.info("hydownloader-test", f"Found expected file: {fname}")
for content in filenames[fname]:
with open(abs_fname) as f:
if re.search(content, f.read()):
log.info("hydownloader-test", "Expected file content found")
else:
log.error("hydownloader-test", f"Expected file content ({content}) NOT found")
result = False
conn = sqlite3.connect(db.get_rootpath()+"/test/anchor.db")
conn.row_factory = sqlite3.Row
c = conn.cursor()
for anchor in anchors:
try:
c.execute('select entry from archive where entry = ?', (anchor,))
if len(c.fetchall()):
log.info("hydownloader-test", f"Expected anchor {anchor} found in database")
else:
log.error("hydownloader-test", f"Expected anchor {anchor} NOT found in database")
result = False
except sqlite3.OperationalError as e:
log.error("hydownloader-test", "Error while trying to query anchor database - download failed?", e)
result = False
return result
def test_internal(sites: str) -> bool:
post_url_data = {
'gelbooru': {
'url':
"https://gelbooru.com/index.php?page=post&s=view&id=6002236",
'filenames': {
"gelbooru/gelbooru_6002236_0ef507cc4c222406da544db3231de323.jpg.json":
["1girl ", "wings", '"rating": "q"', '"tags_general":'],
"gelbooru/gelbooru_6002236_0ef507cc4c222406da544db3231de323.jpg":
[]
},
'anchors': ["gelbooru6002236"]
},
'gelbooru_notes': {
'url':
"https://gelbooru.com/index.php?page=post&s=view&id=5997331",
'filenames': {
"gelbooru/gelbooru_5997331_7726d401af0e6bf5b58809f65d08334e.png.json":
[
'"y": 72', '"x": 35', '"width": 246', '"height": 553',
'"body": "Look over this way when you talk~"'
]
},
'anchors': ["gelbooru5997331"]
},
'danbooru': {
'url': "https://danbooru.donmai.us/posts/4455434",
'filenames': {
"danbooru/danbooru_4455434_e110444217827ef3f82fb33b45e1841f.png.json":
["1girl ", "tail", '"rating": "q"'],
"danbooru/danbooru_4455434_e110444217827ef3f82fb33b45e1841f.png":
[]
},
'anchors': ["danbooru4455434"]
},
'pixiv': {
'url':
"https://www.pixiv.net/en/artworks/88865254",
'filenames': {
"pixiv/3316400 rogia/88865254_p7.jpg.json": [],
"pixiv/3316400 rogia/88865254_p6.jpg.json": [],
"pixiv/3316400 rogia/88865254_p5.jpg.json": [],
"pixiv/3316400 rogia/88865254_p4.jpg.json": [],
"pixiv/3316400 rogia/88865254_p3.jpg.json": [],
"pixiv/3316400 rogia/88865254_p2.jpg.json": [
"Fate/GrandOrder", '"title": "メイドロリンチちゃん"', '"tags":',
'"tags": ['
],
"pixiv/3316400 rogia/88865254_p1.jpg.json": [],
"pixiv/3316400 rogia/88865254_p0.jpg.json": [],
"pixiv/3316400 rogia/88865254_p7.jpg": [],
"pixiv/3316400 rogia/88865254_p6.jpg": [],
"pixiv/3316400 rogia/88865254_p5.jpg": [],
"pixiv/3316400 rogia/88865254_p4.jpg": [],
"pixiv/3316400 rogia/88865254_p3.jpg": [],
"pixiv/3316400 rogia/88865254_p2.jpg": [],
"pixiv/3316400 rogia/88865254_p1.jpg": [],
"pixiv/3316400 rogia/88865254_p0.jpg": []
},
'anchors': [
"pixiv88865254_p00", "pixiv88865254_p01", "pixiv88865254_p02",
"pixiv88865254_p03", "pixiv88865254_p04", "pixiv88865254_p05",
"pixiv88865254_p06", "pixiv88865254_p07"
]
},
'pixiv_ugoira': {
'url': "https://www.pixiv.net/en/artworks/88748768",
'filenames': {
"pixiv/9313418 thaimay704/88748768_p0.zip": [],
"pixiv/9313418 thaimay704/88748768_p0.zip.json": [],
"pixiv/9313418 thaimay704/88748768_p0.webm": []
},
'anchors': ["pixiv88748768"]
},
'lolibooru': {
'url':
'https://lolibooru.moe/post/show/178123/1girl-barefoot-brown_eyes-brown_hair-cameltoe-cove',
'filenames': {
"lolibooru/lolibooru_178123_a77d70e0019fc77c25d0ae563fc9b324.jpg.json":
["1girl ", " swimsuit", '"rating": "q",'],
"lolibooru/lolibooru_178123_a77d70e0019fc77c25d0ae563fc9b324.jpg":
[]
},
'anchors': ["lolibooru178123"]
},
'3dbooru': {
'url':
"http://behoimi.org/post/show/648363/apron-black_legwear-collar-cosplay-hairband-immora",
'filenames': {
"3dbooru/3dbooru_648363_720f344170696293c3fe2640c59d8f41.jpg.json":
["cosplay ", " maid_uniform", '"rating": "s",'],
"3dbooru/3dbooru_648363_720f344170696293c3fe2640c59d8f41.jpg":
[]
},
'anchors': ["3dbooru648363"]
},
'nijie': {
'url': "https://nijie.info/view.php?id=306993",
'filenames': {
"nijie/72870/306993_p0.jpg": [],
"nijie/72870/306993_p1.jpg": [],
"nijie/72870/306993_p0.jpg.json": [],
"nijie/72870/306993_p1.jpg.json":
["\"オリジナル\"", "\"title\": \"朝7時50分の通学路\","]
},
'anchors': ["nijie306993_0", "nijie306993_1"]
},
'patreon': {
'url': "https://www.patreon.com/posts/new-cg-set-on-48042243",
'filenames': {
"patreon/Osiimi Chan/48042243_NEW CG SET on Gumroad!! Ganyu's Hypnotic Rendezvou_01.png":
[]
},
'anchors': ["patreon48042243_1"]
},
'sankaku': {
'url': "https://chan.sankakucomplex.com/post/show/707246",
'filenames': {
"sankaku/sankaku_707246_5da41b5136905c35cad9cbcba89836a3.jpg":
[],
"sankaku/sankaku_707246_5da41b5136905c35cad9cbcba89836a3.jpg.json":
['"kirisame_marisa"', '"3girls"']
},
'anchors': ["sankaku707246"]
},
'idolcomplex': {
'url': "https://idol.sankakucomplex.com/post/show/701724",
'filenames': {
"idolcomplex/idolcomplex_701724_92b853bcf8dbff393c6217839013bcab.jpg":
[],
"idolcomplex/idolcomplex_701724_92b853bcf8dbff393c6217839013bcab.jpg.json":
['"rating": "q",', 'nikumikyo,']
},
'anchors': ["idolcomplex701724"]
},
'artstation': {
'url': "https://www.artstation.com/artwork/W2LROD",
'filenames': {
"artstation/sergey_vasnev/artstation_6721469_24728858_Procession.jpg":
[],
"artstation/sergey_vasnev/artstation_6721469_24728858_Procession.jpg.json":
['"title": "Procession",']
},
'anchors': ["artstation24728858"]
},
'deviantart': {
'url':
"https://www.deviantart.com/squchan/art/Atelier-Ryza-820511154",
'filenames': {
"deviantart/SquChan/deviantart_820511154_Atelier Ryza.jpg": [],
"deviantart/SquChan/deviantart_820511154_Atelier Ryza.jpg.json":
['"is_mature": true,']
},
'anchors': ["deviantart820511154"]
},
'twitter': {
'url':
"https://twitter.com/momosuzunene/status/1380033327680266244",
'filenames': {
"twitter/momosuzunene/1380033327680266244_1.jpg": [],
"twitter/momosuzunene/1380033327680266244_1.jpg.json":
['"name": "momosuzunene",']
},
'anchors': ["twitter1380033327680266244_1"]
},
'webtoons': {
'url':
"https://www.webtoons.com/en/challenge/crawling-dreams/ep-1-nyarla-ghast/viewer?title_no=141539&episode_no=81",
'anchors': [
'webtoons141539_81_1', 'webtoons141539_81_2',
'webtoons141539_81_3', 'webtoons141539_81_4'
],
'filenames': {
"webtoons/crawling-dreams/81-01.jpg": [],
"webtoons/crawling-dreams/81-01.jpg.json":
['"comic": "crawling-dreams"']
}
},
'baraag': {
'url': "https://baraag.net/@pumpkinnsfw/106191173043385531",
'anchors':
['baraag106191139078112401', 'baraag106191139927706653'],
'filenames': {
"mastodon/baraag.net/pumpkinnsfw/baraag_106191173043385531_106191139078112401.png":
[],
"mastodon/baraag.net/pumpkinnsfw/baraag_106191173043385531_106191139078112401.png.json":
['"sensitive": true']
}
},
'hentaifoundry': {
'url':
"https://www.hentai-foundry.com/pictures/user/PalomaP/907277/Rapunzel-loves-creampie",
'anchors': ["hentaifoundry907277"],
'filenames': {
"hentaifoundry/PalomaP/hentaifoundry_907277_Rapunzel loves creampie.jpg":
[],
"hentaifoundry/PalomaP/hentaifoundry_907277_Rapunzel loves creampie.jpg.json":
['"tags": [', '"creampie"']
}
},
'yandere': {
'url': "https://yande.re/post/show/619304",
'anchors': ["yandere619304"],
'filenames': {
'yandere_619304_449a208b7a42f917498a00386e173118.jpg.json': [],
'yandere_619304_449a208b7a42f917498a00386e173118.jpg':
['"tags_artist": "zuima"']
}
}
}
site_set = {site.strip() for site in sites.split(',')}
for site in site_set:
clear_test_env()
log_file = db.get_rootpath() + f"/logs/test-site-{site}-gallery-dl.txt"
should_break = False
if site == 'environment':
log.info("hydownloader-test", "Querying gallery-dl version")
version_str = gallery_dl_utils.run_gallery_dl_with_custom_args(
['--version'], capture_output=True).stdout.strip()
try:
if version_str.endswith("-dev"): version_str = version_str[:-4]
major, minor, patch = tuple(map(int, version_str.split('.')))
if major != 1 or minor < 17 or minor == 17 and patch < 4:
log.error(
'hydownloader-test',
f"Bad gallery-dl version: {version_str}, need 1.17.3 or newer"
)
should_break = True
else:
log.info(
'hydownloader-test',
f"Found gallery-dl version: {version_str}, this is OK")
except ValueError as e:
log.error('hydownloader-test',
"Could not recognize gallery-dl version", e)
should_break = True
try:
ff_result = subprocess.run(['ffmpeg', '-version'],
capture_output=True,
text=True,
check=False).stdout.split('\n')[0]
log.info('hydownloader-test',
f"Found ffmpeg version: {ff_result}")
except FileNotFoundError as e:
log.error('hydownloader-test', "Could not find ffmpeg", e)
should_break = True
try:
yt_result = subprocess.run(['youtube-dl', '--version'],
capture_output=True,
text=True,
check=False).stdout.strip()
log.info('hydownloader-test',
f"Found youtube-dl version: {yt_result}")
except FileNotFoundError as e:
log.error('hydownloader-test', "Could not find youtube-dl", e)
should_break = True
elif site == "gelbooru":
log.info("hydownloader-test", "Testing gelbooru...")
log.info("hydownloader-test",
'Testing search of "sensitive" content')
sensitive_url = "https://gelbooru.com/index.php?page=post&s=list&tags=loli"
result = gallery_dl_utils.run_gallery_dl_with_custom_args(
[
sensitive_url, '--get-urls', '-o', 'image-range="1-10"',
'--write-log', log_file
],
capture_output=True)
sensitive_ok = True
if result.returncode != 0:
status_txt = gallery_dl_utils.check_return_code(
result.returncode)
log.error(
"hydownloader-test",
f'Error returned while trying to download "sensitive" content: return code {result.returncode}, {status_txt}'
)
sensitive_ok = False
should_break = True
sensitive_results_cnt = len(
re.findall("https://.*?gelbooru.com/images", result.stdout))
if sensitive_results_cnt < 10:
log.error(
"hydownloader-test",
f'Failed to find "sensitive" content, insufficient number of results: {sensitive_results_cnt}'
)
sensitive_ok = False
should_break = True
if sensitive_ok:
log.info(
"hydownloader-test",
'Search of "sensitive" content seems to be working OK')
should_break = not check_results_of_post_url(
post_url_data['gelbooru'], site) or should_break
log.info("hydownloader-test", 'Testing note extraction')
should_break = not check_results_of_post_url(
post_url_data['gelbooru_notes'], site) or should_break
elif site == "danbooru":
log.info("hydownloader-test", "Testing danbooru...")
log.info("hydownloader-test",
'Testing search of "sensitive" content')
sensitive_url = "https://danbooru.donmai.us/posts?tags=loli"
result = gallery_dl_utils.run_gallery_dl_with_custom_args(
[
sensitive_url, '--get-urls', '-o', 'image-range="1-10"',
'--write-log', log_file
],
capture_output=True)
sensitive_ok = True
if result.returncode != 0:
status_txt = gallery_dl_utils.check_return_code(
result.returncode)
log.error(
"hydownloader-test",
f'Error returned while trying to download "sensitive" content: return code {result.returncode}, {status_txt}'
)
sensitive_ok = False
should_break = True
sensitive_results_cnt = len(
re.findall("https://danbooru.donmai.us/data", result.stdout))
if sensitive_results_cnt < 10:
log.error(
"hydownloader-test",
f'Failed to find "sensitive" content, insufficient number of results: {sensitive_results_cnt}'
)
sensitive_ok = False
should_break = True
if sensitive_ok:
log.info(
"hydownloader-test",
'Search of "sensitive" content seems to be working OK')
should_break = not check_results_of_post_url(
post_url_data['danbooru'], site) or should_break
elif site == "pixiv":
log.info("hydownloader-test", "Testing pixiv...")
should_break = not check_results_of_post_url(
post_url_data['pixiv'], site) or should_break
log.info("hydownloader-test", 'Testing downloading of ugoira')
should_break = not check_results_of_post_url(
post_url_data['pixiv_ugoira'], site) or should_break
elif site == "lolibooru":
log.info("hydownloader-test", "Testing lolibooru.moe...")
should_break = not check_results_of_post_url(
post_url_data['lolibooru'], site) or should_break
elif site == "3dbooru":
log.info("hydownloader-test", "Testing 3dbooru...")
should_break = not check_results_of_post_url(
post_url_data['3dbooru'], site) or should_break
elif site == "patreon":
log.info("hydownloader-test", "Testing patreon...")
should_break = not check_results_of_post_url(
post_url_data['patreon'], site) or should_break
elif site == "nijie":
log.info("hydownloader-test", "Testing nijie.info...")
should_break = not check_results_of_post_url(
post_url_data['nijie'], site) or should_break
elif site == "sankaku":
log.info("hydownloader-test", "Testing sankaku...")
should_break = not check_results_of_post_url(
post_url_data['sankaku'], site) or should_break
elif site == "idolcomplex":
log.info("hydownloader-test", "Testing idolcomplex...")
should_break = not check_results_of_post_url(
post_url_data['idolcomplex'], site) or should_break
elif site == "artstation":
log.info("hydownloader-test", "Testing artstation...")
should_break = not check_results_of_post_url(
post_url_data['artstation'], site) or should_break
elif site == "twitter":
log.info("hydownloader-test", "Testing twitter...")
should_break = not check_results_of_post_url(
post_url_data['twitter'], site) or should_break
elif site == "deviantart":
log.info("hydownloader-test", "Testing deviantart...")
should_break = not check_results_of_post_url(
post_url_data['deviantart'], site) or should_break
elif site == "webtoons":
log.info("hydownloader-test", "Testing webtoons...")
should_break = not check_results_of_post_url(
post_url_data['webtoons'], site) or should_break
elif site == "baraag":
log.info("hydownloader-test", "Testing baraag...")
should_break = not check_results_of_post_url(
post_url_data['baraag'], site) or should_break
elif site == "hentaifoundry":
log.info("hydownloader-test", "Testing hentaifoundry...")
should_break = not check_results_of_post_url(
post_url_data['hentaifoundry'], site) or should_break
elif site == "yandere":
log.info("hydownloader-test", "Testing yande.re...")
should_break = not check_results_of_post_url(
post_url_data['yandere'], site) or should_break
else:
log.error("hydownloader-test",
f"Site name not recognized: {site}, no testing done")
return False
if should_break:
log.error(
"hydownloader-test",
f"Stopping early due to errors while testing {site}, test environment kept for inspection"
)
return False
clear_test_env()
return True
try:
with open(db.get_rootpath() + "/" + logfname,
'r',
encoding='utf-8-sig') as logf:
log.info("hydownloader", f"Parsing log file: {logfname}")
urls = []
for line in logf:
if m := re.match(
r'(?:\[.+\])* (http.*?)(?::[0-9]+)? "[A-Z]+ (\/.*?) HTTP.*',
line.strip()):
urls.append(m.group(1) + m.group(2))
if m := re.match(r".*Starting DownloadJob for '(.*)'$",
line.strip()):
urls.append(m.group(1))
db.add_known_urls(urls,
subscription_id=subscription_id,
url_id=url_id)
db.remove_log_file_from_parse_queue(db.get_rootpath() + "/" +
logfname)
log.info(
"hydownloader",
f"Finished parsing log file {logfname}, found {len(urls)} URLs"
)
except FileNotFoundError as e:
log.error(
"hydownloader",
f"Log file was in the parse queue, but was not found on the filesystem: {db.get_rootpath()+'/'+logfname}"
)
db.remove_log_file_from_parse_queue(db.get_rootpath() + "/" +
logfname)
def subscription_worker() -> None:
global _sub_worker_ended_flag
proc_id = 'sub worker'
try:
log.info("hydownloader", "Starting subscription worker thread...")
with _worker_lock:
_sub_worker_ended_flag = False
while True:
time.sleep(2)
with _worker_lock:
if _end_threads_flag:
break
subs_due = db.get_due_subscriptions()
if not subs_due:
with _worker_lock:
if _sub_worker_paused_flag:
set_subscription_worker_status("paused")
else:
set_subscription_worker_status("nothing to do: checked for due subscriptions, found none")
sub = subs_due[0] if subs_due else None
while sub:
with _worker_lock:
if _end_threads_flag:
break
if _sub_worker_paused_flag:
set_subscription_worker_status("paused")
break
initial_check = sub['last_check'] is None
url = urls.subscription_data_to_url(sub['downloader'], sub['keywords'])
check_started_time = time.time()
status_msg = f"checking subscription: {sub['id']} (downloader: {sub['downloader']}, keywords: {sub['keywords']})"
set_subscription_worker_status(status_msg)
log.info(f"subscription-{sub['id']}", status_msg.capitalize())
if initial_check:
log.info(f"subscription-{sub['id']}", "This is the first check for this subscription")
result = gallery_dl_utils.run_gallery_dl(
url=url,
ignore_anchor=False,
metadata_only=False,
log_file=db.get_rootpath()+f"/logs/subscription-{sub['id']}-gallery-dl-latest.txt",
old_log_file=db.get_rootpath()+f"/logs/subscription-{sub['id']}-gallery-dl-old.txt",
console_output_file=db.get_rootpath()+f"/temp/subscription-{sub['id']}-gallery-dl-output.txt",
unsupported_urls_file=db.get_rootpath()+f"/logs/subscription-{sub['id']}-unsupported-urls-gallery-dl-latest.txt",
old_unsupported_urls_file=db.get_rootpath()+f"/logs/subscription-{sub['id']}-unsupported-urls-gallery-dl-old.txt",
overwrite_existing=False,
filter_=sub['filter'],
chapter_filter=None,
subscription_mode=True,
abort_after=sub['abort_after'],
max_file_count = sub['max_files_initial'] if initial_check else sub['max_files_regular'],
process_id = proc_id
)
new_sub_data = {
'id': sub['id']
}
if result:
log.warning(f"subscription-{sub['id']}", "Error: "+result)
else:
new_sub_data['last_successful_check'] = check_started_time
new_sub_data['last_check'] = check_started_time
new_files, skipped_files = output_postprocessors.process_additional_data(subscription_id = sub['id'])
output_postprocessors.parse_log_files(False, proc_id)
check_ended_time = time.time()
db.add_subscription_check(sub['id'], new_files=new_files, already_seen_files=skipped_files, time_started=check_started_time, time_finished=check_ended_time, status=result if result else 'ok')
db.add_or_update_subscriptions([new_sub_data])
status_msg = f"finished checking subscription: {sub['id']} (downloader: {sub['downloader']}, keywords: {sub['keywords']}), new files: {new_files}, skipped: {skipped_files}"
set_subscription_worker_status(status_msg)
log.info(f"subscription-{sub['id']}", status_msg.capitalize())
subs_due = db.get_due_subscriptions()
sub = subs_due[0] if subs_due else None
with _worker_lock:
if _end_threads_flag:
break
with _worker_lock:
if _end_threads_flag:
log.info("hydownloader", "Stopping subscription worker thread")
_sub_worker_ended_flag = True
db.close_thread_connections()
set_subscription_worker_status('shut down')
except Exception as e:
log.error("hydownloader", "Uncaught exception in subscription worker thread", e)
with _worker_lock:
_sub_worker_ended_flag = True
db.close_thread_connections()
shutdown()
def url_queue_worker() -> None:
global _url_worker_ended_flag
proc_id = 'url worker'
try:
log.info("hydownloader", "Starting single URL queue worker thread...")
with _worker_lock:
_url_worker_ended_flag = False
while True:
time.sleep(2)
with _worker_lock:
if _end_threads_flag:
break
urls_to_dl = db.get_urls_to_download()
if not urls_to_dl:
with _worker_lock:
if _url_worker_paused_flag:
set_url_worker_status("paused")
else:
set_url_worker_status("nothing to do: checked for queued URLs, found none")
urlinfo = urls_to_dl[0] if urls_to_dl else None
while urlinfo:
with _worker_lock:
if _end_threads_flag:
break
if _url_worker_paused_flag:
set_url_worker_status("paused")
break
check_time = time.time()
status_msg = f"downloading URL: {urlinfo['url']}"
set_url_worker_status(status_msg)
log.info("single url downloader", status_msg.capitalize())
result = gallery_dl_utils.run_gallery_dl(
url=urlinfo['url'],
ignore_anchor=urlinfo['ignore_anchor'],
metadata_only=urlinfo['metadata_only'],
log_file=db.get_rootpath()+f"/logs/single-urls-{urlinfo['id']}-gallery-dl-latest.txt",
old_log_file=db.get_rootpath()+f"/logs/single-urls-{urlinfo['id']}-gallery-dl-old.txt",
console_output_file=db.get_rootpath()+f"/temp/single-url-{urlinfo['id']}-gallery-dl-output.txt",
unsupported_urls_file=db.get_rootpath()+f"/logs/single-urls-{urlinfo['id']}-unsupported-urls-gallery-dl-latest.txt",
old_unsupported_urls_file=db.get_rootpath()+f"/logs/single-urls-{urlinfo['id']}-unsupported-urls-gallery-dl-old.txt",
overwrite_existing=urlinfo['overwrite_existing'],
filter_=urlinfo['filter'],
chapter_filter=None,
subscription_mode=False,
max_file_count = urlinfo['max_files'],
process_id = proc_id
)
new_url_data = {
'id': urlinfo['id']
}
if result:
log.warning("single url downloader", f"Error while downloading {urlinfo['url']}: {result}")
new_url_data['status'] = 1
new_url_data['status_text'] = result
else:
new_url_data['status'] = 0
new_url_data['status_text'] = 'ok'
new_url_data['time_processed'] = check_time
new_files, skipped_files = output_postprocessors.process_additional_data(url_id = urlinfo['id'])
output_postprocessors.parse_log_files(False, proc_id)
new_url_data['new_files'] = new_files
new_url_data['already_seen_files'] = skipped_files
db.add_or_update_urls([new_url_data])
status_msg = f"finished checking URL: {urlinfo['url']}, new files: {new_files}, skipped: {skipped_files}"
set_url_worker_status(status_msg)
log.info("single url downloader", status_msg.capitalize())
urls_to_dl = db.get_urls_to_download()
urlinfo = urls_to_dl[0] if urls_to_dl else None
with _worker_lock:
if _end_threads_flag:
break
with _worker_lock:
if _end_threads_flag:
log.info("hydownloader", "Stopping single URL queue worker thread")
_url_worker_ended_flag = True
db.close_thread_connections()
set_url_worker_status('shut down')
except Exception as e:
log.error("hydownloader", "Uncaught exception in URL worker thread", e)
with _worker_lock:
_url_worker_ended_flag = True
db.close_thread_connections()
shutdown()