def worker():
while not done:
job = queue.get()
job.status = Job.RUNNING
jobshandler.put((Actions.SET_STATUS, (job.id, job.status)))
if job.type == JobType.YDL_DOWNLOAD:
output = io.StringIO() # FIXME intialize this ?
stdout_thread = Thread(target=download_log_update,
args=(job, output))
stdout_thread.start()
try:
job.log = Job.clean_logs(
download(job.url, {'format': job.format}, output, job.id))
job.status = Job.COMPLETED
except Exception as e:
job.status = Job.FAILED
job.log += str(e)
print("Exception during download task:\n" + str(e))
stdout_thread.join()
elif job.type == JobType.YDL_UPDATE:
rc, log = update()
job.log = Job.clean_logs(log)
job.status = Job.COMPLETED if rc == 0 else Job.FAILED
jobshandler.put((Actions.UPDATE, job))
queue.task_done()
def twl_update():
TWL_LOOKBACK_TIME_STRING = request.query.TWL_LOOKBACK_TIME_STRING or None
job = Job("ToWatchList Update", Job.PENDING, "", JobType.TWL_DOWNLOAD,
TWL_LOOKBACK_TIME_STRING, None)
jobshandler.put((Actions.INSERT, job))
return {
"success": True,
"TWL_LOOKBACK_TIME_STRING": TWL_LOOKBACK_TIME_STRING
}
def api_queue_download():
url = request.forms.get("url")
options = {'format': request.forms.get("format")}
if not url:
return {"success": False, "error": "'url' query parameter omitted"}
job = Job(url, Job.PENDING, "", JobType.YDL_DOWNLOAD, request.forms.get("format"), url)
jobshandler.put((Actions.INSERT, job))
print("Added url " + url + " to the download queue")
return {"success": True, "url": url, "options": options}
def resume_pending():
db = JobsDB(readonly=False)
jobs = db.get_all()
not_endeds = [job for job in jobs if job['status'] == "Pending" or job['status'] == 'Running']
for pending in not_endeds:
if int(pending["type"]) == JobType.YDL_UPDATE:
jobshandler.put((Actions.SET_STATUS, (pending["id"], Job.FAILED)))
else:
job = Job(pending["name"], Job.PENDING, "Queue stopped",
int(pending["type"]), pending["format"], pending["url"])
job.id = pending["id"]
jobshandler.put((Actions.RESUME, job))
def resume_pending():
db = JobsDB(readonly=False)
jobs = db.get_all()
not_endeds = [
job for job in jobs
if job['status'] == "Pending" or job['status'] == 'Running'
]
for pending in not_endeds:
job = Job(pending["name"], Job.PENDING, "Queue stopped",
pending["format"])
job.id = pending["id"]
jobshandler.put((Actions.RESUME, job))
def download(url, request_options, output, job_id):
with youtube_dl.YoutubeDL(get_ydl_options(request_options)) as ydl:
ydl.params['extract_flat'] = 'in_playlist'
info = ydl.extract_info(url, download=False)
if 'title' in info and info['title']:
jobshandler.put((Actions.SET_NAME, (job_id, info['title'])))
if '_type' in info and info['_type'] == 'playlist' \
and 'YDL_OUTPUT_TEMPLATE_PLAYLIST' in app_defaults:
ydl.params['outtmpl'] = app_defaults[
'YDL_OUTPUT_TEMPLATE_PLAYLIST']
ydl.params['extract_flat'] = False
# Swap out sys.stdout as ydl's output so we can capture it
ydl._screen_file = output
ydl._err_file = ydl._screen_file
ydl.download([url])
return ydl._screen_file.getvalue()
def api_queue_download():
if (app_config['ydl_server'].get('update_poll_delay_min') and
(datetime.now() - ydlhandler.ydl_last_update).seconds >
app_config['ydl_server'].get('update_poll_delay_min')):
job = Job("Youtube-dl Update", Job.PENDING, "", JobType.YDL_UPDATE, None, None)
jobshandler.put((Actions.INSERT, job))
url = request.forms.get("url")
options = {'format': request.forms.get("format")}
if not url:
return {"success": False, "error": "'url' query parameter omitted"}
job = Job(url, Job.PENDING, "", JobType.YDL_DOWNLOAD, request.forms.get("format"), url)
jobshandler.put((Actions.INSERT, job))
print("Added url " + url + " to the download queue")
return {"success": True, "url": url, "options": options}
def worker():
while not done:
job = queue.get()
job.status = Job.RUNNING
jobshandler.put((Actions.UPDATE, job))
output = io.StringIO() # FIXME intialize this ?
stdout_thread = Thread(target=download_log_update, args=(job, output))
stdout_thread.start()
try:
job.log = Job.clean_logs(
download(job.name, {'format': job.format}, output), )
job.status = Job.COMPLETED
except Exception as e:
job.status = Job.FAILED
job.log += str(e)
print("Exception during download task:\n" + str(e))
stdout_thread.join()
jobshandler.put((Actions.UPDATE, job))
queue.task_done()
def download(job, request_options, output):
ydl_opts = get_ydl_options(app_config.get('ydl_options', {}),
request_options)
cmd = get_ydl_full_cmd(ydl_opts, job.url)
cmd.extend(['-J', '--flat-playlist'])
proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
if proc.wait() != 0:
job.log = Job.clean_logs(stderr.decode())
job.status = Job.FAILED
print("Error during download task:\n" + job.log)
return
metadata = json.loads(stdout)
jobshandler.put((Actions.SET_NAME, (job.id, metadata.get('title',
job.url))))
if metadata.get('_type') == 'playlist':
ydl_opts.update({
'output':
app_config['ydl_server'].get('output_playlist',
ydl_opts.get('output'))
})
cmd = get_ydl_full_cmd(ydl_opts, job.url)
proc = Popen(cmd, stdout=PIPE, stderr=STDOUT)
stdout_thread = Thread(target=download_log_update,
args=(job, proc, output))
stdout_thread.start()
if proc.wait() == 0:
read_proc_stdout(proc, output)
job.log = Job.clean_logs(output.getvalue())
job.status = Job.COMPLETED
else:
read_proc_stdout(proc, output)
job.log = Job.clean_logs(output.getvalue())
job.status = Job.FAILED
print("Error during download task:\n" + output.getvalue())
stdout_thread.join()
def worker():
while not done:
job = queue.get()
job.status = Job.RUNNING
jobshandler.put((Actions.SET_STATUS, (job.id, job.status)))
if job.type == JobType.YDL_DOWNLOAD:
output = io.StringIO()
try:
download(job, {'format': job.format}, output)
except Exception as e:
job.status = Job.FAILED
job.log = "Error during download task"
print("Error during download task:\n{}\n{}".format(
type(e).__name__, str(e)))
elif job.type == JobType.YDL_UPDATE:
rc, log = update()
job.log = Job.clean_logs(log)
job.status = Job.COMPLETED if rc == 0 else Job.FAILED
jobshandler.put((Actions.UPDATE, job))
queue.task_done()
def download_log_update(job, proc, strio):
while job.status == Job.RUNNING:
read_proc_stdout(proc, strio)
job.log = Job.clean_logs(strio.getvalue())
jobshandler.put((Actions.SET_LOG, (job.id, job.log)))
sleep(3)
def download(url, request_options, output, job_id):
with yt_dlp.YoutubeDL(get_ydl_options(request_options)) as ydl:
ydl.params['extract_flat'] = 'in_playlist'
ydl_opts = ChainMap(os.environ, app_defaults)
info = ydl.extract_info(url, download=False)
if 'title' in info and info['title']:
jobshandler.put((Actions.SET_NAME, (job_id, info['title'])))
if '_type' in info and info['_type'] == 'playlist' \
and 'YDL_OUTPUT_TEMPLATE_PLAYLIST' in ydl_opts:
ydl.params['outtmpl'] = ydl_opts['YDL_OUTPUT_TEMPLATE_PLAYLIST']
ydl.params['extract_flat'] = False
# 'YDL_OUTPUT_TEMPLATE': '/youtube-dl/%(title)s [%(id)s].%(ext)s',
# 'YDL_OUTPUT_TEMPLATE_PLAYLIST': '/youtube-dl/%(playlist_title)s/%(title)s [%(id)s].%(ext)s',
if 'YDL_WRITE_NFO' in ydl_opts and ydl_opts['YDL_WRITE_NFO']:
# write NFO file
vidpath = Path(ydl.prepare_filename(info))
nfopath = os.path.join(vidpath.parent, f"{vidpath.stem}.nfo")
if not os.path.isfile(nfopath):
if 'upload_date' in info:
# info['upload_date'] is usually a YYYYMMDD eg 20200906
year = str(info['upload_date'])[:4]
month = str(info['upload_date'])[4:6]
day = str(info['upload_date'])[6:]
with open(nfopath, "w") as nfoF:
# json.dump(info, nfoF)
# nfoF.write(f"{info['title']}\n")
# nfoF.write(f"{ydl.prepare_filename(info)}.nfo\n")
nfoF.write("<musicvideo>\n")
if 'title' in info and info['title']:
nfoF.write(f" <title>{info['title']}</title>\n")
else:
nfoF.write(" <title>Unknown Title</title>\n")
if 'uploader_id' in info and info['uploader_id']:
nfoF.write(
f" <showtitle>{info['uploader']}</showtitle>\n"
)
else:
nfoF.write(
" <showtitle>Unknown Channel</showtitle>\n")
if 'description' in info and info['description']:
nfoF.write(
f" <plot>{info['description']}\n\nUpload Date: {info['upload_date']}</plot>\n"
)
else:
nfoF.write(
f" <plot>Upload Date: {info['upload_date']}</plot>\n"
)
nfoF.write(
f" <runtime>{round(info['duration']/60.0)}</runtime>\n"
)
# split the thumbnail URL and get the filename extension, may be jpg or webp
# nfoF.write(f" <thumb>{vidpath.stem}.{info['thumbnail'].split('.')[-1]}</thumb>\n")
# alternately just link to the original URL, doesn't work with Jellyfin, does work with Kodi
nfoF.write(
f" <thumb aspect=\"thumb\">{info['thumbnail']}</thumb>\n"
)
nfoF.write(f" <videourl>{url}</videourl>\n")
nfoF.write(f" <aired>{year}-{month}-{day}</aired>\n")
nfoF.write("</musicvideo>\n")
else: # no upload date - this could be a playlist instead of a video
pass
# Swap out sys.stdout as ydl's output so we can capture it
ydl._screen_file = output
ydl._err_file = ydl._screen_file
ydl.download([url])
return ydl._screen_file.getvalue()
return ydlhandler.fetch_metadata(url)
@app.route("/api/youtube-dl/update", method="GET")
def ydl_update():
job = Job("Youtube-dl Update", Job.PENDING, "", JobType.YDL_UPDATE, None, None)
jobshandler.put((Actions.INSERT, job))
return {"success": True}
JobsDB.check_db_latest()
JobsDB.init_db()
ydlhandler.start()
print("Started download thread")
jobshandler.start(ydlhandler.queue)
print("Started jobs manager thread")
print("Updating youtube-dl to the newest version")
job = Job("Youtube-dl Update", Job.PENDING, "", JobType.YDL_UPDATE, None, None)
jobshandler.put((Actions.INSERT, job))
ydlhandler.resume_pending()
app.run(host=app_config['ydl_server'].get('host'),
port=app_config['ydl_server'].get('port'),
debug=app_config['ydl_server'].get('debug', False))
ydlhandler.finish()
jobshandler.finish()
ydlhandler.join()
jobshandler.join()
def ydl_update():
job = Job("Youtube-dl Update", Job.PENDING, "", JobType.YDL_UPDATE, None, None)
jobshandler.put((Actions.INSERT, job))
return {"success": True}
def download_log_update(job, stringio):
while job.status == Job.RUNNING:
job.log = Job.clean_logs(stringio.getvalue())
jobshandler.put((Actions.SET_LOG, (job.id, job.log)))
sleep(5)
def api_logs_purge():
jobshandler.put((Actions.PURGE_LOGS, None))
return {"success": True}
def twldownload(url, request_options, output, job_id):
TWL_API_TOKEN = os.getenv("TWL_API_TOKEN", default="unset").strip()
assert TWL_API_TOKEN != "unset", "ERROR: TWL_API_TOKEN should be set in env (and is not)"
ydl_opts = ChainMap(os.environ, app_defaults)
lookbackStr = ydl_opts['TWL_LOOKBACK_TIME_STRING']
if request_options and 'format' in request_options and request_options[
'format']:
# use 'format' as 'TWL_LOOKBACK_TIME_STRING' here
lookbackStr = request_options['format']
r = httpx.get(
f"https://towatchlist.com/api/v1/marks?since={lookbackStr}&uid={TWL_API_TOKEN}"
)
r.raise_for_status()
myMarks = r.json()['marks']
output_dir = Path(ydl_opts['YDL_OUTPUT_TEMPLATE']).parent
with open(os.path.join(output_dir, '.twl.json'), 'w') as filehandle:
json.dump(myMarks, filehandle)
downloadQueueAdd = 0
removedFiles = 0
if 'YDL_WRITE_NFO' in ydl_opts and ydl_opts['YDL_WRITE_NFO']:
targetNumberOfFiles = 2
else:
targetNumberOfFiles = 1
for i in range(len(myMarks)):
# set some values we'll use below
mmeta = {} # mark metadata dict
mmeta['videoURL'] = myMarks[i]['Mark']['source_url']
mmeta['title'] = myMarks[i]['Mark']['title']
mmeta['video_id'] = myMarks[i]['Mark']['video_id']
mmeta['channel_title'] = myMarks[i]['Mark']['channel_title']
mmeta['duration'] = int(myMarks[i]['Mark']['duration']) / 60.0
mmeta['created'] = myMarks[i]['Mark']['created']
existingFiles = listFilesFromID(mmeta['video_id'],
output_dir=output_dir)
if (myMarks[i]['Mark']['watched']) or (myMarks[i]['Mark']['delflag']):
# it's been marked as watched, delete the local copy
for filename in existingFiles:
os.remove(filename)
removedFiles += 1
continue
if len(existingFiles) >= targetNumberOfFiles:
# this file has probably already been downloaded, skip!
continue
try: # a bit more parsing for Kodi
mmeta['description'] = strip_tags(myMarks[i]['Mark']['comment'])
except:
mmeta['description'] = '-Failed to parse-'
downloadQueueAdd += 1
job = Job(mmeta['title'], Job.PENDING, "", JobType.YDL_DOWNLOAD,
ydl_opts['YDL_FORMAT'], mmeta['videoURL'])
jobshandler.put((Actions.INSERT, job))
if removedFiles > 0:
# TODO: clean Kodi library
pass
return f"Processed {len(myMarks)} Marks, Queued {downloadQueueAdd}, Removed {removedFiles} vids/nfos"