def worker():
while not done:
job = queue.get()
job.status = Job.RUNNING
jobshandler.put((Actions.SET_STATUS, (job.id, job.status)))
if job.type == JobType.YDL_DOWNLOAD:
output = io.StringIO() # FIXME intialize this ?
stdout_thread = Thread(target=download_log_update,
args=(job, output))
stdout_thread.start()
try:
job.log = Job.clean_logs(
download(job.url, {'format': job.format}, output, job.id))
job.status = Job.COMPLETED
except Exception as e:
job.status = Job.FAILED
job.log += str(e)
print("Exception during download task:\n" + str(e))
stdout_thread.join()
elif job.type == JobType.YDL_UPDATE:
rc, log = update()
job.log = Job.clean_logs(log)
job.status = Job.COMPLETED if rc == 0 else Job.FAILED
jobshandler.put((Actions.UPDATE, job))
queue.task_done()
def download(job, request_options, output):
ydl_opts = get_ydl_options(app_config.get('ydl_options', {}),
request_options)
cmd = get_ydl_full_cmd(ydl_opts, job.url)
cmd.extend(['-J', '--flat-playlist'])
proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
if proc.wait() != 0:
job.log = Job.clean_logs(stderr.decode())
job.status = Job.FAILED
print("Error during download task:\n" + job.log)
return
metadata = json.loads(stdout)
jobshandler.put((Actions.SET_NAME, (job.id, metadata.get('title',
job.url))))
if metadata.get('_type') == 'playlist':
ydl_opts.update({
'output':
app_config['ydl_server'].get('output_playlist',
ydl_opts.get('output'))
})
cmd = get_ydl_full_cmd(ydl_opts, job.url)
proc = Popen(cmd, stdout=PIPE, stderr=STDOUT)
stdout_thread = Thread(target=download_log_update,
args=(job, proc, output))
stdout_thread.start()
if proc.wait() == 0:
read_proc_stdout(proc, output)
job.log = Job.clean_logs(output.getvalue())
job.status = Job.COMPLETED
else:
read_proc_stdout(proc, output)
job.log = Job.clean_logs(output.getvalue())
job.status = Job.FAILED
print("Error during download task:\n" + output.getvalue())
stdout_thread.join()
def download(self, job, request_options, output):
ydl_opts = self.get_ydl_options(self.app_config.get("ydl_options", {}),
request_options)
cmd = self.get_ydl_full_cmd(ydl_opts, job.url)
rc, metadata = self.fetch_metadata(job.url)
if rc != 0:
job.log = Job.clean_logs(metadata)
job.status = Job.FAILED
raise Exception(job.log)
self.jobshandler.put(
(Actions.SET_NAME, (job.id, metadata.get("title", job.url))))
if metadata.get("_type") == "playlist":
ydl_opts.update({
"output":
self.app_config["ydl_server"].get("output_playlist",
ydl_opts.get("output"))
})
cmd = self.get_ydl_full_cmd(ydl_opts, job.url)
proc = Popen(cmd, stdout=PIPE, stderr=STDOUT)
stdout_thread = Thread(target=self.download_log_update,
args=(job, proc, output))
stdout_thread.start()
if proc.wait() == 0:
read_proc_stdout(proc, output)
job.log = Job.clean_logs(output.getvalue())
job.status = Job.COMPLETED
else:
read_proc_stdout(proc, output)
job.log = Job.clean_logs(output.getvalue())
job.status = Job.FAILED
print("Error during download task:\n" + output.getvalue())
stdout_thread.join()
def worker():
while not done:
job = queue.get()
job.status = Job.RUNNING
jobshandler.put((Actions.UPDATE, job))
output = io.StringIO() # FIXME intialize this ?
stdout_thread = Thread(target=download_log_update, args=(job, output))
stdout_thread.start()
try:
job.log = Job.clean_logs(
download(job.name, {'format': job.format}, output), )
job.status = Job.COMPLETED
except Exception as e:
job.status = Job.FAILED
job.log += str(e)
print("Exception during download task:\n" + str(e))
stdout_thread.join()
jobshandler.put((Actions.UPDATE, job))
queue.task_done()
def worker():
while not done:
job = queue.get()
job.status = Job.RUNNING
jobshandler.put((Actions.SET_STATUS, (job.id, job.status)))
if job.type == JobType.YDL_DOWNLOAD:
output = io.StringIO()
try:
download(job, {'format': job.format}, output)
except Exception as e:
job.status = Job.FAILED
job.log = "Error during download task"
print("Error during download task:\n{}\n{}".format(
type(e).__name__, str(e)))
elif job.type == JobType.YDL_UPDATE:
rc, log = update()
job.log = Job.clean_logs(log)
job.status = Job.COMPLETED if rc == 0 else Job.FAILED
jobshandler.put((Actions.UPDATE, job))
queue.task_done()
def download_log_update(job, proc, strio):
while job.status == Job.RUNNING:
read_proc_stdout(proc, strio)
job.log = Job.clean_logs(strio.getvalue())
jobshandler.put((Actions.SET_LOG, (job.id, job.log)))
sleep(3)
def download_log_update(job, stringio):
while job.status == Job.RUNNING:
job.log = Job.clean_logs(stringio.getvalue())
jobshandler.put((Actions.SET_LOG, (job.id, job.log)))
sleep(5)