def worker(): while not done: job = queue.get() job.status = Job.RUNNING jobshandler.put((Actions.SET_STATUS, (job.id, job.status))) if job.type == JobType.YDL_DOWNLOAD: output = io.StringIO() # FIXME intialize this ? stdout_thread = Thread(target=download_log_update, args=(job, output)) stdout_thread.start() try: job.log = Job.clean_logs( download(job.url, {'format': job.format}, output, job.id)) job.status = Job.COMPLETED except Exception as e: job.status = Job.FAILED job.log += str(e) print("Exception during download task:\n" + str(e)) stdout_thread.join() elif job.type == JobType.YDL_UPDATE: rc, log = update() job.log = Job.clean_logs(log) job.status = Job.COMPLETED if rc == 0 else Job.FAILED jobshandler.put((Actions.UPDATE, job)) queue.task_done()
def download(job, request_options, output): ydl_opts = get_ydl_options(app_config.get('ydl_options', {}), request_options) cmd = get_ydl_full_cmd(ydl_opts, job.url) cmd.extend(['-J', '--flat-playlist']) proc = Popen(cmd, stdout=PIPE, stderr=PIPE) stdout, stderr = proc.communicate() if proc.wait() != 0: job.log = Job.clean_logs(stderr.decode()) job.status = Job.FAILED print("Error during download task:\n" + job.log) return metadata = json.loads(stdout) jobshandler.put((Actions.SET_NAME, (job.id, metadata.get('title', job.url)))) if metadata.get('_type') == 'playlist': ydl_opts.update({ 'output': app_config['ydl_server'].get('output_playlist', ydl_opts.get('output')) }) cmd = get_ydl_full_cmd(ydl_opts, job.url) proc = Popen(cmd, stdout=PIPE, stderr=STDOUT) stdout_thread = Thread(target=download_log_update, args=(job, proc, output)) stdout_thread.start() if proc.wait() == 0: read_proc_stdout(proc, output) job.log = Job.clean_logs(output.getvalue()) job.status = Job.COMPLETED else: read_proc_stdout(proc, output) job.log = Job.clean_logs(output.getvalue()) job.status = Job.FAILED print("Error during download task:\n" + output.getvalue()) stdout_thread.join()
def download(self, job, request_options, output): ydl_opts = self.get_ydl_options(self.app_config.get("ydl_options", {}), request_options) cmd = self.get_ydl_full_cmd(ydl_opts, job.url) rc, metadata = self.fetch_metadata(job.url) if rc != 0: job.log = Job.clean_logs(metadata) job.status = Job.FAILED raise Exception(job.log) self.jobshandler.put( (Actions.SET_NAME, (job.id, metadata.get("title", job.url)))) if metadata.get("_type") == "playlist": ydl_opts.update({ "output": self.app_config["ydl_server"].get("output_playlist", ydl_opts.get("output")) }) cmd = self.get_ydl_full_cmd(ydl_opts, job.url) proc = Popen(cmd, stdout=PIPE, stderr=STDOUT) stdout_thread = Thread(target=self.download_log_update, args=(job, proc, output)) stdout_thread.start() if proc.wait() == 0: read_proc_stdout(proc, output) job.log = Job.clean_logs(output.getvalue()) job.status = Job.COMPLETED else: read_proc_stdout(proc, output) job.log = Job.clean_logs(output.getvalue()) job.status = Job.FAILED print("Error during download task:\n" + output.getvalue()) stdout_thread.join()
def worker(): while not done: job = queue.get() job.status = Job.RUNNING jobshandler.put((Actions.UPDATE, job)) output = io.StringIO() # FIXME intialize this ? stdout_thread = Thread(target=download_log_update, args=(job, output)) stdout_thread.start() try: job.log = Job.clean_logs( download(job.name, {'format': job.format}, output), ) job.status = Job.COMPLETED except Exception as e: job.status = Job.FAILED job.log += str(e) print("Exception during download task:\n" + str(e)) stdout_thread.join() jobshandler.put((Actions.UPDATE, job)) queue.task_done()
def worker(): while not done: job = queue.get() job.status = Job.RUNNING jobshandler.put((Actions.SET_STATUS, (job.id, job.status))) if job.type == JobType.YDL_DOWNLOAD: output = io.StringIO() try: download(job, {'format': job.format}, output) except Exception as e: job.status = Job.FAILED job.log = "Error during download task" print("Error during download task:\n{}\n{}".format( type(e).__name__, str(e))) elif job.type == JobType.YDL_UPDATE: rc, log = update() job.log = Job.clean_logs(log) job.status = Job.COMPLETED if rc == 0 else Job.FAILED jobshandler.put((Actions.UPDATE, job)) queue.task_done()
def download_log_update(job, proc, strio): while job.status == Job.RUNNING: read_proc_stdout(proc, strio) job.log = Job.clean_logs(strio.getvalue()) jobshandler.put((Actions.SET_LOG, (job.id, job.log))) sleep(3)
def download_log_update(job, stringio): while job.status == Job.RUNNING: job.log = Job.clean_logs(stringio.getvalue()) jobshandler.put((Actions.SET_LOG, (job.id, job.log))) sleep(5)