def thread_main(): thread_name = threading.current_thread().name logging.info('%s: start', thread_name) while True: message = cc.statics.message_queue.get() logging.trace('%s: %s', thread_name, message) try: message.process() except Exception: logging.exception('%s: %s', thread_name, message) finally: cc.statics.message_queue.task_done() logging.error('%s: exit (impossible!)', thread_name)
def _downloader_failed(episode_url): logging.error('downloader: error: episode.url=%s', episode_url)
def _download(url, file_name, cwd, prog, download_timeout, monitor_period, cpu_bound, memory_bound, partial_okay): cwd = cwd or os.getcwd() file_name_part = file_name + '.part' output_path = os.path.join(cwd, file_name) output_path_part = os.path.join(cwd, file_name_part) digest = None for retry_exp in itertools.count(): timer = threading.Timer(download_timeout, lambda: None) timer.daemon = True proc = _make_subprocess(url, file_name_part, cwd, prog) timer.start() ret = -1 while True: try: ret = proc.wait(timeout=monitor_period) break except psutil.TimeoutExpired: pass cpu_percent = proc.get_cpu_percent(interval=None) memory_percent = proc.get_memory_percent() logging.trace('rtmp: pid=%d cpu=%.1f memory=%.1f', proc.pid, cpu_percent, memory_percent) if cpu_percent > cpu_bound: logging.error('rtmp: cpu limit exceeded') proc.kill() break if memory_percent > memory_bound: logging.error('rtmp: memory limit exceeded') proc.kill() break if timer.finished.is_set(): logging.error('rtmp: timeout: %s -> %s', url, output_path_part) proc.kill() break timer.cancel() if prog == 'rtmpdump' and ret == RTMPDUMP_INCOMPLETE: if partial_okay: logging.warning( 'rtmp: partial download %s to %s', url, file_name) ret = 0 break with open(output_path_part, 'rb') as output_file: new_digest = hashlib.sha1(output_file.read()).digest() if digest is not None and digest == new_digest: # We made no progress; the download might be completed. # Let's not retry and assume it was. logging.warning( 'rtmp: no progress: url=%s file_name=%s', url, file_name) ret = 0 break digest = new_digest # rtmpdump didn't complete the transfer; resume might get further. retry = 2 ** retry_exp if retry > download_timeout: logging.error('rtmp: retry timeout: %s -> %s', url, output_path_part) else: logging.trace('rtmp: retry=%d url=%s', retry, url) time.sleep(retry) continue if ret is not None and ret != 0: raise cc.Error('Could not download (ret=%s): %s' % (ret, url)) # Okay, we are done. break os.rename(output_path_part, output_path) logging.info('rtmp: success: %s -> %s', url, output_path)