def thread_main():
thread_name = threading.current_thread().name
logging.info('%s: start', thread_name)
while True:
message = cc.statics.message_queue.get()
logging.trace('%s: %s', thread_name, message)
try:
message.process()
except Exception:
logging.exception('%s: %s', thread_name, message)
finally:
cc.statics.message_queue.task_done()
logging.error('%s: exit (impossible!)', thread_name)
def _downloader_failed(episode_url):
logging.error('downloader: error: episode.url=%s', episode_url)
def _download(url, file_name, cwd,
prog,
download_timeout,
monitor_period,
cpu_bound,
memory_bound,
partial_okay):
cwd = cwd or os.getcwd()
file_name_part = file_name + '.part'
output_path = os.path.join(cwd, file_name)
output_path_part = os.path.join(cwd, file_name_part)
digest = None
for retry_exp in itertools.count():
timer = threading.Timer(download_timeout, lambda: None)
timer.daemon = True
proc = _make_subprocess(url, file_name_part, cwd, prog)
timer.start()
ret = -1
while True:
try:
ret = proc.wait(timeout=monitor_period)
break
except psutil.TimeoutExpired:
pass
cpu_percent = proc.get_cpu_percent(interval=None)
memory_percent = proc.get_memory_percent()
logging.trace('rtmp: pid=%d cpu=%.1f memory=%.1f',
proc.pid, cpu_percent, memory_percent)
if cpu_percent > cpu_bound:
logging.error('rtmp: cpu limit exceeded')
proc.kill()
break
if memory_percent > memory_bound:
logging.error('rtmp: memory limit exceeded')
proc.kill()
break
if timer.finished.is_set():
logging.error('rtmp: timeout: %s -> %s', url, output_path_part)
proc.kill()
break
timer.cancel()
if prog == 'rtmpdump' and ret == RTMPDUMP_INCOMPLETE:
if partial_okay:
logging.warning(
'rtmp: partial download %s to %s', url, file_name)
ret = 0
break
with open(output_path_part, 'rb') as output_file:
new_digest = hashlib.sha1(output_file.read()).digest()
if digest is not None and digest == new_digest:
# We made no progress; the download might be completed.
# Let's not retry and assume it was.
logging.warning(
'rtmp: no progress: url=%s file_name=%s', url, file_name)
ret = 0
break
digest = new_digest
# rtmpdump didn't complete the transfer; resume might get further.
retry = 2 ** retry_exp
if retry > download_timeout:
logging.error('rtmp: retry timeout: %s -> %s',
url, output_path_part)
else:
logging.trace('rtmp: retry=%d url=%s', retry, url)
time.sleep(retry)
continue
if ret is not None and ret != 0:
raise cc.Error('Could not download (ret=%s): %s' % (ret, url))
# Okay, we are done.
break
os.rename(output_path_part, output_path)
logging.info('rtmp: success: %s -> %s', url, output_path)
