def _get_url(url):
logging.debug('get_url: url=%s', url)
response = requests.get(url, timeout=60)
if logging.is_enabled_for(logging.TRACE):
for header, value in response.headers.items():
logging.trace('get_url: %s: %s', header, value)
response.raise_for_status()
return response
def _get_manifest_feed_property(manifest_feed_doc, property_names):
'''Extract triforceManifestFeed from doc and read the nested property.'''
match = _PATTERN_MANIFEST_FEED.search(manifest_feed_doc)
if not match:
raise cc.Error('Could not find manifest feed')
manifest_feed = json.loads(match.group(1))
logging.trace('triforceManifestFeed=\n%s',
cc.pformat.PrettyFormatter(manifest_feed))
blob = manifest_feed
for property_name in property_names:
blob = blob[property_name]
return blob
def thread_main():
thread_name = threading.current_thread().name
logging.info('%s: start', thread_name)
while True:
message = cc.statics.message_queue.get()
logging.trace('%s: %s', thread_name, message)
try:
message.process()
except Exception:
logging.exception('%s: %s', thread_name, message)
finally:
cc.statics.message_queue.task_done()
logging.error('%s: exit (impossible!)', thread_name)
def _downloader(episode, output_dir_path, simulate, salvage_dirs):
logging.info('downloader: episode .date=%s .url=%s',
episode.date, episode.url)
logging.trace('downloader: episode...'
'\n url=%s'
'\n dir_name=%s'
'\n date=%s'
'\n videos=\n%s',
episode.url, episode.dir_name, episode.date,
cc.pformat.PrettyFormatter(episode.videos))
# Check/make paths.
dir_path = os.path.join(output_dir_path, episode.dir_name)
if os.path.exists(dir_path):
logging.info('downloader: skip: dir_path=%s', dir_path)
return
salvage = cc.salvage.Salvage(episode.dir_name, salvage_dirs)
if simulate:
tmp_dir_path = os.path.join(
output_dir_path, 'tmpXXXXXXXX-' + episode.dir_name)
else:
tmp_dir_path = tempfile.mkdtemp(
suffix='-'+episode.dir_name, dir=output_dir_path)
logging.debug('downloader: tmp_dir_path=%s', tmp_dir_path)
# Construct actors.
counter = cc.actor.counter.Counter(
functools.partial(_downloader_success,
episode.url,
tmp_dir_path,
dir_path,
simulate),
functools.partial(_downloader_failed,
episode.url))
dlers = _make_dlers(episode, tmp_dir_path, counter, simulate, salvage)
counter.count = len(dlers)
# Start actors.
for dler in dlers:
dler()
def feed(self):
if self._feed is None:
self._feed = cc.http.get_url_json(self.url)
logging.trace('feed.feed=\n%s',
cc.pformat.PrettyFormatter(self._feed))
return self._feed
def _download(url, file_name, cwd,
prog,
download_timeout,
monitor_period,
cpu_bound,
memory_bound,
partial_okay):
cwd = cwd or os.getcwd()
file_name_part = file_name + '.part'
output_path = os.path.join(cwd, file_name)
output_path_part = os.path.join(cwd, file_name_part)
digest = None
for retry_exp in itertools.count():
timer = threading.Timer(download_timeout, lambda: None)
timer.daemon = True
proc = _make_subprocess(url, file_name_part, cwd, prog)
timer.start()
ret = -1
while True:
try:
ret = proc.wait(timeout=monitor_period)
break
except psutil.TimeoutExpired:
pass
cpu_percent = proc.get_cpu_percent(interval=None)
memory_percent = proc.get_memory_percent()
logging.trace('rtmp: pid=%d cpu=%.1f memory=%.1f',
proc.pid, cpu_percent, memory_percent)
if cpu_percent > cpu_bound:
logging.error('rtmp: cpu limit exceeded')
proc.kill()
break
if memory_percent > memory_bound:
logging.error('rtmp: memory limit exceeded')
proc.kill()
break
if timer.finished.is_set():
logging.error('rtmp: timeout: %s -> %s', url, output_path_part)
proc.kill()
break
timer.cancel()
if prog == 'rtmpdump' and ret == RTMPDUMP_INCOMPLETE:
if partial_okay:
logging.warning(
'rtmp: partial download %s to %s', url, file_name)
ret = 0
break
with open(output_path_part, 'rb') as output_file:
new_digest = hashlib.sha1(output_file.read()).digest()
if digest is not None and digest == new_digest:
# We made no progress; the download might be completed.
# Let's not retry and assume it was.
logging.warning(
'rtmp: no progress: url=%s file_name=%s', url, file_name)
ret = 0
break
digest = new_digest
# rtmpdump didn't complete the transfer; resume might get further.
retry = 2 ** retry_exp
if retry > download_timeout:
logging.error('rtmp: retry timeout: %s -> %s',
url, output_path_part)
else:
logging.trace('rtmp: retry=%d url=%s', retry, url)
time.sleep(retry)
continue
if ret is not None and ret != 0:
raise cc.Error('Could not download (ret=%s): %s' % (ret, url))
# Okay, we are done.
break
os.rename(output_path_part, output_path)
logging.info('rtmp: success: %s -> %s', url, output_path)
