def run(self, info): metadata = self._extract_metadata(info) if not metadata: self._downloader.to_screen( '[ffmpeg] There isn\'t any metadata to add') return [], info filename = info['filepath'] temp_filename = prepend_extension(filename, 'temp') if info['ext'] == 'm4a': options = ['-vn', '-acodec', 'copy'] else: options = ['-c', 'copy'] for (name, value) in metadata.items(): options.extend(['-metadata', '%s=%s' % (name, value)]) self._downloader.to_screen('[ffmpeg] Adding metadata to \'%s\'' % filename) self.run_ffmpeg(filename, temp_filename, options) os.remove(encodeFilename(filename)) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) return [], info
def get_audio_codec(self, path): if not self.probe_available: raise PostProcessingError( 'ffprobe or avprobe not found. Please install one.') try: cmd = [ encodeFilename(self.probe_executable, True), encodeArgument('-show_streams'), encodeFilename(self._ffmpeg_filename_argument(path), True) ] # if self._downloader.params.get('verbose', False): self.logger.debug('%s command line: %s' % (self.basename, shell_quote(cmd))) handle = subprocess.Popen(cmd, stderr=compat_subprocess_get_DEVNULL(), stdout=subprocess.PIPE, stdin=subprocess.PIPE) output = handle.communicate()[0] if handle.wait() != 0: return None except (IOError, OSError): return None audio_codec = None for line in output.decode('ascii', 'ignore').split('\n'): if line.startswith('codec_name='): audio_codec = line.split('=')[1].strip() elif line.strip( ) == 'codec_type=audio' and audio_codec is not None: return audio_codec return None
def run_ffmpeg_multiple_files(self, input_paths, out_path, opts): self.check_version() oldest_mtime = min( os.stat(encodeFilename(path)).st_mtime for path in input_paths) # opts += self._configuration_args() -- postprocessor_args files_cmd = [] for path in input_paths: files_cmd.extend([ encodeArgument('-i'), encodeFilename(self._ffmpeg_filename_argument(path), True) ]) cmd = ( [encodeFilename(self.executable, True), encodeArgument('-y')] + files_cmd + [encodeArgument(o) for o in opts] + [encodeFilename(self._ffmpeg_filename_argument(out_path), True)]) # if self._downloader.params.get('verbose', False): self.logger.debug('[debug] ffmpeg command line: %s' % shell_quote(cmd)) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) stdout, stderr = p.communicate() if p.returncode != 0: stderr = stderr.decode('utf-8', 'replace') msg = stderr.strip().split('\n')[-1] raise SaneFFmpegPostProcessorError(msg) self.try_utime(out_path, oldest_mtime, oldest_mtime)
def try_rename(self, old_filename, new_filename): try: if old_filename == new_filename: return os.rename(encodeFilename(old_filename), encodeFilename(new_filename)) except (IOError, OSError): self.report_error(u'unable to rename file')
def run(self, info): self.logger = create_logger(__name__) filename = info['filepath'] temp_filename = prepend_extension(filename, 'sanetemp') remux = ['-c', 'copy', '-map', '0:v:0', '-map', '1:a:0'] args = [remux] if info.get('audio_codec') is not None: encode_audio = ['-c', 'copy', '-map', '0:v:0', '-map', '1:a:0', '-c:1:a:0', info.get('audio_codec')] args.append(encode_audio) if info.get('video_codec') is not None: encode_video = ['-c', 'copy', '-c:0:v:0', info.get('video_codec'), '-map', '0:v:0', '-map', '1:a:0', '-c:1:a:0'] args.append(encode_video) if info.get('video_codec') is not None and info.get('audio_codec') is not None: encode_both = ['-c', 'copy', '-c:0:v:0', info.get('video_codec'), '-map', '0:v:0', '-map', '1:a:0', '-c:1:a:0', info.get('audio_codec')] args.append(encode_both) if info.get('no_remux') is not None: args.pop(0) self.logger.info('[ffmpeg] Merging formats into "%s"' % filename) self.attempt_ffmpeg(info, temp_filename, args) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) return info['__files_to_merge'], info
def _write_slides(self, slides_infos: {}, ydl: YoutubeDL): for slide in slides_infos: if os.path.exists(encodeFilename(slide.path)): self.to_screen('Slide %s is already present' % (slide.filename)) else: self.to_screen('Downloading slide %s...' % (slide.filename)) try_num = 1 try: url_f = ydl.urlopen(slide.url) with open(encodeFilename(slide.path), 'wb') as slide_f: shutil.copyfileobj(url_f, slide_f) self.to_screen('Successfully downloaded to: %s' % (slide.path)) except (compat_urllib_error.URLError, compat_http_client.HTTPException, socket.error) as err: self.report_warning( '(Try %s of 3) Unable to download slide "%s": %s' % (try_num, slide.url, error_to_compat_str(err))) if try_num == 3: self.report_warning( 'Slides are essential. Abort! Please try again later!' ) exit(1) try_num += 1
def download(self, params, ep): params['logger'] = FakeLogger() ydl = YoutubeDL(params) downloader = HttpFD(ydl, params) filename = 'testfile.mp4' try_rm(encodeFilename(filename)) self.assertTrue(downloader.real_download(filename, { 'url': 'http://127.0.0.1:%d/%s' % (self.port, ep), })) self.assertEqual(os.path.getsize(encodeFilename(filename)), TEST_SIZE) try_rm(encodeFilename(filename))
def run_ffmpeg_multiple_files(self, input_paths, out_path, opts, opts_before=[]): self.check_version() # sanitize file path out_path = pathvalidate.sanitize_filepath(out_path) oldest_mtime = min( os.stat(encodeFilename(path)).st_mtime for path in input_paths) opts += self._configuration_args() files_cmd = [] for path in input_paths: files_cmd.extend([ encodeArgument('-i'), encodeFilename(self._ffmpeg_filename_argument(path), True) ]) cmd = [ encodeFilename(self.executable, True), encodeArgument('-y'), ] # without -y there is a error callen, if the file exists if self.basename == 'ffmpeg': cmd += [encodeArgument('-loglevel'), encodeArgument('repeat+info')] cmd += ( [encodeArgument(o) for o in opts_before] + files_cmd + [encodeArgument(o) for o in opts] + [encodeFilename(self._ffmpeg_filename_argument(out_path), True)]) if self._downloader.params.get('verbose', False): self._downloader.to_screen('[debug] ffmpeg command line: %s' % shell_quote(cmd)) p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, universal_newlines=True) last_line = '' for line in p.stderr: # line = line.decode('utf-8', 'replace') if line.find('time=') > 0: print('\033[K' + line.replace('\n', '') + '\r', end='') last_line = line print('') std_out, std_err = p.communicate() if p.returncode != 0: msg = last_line.strip().split('\n')[-1] raise FFmpegPostProcessorError(msg) self.try_utime(out_path, oldest_mtime, oldest_mtime)
def run(self, info): filename = info['filepath'] if self.get_audio_codec(filename) == 'aac': temp_filename = prepend_extension(filename, 'temp') options = ['-c', 'copy', '-f', 'mp4', '-bsf:a', 'aac_adtstoasc'] self._downloader.to_screen('[ffmpeg] Fixing malformed AAC bitstream in "%s"' % filename) self.run_ffmpeg(filename, temp_filename, options) os.remove(encodeFilename(filename)) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) return [], info
def run(self, information): if information['ext'] not in ('mp4', 'webm', 'mkv'): self._downloader.to_screen('[ffmpeg] Subtitles can only be embedded in mp4, webm or mkv files') return [], information subtitles = information.get('requested_subtitles') if not subtitles: self._downloader.to_screen('[ffmpeg] There aren\'t any subtitles to embed') return [], information filename = information['filepath'] ext = information['ext'] sub_langs = [] sub_filenames = [] webm_vtt_warn = False for lang, sub_info in subtitles.items(): sub_ext = sub_info['ext'] if ext != 'webm' or ext == 'webm' and sub_ext == 'vtt': sub_langs.append(lang) sub_filenames.append(subtitles_filename(filename, lang, sub_ext)) else: if not webm_vtt_warn and ext == 'webm' and sub_ext != 'vtt': webm_vtt_warn = True self._downloader.to_screen('[ffmpeg] Only WebVTT subtitles can be embedded in webm files') if not sub_langs: return [], information input_files = [filename] + sub_filenames opts = [ '-map', '0', '-c', 'copy', # Don't copy the existing subtitles, we may be running the # postprocessor a second time '-map', '-0:s', ] if information['ext'] == 'mp4': opts += ['-c:s', 'mov_text'] for (i, lang) in enumerate(sub_langs): opts.extend(['-map', '%d:0' % (i + 1)]) lang_code = ISO639Utils.short2long(lang) if lang_code is not None: opts.extend(['-metadata:s:s:%d' % i, 'language=%s' % lang_code]) temp_filename = prepend_extension(filename, 'temp') self._downloader.to_screen('[ffmpeg] Embedding subtitles in \'%s\'' % filename) self.run_ffmpeg_multiple_files(input_files, temp_filename, opts) os.remove(encodeFilename(filename)) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) return sub_filenames, information
def run(self, info): if info.get('container') != 'm4a_dash': return [], info filename = info['filepath'] temp_filename = prepend_extension(filename, 'temp') options = ['-c', 'copy', '-f', 'mp4'] self._downloader.to_screen('[ffmpeg] Correcting container in "%s"' % filename) self.run_ffmpeg(filename, temp_filename, options) os.remove(encodeFilename(filename)) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) return [], info
def _download_with_mplayer(self, filename, url): self.report_destination(filename) tmpfilename = self.temp_name(filename) args = ['mplayer', '-really-quiet', '-vo', 'null', '-vc', 'dummy', '-dumpstream', '-dumpfile', tmpfilename, url] # Check for mplayer first try: subprocess.call(['mplayer', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT) except (OSError, IOError): self.report_error(u'MMS or RTSP download detected but "%s" could not be run' % args[0] ) return False # Download using mplayer. retval = subprocess.call(args) if retval == 0: fsize = os.path.getsize(encodeFilename(tmpfilename)) self.to_screen(u'\r[%s] %s bytes' % (args[0], fsize)) self.try_rename(tmpfilename, filename) self._hook_progress({ 'downloaded_bytes': fsize, 'total_bytes': fsize, 'filename': filename, 'status': 'finished', }) return True else: self.to_stderr(u"\n") self.report_error(u'mplayer exited with code %d' % retval) return False
def run(self, info): stretched_ratio = info.get('stretched_ratio') if stretched_ratio is None or stretched_ratio == 1: return [], info filename = info['filepath'] temp_filename = prepend_extension(filename, 'temp') options = ['-c', 'copy', '-aspect', '%f' % stretched_ratio] self._downloader.to_screen('[ffmpeg] Fixing aspect ratio in "%s"' % filename) self.run_ffmpeg(filename, temp_filename, options) os.remove(encodeFilename(filename)) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) return [], info
def post_process(self, filename, ie_info): """Run all the postprocessors on the given file.""" try: info = dict(ie_info) info['filepath'] = filename pps_chain = [] if ie_info.get('__postprocessors') is not None: pps_chain.extend(ie_info['__postprocessors']) pps_chain.extend(self._pps) for pp in pps_chain: files_to_delete = [] try: files_to_delete, info = pp.run(info) except PostProcessingError as e: self.report_error(e.msg) if files_to_delete and not self.params.get('keepvideo', False): for old_filename in files_to_delete: self.to_screen( 'Deleting original file %s (pass -k to keep)' % old_filename) try: os.remove(encodeFilename(old_filename)) except (IOError, OSError): self.report_warning( 'Unable to remove downloaded original file') except BaseException: print("Problem postprocessing")
def _download_m3u8_with_ffmpeg(self, filename, url): self.report_destination(filename) tmpfilename = self.temp_name(filename) args = ['-y', '-i', url, '-f', 'mp4', '-c', 'copy', '-bsf:a', 'aac_adtstoasc', tmpfilename] for program in ['avconv', 'ffmpeg']: try: subprocess.call([program, '-version'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT) break except (OSError, IOError): pass else: self.report_error(u'm3u8 download detected but ffmpeg or avconv could not be found') cmd = [program] + args retval = subprocess.call(cmd) if retval == 0: fsize = os.path.getsize(encodeFilename(tmpfilename)) self.to_screen(u'\r[%s] %s bytes' % (args[0], fsize)) self.try_rename(tmpfilename, filename) self._hook_progress({ 'downloaded_bytes': fsize, 'total_bytes': fsize, 'filename': filename, 'status': 'finished', }) return True else: self.to_stderr(u"\n") self.report_error(u'ffmpeg exited with code %d' % retval) return False
def run(self, info): metadata = self._extract_metadata(info) if not metadata: self._downloader.to_screen('[ffmpeg] There isn\'t any metadata to add') return [], info filename = info['filepath'] temp_filename = prepend_extension(filename, 'temp') if info['ext'] == 'm4a': options = ['-vn', '-acodec', 'copy'] else: options = ['-c', 'copy'] for (name, value) in metadata.items(): options.extend(['-metadata', '%s=%s' % (name, value)]) self._downloader.to_screen('[ffmpeg] Adding metadata to \'%s\'' % filename) self.run_ffmpeg(filename, temp_filename, options) os.remove(encodeFilename(filename)) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) return [], info
def try_utime(self, filename, last_modified_hdr): """Try to set the last-modified time of the given file.""" if last_modified_hdr is None: return if not os.path.isfile(encodeFilename(filename)): return timestr = last_modified_hdr if timestr is None: return filetime = timeconvert(timestr) if filetime is None: return filetime # Ignore obviously invalid dates if filetime == 0: return try: os.utime(filename, (time.time(), filetime)) except: pass return filetime
def test_shell_quote(self): args = ["ffmpeg", "-i", encodeFilename("ñ€ß'.mp4")] self.assertEqual(shell_quote(args), """ffmpeg -i 'ñ€ß'"'"'.mp4'""")
def test_shell_quote(self): args = ['ffmpeg', '-i', encodeFilename('ñ€ß\'.mp4')] self.assertEqual(shell_quote(args), """ffmpeg -i 'ñ€ß'"'"'.mp4'""")
def run(self, info, custom_map=None): self.logger = create_logger(__name__) metadata = {} def add(meta_list, info_list=None): if not info_list: info_list = meta_list if not isinstance(meta_list, (list, tuple)): meta_list = (meta_list, ) if not isinstance(info_list, (list, tuple)): info_list = (info_list, ) for info_f in info_list: if info.get(info_f) is not None: for meta_f in meta_list: metadata[meta_f] = info[info_f] break # if custom_map is None: # add('title', ('track', 'title')) # add('date', 'upload_date') # add(('description', 'comment'), 'description') # add('purl', 'webpage_url') # add('track', 'track_number') # add('artist', ('artist', 'creator', 'uploader', 'uploader_id')) # add('genre') # add('album') # add('album_artist') # add('disc', 'disc_number') # else: for entry in info: if entry not in info['not_a_tag']: add(entry) if not metadata: self.logger.warning('[ffmpeg] There isn\'t any metadata to add') return [], info filename = info['filepath'] temp_filename = prepend_extension(filename, 'temp') in_filenames = [filename] options = [] if info['ext'] == 'm4a': options.extend(['-vn', '-acodec', 'copy']) else: options.extend(['-c', 'copy']) for (name, value) in metadata.items(): options.extend(['-metadata', '%s=%s' % (name, value)]) chapters = info.get('chapters', []) if chapters: metadata_filename = replace_extension(filename, 'meta') with io.open(metadata_filename, 'wt', encoding='utf-8') as f: def ffmpeg_escape(text): return re.sub(r'(=|;|#|\\|\n)', r'\\\1', text) metadata_file_content = ';FFMETADATA1\n' for chapter in chapters: metadata_file_content += '[CHAPTER]\nTIMEBASE=1/1000\n' metadata_file_content += 'START=%d\n' % ( chapter['start_time'] * 1000) metadata_file_content += 'END=%d\n' % ( chapter['end_time'] * 1000) chapter_title = chapter.get('title') if chapter_title: metadata_file_content += 'title=%s\n' % ffmpeg_escape( chapter_title) f.write(metadata_file_content) in_filenames.append(metadata_filename) options.extend(['-map_metadata', '1']) self.logger.info('[ffmpeg] Adding metadata to \'%s\'' % filename) self.run_ffmpeg_multiple_files(in_filenames, temp_filename, options) if chapters: os.remove(metadata_filename) os.remove(encodeFilename(filename)) os.rename(encodeFilename(temp_filename), encodeFilename(filename)) return [], info
def run(self, information): path = information['filepath'] filecodec = self.get_audio_codec(path) if filecodec is None: raise PostProcessingError( 'WARNING: unable to obtain file audio codec with ffprobe') more_opts = [] if self._preferredcodec == 'best' or self._preferredcodec == filecodec or ( self._preferredcodec == 'm4a' and filecodec == 'aac'): if filecodec == 'aac' and self._preferredcodec in ['m4a', 'best']: # Lossless, but in another container acodec = 'copy' extension = 'm4a' more_opts = ['-bsf:a', 'aac_adtstoasc'] elif filecodec in ['aac', 'flac', 'mp3', 'vorbis', 'opus']: # Lossless if possible acodec = 'copy' extension = filecodec if filecodec == 'aac': more_opts = ['-f', 'adts'] if filecodec == 'vorbis': extension = 'ogg' else: # MP3 otherwise. acodec = 'libmp3lame' extension = 'mp3' more_opts = [] if self._preferredquality is not None: if int(self._preferredquality) < 10: more_opts += ['-q:a', self._preferredquality] else: more_opts += ['-b:a', self._preferredquality + 'k'] else: # We convert the audio (lossy if codec is lossy) acodec = ACODECS[self._preferredcodec] extension = self._preferredcodec more_opts = [] if self._preferredquality is not None: # The opus codec doesn't support the -aq option if int(self._preferredquality) < 10 and extension != 'opus': more_opts += ['-q:a', self._preferredquality] else: more_opts += ['-b:a', self._preferredquality + 'k'] if self._preferredcodec == 'aac': more_opts += ['-f', 'adts'] if self._preferredcodec == 'm4a': more_opts += ['-bsf:a', 'aac_adtstoasc'] if self._preferredcodec == 'vorbis': extension = 'ogg' if self._preferredcodec == 'wav': extension = 'wav' more_opts += ['-f', 'wav'] prefix, sep, ext = path.rpartition( '.' ) # not os.path.splitext, since the latter does not work on unicode in all setups new_path = prefix + sep + extension information['filepath'] = new_path information['ext'] = extension # If we download foo.mp3 and convert it to... foo.mp3, then don't delete foo.mp3, silly. if (new_path == path or (self._nopostoverwrites and os.path.exists(encodeFilename(new_path)))): self._downloader.to_screen( '[ffmpeg] Post-process file %s exists, skipping' % new_path) return [], information try: self._downloader.to_screen('[ffmpeg] Destination: ' + new_path) self.run_ffmpeg(path, new_path, acodec, more_opts) except AudioConversionError as e: raise PostProcessingError('audio conversion failed: ' + e.msg) except Exception: raise PostProcessingError('error running ' + self.basename) # Try to update the date time for extracted audio file. if information.get('filetime') is not None: self.try_utime(new_path, time.time(), information['filetime'], errnote='Cannot update utime of audio file') return [path], information
def temp_name(self, filename): """Returns a temporary filename for the given filename.""" if self.params.get('nopart', False) or filename == u'-' or \ (os.path.exists(encodeFilename(filename)) and not os.path.isfile(encodeFilename(filename))): return filename return filename + u'.part'
def _download_with_rtmpdump(self, filename, url, player_url, page_url, play_path, tc_url, live, conn): def run_rtmpdump(args): start = time.time() resume_percent = None resume_downloaded_data_len = None proc = subprocess.Popen(args, stderr=subprocess.PIPE) cursor_in_new_line = True proc_stderr_closed = False while not proc_stderr_closed: # read line from stderr line = u'' while True: char = proc.stderr.read(1) if not char: proc_stderr_closed = True break if char in [b'\r', b'\n']: break line += char.decode('ascii', 'replace') if not line: # proc_stderr_closed is True continue mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec \(([0-9]{1,2}\.[0-9])%\)', line) if mobj: downloaded_data_len = int(float(mobj.group(1))*1024) percent = float(mobj.group(2)) if not resume_percent: resume_percent = percent resume_downloaded_data_len = downloaded_data_len eta = self.calc_eta(start, time.time(), 100-resume_percent, percent-resume_percent) speed = self.calc_speed(start, time.time(), downloaded_data_len-resume_downloaded_data_len) data_len = None if percent > 0: data_len = int(downloaded_data_len * 100 / percent) data_len_str = u'~' + format_bytes(data_len) self.report_progress(percent, data_len_str, speed, eta) cursor_in_new_line = False self._hook_progress({ 'downloaded_bytes': downloaded_data_len, 'total_bytes': data_len, 'tmpfilename': tmpfilename, 'filename': filename, 'status': 'downloading', 'eta': eta, 'speed': speed, }) else: # no percent for live streams mobj = re.search(r'([0-9]+\.[0-9]{3}) kB / [0-9]+\.[0-9]{2} sec', line) if mobj: downloaded_data_len = int(float(mobj.group(1))*1024) time_now = time.time() speed = self.calc_speed(start, time_now, downloaded_data_len) self.report_progress_live_stream(downloaded_data_len, speed, time_now - start) cursor_in_new_line = False self._hook_progress({ 'downloaded_bytes': downloaded_data_len, 'tmpfilename': tmpfilename, 'filename': filename, 'status': 'downloading', 'speed': speed, }) elif self.params.get('verbose', False): if not cursor_in_new_line: self.to_screen(u'') cursor_in_new_line = True self.to_screen(u'[rtmpdump] '+line) proc.wait() if not cursor_in_new_line: self.to_screen(u'') return proc.returncode self.report_destination(filename) tmpfilename = self.temp_name(filename) test = self.params.get('test', False) # Check for rtmpdump first try: subprocess.call(['rtmpdump', '-h'], stdout=(open(os.path.devnull, 'w')), stderr=subprocess.STDOUT) except (OSError, IOError): self.report_error(u'RTMP download detected but "rtmpdump" could not be run') return False # Download using rtmpdump. rtmpdump returns exit code 2 when # the connection was interrumpted and resuming appears to be # possible. This is part of rtmpdump's normal usage, AFAIK. basic_args = ['rtmpdump', '--verbose', '-r', url, '-o', tmpfilename] if player_url is not None: basic_args += ['--swfVfy', player_url] if page_url is not None: basic_args += ['--pageUrl', page_url] if play_path is not None: basic_args += ['--playpath', play_path] if tc_url is not None: basic_args += ['--tcUrl', url] if test: basic_args += ['--stop', '1'] if live: basic_args += ['--live'] if conn: basic_args += ['--conn', conn] args = basic_args + [[], ['--resume', '--skip', '1']][self.params.get('continuedl', False)] if sys.platform == 'win32' and sys.version_info < (3, 0): # Windows subprocess module does not actually support Unicode # on Python 2.x # See http://stackoverflow.com/a/9951851/35070 subprocess_encoding = sys.getfilesystemencoding() args = [a.encode(subprocess_encoding, 'ignore') for a in args] else: subprocess_encoding = None if self.params.get('verbose', False): if subprocess_encoding: str_args = [ a.decode(subprocess_encoding) if isinstance(a, bytes) else a for a in args] else: str_args = args try: import pipes shell_quote = lambda args: ' '.join(map(pipes.quote, str_args)) except ImportError: shell_quote = repr self.to_screen(u'[debug] rtmpdump command line: ' + shell_quote(str_args)) retval = run_rtmpdump(args) while (retval == 2 or retval == 1) and not test: prevsize = os.path.getsize(encodeFilename(tmpfilename)) self.to_screen(u'[rtmpdump] %s bytes' % prevsize) time.sleep(5.0) # This seems to be needed retval = run_rtmpdump(basic_args + ['-e'] + [[], ['-k', '1']][retval == 1]) cursize = os.path.getsize(encodeFilename(tmpfilename)) if prevsize == cursize and retval == 1: break # Some rtmp streams seem abort after ~ 99.8%. Don't complain for those if prevsize == cursize and retval == 2 and cursize > 1024: self.to_screen(u'[rtmpdump] Could not download the whole video. This can happen for some advertisements.') retval = 0 break if retval == 0 or (test and retval == 2): fsize = os.path.getsize(encodeFilename(tmpfilename)) self.to_screen(u'[rtmpdump] %s bytes' % fsize) self.try_rename(tmpfilename, filename) self._hook_progress({ 'downloaded_bytes': fsize, 'total_bytes': fsize, 'filename': filename, 'status': 'finished', }) return True else: self.to_stderr(u"\n") self.report_error(u'rtmpdump exited with code %d' % retval) return False
def _do_download(self, filename, info_dict): url = info_dict['url'] # Check file already present if self.params.get('continuedl', False) and os.path.isfile(encodeFilename(filename)) and not self.params.get('nopart', False): self.report_file_already_downloaded(filename) self._hook_progress({ 'filename': filename, 'status': 'finished', 'total_bytes': os.path.getsize(encodeFilename(filename)), }) return True # Attempt to download using rtmpdump if url.startswith('rtmp'): return self._download_with_rtmpdump(filename, url, info_dict.get('player_url', None), info_dict.get('page_url', None), info_dict.get('play_path', None), info_dict.get('tc_url', None), info_dict.get('rtmp_live', False), info_dict.get('rtmp_conn', None)) # Attempt to download using mplayer if url.startswith('mms') or url.startswith('rtsp'): return self._download_with_mplayer(filename, url) # m3u8 manifest are downloaded with ffmpeg if determine_ext(url) == u'm3u8': return self._download_m3u8_with_ffmpeg(filename, url) tmpfilename = self.temp_name(filename) stream = None # Do not include the Accept-Encoding header headers = {'Youtubedl-no-compression': 'True'} if 'user_agent' in info_dict: headers['Youtubedl-user-agent'] = info_dict['user_agent'] basic_request = compat_urllib_request.Request(url, None, headers) request = compat_urllib_request.Request(url, None, headers) if self.params.get('test', False): request.add_header('Range','bytes=0-10240') # Establish possible resume length if os.path.isfile(encodeFilename(tmpfilename)): resume_len = os.path.getsize(encodeFilename(tmpfilename)) else: resume_len = 0 open_mode = 'wb' if resume_len != 0: if self.params.get('continuedl', False): self.report_resuming_byte(resume_len) request.add_header('Range','bytes=%d-' % resume_len) open_mode = 'ab' else: resume_len = 0 count = 0 retries = self.params.get('retries', 0) while count <= retries: # Establish connection try: if count == 0 and 'urlhandle' in info_dict: data = info_dict['urlhandle'] data = compat_urllib_request.urlopen(request) break except (compat_urllib_error.HTTPError, ) as err: if (err.code < 500 or err.code >= 600) and err.code != 416: # Unexpected HTTP error raise elif err.code == 416: # Unable to resume (requested range not satisfiable) try: # Open the connection again without the range header data = compat_urllib_request.urlopen(basic_request) content_length = data.info()['Content-Length'] except (compat_urllib_error.HTTPError, ) as err: if err.code < 500 or err.code >= 600: raise else: # Examine the reported length if (content_length is not None and (resume_len - 100 < int(content_length) < resume_len + 100)): # The file had already been fully downloaded. # Explanation to the above condition: in issue #175 it was revealed that # YouTube sometimes adds or removes a few bytes from the end of the file, # changing the file size slightly and causing problems for some users. So # I decided to implement a suggested change and consider the file # completely downloaded if the file size differs less than 100 bytes from # the one in the hard drive. self.report_file_already_downloaded(filename) self.try_rename(tmpfilename, filename) self._hook_progress({ 'filename': filename, 'status': 'finished', }) return True else: # The length does not match, we start the download over self.report_unable_to_resume() open_mode = 'wb' break # Retry count += 1 if count <= retries: self.report_retry(count, retries) if count > retries: self.report_error(u'giving up after %s retries' % retries) return False data_len = data.info().get('Content-length', None) if data_len is not None: data_len = int(data_len) + resume_len min_data_len = self.params.get("min_filesize", None) max_data_len = self.params.get("max_filesize", None) if min_data_len is not None and data_len < min_data_len: self.to_screen(u'\r[download] File is smaller than min-filesize (%s bytes < %s bytes). Aborting.' % (data_len, min_data_len)) return False if max_data_len is not None and data_len > max_data_len: self.to_screen(u'\r[download] File is larger than max-filesize (%s bytes > %s bytes). Aborting.' % (data_len, max_data_len)) return False data_len_str = format_bytes(data_len) byte_counter = 0 + resume_len block_size = self.params.get('buffersize', 1024) start = time.time() while self._go_on: # Download and write before = time.time() data_block = data.read(block_size) after = time.time() if len(data_block) == 0: break byte_counter += len(data_block) # Open file just in time if stream is None: try: (stream, tmpfilename) = sanitize_open(tmpfilename, open_mode) assert stream is not None filename = self.undo_temp_name(tmpfilename) self.report_destination(filename) except (OSError, IOError) as err: self.report_error(u'unable to open for writing: %s' % str(err)) return False try: stream.write(data_block) except (IOError, OSError) as err: self.to_stderr(u"\n") self.report_error(u'unable to write data: %s' % str(err)) return False if not self.params.get('noresizebuffer', False): block_size = self.best_block_size(after - before, len(data_block)) # Progress message speed = self.calc_speed(start, time.time(), byte_counter - resume_len) if data_len is None: eta = percent = None else: percent = self.calc_percent(byte_counter, data_len) eta = self.calc_eta(start, time.time(), data_len - resume_len, byte_counter - resume_len) self.report_progress(percent, data_len_str, speed, eta) self._hook_progress({ 'downloaded_bytes': byte_counter, 'total_bytes': data_len, 'tmpfilename': tmpfilename, 'filename': filename, 'status': 'downloading', 'eta': eta, 'speed': speed, }) # Apply rate limit self.slow_down(start, byte_counter - resume_len) if stream is None: self.to_stderr(u"\n") self.report_error(u'Did not get any data blocks') return False stream.close() self.report_finish(data_len_str, (time.time() - start)) if data_len is not None and byte_counter != data_len: raise ContentTooShortError(byte_counter, int(data_len)) self.try_rename(tmpfilename, filename) # Update file modification time if self.params.get('updatetime', True): info_dict['filetime'] = self.try_utime(filename, data.info().get('last-modified', None)) self._hook_progress({ 'downloaded_bytes': byte_counter, 'total_bytes': byte_counter, 'filename': filename, 'status': 'finished', }) return True