def test_downloader_resp_headers(self): with open('mpdstub/mpd/17875351285037717.mpd', 'r') as f: mpd_content = f.read() with responses.RequestsMock( assert_all_requests_are_fired=True) as rsps: rsps.add(responses.GET, self.TEST_MPD_URL, body=mpd_content) rsps.add(responses.GET, self.TEST_MPD_URL, body=mpd_content, headers={'Cache-Control': 'max-age=1'}) rsps.add(responses.GET, self.TEST_MPD_URL, body=mpd_content, headers={'X-FB-Video-Broadcast-Ended': '1'}) dl = live.Downloader(mpd=self.TEST_MPD_URL, output_dir='output_respheaders') dl.run() with responses.RequestsMock( assert_all_requests_are_fired=True) as rsps: rsps.add(responses.GET, self.TEST_MPD_URL, body=mpd_content, headers={'Cache-Control': 'max-age=1'}) rsps.add(responses.GET, self.TEST_MPD_URL, body=mpd_content, headers={'Cache-Control': 'max-age=1000'}) dl = live.Downloader(mpd=self.TEST_MPD_URL, output_dir='output_respheaders') dl.run()
def test_downloader_single_threaded(self): dl = live.Downloader(mpd=self.TEST_MPD_URL, output_dir='output_singlethreaded', duplicate_etag_retry=10, singlethreaded=True) dl.run() output_file = 'output_singlethreaded.mp4' dl.stitch(output_file, cleartempfiles=True) self.assertTrue(os.path.isfile(output_file), '{0!s} not generated'.format(output_file))
def test_downloader(self): def check_status(): return True dl = live.Downloader(mpd=self.TEST_MPD_URL, output_dir='output', duplicate_etag_retry=10, callback_check=check_status) dl.run() output_file = 'output.mp4' dl.stitch(output_file, cleartempfiles=False) self.assertTrue(os.path.isfile(output_file), '{0!s} not generated'.format(output_file))
def test_downloader_404(self): def check_status(): return True dl = live.Downloader(mpd=self.TEST_MPD_URL + 'x', output_dir='output_404', max_connection_error_retry=2, callback_check=check_status) dl.run() output_file = 'output_404.mp4' with self.assertRaises(Exception): dl.stitch(output_file, cleartempfiles=False) self.assertFalse(os.path.isfile(output_file), '{0!s} is generated'.format(output_file))
def test_downloader_fragment_dl_error(self): exception = ConnectionError() fragments = [ 'dash-hd1/17875351285037717-init.m4v', 'dash-hd1/17875351285037717-281033.m4v', 'dash-hd1/17875351285037717-282033.m4v', 'dash-hd1/17875351285037717-283033.m4v', 'dash-hd1/17875351285037717-284033.m4v', 'dash-hd1/17875351285037717-285033.m4v', 'dash-hd1/17875351285037717-286033.m4v', 'dash-hd1/17875351285037717-287033.m4v', 'dash-hd1/17875351285037717-288033.m4v', 'dash-hd1/17875351285037717-289033.m4v', 'dash-hd1/17875351285037717-290033.m4v', 'dash-ld/17875351285037717-init.m4a', 'dash-ld/17875351285037717-281033.m4a', 'dash-ld/17875351285037717-282033.m4a', 'dash-ld/17875351285037717-283033.m4a', 'dash-ld/17875351285037717-284033.m4a', 'dash-ld/17875351285037717-285033.m4a', 'dash-ld/17875351285037717-286033.m4a', 'dash-ld/17875351285037717-287033.m4a', 'dash-ld/17875351285037717-288033.m4a', 'dash-ld/17875351285037717-289033.m4a', 'dash-ld/17875351285037717-290033.m4a', ] with open('mpdstub/mpd/17875351285037717.mpd', 'r') as f: mpd_content = f.read() with responses.RequestsMock( assert_all_requests_are_fired=True) as rsps: rsps.add(responses.GET, self.TEST_MPD_URL, body=mpd_content) max_retry = 1 for fragment in fragments: for _ in range(max_retry + 1): rsps.add(responses.GET, 'http://127.0.01:8000/' + fragment, body=exception) dl = live.Downloader(mpd=self.TEST_MPD_URL, output_dir='output_fragment_connerror', duplicate_etag_retry=2, singlethreaded=True, max_connection_error_retry=max_retry) dl.run() dl.stream_id = '17875351285037717' output_file = 'output_fragment_connerror.mp4' dl.stitch(output_file, cleartempfiles=True) self.assertFalse(os.path.isfile(output_file), '{0!s} not generated'.format(output_file))
def test_downloader_http_errors(self): with responses.RequestsMock( assert_all_requests_are_fired=True) as rsps: rsps.add(responses.GET, self.TEST_MPD_URL, status=500) rsps.add(responses.GET, self.TEST_MPD_URL, status=404) dl = live.Downloader(mpd=self.TEST_MPD_URL, output_dir='output_httperrors', duplicate_etag_retry=2, singlethreaded=True) dl.run() dl.stream_id = '17875351285037717' output_file = 'output_httperrors.mp4' dl.stitch(output_file, cleartempfiles=True) self.assertFalse(os.path.isfile(output_file), '{0!s} is generated'.format(output_file))
def test_downloader_conn_error(self): exception = ConnectionError() with responses.RequestsMock( assert_all_requests_are_fired=True) as rsps: max_retry = 3 for _ in range(max_retry + 1): rsps.add(responses.GET, self.TEST_MPD_URL, body=exception) dl = live.Downloader(mpd=self.TEST_MPD_URL, output_dir='output_connerror', duplicate_etag_retry=2, singlethreaded=True, max_connection_error_retry=max_retry) dl.run() dl.stream_id = '17875351285037717' output_file = 'output_connerror.mp4' dl.stitch(output_file, cleartempfiles=True) self.assertFalse(os.path.isfile(output_file), '{0!s} not generated'.format(output_file))
def download_livestream(broadcast): try: def print_status(sep=True): heartbeat_info = instagram_api.broadcast_heartbeat_and_viewercount( broadcast.get('id')) viewers = broadcast.get('viewer_count', 0) if sep: log_seperator() log_info_green('Viewers : {:s} watching'.format( str(int(viewers)))) log_info_green('Airing time : {:s}'.format( get_stream_duration(broadcast.get('published_time')))) log_info_green('Status : {:s}'.format( heartbeat_info.get('broadcast_status').title())) return heartbeat_info.get('broadcast_status') not in [ 'active', 'interrupted' ] mpd_url = (broadcast.get('dash_manifest') or broadcast.get('dash_abr_playback_url') or broadcast.get('dash_playback_url')) output_dir = '{}{}_{}_{}_{}_live_downloads'.format( settings.save_path, settings.current_date, user_to_download, broadcast.get('id'), settings.current_time) broadcast_downloader = live.Downloader( mpd=mpd_url, output_dir=output_dir, user_agent=instagram_api.user_agent, max_connection_error_retry=3, duplicate_etag_retry=30, callback_check=print_status, mpd_download_timeout=3, download_timeout=3) except Exception as e: log_error('Could not start downloading livestream: {:s}'.format( str(e))) log_seperator() try: os.remove( os.path.join(settings.save_path, user_to_download + '.lock')) except Exception: pass sys.exit(1) try: log_info_green('Livestream found, beginning download...') broadcast_owner = broadcast.get('broadcast_owner', {}).get('username') try: broadcast_guest = broadcast.get('cobroadcasters', {})[0].get('username') except Exception: broadcast_guest = None if (broadcast_owner != user_to_download): log_info_blue( 'This livestream is a dual-live, the owner is "{}".'.format( broadcast_owner)) broadcast_guest = None if broadcast_guest: log_info_blue( 'This livestream is a dual-live, the current guest is "{}".'. format(broadcast_guest)) log_seperator() log_info_green('Username : {:s}'.format(user_to_download)) print_status(False) log_info_green('MPD URL : {:s}'.format(mpd_url)) log_seperator() open(os.path.join(output_dir, 'folder.lock'), 'a').close() log_info_green('Downloading livestream... press [CTRL+C] to abort.') if (settings.run_at_start is not "None"): try: thread = threading.Thread(target=run_command, args=(settings.run_at_start, )) thread.daemon = True thread.start() log_info_green("Command executed: \033[94m{:s}".format( settings.run_at_start)) except Exception as e: log_warn('Could not execute command: {:s}'.format(str(e))) comment_thread_worker = None if settings.save_comments.title() == "True": try: comments_json_file = os.path.join( output_dir, '{}_{}_{}_{}_live_comments.json'.format( settings.current_date, user_to_download, broadcast.get('id'), settings.current_time)) comment_thread_worker = threading.Thread( target=get_live_comments, args=( instagram_api, broadcast, comments_json_file, broadcast_downloader, )) comment_thread_worker.start() except Exception as e: log_error('An error occurred while downloading comments: {:s}'. format(str(e))) broadcast_downloader.run() log_seperator() log_info_green('Download duration : {}'.format( get_stream_duration(int(settings.current_time)))) log_info_green('Stream duration : {}'.format( get_stream_duration(broadcast.get('published_time')))) log_info_green('Missing (approx.) : {}'.format( get_stream_duration(int(settings.current_time), broadcast))) log_seperator() stitch_video(broadcast_downloader, broadcast, comment_thread_worker) except KeyboardInterrupt: log_seperator() log_info_blue('The download has been aborted by the user.') log_seperator() log_info_green('Download duration : {}'.format( get_stream_duration(int(settings.current_time)))) log_info_green('Stream duration : {}'.format( get_stream_duration(broadcast.get('published_time')))) log_info_green('Missing (approx.) : {}'.format( get_stream_duration(int(settings.current_time), broadcast))) log_seperator() if not broadcast_downloader.is_aborted: broadcast_downloader.stop() stitch_video(broadcast_downloader, broadcast, comment_thread_worker) except Exception as e: log_error("Could not download livestream: {:s}".format(str(e))) try: os.remove(os.path.join(output_dir, 'folder.lock')) except Exception: pass try: os.remove( os.path.join(settings.save_path, user_to_download + '.lock')) except Exception: pass
# reuse auth settings api = Client( username, password, settings=cached_settings) except (ClientCookieExpiredError, ClientLoginRequiredError) as e: print('ClientCookieExpiredError/ClientLoginRequiredError: {0!s}'.format(e)) print('run savesettings_logincallback') exit() for user in users: time.sleep(random.randrange(3, 5)) if api.user_broadcast(user) is not None: broadcast_id = api.user_story_feed(user).get('broadcast').get('id') time.sleep(random.randrange(3, 5)) broadcast = api.broadcast_info(broadcast_id) time.sleep(random.randrange(3, 5)) dl = live.Downloader(mpd=broadcast['dash_playback_url'], output_dir='output_{}/'.format(broadcast['id']), user_agent=api.user_agent) time.sleep(random.randrange(3, 5)) try: dl.run() except KeyboardInterrupt: if not dl.is_aborted: dl.stop() finally: # combine the downloaded files # Requires ffmpeg installed. If you prefer to use avconv # for example, omit this step and do it manually dl.stitch('my_video' + broadcast.get(('username') + broadcast.get(('published_time') + '.mp4')))
def download_livestream(): try: def print_status(sep=True): if pil.do_heartbeat: heartbeat_info = pil.ig_api.broadcast_heartbeat_and_viewercount( pil.livestream_obj.get('id')) viewers = pil.livestream_obj.get('viewer_count', 0) if sep: logger.separator() else: logger.info('Username : {:s}'.format(pil.dl_user)) logger.info('Viewers : {:s} watching'.format(str( int(viewers)))) logger.info('Airing time : {:s}'.format(get_stream_duration(0))) if pil.do_heartbeat: logger.info('Status : {:s}'.format( heartbeat_info.get('broadcast_status').title())) return heartbeat_info.get('broadcast_status') not in [ 'active', 'interrupted' ] else: return None mpd_url = (pil.livestream_obj.get('dash_manifest') or pil.livestream_obj.get('dash_abr_playback_url') or pil.livestream_obj.get('dash_playback_url')) pil.live_folder_path = '{}{}_{}_{}_{}_live_downloads'.format( pil.dl_path, pil.datetime_compat, pil.dl_user, pil.livestream_obj.get('id'), pil.epochtime) pil.broadcast_downloader = live.Downloader( mpd=mpd_url, output_dir=pil.live_folder_path, user_agent=pil.ig_api.user_agent, max_connection_error_retry=3, duplicate_etag_retry=30, callback_check=print_status, mpd_download_timeout=3, download_timeout=3, ffmpeg_binary=pil.ffmpeg_path) except Exception as e: logger.error('Could not start downloading livestream: {:s}'.format( str(e))) logger.separator() helpers.remove_lock() try: broadcast_owner = pil.livestream_obj.get('broadcast_owner', {}).get('username') try: broadcast_guest = pil.livestream_obj.get('cobroadcasters', {})[0].get('username') except Exception: broadcast_guest = None if broadcast_owner != pil.dl_user: logger.binfo( 'This livestream is a dual-live, the owner is "{}".'.format( broadcast_owner)) broadcast_guest = None if broadcast_guest: logger.binfo( 'This livestream is a dual-live, the current guest is "{}".'. format(broadcast_guest)) pil.has_guest = broadcast_guest logger.separator() print_status(False) logger.separator() helpers.create_lock_folder() pil.segments_json_thread_worker = threading.Thread( target=helpers.generate_json_segments) pil.segments_json_thread_worker.start() logger.info('Downloading livestream, press [CTRL+C] to abort.') if pil.run_at_start: try: thread = threading.Thread(target=helpers.run_command, args=(pil.run_at_start, )) thread.daemon = True thread.start() logger.binfo("Launched start command: {:s}".format( pil.run_at_start)) except Exception as e: logger.warn('Could not launch command: {:s}'.format(str(e))) if pil.dl_comments: try: comments_json_file = '{}{}_{}_{}_{}_live_comments.json'.format( pil.dl_path, pil.datetime_compat, pil.dl_user, pil.livestream_obj.get('id'), pil.epochtime) pil.comment_thread_worker = threading.Thread( target=get_live_comments, args=(comments_json_file, )) pil.comment_thread_worker.start() except Exception as e: logger.error( 'An error occurred while downloading comments: {:s}'. format(str(e))) pil.broadcast_downloader.run() logger.separator() logger.info("The livestream has been ended by the user.") logger.separator() logger.info('Airtime duration : {}'.format(get_stream_duration(0))) logger.info('Download duration : {}'.format(get_stream_duration(1))) logger.info('Missing (approx.) : {}'.format(get_stream_duration(2))) logger.separator() merge_segments() except KeyboardInterrupt: logger.separator() logger.binfo('The download has been aborted.') logger.separator() logger.info('Airtime duration : {}'.format(get_stream_duration(0))) logger.info('Download duration : {}'.format(get_stream_duration(1))) logger.info('Missing (approx.) : {}'.format(get_stream_duration(2))) logger.separator() if not pil.broadcast_downloader.is_aborted: pil.broadcast_downloader.stop() merge_segments()
def download_livestream(broadcast): try: def print_status(sep=True): heartbeat_info = instagram_api.broadcast_heartbeat_and_viewercount(broadcast.get('id')) viewers = broadcast.get('viewer_count', 0) if sep: seperator("GREEN") log('[I] Viewers : {:s} watching'.format(str(int(viewers))), "GREEN") log('[I] Airing time : {:s}'.format(get_stream_duration(broadcast.get('published_time'))), "GREEN") log('[I] Status : {:s}'.format(heartbeat_info.get('broadcast_status').title()), "GREEN") return heartbeat_info.get('broadcast_status') not in ['active', 'interrupted'] mpd_url = (broadcast.get('dash_manifest') or broadcast.get('dash_abr_playback_url') or broadcast.get('dash_playback_url')) output_dir = settings.save_path + '{}_{}_{}_{}_live_downloads'.format(settings.current_date, user_to_record, broadcast.get('id'), settings.current_time) broadcast_downloader = live.Downloader( mpd=mpd_url, output_dir=output_dir, user_agent=instagram_api.user_agent, max_connection_error_retry=3, duplicate_etag_retry=30, callback_check=print_status, mpd_download_timeout=3, download_timeout=3) except Exception as e: log('[E] Could not start downloading livestream: {:s}'.format(str(e)), "RED") seperator("GREEN") sys.exit(1) try: log('[I] Livestream found, beginning download...', "GREEN") broadcast_owner = broadcast.get('broadcast_owner', {}).get('username') try: broadcast_guest = broadcast.get('cobroadcasters', {})[0].get('username') except: broadcast_guest = None if (broadcast_owner != user_to_record): log('[I] This livestream is a dual-live, the owner is "{}".'.format(broadcast_owner), "BLUE") broadcast_guest = None if broadcast_guest: log('[I] This livestream is a dual-live, the guest is "{}".'.format(broadcast_guest), "BLUE") seperator("GREEN") log('[I] Username : {:s}'.format(user_to_record), "GREEN") print_status(False) log('[I] MPD URL : {:s}'.format(mpd_url), "GREEN") seperator("GREEN") log('[I] Downloading livestream... press [CTRL+C] to abort.', "GREEN") if (settings.run_at_start is not "None"): try: thread = threading.Thread(target=run_command, args=(settings.run_at_start,)) thread.daemon = True thread.start() log("[I] Command executed: \033[94m{:s}".format(settings.run_at_start), "GREEN") except Exception as e: log('[W] Could not execute command: {:s}'.format(str(e)), "YELLOW") comment_thread_worker = None if settings.save_comments.title() == "True": try: comments_json_file = settings.save_path + '{}_{}_{}_{}_live_comments.json'.format(settings.current_date, user_to_record, broadcast.get('id'), settings.current_time) comment_thread_worker = threading.Thread(target=get_live_comments, args=(instagram_api, broadcast, comments_json_file, broadcast_downloader,)) comment_thread_worker.start() except Exception as e: log('[E] An error occurred while checking comments: {:s}'.format(str(e)), "RED") broadcast_downloader.run() seperator("GREEN") log('[I] The livestream has ended.\n[I] Time recorded : {}\n[I] Stream duration : {}\n[I] Missing (approx.) : {}'.format(get_stream_duration(int(settings.current_time)), get_stream_duration(broadcast.get('published_time')), get_stream_duration(int(settings.current_time), broadcast)), "YELLOW") seperator("GREEN") stitch_video(broadcast_downloader, broadcast, comment_thread_worker) except KeyboardInterrupt: seperator("GREEN") log('[I] The download has been aborted by the user.\n[I] Time recorded : {}\n[I] Stream duration : {}\n[I] Missing (approx.) : {}'.format(get_stream_duration(int(settings.current_time)), get_stream_duration(broadcast.get('published_time')), get_stream_duration(int(settings.current_time), broadcast)), "YELLOW") seperator("GREEN") if not broadcast_downloader.is_aborted: broadcast_downloader.stop() stitch_video(broadcast_downloader, broadcast, comment_thread_worker) except Exception as e: log("[E] Could not download livestream: {:s}".format(str(e)), "RED")