def test_downloader_resp_headers(self):
with open('mpdstub/mpd/17875351285037717.mpd', 'r') as f:
mpd_content = f.read()
with responses.RequestsMock(
assert_all_requests_are_fired=True) as rsps:
rsps.add(responses.GET, self.TEST_MPD_URL, body=mpd_content)
rsps.add(responses.GET,
self.TEST_MPD_URL,
body=mpd_content,
headers={'Cache-Control': 'max-age=1'})
rsps.add(responses.GET,
self.TEST_MPD_URL,
body=mpd_content,
headers={'X-FB-Video-Broadcast-Ended': '1'})
dl = live.Downloader(mpd=self.TEST_MPD_URL,
output_dir='output_respheaders')
dl.run()
with responses.RequestsMock(
assert_all_requests_are_fired=True) as rsps:
rsps.add(responses.GET,
self.TEST_MPD_URL,
body=mpd_content,
headers={'Cache-Control': 'max-age=1'})
rsps.add(responses.GET,
self.TEST_MPD_URL,
body=mpd_content,
headers={'Cache-Control': 'max-age=1000'})
dl = live.Downloader(mpd=self.TEST_MPD_URL,
output_dir='output_respheaders')
dl.run()
def test_downloader_single_threaded(self):
dl = live.Downloader(mpd=self.TEST_MPD_URL,
output_dir='output_singlethreaded',
duplicate_etag_retry=10,
singlethreaded=True)
dl.run()
output_file = 'output_singlethreaded.mp4'
dl.stitch(output_file, cleartempfiles=True)
self.assertTrue(os.path.isfile(output_file),
'{0!s} not generated'.format(output_file))
def test_downloader(self):
def check_status():
return True
dl = live.Downloader(mpd=self.TEST_MPD_URL,
output_dir='output',
duplicate_etag_retry=10,
callback_check=check_status)
dl.run()
output_file = 'output.mp4'
dl.stitch(output_file, cleartempfiles=False)
self.assertTrue(os.path.isfile(output_file),
'{0!s} not generated'.format(output_file))
def test_downloader_404(self):
def check_status():
return True
dl = live.Downloader(mpd=self.TEST_MPD_URL + 'x',
output_dir='output_404',
max_connection_error_retry=2,
callback_check=check_status)
dl.run()
output_file = 'output_404.mp4'
with self.assertRaises(Exception):
dl.stitch(output_file, cleartempfiles=False)
self.assertFalse(os.path.isfile(output_file),
'{0!s} is generated'.format(output_file))
def test_downloader_fragment_dl_error(self):
exception = ConnectionError()
fragments = [
'dash-hd1/17875351285037717-init.m4v',
'dash-hd1/17875351285037717-281033.m4v',
'dash-hd1/17875351285037717-282033.m4v',
'dash-hd1/17875351285037717-283033.m4v',
'dash-hd1/17875351285037717-284033.m4v',
'dash-hd1/17875351285037717-285033.m4v',
'dash-hd1/17875351285037717-286033.m4v',
'dash-hd1/17875351285037717-287033.m4v',
'dash-hd1/17875351285037717-288033.m4v',
'dash-hd1/17875351285037717-289033.m4v',
'dash-hd1/17875351285037717-290033.m4v',
'dash-ld/17875351285037717-init.m4a',
'dash-ld/17875351285037717-281033.m4a',
'dash-ld/17875351285037717-282033.m4a',
'dash-ld/17875351285037717-283033.m4a',
'dash-ld/17875351285037717-284033.m4a',
'dash-ld/17875351285037717-285033.m4a',
'dash-ld/17875351285037717-286033.m4a',
'dash-ld/17875351285037717-287033.m4a',
'dash-ld/17875351285037717-288033.m4a',
'dash-ld/17875351285037717-289033.m4a',
'dash-ld/17875351285037717-290033.m4a',
]
with open('mpdstub/mpd/17875351285037717.mpd', 'r') as f:
mpd_content = f.read()
with responses.RequestsMock(
assert_all_requests_are_fired=True) as rsps:
rsps.add(responses.GET, self.TEST_MPD_URL, body=mpd_content)
max_retry = 1
for fragment in fragments:
for _ in range(max_retry + 1):
rsps.add(responses.GET,
'http://127.0.01:8000/' + fragment,
body=exception)
dl = live.Downloader(mpd=self.TEST_MPD_URL,
output_dir='output_fragment_connerror',
duplicate_etag_retry=2,
singlethreaded=True,
max_connection_error_retry=max_retry)
dl.run()
dl.stream_id = '17875351285037717'
output_file = 'output_fragment_connerror.mp4'
dl.stitch(output_file, cleartempfiles=True)
self.assertFalse(os.path.isfile(output_file),
'{0!s} not generated'.format(output_file))
def test_downloader_http_errors(self):
with responses.RequestsMock(
assert_all_requests_are_fired=True) as rsps:
rsps.add(responses.GET, self.TEST_MPD_URL, status=500)
rsps.add(responses.GET, self.TEST_MPD_URL, status=404)
dl = live.Downloader(mpd=self.TEST_MPD_URL,
output_dir='output_httperrors',
duplicate_etag_retry=2,
singlethreaded=True)
dl.run()
dl.stream_id = '17875351285037717'
output_file = 'output_httperrors.mp4'
dl.stitch(output_file, cleartempfiles=True)
self.assertFalse(os.path.isfile(output_file),
'{0!s} is generated'.format(output_file))
def test_downloader_conn_error(self):
exception = ConnectionError()
with responses.RequestsMock(
assert_all_requests_are_fired=True) as rsps:
max_retry = 3
for _ in range(max_retry + 1):
rsps.add(responses.GET, self.TEST_MPD_URL, body=exception)
dl = live.Downloader(mpd=self.TEST_MPD_URL,
output_dir='output_connerror',
duplicate_etag_retry=2,
singlethreaded=True,
max_connection_error_retry=max_retry)
dl.run()
dl.stream_id = '17875351285037717'
output_file = 'output_connerror.mp4'
dl.stitch(output_file, cleartempfiles=True)
self.assertFalse(os.path.isfile(output_file),
'{0!s} not generated'.format(output_file))
def download_livestream(broadcast):
try:
def print_status(sep=True):
heartbeat_info = instagram_api.broadcast_heartbeat_and_viewercount(
broadcast.get('id'))
viewers = broadcast.get('viewer_count', 0)
if sep:
log_seperator()
log_info_green('Viewers : {:s} watching'.format(
str(int(viewers))))
log_info_green('Airing time : {:s}'.format(
get_stream_duration(broadcast.get('published_time'))))
log_info_green('Status : {:s}'.format(
heartbeat_info.get('broadcast_status').title()))
return heartbeat_info.get('broadcast_status') not in [
'active', 'interrupted'
]
mpd_url = (broadcast.get('dash_manifest')
or broadcast.get('dash_abr_playback_url')
or broadcast.get('dash_playback_url'))
output_dir = '{}{}_{}_{}_{}_live_downloads'.format(
settings.save_path, settings.current_date, user_to_download,
broadcast.get('id'), settings.current_time)
broadcast_downloader = live.Downloader(
mpd=mpd_url,
output_dir=output_dir,
user_agent=instagram_api.user_agent,
max_connection_error_retry=3,
duplicate_etag_retry=30,
callback_check=print_status,
mpd_download_timeout=3,
download_timeout=3)
except Exception as e:
log_error('Could not start downloading livestream: {:s}'.format(
str(e)))
log_seperator()
try:
os.remove(
os.path.join(settings.save_path, user_to_download + '.lock'))
except Exception:
pass
sys.exit(1)
try:
log_info_green('Livestream found, beginning download...')
broadcast_owner = broadcast.get('broadcast_owner', {}).get('username')
try:
broadcast_guest = broadcast.get('cobroadcasters',
{})[0].get('username')
except Exception:
broadcast_guest = None
if (broadcast_owner != user_to_download):
log_info_blue(
'This livestream is a dual-live, the owner is "{}".'.format(
broadcast_owner))
broadcast_guest = None
if broadcast_guest:
log_info_blue(
'This livestream is a dual-live, the current guest is "{}".'.
format(broadcast_guest))
log_seperator()
log_info_green('Username : {:s}'.format(user_to_download))
print_status(False)
log_info_green('MPD URL : {:s}'.format(mpd_url))
log_seperator()
open(os.path.join(output_dir, 'folder.lock'), 'a').close()
log_info_green('Downloading livestream... press [CTRL+C] to abort.')
if (settings.run_at_start is not "None"):
try:
thread = threading.Thread(target=run_command,
args=(settings.run_at_start, ))
thread.daemon = True
thread.start()
log_info_green("Command executed: \033[94m{:s}".format(
settings.run_at_start))
except Exception as e:
log_warn('Could not execute command: {:s}'.format(str(e)))
comment_thread_worker = None
if settings.save_comments.title() == "True":
try:
comments_json_file = os.path.join(
output_dir, '{}_{}_{}_{}_live_comments.json'.format(
settings.current_date, user_to_download,
broadcast.get('id'), settings.current_time))
comment_thread_worker = threading.Thread(
target=get_live_comments,
args=(
instagram_api,
broadcast,
comments_json_file,
broadcast_downloader,
))
comment_thread_worker.start()
except Exception as e:
log_error('An error occurred while downloading comments: {:s}'.
format(str(e)))
broadcast_downloader.run()
log_seperator()
log_info_green('Download duration : {}'.format(
get_stream_duration(int(settings.current_time))))
log_info_green('Stream duration : {}'.format(
get_stream_duration(broadcast.get('published_time'))))
log_info_green('Missing (approx.) : {}'.format(
get_stream_duration(int(settings.current_time), broadcast)))
log_seperator()
stitch_video(broadcast_downloader, broadcast, comment_thread_worker)
except KeyboardInterrupt:
log_seperator()
log_info_blue('The download has been aborted by the user.')
log_seperator()
log_info_green('Download duration : {}'.format(
get_stream_duration(int(settings.current_time))))
log_info_green('Stream duration : {}'.format(
get_stream_duration(broadcast.get('published_time'))))
log_info_green('Missing (approx.) : {}'.format(
get_stream_duration(int(settings.current_time), broadcast)))
log_seperator()
if not broadcast_downloader.is_aborted:
broadcast_downloader.stop()
stitch_video(broadcast_downloader, broadcast,
comment_thread_worker)
except Exception as e:
log_error("Could not download livestream: {:s}".format(str(e)))
try:
os.remove(os.path.join(output_dir, 'folder.lock'))
except Exception:
pass
try:
os.remove(
os.path.join(settings.save_path, user_to_download + '.lock'))
except Exception:
pass
# reuse auth settings
api = Client(
username, password,
settings=cached_settings)
except (ClientCookieExpiredError, ClientLoginRequiredError) as e:
print('ClientCookieExpiredError/ClientLoginRequiredError: {0!s}'.format(e))
print('run savesettings_logincallback')
exit()
for user in users:
time.sleep(random.randrange(3, 5))
if api.user_broadcast(user) is not None:
broadcast_id = api.user_story_feed(user).get('broadcast').get('id')
time.sleep(random.randrange(3, 5))
broadcast = api.broadcast_info(broadcast_id)
time.sleep(random.randrange(3, 5))
dl = live.Downloader(mpd=broadcast['dash_playback_url'], output_dir='output_{}/'.format(broadcast['id']), user_agent=api.user_agent)
time.sleep(random.randrange(3, 5))
try:
dl.run()
except KeyboardInterrupt:
if not dl.is_aborted:
dl.stop()
finally:
# combine the downloaded files
# Requires ffmpeg installed. If you prefer to use avconv
# for example, omit this step and do it manually
dl.stitch('my_video' + broadcast.get(('username') + broadcast.get(('published_time') + '.mp4')))
def download_livestream():
try:
def print_status(sep=True):
if pil.do_heartbeat:
heartbeat_info = pil.ig_api.broadcast_heartbeat_and_viewercount(
pil.livestream_obj.get('id'))
viewers = pil.livestream_obj.get('viewer_count', 0)
if sep:
logger.separator()
else:
logger.info('Username : {:s}'.format(pil.dl_user))
logger.info('Viewers : {:s} watching'.format(str(
int(viewers))))
logger.info('Airing time : {:s}'.format(get_stream_duration(0)))
if pil.do_heartbeat:
logger.info('Status : {:s}'.format(
heartbeat_info.get('broadcast_status').title()))
return heartbeat_info.get('broadcast_status') not in [
'active', 'interrupted'
]
else:
return None
mpd_url = (pil.livestream_obj.get('dash_manifest')
or pil.livestream_obj.get('dash_abr_playback_url')
or pil.livestream_obj.get('dash_playback_url'))
pil.live_folder_path = '{}{}_{}_{}_{}_live_downloads'.format(
pil.dl_path, pil.datetime_compat, pil.dl_user,
pil.livestream_obj.get('id'), pil.epochtime)
pil.broadcast_downloader = live.Downloader(
mpd=mpd_url,
output_dir=pil.live_folder_path,
user_agent=pil.ig_api.user_agent,
max_connection_error_retry=3,
duplicate_etag_retry=30,
callback_check=print_status,
mpd_download_timeout=3,
download_timeout=3,
ffmpeg_binary=pil.ffmpeg_path)
except Exception as e:
logger.error('Could not start downloading livestream: {:s}'.format(
str(e)))
logger.separator()
helpers.remove_lock()
try:
broadcast_owner = pil.livestream_obj.get('broadcast_owner',
{}).get('username')
try:
broadcast_guest = pil.livestream_obj.get('cobroadcasters',
{})[0].get('username')
except Exception:
broadcast_guest = None
if broadcast_owner != pil.dl_user:
logger.binfo(
'This livestream is a dual-live, the owner is "{}".'.format(
broadcast_owner))
broadcast_guest = None
if broadcast_guest:
logger.binfo(
'This livestream is a dual-live, the current guest is "{}".'.
format(broadcast_guest))
pil.has_guest = broadcast_guest
logger.separator()
print_status(False)
logger.separator()
helpers.create_lock_folder()
pil.segments_json_thread_worker = threading.Thread(
target=helpers.generate_json_segments)
pil.segments_json_thread_worker.start()
logger.info('Downloading livestream, press [CTRL+C] to abort.')
if pil.run_at_start:
try:
thread = threading.Thread(target=helpers.run_command,
args=(pil.run_at_start, ))
thread.daemon = True
thread.start()
logger.binfo("Launched start command: {:s}".format(
pil.run_at_start))
except Exception as e:
logger.warn('Could not launch command: {:s}'.format(str(e)))
if pil.dl_comments:
try:
comments_json_file = '{}{}_{}_{}_{}_live_comments.json'.format(
pil.dl_path, pil.datetime_compat, pil.dl_user,
pil.livestream_obj.get('id'), pil.epochtime)
pil.comment_thread_worker = threading.Thread(
target=get_live_comments, args=(comments_json_file, ))
pil.comment_thread_worker.start()
except Exception as e:
logger.error(
'An error occurred while downloading comments: {:s}'.
format(str(e)))
pil.broadcast_downloader.run()
logger.separator()
logger.info("The livestream has been ended by the user.")
logger.separator()
logger.info('Airtime duration : {}'.format(get_stream_duration(0)))
logger.info('Download duration : {}'.format(get_stream_duration(1)))
logger.info('Missing (approx.) : {}'.format(get_stream_duration(2)))
logger.separator()
merge_segments()
except KeyboardInterrupt:
logger.separator()
logger.binfo('The download has been aborted.')
logger.separator()
logger.info('Airtime duration : {}'.format(get_stream_duration(0)))
logger.info('Download duration : {}'.format(get_stream_duration(1)))
logger.info('Missing (approx.) : {}'.format(get_stream_duration(2)))
logger.separator()
if not pil.broadcast_downloader.is_aborted:
pil.broadcast_downloader.stop()
merge_segments()
def download_livestream(broadcast):
try:
def print_status(sep=True):
heartbeat_info = instagram_api.broadcast_heartbeat_and_viewercount(broadcast.get('id'))
viewers = broadcast.get('viewer_count', 0)
if sep:
seperator("GREEN")
log('[I] Viewers : {:s} watching'.format(str(int(viewers))), "GREEN")
log('[I] Airing time : {:s}'.format(get_stream_duration(broadcast.get('published_time'))), "GREEN")
log('[I] Status : {:s}'.format(heartbeat_info.get('broadcast_status').title()), "GREEN")
return heartbeat_info.get('broadcast_status') not in ['active', 'interrupted']
mpd_url = (broadcast.get('dash_manifest')
or broadcast.get('dash_abr_playback_url')
or broadcast.get('dash_playback_url'))
output_dir = settings.save_path + '{}_{}_{}_{}_live_downloads'.format(settings.current_date, user_to_record, broadcast.get('id'), settings.current_time)
broadcast_downloader = live.Downloader(
mpd=mpd_url,
output_dir=output_dir,
user_agent=instagram_api.user_agent,
max_connection_error_retry=3,
duplicate_etag_retry=30,
callback_check=print_status,
mpd_download_timeout=3,
download_timeout=3)
except Exception as e:
log('[E] Could not start downloading livestream: {:s}'.format(str(e)), "RED")
seperator("GREEN")
sys.exit(1)
try:
log('[I] Livestream found, beginning download...', "GREEN")
broadcast_owner = broadcast.get('broadcast_owner', {}).get('username')
try:
broadcast_guest = broadcast.get('cobroadcasters', {})[0].get('username')
except:
broadcast_guest = None
if (broadcast_owner != user_to_record):
log('[I] This livestream is a dual-live, the owner is "{}".'.format(broadcast_owner), "BLUE")
broadcast_guest = None
if broadcast_guest:
log('[I] This livestream is a dual-live, the guest is "{}".'.format(broadcast_guest), "BLUE")
seperator("GREEN")
log('[I] Username : {:s}'.format(user_to_record), "GREEN")
print_status(False)
log('[I] MPD URL : {:s}'.format(mpd_url), "GREEN")
seperator("GREEN")
log('[I] Downloading livestream... press [CTRL+C] to abort.', "GREEN")
if (settings.run_at_start is not "None"):
try:
thread = threading.Thread(target=run_command, args=(settings.run_at_start,))
thread.daemon = True
thread.start()
log("[I] Command executed: \033[94m{:s}".format(settings.run_at_start), "GREEN")
except Exception as e:
log('[W] Could not execute command: {:s}'.format(str(e)), "YELLOW")
comment_thread_worker = None
if settings.save_comments.title() == "True":
try:
comments_json_file = settings.save_path + '{}_{}_{}_{}_live_comments.json'.format(settings.current_date, user_to_record, broadcast.get('id'), settings.current_time)
comment_thread_worker = threading.Thread(target=get_live_comments, args=(instagram_api, broadcast, comments_json_file, broadcast_downloader,))
comment_thread_worker.start()
except Exception as e:
log('[E] An error occurred while checking comments: {:s}'.format(str(e)), "RED")
broadcast_downloader.run()
seperator("GREEN")
log('[I] The livestream has ended.\n[I] Time recorded : {}\n[I] Stream duration : {}\n[I] Missing (approx.) : {}'.format(get_stream_duration(int(settings.current_time)), get_stream_duration(broadcast.get('published_time')), get_stream_duration(int(settings.current_time), broadcast)), "YELLOW")
seperator("GREEN")
stitch_video(broadcast_downloader, broadcast, comment_thread_worker)
except KeyboardInterrupt:
seperator("GREEN")
log('[I] The download has been aborted by the user.\n[I] Time recorded : {}\n[I] Stream duration : {}\n[I] Missing (approx.) : {}'.format(get_stream_duration(int(settings.current_time)), get_stream_duration(broadcast.get('published_time')), get_stream_duration(int(settings.current_time), broadcast)), "YELLOW")
seperator("GREEN")
if not broadcast_downloader.is_aborted:
broadcast_downloader.stop()
stitch_video(broadcast_downloader, broadcast, comment_thread_worker)
except Exception as e:
log("[E] Could not download livestream: {:s}".format(str(e)), "RED")