def getTwitchStreamInfo(self) -> List[Union[bool, HLS]]:
arguments = {
'allow_source': 'true',
'allow_spectre': 'true',
'p': randint(1000000, 10000000),
'sig': self.access_token['sig'].encode('utf-8'),
'supported_codecs': 'avc1',
'token': self.access_token['token'].encode('utf-8'),
'cdm': 'wv',
}
manifest_url = '{0}/api/channel/hls/{1}.m3u8?{2}'.format(
self._USHER_START_PATH, self.channel_name, urlencode(arguments))
download_object = download_website(
manifest_url,
CookieDict=self.sharedCookieDict,
headers={
'Accept':
'application/x-mpegURL, application/vnd.apple.mpegurl, application/json, text/plain'
})
if download_object.status_code == 404:
# TRANSCODE DOESNT EXIST. THAT IS NORMAL. SO NOT LIVE
return [False, None]
elif download_object.response_headers[
'Content-Type'] == "application/vnd.apple.mpegurl":
return [True, download_object.parse_m3u8_formats()]
print("Unable To Handle Status: {0} For Twitch.".format(
download_object.status_code))
return [False, None]
def __callAPI__(self, path):
url_referer = 'https://www.twitch.tv/{0}'.format(self.channel_name)
download_object = download_website(
'{0}/{1}'.format(self._API_START_PATH, path),
headers={
'DNT': '1',
'Referer': url_referer,
'Sec-Fetch-Mode': 'cors',
'Client-ID': self.globalVariables.get("client_id")
},
CookieDict=self.sharedCookieDict)
return download_object.parse_json()
def loadVideoData(self):
url = "https://www.twitch.tv/{0}".format(self.channel_name)
download_object = download_website(url,
CookieDict=self.sharedCookieDict)
if download_object.status_code == 404:
return [
False,
"Failed getting Twitch Data! \"{0}\" doesn't exist as a channel name!"
.format(self.channel_name)
]
if download_object.text is None:
return [
False, "Failed getting Youtube Data from the internet! "
"This means there is no good internet available!"
]
if self.globalVariables.get("client_id") is None:
verbose("Getting Client ID. [TWITCH]")
okay, client_id = find_client_id(download_object.text)
if okay is False:
warning(client_id)
self.globalVariables.set("client_id", client_id)
verbose('Getting Channel ID. [TWITCH]')
self.access_token = self.__callAPI__(
'api/channels/{0}/access_token?{1}&oauth_token'.format(
self.channel_name,
urlencode({
'need_https': 'true',
'platform': 'web',
'player_backend': 'mediaplayer',
'player_type': 'site'
})))
token = parse_json(self.access_token['token'])
self.channel_id = try_get(token, lambda x: x['channel_id'])
# website_dict = self.__callAPI__('kraken/channels/{0}'.format(
# self.channel_name))
# self.channel_image = try_get(website_dict, lambda x: x['logo'])
# self.channel_id = try_get(website_dict, lambda x: x['_id'], int)
self.live_streaming, hls = self.getTwitchStreamInfo()
if hls is not None:
self.StreamFormat = get_format_from_data(
hls, self.cachedDataHandler.getValue('recordingResolution'))
if not self.channel_id:
return [False, "Unable to find Channel ID."]
return [True, "OK"]
def get_sponsor_channel(self, html_code=None):
# from .. import is_google_account_login_in
if True:
verbose("Checking if account sponsored {0}.".format(
self.channel_name))
if html_code is None:
html_code = download_website(
"https://www.youtube.com/channel/{0}/live".format(
self.channel_id),
CookieDict=self.sharedCookieDict)
if html_code is None:
return None
html_code = str(html_code)
array = re.findall(
'/channel/{0}/membership'.format(self.channel_id), html_code)
if array:
return True
return False
def searchChannels(search, CookieDict, globalVariables):
"""
:type CookieDict: dict
:type globalVariables: GlobalVariables
"""
def formatChannel(channel):
return {
'channel_identifier':
try_get(channel, lambda x: x['displayName'], str),
'channel_name':
try_get(channel, lambda x: x['displayName'], str),
'channel_image':
try_get(channel, lambda x: x['profileImageURL'], str),
'follower_count':
str(try_get(channel, lambda x: x['followers']['totalCount'], int)),
'platform':
'TWITCH'
}
referer = 'https://www.twitch.tv/search?term={0}&type=channels'.format(
search)
client_id = globalVariables.get("client_id")
if client_id is None:
verbose("Getting Client ID. [TWITCH]")
downloadOBJECT = download_website(referer, CookieDict=CookieDict)
okay, client_id = find_client_id(downloadOBJECT.text)
if okay is False:
warning(client_id)
globalVariables.set("client_id", client_id)
data = [{
"operationName": "SearchResultsPage_SearchResults",
"variables": {
"query": search,
"options": {
"targets": [{
"index": "CHANNEL"
}]
}
},
"extensions": {
"persistedQuery": {
"version":
1,
"sha256Hash":
'1d3ca64005f07f8e34a33677d119ba8601e7deac745ea127e67b7925535ed735'
}
}
}] # TODO: reverse the sha265Hash. :p
downloadOBJECT = download_website('https://gql.twitch.tv/gql',
RequestMethod='POST',
data=data,
headers={
'Origin': 'https://www.twitch.tv',
'Referer': referer,
'Client-Id': client_id,
'Content-Length': '225'
},
CookieDict=CookieDict)
data = try_get(downloadOBJECT.parse_json(), lambda x: x[0]['data'], dict)
items = try_get(data, lambda x: x['searchFor']['channels']['items'], list)
channels = list(map(formatChannel, items))
return [True, {'channels': channels}]
def is_live(channel_Class, CookieDict=None, globalVariables=None, json=None):
"""
Checks if channel is live using the normal Youtube heartbeat.
Also sets heartbeat related variables.
:type CookieDict: dict
:type channel_Class: TemplateChannel
:type globalVariables: GlobalVariables
"""
if globalVariables is None:
globalVariables = TempGlobalVariables()
try:
if json is None:
referer_url = 'https://www.youtube.com/channel/{0}/live'.format(
channel_Class.channel_id)
headers = {
'Accept': "*/*",
'Accept-Language': 'en-US,en;q=0.9',
'dnt': '1',
'referer': referer_url,
'x-youtube-client-name': '1'
}
url_arguments = {
'video_id':
channel_Class.video_id,
'heartbeat_token':
'',
'c':
(globalVariables.get("client_name")
if globalVariables.get("client_name") is not None else 'WEB'),
'sequence_number':
str(channel_Class.sequence_number)
}
if globalVariables.get("account_playback_token") is not None:
headers.update({
'x-youtube-identity-token':
globalVariables.get("account_playback_token")
})
if globalVariables.get("page_build_label") is not None:
headers.update({
'x-youtube-page-label':
globalVariables.get("page_build_label")
})
if globalVariables.get("page_cl") is not None:
headers.update(
{'x-youtube-page-cl': globalVariables.get("page_cl")})
if globalVariables.get("variants_checksum") is not None:
headers.update({
'x-youtube-variants-checksum':
globalVariables.get("variants_checksum")
})
if globalVariables.get("utf_offset") is not None:
headers.update({
'x-youtube-utc-offset':
str(globalVariables.get("utf_offset"))
})
url_arguments.update({
'utc_offset_minutes':
str(globalVariables.get("utf_offset"))
})
if globalVariables.get("client_version") is not None:
headers.update({
'x-youtube-client-version':
globalVariables.get("client_version")
})
url_arguments.update(
{'cver': str(globalVariables.get("client_version"))})
if globalVariables.get("timezone") is not None:
url_arguments.update(
{'time_zone': str(globalVariables.get("timezone"))})
if channel_Class.cpn is not None:
url_arguments.update({'cpn': channel_Class.cpn})
websiteClass = download_website(
'https://www.youtube.com/heartbeat?{0}'.format(
urlencode(url_arguments)),
headers=headers,
CookieDict=CookieDict)
CookieDict.update(websiteClass.cookies)
if websiteClass.text is None:
return None
json = websiteClass.parse_json()
channel_Class.sequence_number += 1
reply('FROM YOUTUBE -> {0}'.format(json))
# SETTING VARIABLES
liveStreamAbilityRenderer = try_get(
json,
lambda x: x['liveStreamability']['liveStreamabilityRenderer'],
dict)
if liveStreamAbilityRenderer:
thumbnail = get_thumbnail(liveStreamAbilityRenderer)
if thumbnail:
channel_Class.thumbnail_url = thumbnail
channel_Class.pollDelayMs = get_poll_delay_ms(
liveStreamAbilityRenderer, channel_Class)
channel_Class.live_scheduled = get_unix_schedule_time(
liveStreamAbilityRenderer) is not None
channel_Class.broadcast_id = get_broadcast_id(
liveStreamAbilityRenderer)
video_id = get_video_id(liveStreamAbilityRenderer)
if video_id:
if video_id != channel_Class.video_id:
channel_Class.add_youtube_queue(
) # just in case something happens.
channel_Class.video_id = video_id
if channel_Class.live_scheduled is True:
channel_Class.live_scheduled_timeString = get_schedule_time(
liveStreamAbilityRenderer)
unix_time = get_unix_schedule_time(liveStreamAbilityRenderer)
if unix_time:
channel_Class.live_scheduled_time = datetime.fromtimestamp(
unix_time)
if 'stop_heartbeat' in json:
channel_Class.add_youtube_queue()
channel_Class.loadVideoData()
return False
if try_get(liveStreamAbilityRenderer, lambda x: x['displayEndscreen'],
bool):
last_video_id = channel_Class.video_id
channel_Class.loadVideoData()
# CHECK IF A VIDEO ID CHANGE BEFORE ADDING.
if last_video_id != channel_Class.video_id:
channel_Class.add_youtube_queue()
return False
status = try_get(json, lambda x: x['status'], str) # type: str
if status: # Sometimes status is removed and causes an error.
if "OK" in status.upper():
return True
if "STOP" in status.upper():
channel_Class.add_youtube_queue()
channel_Class.loadVideoData()
return False
if "ERROR" in status.upper():
warning("Getting the Live Data, failed on Youtube's Side. "
"Youtube Replied with: {0}.".format(json['reason']))
return False
if "LIVE_STREAM_OFFLINE" in status.upper():
return False
warning(
"The Program couldn't find any value that matches the normal heartbeat. Returning False."
)
return False
except KeyboardInterrupt:
pass
except BrokenPipeError:
exit()
except Exception:
warning("Error occurred when doing Heartbeat.")
error_warning(traceback.format_exc())
return -1
def loadVideoData(self, video_id=None):
self.video_id = video_id
if video_id:
url = "https://www.youtube.com/watch?v={0}".format(video_id)
else:
url = "https://www.youtube.com/channel/{0}/live".format(
self.channel_id)
website_object = download_website(url,
CookieDict=self.sharedCookieDict)
# self.sharedCookieDict.update(websiteClass.cookies)
if website_object.text is None:
return [
False, "Failed getting Youtube Data from the internet! "
"This means there is no good internet available!"
]
if website_object.status_code == 404:
return [
False,
"Failed getting Youtube Data! \"{0}\" doesn't exist as a channel id!"
.format(self.channel_id)
]
website_string = website_object.text
endpoint_type = get_endpoint_type(website_string)
if endpoint_type:
if endpoint_type == 'browse':
array = re.findall(r'property="og:title" content="(.+?)"',
website_string)
if array:
channel_name = array[0]
warning(
"{0} has the live stream "
"currently unlisted or private, or only for members. "
"Using safeguard. This may not be the best to leave on.\n"
.format(channel_name))
self.channel_name = channel_name
self.video_id = None
self.privateStream = True
else:
if not endpoint_type == 'watch':
warning("Unrecognized endpoint type. Endpoint Type: {0}.".
format(endpoint_type))
verbose("Getting Video ID.")
youtube_initial_data = get_yt_initial_data(website_string)
yt_player_config = try_get(
get_yt_player_config(website_string), lambda x: x, dict)
player_response = parse_json(
try_get(yt_player_config,
lambda x: x['args']['player_response'], str))
videoDetails = try_get(player_response,
lambda x: x['videoDetails'], dict)
if yt_player_config and videoDetails:
if "isLiveContent" in videoDetails and \
videoDetails['isLiveContent'] and \
("isLive" in videoDetails or "isUpcoming" in videoDetails):
self.channel_name = try_get(videoDetails,
lambda x: x['author'], str)
self.video_id = try_get(videoDetails,
lambda x: x['videoId'], str)
self.privateStream = False
if not self.channel_id:
self.channel_id = try_get(videoDetails,
lambda x: x['channelId'],
str)
else:
return [
False,
"Found a stream, the stream seemed to be a non-live stream."
]
else:
return [
False,
"Unable to get yt player config, and videoDetails."
]
contents = try_get(
youtube_initial_data,
lambda x: x['contents']['twoColumnWatchNextResults'][
'results']['results']['contents'], list)
videoSecondaryInfoRenderer = try_get([
content for content in contents
if content.get("videoSecondaryInfoRenderer") is not None
], lambda x: x[0], dict).get("videoSecondaryInfoRenderer")
channelImageFormats = try_get(
videoSecondaryInfoRenderer, lambda x: x['owner'][
'videoOwnerRenderer']['thumbnail']['thumbnails'], list)
if channelImageFormats is not None:
self.channel_image = max(
channelImageFormats,
key=lambda x: x.get("height")).get("url")
if not self.privateStream:
# TO AVOID REPEATING REQUESTS.
if player_response:
# playabilityStatus is legit heartbeat all over again..
playabilityStatus = try_get(
player_response, lambda x: x['playabilityStatus'],
dict)
status = try_get(playabilityStatus,
lambda x: x['status'],
str) # type: str
reason = try_get(playabilityStatus,
lambda x: x['reason'],
str) # type: str
if playabilityStatus and status:
if 'OK' in status.upper():
if reason and 'ended' in reason:
return [False, reason]
streamingData = try_get(
player_response,
lambda x: x['streamingData'], dict)
if streamingData:
if 'licenseInfos' in streamingData:
licenseInfo = streamingData.get(
'licenseInfos')
drmFamilies = map(
lambda x: x.get('drmFamily'),
licenseInfo)
return [
False,
"This live stream contains DRM and cannot be recorded.\n"
"DRM Families: {0}".format(
', '.join(drmFamilies))
]
manifest_url = str(
try_get(streamingData,
lambda x: x['hlsManifestUrl'],
str))
if not manifest_url:
return [
False,
"Unable to find HLS Manifest URL."
]
downloadOBJECT = download_website(
manifest_url,
CookieDict=self.sharedCookieDict)
hls = downloadOBJECT.parse_m3u8_formats()
if len(hls.formats) == 0:
return [
False,
"There were no formats found! Even when the streamer is live."
]
format_ = get_format_from_data(
hls,
self.cachedDataHandler.getValue(
'recordingResolution'))
if not videoDetails:
videoDetails = try_get(
player_response,
lambda x: x['videoDetails'], dict)
thumbnails = try_get(
videoDetails,
lambda x: x['thumbnail']['thumbnails'],
list)
if thumbnails:
self.thumbnail_url = get_highest_thumbnail(
thumbnails)
self.dvr_enabled = try_get(
videoDetails,
lambda x: x['isLiveDvrEnabled'], bool)
self.StreamFormat = format_
self.title = try_get(
videoDetails, lambda x: x['title'],
str)
self.description = videoDetails[
'shortDescription']
else:
return [
False,
"No StreamingData, YouTube bugged out!"
]
self.live_streaming = self.is_live(
json=playabilityStatus)
# GET YOUTUBE GLOBAL VARIABLES
if self.globalVariables.get(
"checkedYouTubeVariables") is None:
def getSettingsValue(ServiceSettings,
settings_nameLook,
name=None):
for service in ServiceSettings:
service_name = try_get(service,
lambda x: x['key'], str)
if service_name is not None and service_name in settings_nameLook:
value = try_get(service,
lambda x: x['value'], str)
if name:
if not value:
warning(
"Something happened when finding the "
+ name)
return None
return value
return None
def getServiceSettings(serviceTrackingParamsList,
service_nameLook):
if serviceTrackingParamsList:
for service in serviceTrackingParamsList:
service_name = try_get(
service, lambda x: x['service'], str)
if service_name is not None and service_name in service_nameLook:
return service
return None
if self.globalVariables.get(
"alreadyChecked"
) is False or self.globalVariables.get(
"alreadyChecked") is None:
verbose("Getting Global YouTube Variables.")
e_catcher = getServiceSettings(
try_get(
youtube_initial_data,
lambda x: x['responseContext'][
'serviceTrackingParams'], list),
"ECATCHER")
account_playback_token = try_get(
yt_player_config, lambda x: x['args'][
'account_playback_token'][:-1], str)
ps = try_get(yt_player_config,
lambda x: x['args']['ps'], str)
cbr = try_get(yt_player_config,
lambda x: x['args']['cbr'])
client_os = try_get(yt_player_config,
lambda x: x['args']['cos'])
client_os_version = try_get(
yt_player_config,
lambda x: x['args']['cosver'])
if account_playback_token is None:
warning(
"Unable to find account playback token in the YouTube player config."
)
if ps is None:
warning(
"Unable to find ps in the YouTube player config."
)
if cbr is None:
warning(
"Unable to find cbr in the YouTube player config."
)
if client_os is None:
warning(
"Unable to find Client OS in the YouTube player config."
)
if client_os_version is None:
warning(
"Unable to find Client OS Version in the YouTube player config."
)
self.globalVariables.set("checkedYouTubeVariables",
None)
if not youtube_initial_data:
warning(
"Unable to get Youtube Initial Data. Cannot find all Youtube Variables."
)
elif e_catcher is None:
warning(
"Unable to get ECATCHER service data in Youtube Initial Data. "
"Cannot find all Youtube Variables.")
else:
params = try_get(e_catcher,
lambda x: x['params'], list)
page_build_label = getSettingsValue(
params,
'innertube.build.label',
name="Page Build Label")
page_cl = getSettingsValue(
params,
'innertube.build.changelist',
name="Page CL")
variants_checksum = getSettingsValue(
params,
'innertube.build.variants.checksum',
name="Variants Checksum")
client_version = getSettingsValue(
params,
'client.version',
name="Client Version")
client_name = getSettingsValue(
params, 'client.name', name="Client Name")
self.globalVariables.set(
"page_build_label", page_build_label)
self.globalVariables.set("page_cl", page_cl)
self.globalVariables.set(
"client_version", client_version)
self.globalVariables.set(
"client_name", client_name)
self.globalVariables.set(
"variants_checksum", variants_checksum)
self.globalVariables.set("ps", ps)
self.globalVariables.set("cbr", cbr)
self.globalVariables.set("client_os", client_os)
self.globalVariables.set("client_os_version",
client_os_version)
self.globalVariables.set("account_playback_token",
account_playback_token)
self.globalVariables.set("utf_offset",
get_utc_offset())
self.globalVariables.set("timezone", getTimeZone())
self.globalVariables.set("alreadyChecked", True)
# ONLY WORKS IF LOGGED IN
self.sponsor_on_channel = self.get_sponsor_channel(
html_code=website_string)
self.cpn = self.generate_cpn()
return [True, "OK"]
def searchChannels(search, CookieDict):
def formatChannel(channelContent):
channelRenderer = channelContent.get("channelRenderer")
channel_image = None
channelImageFormats = try_get(channelRenderer,
lambda x: x['thumbnail']['thumbnails'],
list)
if channelImageFormats is not None:
channel_image = "https:{0}".format(
max(channelImageFormats,
key=lambda x: x.get("height")).get("url"))
channel = {
'channel_identifier':
try_get(channelRenderer, lambda x: x['channelId'], str),
'channel_name':
try_get(channelRenderer, lambda x: x['title']['simpleText'], str),
'follower_count':
try_get(channelRenderer,
lambda x: x['subscriberCountText']['simpleText'], str),
'channel_image':
channel_image,
'platform':
'YOUTUBE'
}
return channel
def formatDidYouMean(didYouMeanContent):
didYouMeanRenderer = didYouMeanContent.get("didYouMeanRenderer")
didYouMean = {
'didYouMean':
try_get(didYouMeanRenderer,
lambda x: x['didYouMean']['runs'][0]['text'], str),
'correctedQuery':
try_get(didYouMeanRenderer,
lambda x: x['correctedQuery']['runs'][0]['text'], str)
}
return didYouMean
# For some stupid reason, YouTube will only provide Searches on BR Content Encoding
downloadOBJECT = download_website(
"https://www.youtube.com/results?{0}".format(
urlencode({
'search_query': search,
'sp': 'EgIQAg%3D%3D'
})),
headers={
'Accept-Encoding': 'gzip, deflate, br',
'Accept': '*/*',
},
CookieDict=CookieDict)
if downloadOBJECT.response_headers.get('Content-Encoding') == 'br':
# Check for support for that.
try:
import brotli
except ImportError:
return [
False,
'No Support for BR Content Encoding on Server. Required Packages: requests, brotlipy.'
]
websiteString = downloadOBJECT.text
youtube_initial_data = get_yt_initial_data(websiteString)
if youtube_initial_data is None:
return [False, 'Unable to find YouTube initial data.']
contents = try_get(
youtube_initial_data,
lambda x: x['contents']['twoColumnSearchResultsRenderer'][
'primaryContents']['sectionListRenderer']['contents'], list)
if contents is None:
return [False, 'Unable to find contents.']
itemSectionRenderer = try_get([
content for content in contents
if content.get("itemSectionRenderer") is not None
] or [], lambda x: x[0], dict).get("itemSectionRenderer")
if itemSectionRenderer is None:
return [False, 'Unable to find itemSectionRenderer.']
contents = itemSectionRenderer.get("contents")
if itemSectionRenderer is None:
return [False, "Unable to find itemSectionRenderer contents."]
channels = list(
map(formatChannel, [x for x in contents if 'channelRenderer' in x]))
response = {'channels': channels}
didYouMean = list(
map(formatDidYouMean,
[x for x in contents if 'didYouMeanRenderer' in x]))
if len(didYouMean) > 0:
response.update({'didYouMean': didYouMean})
return [True, response]
def get_video_info(self):
"""
Gets the stream info from channelClass.
Looked at for reference:
https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L1675
"""
url_arguments = {'html5': 1, 'video_id': self.video_id}
if self.globalVariables.get("ps") is not None:
url_arguments.update({'ps': self.globalVariables.get("ps")})
url_arguments.update({'eurl': ''})
url_arguments.update({'hl': 'en_US'})
if self.globalVariables.get("client_name") is not None:
url_arguments.update(
{'c': self.globalVariables.get("client_name")})
if self.globalVariables.get("cbr") is not None:
url_arguments.update({'cbr': self.globalVariables.get("cbr")})
if self.globalVariables.get("client_version") is not None:
url_arguments.update(
{'cver': self.globalVariables.get("client_version")})
if self.globalVariables.get("client_os") is not None:
url_arguments.update(
{'cos': self.globalVariables.get("client_os")})
if self.globalVariables.get("client_os_version") is not None:
url_arguments.update(
{'cosver': self.globalVariables.get("client_os_version")})
if self.cpn is not None:
url_arguments.update({'cpn': self.cpn})
downloadClass = download_website(
'https://www.youtube.com/get_video_info?{0}'.format(
urlencode(url_arguments)),
CookieDict=self.sharedCookieDict)
video_info_website = downloadClass.text
video_info = parse_qs(video_info_website)
player_response = parse_json(
try_get(video_info, lambda x: x['player_response'][0], str))
if player_response:
video_details = try_get(player_response,
lambda x: x['videoDetails'], dict)
if "streamingData" not in player_response:
warning("No StreamingData, Youtube bugged out!")
return None
manifest_url = str(
try_get(player_response,
lambda x: x['streamingData']['hlsManifestUrl'], str))
if not manifest_url:
warning("Unable to find HLS Manifest URL.")
return None
downloadOBJECT = download_website(manifest_url,
CookieDict=self.sharedCookieDict)
if downloadOBJECT.status_code != 200:
return None
hls = downloadOBJECT.parse_m3u8_formats()
if len(hls.formats) == 0:
warning(
"There were no formats found! Even when the streamer is live."
)
return None
return {
'formats': hls,
'manifest_url': manifest_url,
'video_details': video_details,
}
return None