def youtube(request):
api = Api(api_key="AIzaSyDHAS3sDLVtUqM1vx-kxykrBHMVSi0BLJI")
query = request.session['user-input']
res = api.search_by_keywords(q=query,
search_type=["channel"],
count=25,
limit=8)
res = res.to_dict()
res_items = res["items"]
result = []
for data in res_items:
temp = {
"channel_name":
data["snippet"]["title"],
"channel_url":
"https:/www.youtube.com/channel/" +
str(data["snippet"]["channelId"]),
"channel_logo":
data["snippet"]["thumbnails"]["default"]["url"]
}
result.append(temp)
context = {"result": result, "text": query}
return render(request, 'youtube.html', context)
def get_latest_video(self, youtubeChannelName):
res = {}
apiKey = 'AIzaSyBOCLFDDz4wHFmatH-fPxsjjRnBfPzcOFQ'
try:
api = YoutubeApi(api_key=apiKey)
r = api.search_by_keywords(q=youtubeChannelName,
search_type=["channel"],
count=2,
limit=2)
idChannel = r.items[0].snippet.channelId
link = 'https://www.googleapis.com/youtube/v3/search?key=' + apiKey + '&channelId=' + idChannel + '&part=snippet,id&order=date&maxResults=2'
resp = requests.get(url=link)
data = resp.json()
if (data):
res['result'] = data['items'][0]
else:
res['result'] = {
'error': 'Youtube url not updated',
'msg': 'Channel not found.'
}
return jsonify(res)
except Exception as e:
print("error : get_channel_ID\n", str(e), flush=True)
res['error'] = "internal error"
res['message'] = 'Youtube Data API exceded'
return res
def get_number_subscribers_youtube_channel(self, youtubeChannelName):
res = {}
apiKey = 'AIzaSyBOCLFDDz4wHFmatH-fPxsjjRnBfPzcOFQ'
try:
api = YoutubeApi(api_key=apiKey)
r = api.search_by_keywords(q=youtubeChannelName,
search_type=["channel"],
count=2,
limit=2)
idChannel = r.items[0].snippet.channelId
channel = api.get_channel_info(channel_id=idChannel)
if (channel.items):
res['result'] = {
'subscriberCount':
channel.items[0].statistics.subscriberCount,
'channel': channel.items[0].snippet
}
else:
res['result'] = {
'error': 'Youtube url not updated',
'msg': 'Channel not found.'
}
return jsonify(res)
except Exception as e:
print("error : get_number_subscribers_youtube_channel\n",
str(e),
flush=True)
res['error'] = "internal error"
res['message'] = 'Youtube Data API exceded'
return res
def get_all_comments(YOUTUBE_API_KEY,
query,
count_video=10,
limit=30,
maxResults=10,
nextPageToken=''):
"""
Выгрузка maxResults комментариев
"""
api = Api(api_key=YOUTUBE_API_KEY)
video_by_keywords = api.search_by_keywords(q=query,
search_type=["video"],
count=count_video,
limit=limit)
videoId = [x.id.videoId for x in video_by_keywords.items]
comments_all = []
for id_video in videoId:
try:
data = get_data(YOUTUBE_API_KEY,
id_video,
maxResults=maxResults,
nextPageToken=nextPageToken)
comment = list(get_text_of_comment(data))
comments_all.append(comment)
except:
continue
comments = sum(comments_all, [])
return comments
class YTSearch():
def __init__(self, api_key=API_KEY):
self.key = api_key
self.api = Api(api_key=self.key)
def check_video_eligibility(self, ids, expected_type):
like_max = {"yt_id": "", "youtube_rating": 0.0}
for id in ids:
video = list(
self.api.get_video_by_id(video_id=id).to_dict().items())
try:
vid = video[5][1][0]
duration = isodate.parse_duration(
vid['contentDetails']['duration']).total_seconds()
definition = vid['contentDetails']['definition']
like_count = float(vid["statistics"]["likeCount"])
dislike_count = float(vid["statistics"]["dislikeCount"])
duration_min = Duration[expected_type][0]
duration_max = Duration[expected_type][1]
like_percentage = like_count / (like_count + dislike_count)
channel_title = vid["snippet"]["channelTitle"]
if like_percentage > like_max["youtube_rating"] and (duration_min < duration <= duration_max) and \
channel_title != 'YouTube Movies':
like_max["yt_id"] = id
like_max["yt_duration"] = duration
like_max["yt_likes"] = like_count
like_max["youtube_rating"] = like_percentage
like_max["yt_definition"] = definition
except Exception as e:
print("ignoring the error")
continue
return like_max
def obsolete_get_stats(self,
key_word,
expected_type="movie",
max_results=10):
videos = list(
self.api.search_by_keywords(q=key_word,
search_type=["video"]).items)
vids = [vid.to_dict()["id"]['videoId'] for vid in videos]
ret = self.check_video_eligibility(vids, expected_type)
print(ret)
def get_youtube_stats(self,
key_word,
expected_type="movie",
max_results=10):
get_mov_stats = YoutubeSearch(key_word,
max_results=max_results).to_dict()
ids = [d['id'] for d in get_mov_stats]
ret = self.check_video_eligibility(ids, expected_type)
for mov in get_mov_stats:
if mov["id"] == ret["yt_id"]:
ret["youtube_link"] = "https://youtube.com" + mov["link"]
return (ret)
async def find_first_youtube_match(keyword: str):
youtube = Api(api_key=os.environ['YOUTUBE_TOKEN'])
results = youtube.search_by_keywords(
q=keyword,
search_type=('video', ),
count=1,
).items
if len(results):
msg = f'https://www.youtube.com/watch?v={results[0].id.videoId}'
else:
msg = 'I have found nothing'
return msg
def get_channel_ID(self, youtubeChannelName):
res = {}
apiKey = 'AIzaSyBOCLFDDz4wHFmatH-fPxsjjRnBfPzcOFQ'
try:
api = YoutubeApi(api_key=apiKey)
r = api.search_by_keywords(q=youtubeChannelName,
search_type=["channel"],
count=2,
limit=2)
idChannel = r.items[0].snippet.channelId
if (idChannel):
res['result'] = idChannel
else:
res['result'] = {
'error': 'Youtube url not updated',
'msg': 'Channel not found.'
}
return jsonify(res)
except Exception as e:
print("error : get_channel_ID\n", str(e), flush=True)
res['error'] = "internal error"
res['message'] = 'Youtube Data API exceded'
return res
class Video_Search_Json:
"""Class to retrieve videos
This class helps query and search our video database
using an API YouTube key if needed to pull from Youtube.
Automatically updates the json file.
"""
def __init__(self, API_KEY, JSON_PATH):
# Declare an API object using our API_KEY
self.api = Api(api_key=API_KEY)
self.data_path = JSON_PATH
with open(JSON_PATH) as f:
self.database = json.load(f)
def search_by_keywords(self, subtopic, channels=[Channel['MIT']], num_videos=10, force_query=False, include_transcripts=True):
"""Searches for videos by subtopic
Takes in a query string to search for on youtube.
Returns a JSON.
Parameters:
subtopic -- the subtopic to search for in the query
channels -- a list of Channel enums to specify which channels that must be included (default MIT OpenCourseWare)
num_vidoes -- minimum number of videos to include (default 10)
force_query -- whether or not to query regardless of inclusion in json (default False)
include_transcripts -- whether or not to include transcripts (default True)
"""
if subtopic not in self.database.keys() and not force_query:
# YouTube retrieve
self.query_youtube(subtopic, channels, num_videos=num_videos, include_transcripts=include_transcripts)
self.write_to_json(self.database)
return self.database[subtopic]
def query_youtube(self, subtopic, channels=[], num_videos=5, include_transcripts=True, search_count=50):
"""Query Youtube for a subtopic
Queries the YouTube database for 5 videos pertaining to a certain subtopic.
Automatically re-queries if a video doesn't include a transcript if transcripts
are required.
Parameters:
subtopic -- the topic to search for
channels -- specifically which Channel enums to also include from the Channels class (default empty)
num_videos -- the number of videos to return. Could be more if a channels argument is non-empty (default 5)
include_transcripts -- requires videos to have transcripts (default True)
count -- the number if videos to query for everytime (default 50)
"""
assert num_videos >= 0 # Number of videos cannot be negative
# Query Youtube and add videos pertaining to the subtopic
r = self.api.search_by_keywords(q=subtopic, search_type=["video"], count=search_count, limit=search_count, video_caption="any", video_duration=["any"])
# Add list of videos to database
videos = []
# Creates deep copy of Channel(s) to keep track of which ones are included
includes_channels = []
for channel_enum in channels:
includes_channels.append(channel_enum.name)
# Maximum amount of videos to include (one video from each channel plus top 5 from results)
video_counter = num_videos
for vid in r.items:
should_append = False
# Check to see if max number of videos has been reached
if video_counter <= 0 and len(includes_channels) == 0:
break
# Filter the video from the YouTube API
filtered_video = self.filter_video_information(vid.to_dict())
# Conditions for when to add a video
if filtered_video["channelId"] in includes_channels:
# Remove minimum one video from channel requirement if video found
includes_channels.remove(filtered_video["channelId"])
should_append = True
elif video_counter > 0:
# Add video if we still need to add videos to reach minimum number of videos
should_append = True
if not should_append:
continue
# Include transcripts if specified
if include_transcripts and should_append:
filtered_video["transcript"] = self.get_youtube_transcript(filtered_video["videoId"])
if filtered_video["transcript"] == None:
continue
if should_append:
# Add in other fields
filtered_video["url"] = "www.youtube.com/watch?v=" + filtered_video["videoId"]
filtered_video["source"] = "Youtube"
filtered_video["difficulty"] = 3 # Default difficulty level
# Add video to list
videos.append(filtered_video)
video_counter -= 1 # Decrement video_counter for minimum number of videos to include
# Add filtered videos into the database (mutates)
self.database[subtopic] = videos
def filter_video_information(self, video, keys=["publishedAt", "channelId", "title", "description", "channelTitle", "videoId"]):
"""Filters video dict for certain keys
Filters a YouTube Video entry to only include a certain number of keys
specified by a keys list taken from the YouTube API.
Parameters:
video -- the video information as a dictionary to filter through
keys -- the keys to include (default ["publishedAt", "channelId", "title", "description", "channelTitle", "videoId])
"""
new_video = {}
self.recur_dict(video, new_video, keys) # Recursively loop through nested dictionaries and put everything on first layer
return new_video
def get_youtube_transcript(self, video_id):
"""Returns video's transcript from YouTube
Returns the video's transcripts given the video_id on YouTube. Returns
None if no transcript was found. This functionality is included in order
to check whether or not a video holds a transcript.
Parameters:
video_id -- the id of the video on YouTube. Can be found after the "v=" part in the link.
"""
# Try grabbing the raw translation using the YouTubeTranscriptApi
raw_trans = []
try:
raw_trans = YouTubeTranscriptApi.get_transcript(video_id, languages=["en"])
except (Exception):
print(video_id, "excluded")
return None
# Use only the text portion of transcript and throwaway time
transcript = ""
for i in raw_trans:
transcript += i["text"] + " "
return transcript
def recur_dict(self, data, output, keys_to_include):
"""Recursively loop through nested dict
Recursive function for reading nested dictionaries and
retrieving the keys to a one-layer dictionary.
"""
# Loop through key value pair in dictionary
for key, value in data.items():
if isinstance(value, dict):
# If the value is a dict, recursively loop
self.recur_dict(value, output, keys_to_include)
elif key in keys_to_include:
# If value is not a dictionary, add to new dict
output[key] = value
def write_to_json(self, data_dict):
"""Write dictionary to JSON file
Writes the file into a JSON and includes Exception
protection and null dictionary protection.
Parameters:
data_dict -- the dictionary to write to json
"""
try:
# Only write data to JSON if it is non Null
if data_dict:
with open(self.data_path, 'w') as json_file:
json.dump(data_dict, json_file)
except (json.decoder.JSONDecodeError):
print("Error Writing to Json File, Dictionary improperly formatted.")
def get(self,encodedWebSearchTerm):
api_key_="AIzaSyCy8jgQFIVEq2qLBdZMSaHQOiDGAggQTeQ"
api = API_YOUTUBE(api_key=api_key_)
r=api.search_by_keywords(q=encodedWebSearchTerm, search_type=["video"], count=15, limit=15)
return jsonify(r)
