def test_init(self):
'''#Verified by Megan Brown on 11/30/2018'''
with self.assertRaisesRegex(ValueError, 'No API key used to initate the class.'):
yt = YoutubeDataApi('')
with self.assertRaisesRegex(ValueError, 'The API Key is invalid'):
yt = YoutubeDataApi(self.wrong_key)
def Youtube_Scrape(api_key, vid):
"""
Scrape metadata details of the video.
"""
yt = YoutubeDataApi(api_key)
TomScott = yt.get_video_metadata(vid)
return TomScott
def test_init(self):
with self.assertRaisesRegex(ValueError,
'No API key used to initate the class.'):
yt = YoutubeDataApi('')
with self.assertRaisesRegex(ValueError, 'The API Key is invalid'):
yt = YoutubeDataApi(self.wrong_key)
class TestVideo(unittest.TestCase):
@classmethod
def setUpClass(self):
self.key = os.environ.get('YT_KEY')
self.yt = YoutubeDataApi(self.key)
self.video_id = 'wmxDZeh8W34'
self.video_id_list = ['wmxDZeh8W34', 'PIXQdfiRZNk', 'nvEFb_dWJdQ']
self.fake_vid = '12345'
def test_valid_caption(self):
''' #Verified by Megan Brown on 11/30/2018'''
resp = self.yt.get_captions(self.video_id)
self.assertEqual(type(resp), dict)
self.assertEqual(type(resp['video_id']), str)
def test_valid_list_of_captions(self):
'''#Written by Megan Brown on 11/30/2018'''
resp = self.yt.get_captions(self.video_id_list)
self.assertEqual(type(resp), list)
self.assertEqual(type(resp[0]['video_id']), str)
def test_invalid_short_caption(self):
'''#Written by Megan Brown on 11/30/2018'''
with self.assertRaises(Exception):
resp = self.yt.get_captions(self.fake_vid)
def test_list_of_captions_with_invalid_string(self):
'''#Written by Megan Brown on 11/30/2018'''
error_list = self.video_id_list.copy()
error_list.append(self.fake_vid)
with self.assertRaises(Exception):
resp = self.yt.get_captions(error_list)
def loadPlaylist(playlist_id):
YT_KEY = yc.load_creds()
yt = YoutubeDataApi(YT_KEY)
playlist = yt.get_videos_from_playlist_id(playlist_id=playlist_id)
video_ids = []
for video in playlist:
video_ids.append(video['video_id'])
return (video_ids)
def getResults(query):
streamableLinks=[]
from youtube_api import YoutubeDataApi
yt=YoutubeDataApi(FREE_API_KEY)
searches=yt.search(q=query,max_results=10)
for i in searches:
streamableLinks.append(makeStreamable(i["video_id"]))
return streamableLinks
def watchtrailer(request):
movie_title = request.GET['movie_title']
movie_year = request.GET['movie_year']
#print(movie_year)
#print(movie_title)
yt = YoutubeDataApi(youtube_APIKEY)
movie_trailer_search = yt.search(movie_title + ' ' + str(movie_year) +
' trailer',
max_results=1)
movie_trailer_search = movie_trailer_search[0]
movie_trailer_id = movie_trailer_search['video_id']
return redirect('https://www.youtube.com/watch?v=' + movie_trailer_id)
class TestVideo(unittest.TestCase):
def __init__():
self.key = os.environ.get('YT_KEY')
self.yt = YoutubeDataApi(self.key)
self.channel_id = 'UC3XTzVzaHQEd30rQbuvCtTQ'
self.channel_title = 'LastWeekTonight'
#written by Megan Brown on 11/30/2018
def test_channel_id(self):
resp = self.yt.get_channel_id_from_user(channel_title)
self.assertEqual(resp, self.channel_id)
def test_get_channel_metadata_channel_id(self):
resp = self.yt.get_channel_metadata(self.channel_id)
self.assertEqual(resp['channel_id'], self.channel_id)
self.assertEqual(resp['channel_title'])
def setUpClass(cls):
cls.key = os.environ.get('YT_KEY')
cls.yt = YoutubeDataApi(cls.key)
cls.video_id = 'QE5KOfjKLy0'
cls.channel_id = 'UC_x5XG1OV2P6uZZ5FSM9Ttw'
#cls.vid_publish = datetime.datetime(2018, 11, 28, 10, 0, 3)
cls.search_term = 'John Oliver'
cls.list_of_videos = ['ximgPmJ9A5s', 'gl1aHhXnN1k']
def setUpClass(cls):
cls.key = os.environ.get('YT_KEY')
cls.yt = YoutubeDataApi(cls.key)
cls.video_id = 'RdjRMDADpcg'
cls.channel_id = 'UC8-Th83bH_thdKZDJCrn88g'
cls.vid_publish = datetime.datetime(2018, 11, 28, 10, 0, 3)
cls.search_term = 'John Oliver'
cls.list_of_videos = ['ximgPmJ9A5s', 'RdjRMDADpcg']
def setUpClass(cls):
cls.key = os.environ.get('YT_KEY')
cls.yt = YoutubeDataApi(cls.key)
cls.search_term = 'John Oliver'
cls.channel_id = 'UC3XTzVzaHQEd30rQbuvCtTQ'
cls.max_results = 10
cls.published_after = datetime.datetime(2018,11,30)
cls.published_before = datetime.datetime(2018,12,1)
cls.topic_search = 'ted talks'
cls.topic_id = 22
class YTQuerry:
def __init__(self, token):
self.token = token
self.api = YoutubeDataApi(token)
def get_url(self, video_id):
return BASE_URL.format(video_id)
def search_videos(self, q, results=5):
return self.api.search(q=q, max_results=results)
def setUpClass(cls):
cls.key = os.environ.get('YT_KEY')
cls.yt = YoutubeDataApi(cls.key)
cls.channel_id = 'UC3XTzVzaHQEd30rQbuvCtTQ'
cls.upload_id = 'UU3XTzVzaHQEd30rQbuvCtTQ'
cls.liked_id = 'LL3XTzVzaHQEd30rQbuvCtTQ'
cls.date = '2018-03-14T20:53:14.000Z'
cls.datetime_date = datetime.datetime(2018, 3, 14, 20, 53, 14)
cls.user_url = 'https://www.youtube.com/user/LastWeekTonight'
cls.channel_url = 'https://www.youtube.com/channel/UC3XTzVzaHQEd30rQbuvCtTQ'
cls.video_id = '481YX6T9Xzs'
cls.video_url = 'https://youtube.com/watch?v=481YX6T9Xzs'
class TestVideo(unittest.TestCase):
def __init__():
self.key = os.environ.get('YT_KEY')
self.yt = YoutubeDataApi(self.key)
self.video_id = 'wmxDZeh8W34'
self.video_id_list = ['wmxDZeh8W34', 'PIXQdfiRZNk','nvEFb_dWJdQ']
self.fake_vid = '12345'
#Verified by Megan Brown on 11/30/2018
def test_valid_caption(self):
resp = self.yt.get_captions(self.video_id)
self.assertEqual(type(resp), dict)
self.assertEqual(type(resp['video_id']), str)
#Written by Megan Brown on 11/30/2018
def test_valid_list_of_captions(self):
resp = self.yt.get_captions(self.video_id_list)
self.assertEqual(type(resp), list)
self.assertEqual(type(resp[0]['video_id']), str)
#Written by Megan Brown on 11/30/2018
def test_invalid_short_caption(self):
with self.assertRaises(Exception):
resp = self.yt.get_captions(self.fake_vid)
#Written by Megan Brown on 11/30/2018
def test_list_of_captions_with_invalid_string(self):
error_list = self.video_id_list
error_list.append(self.fake_vid)
with self.assertRaises(Exception):
resp = self.yt.get_captions(error_list)
#Written by Megan Brown on 12/20/2018
def test_caption_parser_w_raw_json(self):
captions = self.yt.get_captions(self.video_id, parser=P.raw_json)
self.assertTrue(isinstance(self.captions, dict))
async def yt(ctx, *, query):
yt = YoutubeDataApi(ytid)
searches = yt.search(str(query), max_results=3)
result = yt.get_video_metadata(
searches[0]["video_id"],
part=['id', 'snippet', 'contentDetails', 'statistics'])
searches.clear()
del yt
url = 'https://www.youtube.com/watch?v=' + result["video_id"]
desc = result["video_description"].split('\n')[0]
if len(desc) > 300:
desc = desc[:300] + "..."
embed = discord.Embed(colour=0xff0000,
title=result["video_title"],
description=desc,
url=url)
embed.set_author(name=result["channel_title"])
embed.set_thumbnail(url=result["video_thumbnail"])
embed.add_field(name="Views",
value=str(result["video_view_count"]),
inline=True)
embed.add_field(name="Comments",
value=str(result["video_comment_count"]),
inline=True)
embed.add_field(name="Duration",
value=str(result["duration"])[2:-1].replace("M",
":").replace(
"H", ":"),
inline=True)
embed.add_field(name="Likes",
value=str(result["video_like_count"]),
inline=True)
embed.add_field(name="Dislikes",
value=str(result["video_dislike_count"]),
inline=True)
embed.set_footer(text=datetime.utcfromtimestamp(
int(result["video_publish_date"])).strftime('%Y-%m-%d %H:%M:%S'))
await ctx.send(embed=embed)
def setUpClass(cls):
cls.key = config.key
cls.yt = YoutubeDataApi(config.key)
cls.defunct_playlist_id = 'UUvsye7V9psc-APX6wV1twLg' #alex jones
cls.playlist_id = 'UU3XTzVzaHQEd30rQbuvCtTQ' #lastweektonight
cls.future_date = datetime.datetime(2200, 1, 1)
cls.cutoff_date = datetime.datetime(2018, 1, 1)
cls.default_parser_output_keys = [
'channel_id', 'collection_date', 'publish_date', 'video_id'
]
cls.raw_json_output_keys = [
'snippet.thumbnails.medium.width',
'snippet.thumbnails.maxres.height',
'snippet.channelId',
'snippet.thumbnails.standard.height',
'snippet.thumbnails.high.url',
'collection_date',
'snippet.playlistId',
'snippet.publishedAt',
'snippet.thumbnails.default.height',
'kind',
'channel_id',
'snippet.thumbnails.maxres.url',
'etag',
'snippet.description',
'snippet.resourceId.kind',
'snippet.thumbnails.high.width',
'snippet.title',
'snippet.thumbnails.medium.height',
'publish_date',
'snippet.thumbnails.standard.width',
'snippet.channelTitle',
'snippet.position',
'video_id',
'snippet.thumbnails.default.width',
'snippet.thumbnails.medium.url',
'snippet.thumbnails.standard.url',
'id',
'snippet.thumbnails.high.height',
'snippet.thumbnails.default.url',
'snippet.thumbnails.maxres.width',
'snippet.resourceId.videoId'
]
def youtube_data(group):
"""Runs all the YouTube related tasks
It scrapes data from YouTube for the whole group and the single artists
Args:
group: dictionary with the data of the group to scrape
Returns:
the same group dictionary with updated data
"""
print("[{}] Starting tasks...".format(module))
yt = YoutubeDataApi(youtube_api_key)
# Getting channel data and stats
channel_data = youtube_get_channel(yt, group["youtube"]["url"])
group["youtube"] = youtube_check_channel_change(group["youtube"],
channel_data)
# Getting video data and stats
videos = youtube_get_videos(yt, group["youtube"]["playlist"],
group["youtube"]["name"])
group["youtube"]["videos"] = youtube_check_videos_change(
group["name"], group["youtube"]["videos_scale"],
group["youtube"]["videos"], videos)
# Getting Youtube data for each member
for member in group["members"]:
if "youtube" in member:
channel_data = youtube_get_channel(yt, member["youtube"]["url"])
member["youtube"] = youtube_check_channel_change(
member["youtube"], channel_data)
videos = youtube_get_videos(yt, member["youtube"]["playlist"],
member["youtube"]["name"])
member["youtube"]["videos"] = youtube_check_videos_change(
member["name"], member["youtube"]["videos_scale"],
member["youtube"]["videos"], videos)
print()
return group
class TestAPI(unittest.TestCase):
@classmethod
def setUpClass():
self.key = os.environ.get('YT_KEY')
self.wrong_key = 'xxxxxxxxx'
self.yt = YoutubeDataApi(cls.key)
#Verified by Megan Brown on 11/30/2018
def test_init(self):
with self.assertRaisesRegex(ValueError,
'No API key used to initate the class.'):
yt = YoutubeDataApi('')
with self.assertRaisesRegex(ValueError, 'The API Key is invalid'):
yt = YoutubeDataApi(self.wrong_key)
#verified by Megan Brown on 11/30/2018
@patch('requests.get')
def test_verify(self, mock_request):
mock_resp = requests.models.Response()
mock_resp.status_code = 404
mock_request.return_value = mock_resp
self.assertEqual(self.yt.verify_key(), False)
from youtube_api import YoutubeDataApi
import pandas as pd
from utils import search_dict
from utils import API_KEY
query = 'Juul'
youtube = YoutubeDataApi(key=API_KEY)
searches = youtube.search(q=query)
print(len(searches))
searches = [search_dict(search) for search in searches]
searches_df = pd.DataFrame(searches)
searches_df.to_csv('{}_searches.csv'.format(query), index=False)
def __init__():
self.key = os.environ.get('YT_KEY')
self.yt = YoutubeDataApi(self.key)
self.channel_id = 'UC3XTzVzaHQEd30rQbuvCtTQ'
self.channel_title = 'LastWeekTonight'
def setUpClass():
self.key = os.environ.get('YT_KEY')
self.wrong_key = 'xxxxxxxxx'
self.yt = YoutubeDataApi(cls.key)
import youtube_dl
import youtube_api
from spotipy.oauth2 import SpotifyClientCredentials
from youtube_api import YoutubeDataApi
# TODO: modularize the login and authentication
# Set the API keys
spotify_client_id = config.SPOTIFY_CLIENT_ID
spotify_client_secret = config.SPOTIFY_API_SECRET
youtube_data_api_key = config.YOUTUBE_API_SECRET
youtube_dl_options = config.YOUTUBE_DL_OPTIONS
# Authenticate
sp = spotipy.Spotify(client_credentials_manager=SpotifyClientCredentials(
spotify_client_id, spotify_client_secret))
yt = YoutubeDataApi(youtube_data_api_key)
class TestPlaylistDownloader(unittest.TestCase):
def test_video_id(self):
# Test 1
video_id = yt.search('Rick Astley Never Gonna Give You Up',
max_results=5)[0]['video_id']
self.assertEqual(video_id, 'dQw4w9WgXcQ',
'@method test_video_id TEST 1: Video ID Error.')
# Test 2
video_id = yt.search('Michael Jackson Billie Jean',
max_results=5)[0]['video_id']
self.assertEqual(video_id, 'Zi_XLOBDo_Y',
'@method test_video_id TEST 2: Video ID Error.')
from youtube_api import YoutubeDataApi
from dotenv import load_dotenv
import os
load_dotenv()
yt_token = os.getenv('youtube_token')
yt_watch_base_url = 'https://www.youtube.com/watch?v='
yt = YoutubeDataApi(yt_token)
def get_trailer_url(movie_title):
search_result = yt.search(q=movie_title + ' trailer',
max_results=1,
search_type="video")
if search_result and len(search_result) > 0:
trailer_url = yt_watch_base_url + search_result[0]['video_id']
return trailer_url
else:
return None
def setUpClass(cls):
cls.key = os.environ.get('YT_KEY')
cls.yt = YoutubeDataApi(cls.key)
cls.channel_id = 'UC3XTzVzaHQEd30rQbuvCtTQ'
cls.channel_title = 'LastWeekTonight'
def setUpClass(self):
self.key = os.environ.get('YT_KEY')
self.yt = YoutubeDataApi(self.key)
self.video_id = 'wmxDZeh8W34'
self.video_id_list = ['wmxDZeh8W34', 'PIXQdfiRZNk', 'nvEFb_dWJdQ']
self.fake_vid = '12345'
def download_thumbnails(data_dir, api_key):
""" Download youtube thumbnails for the top 50 results of each search term.
Args:
data_dir: (str) Directory to save images to.
api_key: (str) Your personal YouTube API key. Follow the steps to get
one here https://developers.google.com/youtube/v3/getting-started.
Returns: True if completed successfully.
"""
with open(api_key, "r") as f:
api_key = f.read()
os.makedirs(data_dir, exist_ok=True)
yt = YoutubeDataApi(api_key)
search_number = 50
search_terms = [
"makeup",
"grwm",
"GRWM",
"makeup haul",
"plt",
"pretty little thing",
"first impressions",
"beauty",
"kbeauty",
"instabaddie",
"makeup tutorial",
"everyday makeup",
"best makeup of 2019",
"the power of makeup",
"glam makeup",
"full face of",
"eyeshadow palette",
"beauty products",
"makeup routine",
"get ready with me",
"missguided",
"iluvsarahii",
"jeffree star makeup",
"nikkietutorials",
"instagram baddie",
"0-100 makeup transformation",
"glow up",
"best makeup of 2018",
"best makeup of 2017",
"best makeup transformations 2019",
"best makeup transformations 2018",
"best makeup transformations 2017",
"full face of",
"makeup i hate",
"nye grwm",
"nye glam",
"smoky eye tutorial",
"drugstore makeup tutorial",
"drugstore makeup 2019",
"drugstore makeup 2018",
"mmmmitchell",
"catfish makeup",
"no makeup makeup",
"boyfriend choose my makeup",
"kkw x mario",
"roxxsaurus makeup revolution",
"nikita dragun makeup",
"holiday makeup",
"makeup hacks",
"2020 grwm",
"24hr glow up",
"full face drugstore makeup",
"makeup for school",
"everyday makeup routine 2018",
"hd brows foundation",
"grunge makeup",
"natural soft makeup",
"autumn makeup",
"jamie genevieve",
]
if len(search_terms) == 0:
print("Exiting...")
return None
print(
f"Searching for the top {search_number} results ("
f"{search_number * len(search_terms)} videos)"
)
for search_for in tqdm(search_terms):
response = yt.search(
q=search_for, max_results=search_number, parser=None
)
if len(response) < 1:
continue
for i, item in enumerate(response):
if "medium" in item["snippet"]["thumbnails"].keys():
thumbnail = item["snippet"]["thumbnails"]["medium"]["url"]
elif "default" in item["snippet"]["thumbnails"].keys():
thumbnail = item["snippet"]["thumbnails"]["default"]["url"]
else:
continue
fpath = os.path.join(
data_dir,
f"{str(i)}_{search_for}_" + os.path.basename(thumbnail),
)
urllib.request.urlretrieve(thumbnail, fpath)
with open(fpath.replace("jpg", "json"), "w") as fname:
json.dump(item, fname)
return True
from youtube_api import YoutubeDataApi
import pandas as pd
from utils import API_KEY
from utils import video_metadata_dict
from utils import tags
youtube = YoutubeDataApi(key=API_KEY)
for tag in tags:
try:
videos_df = pd.read_csv('data/{}/videos.csv'.format(tag))
except Exception:
print('Video file doesn\'t exist for tag {}'.format(tag))
continue
videos = videos_df.values.tolist()
metas = []
for video in videos:
metadata = youtube.get_video_metadata(video[0])
metadata = video_metadata_dict(metadata, video)
metas.append(metadata)
video_df = pd.DataFrame(metas)
video_df.to_csv('data/{}/videos.csv'.format(tag), index=False)
import pandas as pd
import time
from youtube_api import YoutubeDataApi
from utils import API_KEY
from utils import video_playlist_dict2
from utils import tags
from utils import channelIds
youtube = YoutubeDataApi(key=API_KEY)
for channelId, tag in zip(channelIds, tags):
print(tag)
meta = youtube.get_channel_metadata(channelId)
meta['account_creation_date'] = time.strftime(
'%Y-%m-%d %H:%M:%S', time.localtime(meta['account_creation_date']))
meta_df = pd.DataFrame([meta])
meta_df.to_csv('data/{}/metadata.csv'.format(tag), index=False)
playlist_id_uploads = meta['playlist_id_uploads']
try:
videos = youtube.get_videos_from_playlist_id(playlist_id_uploads)
videos = [video_playlist_dict2(video) for video in videos]
# playlists = youtube.get_playlists(channelId)
# # print(len(playlists))
# playlists = [playlist_dict(playlist) for playlist in playlists]
# all_videos = []
# for playlist in playlists:
# videos = youtube.get_videos_from_playlist_id(playlist['playlist_id'])
# # playlists_df = pd.DataFrame(playlists)
# # playlists_df.to_csv('{}_channel_playlists.csv'.format(channelId), index=False)
# videos = [video_playlist_dict(video, playlist['playlist_id']) for video in videos]
# # print(len(videos))
def setUpClass(cls):
cls.key = os.environ.get('YT_KEY')
cls.yt = YoutubeDataApi(cls.key)
return filename
def check_args():
if len(sys.argv) > 1:
if sys.argv[1] == "-no-tweet":
print(
"-no-tweet parameter passed!\nTest mode enabled: the bot won't tweet anything\n"
)
global test_mode
test_mode = True
if __name__ == '__main__':
check_args()
group = load_group()
check_birthdays(group)
youtube = YoutubeDataApi(youtube_api_key)
group = youtube_data(youtube, group)
group["twitter"] = twitter_repost(group["twitter"], test=test_mode)
#group = instagram_data(group)
write_group(group)
