def video_info():
if request.method == 'POST':
# specific for 'try it' on /
# since it is my own api_key used for now...
if request.form.get('api_key_test') is not None:
api_key = app.config['api_key_test']
elif 'api_key' in session:
api_key = session['api_key']
else:
return render_template('explore.html',
message="""
<h4><strong class="text-danger">You can try Pytheas but to go further you will need to get an api_key from Google services
<br>Please go to <a href="./config" class="btn btn-primary btn-lg active" role="button" aria-pressed="true">Config</a> and follow guidelines</strong></h4>
""")
id_video = request.form.get('unique_id_video')
id_video = YouTube.cleaning_video(id_video)
part = ', '.join(request.form.getlist('part'))
api = YouTube(api_key=api_key)
video_result = api.get_query('videos', id=id_video, part=part)
video_result_string = json.dumps(video_result,
sort_keys=True,
indent=2,
separators=(',', ': '))
return render_template('methods/view_results.html',
result=video_result,
string=video_result_string)
return render_template('explore.html')
def channel_info():
if request.method == 'POST':
if 'api_key' in session:
id_channel = request.form.get('unique_id_channel')
id_username = request.form.get('unique_user_channel')
part = ', '.join(request.form.getlist('part'))
api = YouTube(api_key=session['api_key'])
if 'youtube.com/channel/' in id_channel:
id_channel = YouTube.cleaning_channel(id_channel)
channel_result = api.get_query('channels',
id=id_channel,
part=part)
elif id_username != '':
id_username = YouTube.cleaning_channel(id_username)
channel_result = api.get_query('channels',
forUsername=id_username,
part=part)
else:
channel_result = api.get_query('channels',
id=id_channel,
part=part)
channel_result_string = json.dumps(channel_result,
sort_keys=True,
indent=2,
separators=(',', ': '))
return render_template('methods/view_results.html',
result=channel_result,
string=channel_result_string)
else:
return render_template('explore.html', message='api key not set')
return render_template('explore.html')
def add_comments(user_id, query_id):
current_comment_thread = Comment(mongo_curs, query_id)
param = request.get_json()
api = YouTube(api_key=param['api_key'])
list_vid = param['list_vid']
for id_video in list_vid:
commentThreads_result = api.get_query('commentThreads',
videoId=id_video,
part='id, replies, snippet')
current_comment_thread.create_comment_for_each(commentThreads_result)
# Loop and save while there is content
while 'nextPageToken' in commentThreads_result:
commentThreads_result = api.get_query(
'commentThreads',
videoId=id_video,
part='id, replies, snippet',
pageToken=commentThreads_result['nextPageToken'])
current_comment_thread.create_comment_for_each(
commentThreads_result)
count_comments = int(
mongo_curs.db.comments.find({
'query_id': query_id
}).count())
mongo_curs.db.queries.update_one(
{'query_id': query_id},
{'$set': {
'count_comments': count_comments
}})
return 'POST REQUEST add_comments IS RECEIVED'
def run(self):
""" run """
count = 1
while count <= self.opts.visits:
youtube = YouTube(url=self.opts.url,
proxy=self.opts.proxy,
verbose=self.opts.verbose)
youtube.get_url()
title = youtube.get_title()
if self.opts.visits > 1 and title:
length = (len(title) + 4 - len(str(count)))
print('[{0}] {1}'.format(count, '-' * length))
ip_address = utils.get_ipaddr(proxy=self.opts.proxy)
if ip_address:
print('external IP address:', ip_address)
if title:
print('title:', title)
youtube.play_video()
youtube.get_views()
video_duration = youtube.time_duration()
if video_duration:
print('video duration time:', video_duration)
seconds = utils.to_seconds(duration=video_duration.split(':'))
if seconds:
sleep_time = randrange(seconds)
if self.opts.verbose:
print('video duration time in seconds:', seconds)
print('stopping video in %s seconds' % sleep_time)
time.sleep(sleep_time)
youtube.disconnect()
count += 1
def youtube():
try:
req_data = request.get_json()
search = req_data['search']
continued = int(req_data['more'])
prevList=[]
if continued==1:
prevList = list(req_data['prev'])
yt = YouTube()
yt.NewMain(search,prevList)
yturl = list(yt.getLinks())
try :
maax = int(req_data['items'])
if maax > 0 and maax < len(yturl):
yturl=yturl[:maax]
except :
return {"error" : "Invalid json data. items field compulsory"}
lst = {}
for url in yturl:
dcdr = Decoder(url)
lst.update({url : dcdr.getDat()})
data= {"data" : lst}
data.update({"search" : search})
data.update({"items" : len(lst)})
return data
except Exception as ex:
return({"error":ex})
def add_related(user_id, query_id):
current_relatedVideos = RelatedVideos(mongo_curs, query_id)
param = request.get_json()
api = YouTube(api_key=param['api_key'])
list_vid = param['list_vid']
for id_video in list_vid:
search_results = api.get_query(
'search',
part='id,snippet',
maxResults=50,
relatedToVideoId=id_video,
type='video',
)
for each in search_results['items']:
each.update({'query_id': query_id})
each.update({'videoId': id_video})
mongo_curs.db.relatedVideos.insert_one(each)
## Loop and save
while 'nextPageToken' in search_results:
search_results = api.get_query(
'search',
part='id,snippet',
maxResults=50,
relatedToVideoId=id_video,
type='video',
pageToken=search_results['nextPageToken'])
if not search_results['items']:
return redirect(url_for('manage'))
else:
for each in search_results['items']:
each.update({'query_id': query_id})
each.update({'belongTo': id_video})
mongo_curs.db.relatedVideos.insert_one(each)
# add metrics for query in json
count_videos = int(
mongo_curs.db.relatedVideos.find({
'query_id': query_id
}).count())
mongo_curs.db.queries.update_one(
{'query_id': query_id},
{'$set': {
'count_relatedVideos': count_videos
}})
# add part as indicator
part_value = mongo_curs.db.queries.find_one_or_404({'query_id': query_id})
mongo_curs.db.queries.update_one(
{'query_id': query_id},
{'$set': {
'part': part_value['part'] + ", relatedVideos",
}},
upsert=False)
return 'POST REQUEST add_related IS RECEIVED'
def download(youtube: Union[YouTube, None], playlist: Union[Playlist, None],
stream: str, path: str) -> None:
"""
Function to handle download for a video/playlist.
Parameters
----------
stream : `str`
Selected value of the window stream
path: `str`
Path where the video will be downloaded
"""
if not playlist and not youtube:
window.s_append('You must search for a video before download!')
return
if playlist:
for video_url in playlist.video_urls:
# A try here is needed since it would
# stop downloading the playlist after
# an exception.
try:
youtube = YouTube(path=path, url=video_url, window=window)
if 'Audio' in stream:
youtube.download_audio(title=True)
else:
youtube.download_video()
except Exception as e:
window.s_append('An unexpected pytube library error occured,'
' could not download.')
print(f'Exception {e}')
window.s_append('All downloads finished!')
if youtube:
if youtube.path != path:
youtube.path = path
quality = stream[:5].strip()
fps = int(stream[-7:-5])
if 'kb' in quality:
youtube.download_audio(bitrate=stream)
else:
youtube.download_video(res=quality, fps=fps)
window.s_append(f'{youtube.title} download finished!')
def __init__(self, engine):
super(YouTubeWatcher, self).__init__(engine)
# db.upsert('youtube', [
# ('channel', 'varchar(64)'),
# ('video_id', 'varchar(64)')
# ])
self.yt = YouTube()
self.start_later(1)
def __init__(self, username):
config = ConfigParser.ConfigParser()
config.read('./config.properties')
self.pinterest = Pinterest(username, config)
self.googleplus = GooglePlus(username, config)
self.youtube = YouTube(username, config)
self.twitch = Twitch(username, config)
self.vimeo = Vimeo(username, config)
self.behance = Behance(username, config)
self.instagram = Instagram(username, config)
self.twitter = Twitter(username, config)
self.github = Github(username, config)
self.dict = dict()
def add_captions(user_id, query_id):
current_captions = Caption(mongo_curs, query_id)
param = request.get_json()
api = YouTube(api_key=param['api_key'])
list_vid = param['list_vid']
for id_video in list_vid:
worker.logger.debug(id_video)
current_captions.create_if_not_exist(id_video)
# counting captions
current_captions.count_captions()
return 'POST REQUEST add_captions IS RECEIVED'
def search(
url: str,
path: str,
) -> Tuple[Union[YouTube, None], Union[None, Playlist]]:
"""
Function to handle search for a video/playlist.
Parameters
----------
url : `str`
Searched video/playlist url.
path: `str`
Path where the video will be downloaded
Returns
-------
It can returns a YouTube or Playlist object,
or nothing if any exception was raised
"""
window.s_append(f'Searching for {url}...')
youtube = playlist = None
if 'playlist?' in url:
playlist = Playlist(url=url, window=window)
window.s_append('Playlist found! Select a download mode on '
'the streams below!')
window['stream'].update(
values=['Video (Max quality)', 'Audio (Max quality)'])
else:
youtube = YouTube(path=path, url=url, window=window)
v = [f'{v.resolution} ({v.fps} FPS)' for v in youtube['videos']]
a = [a.abr for a in youtube['audios']]
window['stream'].update(values=v + a)
window.s_append(f'{len(youtube)} downloadable streams found!'
' Select one below!')
return youtube, playlist
def download(url, tempdir):
yt = YouTube()
yt.url = url
video = None
if len(yt.videos) == 0:
raise NotFound(url)
if len(yt.filter('webm')) > 0:
video = yt.filter('webm')[-1] # Best resolution
elif len(yt.filter('mp4')) > 0:
video = yt.filter('mp4')[-1] # Best resolution
else:
video = yt.videos[-1] # Best resolution
video.download(tempdir)
return video.filename
def main():
# Disable OAuthlib's HTTPS verification when running locally.
# *DO NOT* leave this option enabled in production.
os.environ["OAUTHLIB_INSECURE_TRANSPORT"] = "1"
# Get the API key as a CLI arg.
api_key = sys.argv[1]
if not api_key:
raise Exception("No API key provided.")
# Get credentials and create an API client
youtube = YouTube(api_key)
# Do stuff.
da = DataAccess()
pitems_dict = da.get_pitems_dict(OTHER_PLAYLIST_IDS)
current_vid = "lM28rfsHge0"
# save_threads(youtube, da, from_vid=current_vid, dry_run=False)
# save_all_playlist_items(youtube, OTHER_PLAYLIST_IDS, dry_run=False)
save_all_videos(youtube, pitems_dict, dry_run=False)
def playlist_info():
if request.method == 'POST':
if 'api_key' in session:
id_playlist = request.form.get('unique_id_playlist')
# remember to make same as for cleanning_ytb
if 'youtube.com/watch?v=' in id_playlist:
f = furl(id_playlist)
id_playlist = f.args['list']
part = ', '.join(request.form.getlist('part'))
api = YouTube(api_key=session['api_key'])
playlist_info = api.get_query('playlists',
id=id_playlist,
part=part)
playlist_info_string = json.dumps(playlist_info,
sort_keys=True,
indent=2,
separators=(',', ': '))
return render_template('methods/view_results.html',
result=playlist_info,
string=playlist_info_string)
else:
return render_template('explore.html', message='api key not set')
return render_template('explore.html')
key = os.getenv('KEY')
idVideo = os.getenv('ID_VIDEO')
idChannel = os.getenv('ID_CHANNEL')
client = os.getenv('PAST_CLIENT')
category = os.getenv('CATEGORY')
refresh = os.getenv('REFRESH')
access = os.getenv('ACCESS')
scopes = [
"https://www.googleapis.com/auth/youtubepartner",
"https://www.googleapis.com/auth/youtube",
"https://www.googleapis.com/auth/youtube.force-ssl"
]
# YouTube object that will hold details of the video
video = YouTube(idVideo)
thumbnail_path = "thumbnail_base.png"
# Function: main()
# The function that will create an API client and use the logic to update the
# title and description.
def main():
api_service = 'youtube'
api_version = 'v3'
client_secrets_file = client
# Getting credentials and create API client
flow = google_auth_oauthlib.flow.InstalledAppFlow.from_client_secrets_file(
def __init__(self):
self.Twitter = Twitter("", "")
self.YouTube = YouTube()
# -*- coding: utf-8 -*-
from bs4 import BeautifulSoup
import requests
import time
import pandas as pd
from tqdm import tqdm
import argparse
from youtube import YouTube, video_id_to_channel_id
BASE_URL = 'https://virtual-youtuber.userlocal.jp'
RANKING_URL = BASE_URL + '/document/ranking'
YouTube = YouTube()
def get_ranking_page(page: int = 1):
html = requests.get(RANKING_URL + "?page={}".format(page)).text
return BeautifulSoup(html, 'html.parser')
def ranking(n: int = 100):
for page in range(int(n / 50 + 1)):
soup = get_ranking_page(page + 1)
for i, tr in enumerate(soup.find_all("tr")):
if 50 * page + i >= n:
break
yield 50 * page + i + 1, tr.a.img.get(
"alt"), BASE_URL + tr.get("data-href")
def add_video(user_id, query_id, query_type):
param = request.get_json()
uid = param['query_id']
query_id = param['query_id']
query = param['query']
api_key = param['api_key']
api = YouTube(api_key=api_key)
maxResults = 50
###############################
# RESULTS FOR channel
###############################
# clean empty fields here...
param["channel_username"] = [x for x in param["channel_username"] if x]
param["channel_id"] = [x for x in param["channel_id"] if x]
if query_type == 'channel':
# looking for IDs & usernames
# username need to retrieve ID first
if param['channel_username']:
# HAVE TO PUT ALL OF THIS IN A method FUNCT()
######################################
param_query = param.copy()
rm_list = [
'query', 'query_id', 'author_id', 'channel_id',
'channel_username'
]
[param_query.pop(key) for key in rm_list]
channel_usernames = re.sub("\r", "", param['channel_username'][0])
channel_usernames = re.sub("\n", "", channel_usernames)
channel_usernames = channel_usernames.split(',')
for channel_username in channel_usernames:
param_query['forUsername'] = channel_username
find_channel_id = api.get_query('channels', **param_query)
worker.logger.debug("------> find_channel_id")
worker.logger.debug(find_channel_id)
########################################################
if find_channel_id['items']:
param['query'] = query
param['query_id'] = uid
param['channel_id'] += str(
', ' + find_channel_id['items'][0]['id'])
api.get_channel_videos(mongo_curs,
find_channel_id['items'][0]['id'],
param)
# then for ID
if param['channel_id']:
# worker.logger.debug(channel_ids)
# channel_ids = re.sub("\r", "", param['channel_id'][0])
# channel_ids = re.sub("\n", "", channel_ids)
# channel_ids = channel_ids.split(',')
for channel_id in param['channel_id']:
api.get_channel_videos(mongo_curs, channel_id, param)
# finally add metrics for query in json
count_videos = int(
mongo_curs.db.videos.find({
'query_id': query_id
}).count())
mongo_curs.db.queries.update_one(
{'query_id': query_id},
{'$set': {
'count_videos': count_videos,
'status': 'done'
}})
###############################
## RESULTS FOR searchResults ##
###############################
elif query_type == 'search':
if 'mode' in param:
# FOR NEXT DATE SEARCH INTEGRATION
# param_query = {
# 'q': param['query'],
# 'part': param['part'],
# 'relevenceLanguage': param['language'],
# 'maxResults': param['maxResults'],
# 'order': param['order'],
# 'publishedAfter' : param['publishedAfter'],
# 'publishedBefore' : param['publishedBefore'],
# }
# date_results = api.get_query(
# 'search',
# **param_query,
# )
# for each in date_results['items']:
# each.update({'query_id': query_id})
# each = YouTube.cleaning_each(each)
# mongo_curs.db.videos.insert_one(each)
# while 'nextPageToken' in date_results and len(date_results['items']) != 0:
# worker.logger.debug(date_results['items'][-1]['snippet']['publishedAt'])
# param_query['publishedAfter'] = date_results['items'][-1]['snippet']['publishedAt']
# date_results = api.get_query(
# 'search',
# **param_query,
# pageToken = date_results['nextPageToken'],
# )
# for each in date_results['items']:
# each.update({'query_id': query_id})
# each = YouTube.cleaning_each(each)
# mongo_curs.db.videos.insert_one(each)
# Parse date time from form
d_start = datetime.datetime.strptime(param['publishedAfter'],
"%Y-%m-%dT%H:%M:%SZ")
d_end = datetime.datetime.strptime(param['publishedBefore'],
"%Y-%m-%dT%H:%M:%SZ")
r_after = time.parse(param['publishedAfter'])
r_before = time.parse(param['publishedBefore'])
delta = r_before - r_after
delta_days = delta.days + 1
param_query = {
'q': param['query'],
'part': param['part'],
'maxResults': param['maxResults'],
'order': param['order']
}
if not param['language'] == 'None':
param_query['language'] = param['language']
# worker.logger.debug(param_query)
# Then iterate for each days
for n in range(delta.days + 1):
# increment one day later to get a one-day period
r_after_next = r_after + dt.timedelta(days=1)
st_point = r_after.isoformat()
ed_point = r_after_next.isoformat()
# Querying
date_results = api.get_query(
'search',
**param_query,
publishedAfter=st_point,
publishedBefore=ed_point,
)
# saving
for each in date_results['items']:
each.update({'query_id': query_id})
each = YouTube.cleaning_each(each)
mongo_curs.db.videos.insert_one(each)
# loop
while 'nextPageToken' in date_results and len(
date_results['items']) != 0:
date_results = api.get_query(
'search',
**param_query,
publishedAfter=st_point,
publishedBefore=ed_point,
pageToken=date_results['nextPageToken'])
if date_results['items']:
for each in date_results['items']:
each.update({'query_id': query_id})
each = YouTube.cleaning_each(each)
mongo_curs.db.videos.insert_one(each)
else:
break
# finally increment next after day
r_after += dt.timedelta(days=1)
else:
search_results = api.get_query('search',
q=param['query'],
part=param['part'],
relevanceLanguage=param['language'],
maxResults=param['maxResults'],
order=param['order'])
# insert videos
for each in search_results['items']:
each.update({'query_id': uid})
each = YouTube.cleaning_each(each)
mongo_curs.db.videos.insert_one(each)
# Loop and save
while 'nextPageToken' in search_results:
search_results = api.get_query(
'search',
q=param['query'],
part=param['part'],
relevanceLanguage=param['language'],
maxResults=param['maxResults'],
order=param['order'],
pageToken=search_results['nextPageToken'])
if search_results['items']:
# insert video-info
for each in search_results['items']:
each.update({'query_id': uid})
each = YouTube.cleaning_each(each)
mongo_curs.db.videos.insert_one(each)
else:
# add metrics for query in json
count_videos = int(
mongo_curs.db.videos.find({
'query_id': uid
}).count())
mongo_curs.db.queries.update_one(
{'query_id': uid},
{'$set': {
'count_videos': count_videos
}})
break
count_videos = int(
mongo_curs.db.videos.find({
'query_id': uid
}).count())
mongo_curs.db.queries.update_one(
{'query_id': uid},
{'$set': {
'count_videos': count_videos,
'status': 'done'
}})
###############################
# RESULTS FOR SET OF videosList
###############################
elif query_type == 'videos':
param['videos'] = [x.strip() for x in param['videos'][0].split(',')]
param['videos'] = [i for i in param['videos'] if i]
for each in param['videos']:
video_result = api.get_query('videos', id=each, part=param['part'])
video_result = video_result['items'][0]
video_result.update({'query_id': uid})
mongo_curs.db.videos.insert_one(video_result)
count_videos = int(
mongo_curs.db.videos.find({
'query_id': uid
}).count())
mongo_curs.db.queries.update_one(
{'query_id': uid},
{'$set': {
'count_videos': count_videos,
'status': 'done'
}})
###############################
# RESULTS FOR PLAYLISTITEM
###############################
elif query_type == 'playlist':
for playlist_id in param['playlist_id']:
param.update({'playlist_id': playlist_id})
# call request
playlist_results = api.get_playlist(mongo_curs, param)
# add metrics for query in json
count_videos = int(
mongo_curs.db.videos.find({
'query_id': query_id
}).count())
mongo_curs.db.queries.update_one(
{'query_id': query_id},
{'$set': {
'count_videos': count_videos,
'status': 'done'
}})
return 'videos added'
def play_with_res(video,res,config,f):
print("Launching browser...")
d = get_driver(config['driver'])
y = YouTube(video,res, d, config['youtube'], f)
y.play()
time.sleep(2)
from database import Database from youtube import YouTube from clusterer import Clusterer env = 'desktop' db_name = 'comment_sense_3' db = Database(env, db_name) yt = YouTube() videoId = 'kQibkV_V8-c' video_data = yt.video(videoId) comment_topics = db.comment_topics(videoId) cl = Clusterer(video_data, db) topics = cl.cluster(comment_topics) print(topics)
def search():
# to integrate later to get global dated search (not only one day/one day)
if request.method == 'POST':
if not 'api_key' in session:
return render_template('download/search.html',
message='api key not set')
query_id = str(uuid4())
api = YouTube(api_key=session['api_key'])
user_id = session['profil']['id']
query = str(request.form.get('query'))
part = ','.join(request.form.getlist('part'))
order = str(request.form.get('order'))
language = str(request.form.get('language'))
param = {
'author_id': session['profil']['id'],
'api_key': session['api_key'],
'query_id': query_id,
'query': query,
'part': part,
'order': order,
'language': language,
'maxResults': maxResults,
'kind': 'searchResults',
}
if request.form.get('advanced'):
# get date points & convert them
st_point = request.form.get('startpoint') + 'T00:00:00Z'
ed_point = request.form.get('endpoint') + 'T00:00:00Z'
param = {
**param,
'mode': 'advanced',
'publishedAfter': st_point,
'publishedBefore': ed_point,
}
r_query = requests.post("http://restapp:" +
app.config['REST_PORT'] + "/" + user_id +
"/add_query/" + query_id,
json=param)
def send_request():
requests.post("http://restapp:" + app.config['REST_PORT'] +
"/" + user_id + "/query/" + query_id +
"/add_video/search",
json=param)
Thread(target=send_request).start()
else:
r_query = requests.post("http://restapp:" +
app.config['REST_PORT'] + "/" + user_id +
"/add_query/" + query_id,
json=param)
def send_request():
requests.post("http://restapp:" + app.config['REST_PORT'] +
"/" + user_id + "/query/" + query_id +
"/add_video/search",
json=param)
Thread(target=send_request).start()
return render_template('methods/download_process.html')
def resolve(self, link):
domain = urlparse(link).netloc
if domain in self.youtube_domains:
return YouTube()
# Parameters
spotify_playlist = "Quickify" # Set to None to sync all tracks
youtube_playlist = "Quickify"
# read config file
with open('config.json', 'r') as myfile:
data = myfile.read()
# parse config file
config_data = json.loads(data)
spotify = Spotify(config_data['spotify_username'],
config_data['spotify_client_id'],
config_data['spotify_client_secret'])
# Grab all tracks from saved tracks or specified playlist
if spotify_playlist is None:
tracks = spotify.get_saved_tracks()
else:
tracks = spotify.get_playlist_tracks(spotify_playlist)
if not tracks:
print("Spotify playlist does not exist")
exit()
# Sync playlists
print(f"Spotify playlist contains {len(tracks)} tracks")
youtube = YouTube(youtube_playlist)
print(f"YouTube playlist contains {len(youtube.videos)} videos")
youtube.sync_playlist(tracks, youtube.playlist)
