def add_comments(user_id, query_id):
current_comment_thread = Comment(mongo_curs, query_id)
param = request.get_json()
api = YouTube(api_key=param['api_key'])
list_vid = param['list_vid']
for id_video in list_vid:
commentThreads_result = api.get_query('commentThreads',
videoId=id_video,
part='id, replies, snippet')
current_comment_thread.create_comment_for_each(commentThreads_result)
# Loop and save while there is content
while 'nextPageToken' in commentThreads_result:
commentThreads_result = api.get_query(
'commentThreads',
videoId=id_video,
part='id, replies, snippet',
pageToken=commentThreads_result['nextPageToken'])
current_comment_thread.create_comment_for_each(
commentThreads_result)
count_comments = int(
mongo_curs.db.comments.find({
'query_id': query_id
}).count())
mongo_curs.db.queries.update_one(
{'query_id': query_id},
{'$set': {
'count_comments': count_comments
}})
return 'POST REQUEST add_comments IS RECEIVED'
def channel_info():
if request.method == 'POST':
if 'api_key' in session:
id_channel = request.form.get('unique_id_channel')
id_username = request.form.get('unique_user_channel')
part = ', '.join(request.form.getlist('part'))
api = YouTube(api_key=session['api_key'])
if 'youtube.com/channel/' in id_channel:
id_channel = YouTube.cleaning_channel(id_channel)
channel_result = api.get_query('channels',
id=id_channel,
part=part)
elif id_username != '':
id_username = YouTube.cleaning_channel(id_username)
channel_result = api.get_query('channels',
forUsername=id_username,
part=part)
else:
channel_result = api.get_query('channels',
id=id_channel,
part=part)
channel_result_string = json.dumps(channel_result,
sort_keys=True,
indent=2,
separators=(',', ': '))
return render_template('methods/view_results.html',
result=channel_result,
string=channel_result_string)
else:
return render_template('explore.html', message='api key not set')
return render_template('explore.html')
def add_related(user_id, query_id):
current_relatedVideos = RelatedVideos(mongo_curs, query_id)
param = request.get_json()
api = YouTube(api_key=param['api_key'])
list_vid = param['list_vid']
for id_video in list_vid:
search_results = api.get_query(
'search',
part='id,snippet',
maxResults=50,
relatedToVideoId=id_video,
type='video',
)
for each in search_results['items']:
each.update({'query_id': query_id})
each.update({'videoId': id_video})
mongo_curs.db.relatedVideos.insert_one(each)
## Loop and save
while 'nextPageToken' in search_results:
search_results = api.get_query(
'search',
part='id,snippet',
maxResults=50,
relatedToVideoId=id_video,
type='video',
pageToken=search_results['nextPageToken'])
if not search_results['items']:
return redirect(url_for('manage'))
else:
for each in search_results['items']:
each.update({'query_id': query_id})
each.update({'belongTo': id_video})
mongo_curs.db.relatedVideos.insert_one(each)
# add metrics for query in json
count_videos = int(
mongo_curs.db.relatedVideos.find({
'query_id': query_id
}).count())
mongo_curs.db.queries.update_one(
{'query_id': query_id},
{'$set': {
'count_relatedVideos': count_videos
}})
# add part as indicator
part_value = mongo_curs.db.queries.find_one_or_404({'query_id': query_id})
mongo_curs.db.queries.update_one(
{'query_id': query_id},
{'$set': {
'part': part_value['part'] + ", relatedVideos",
}},
upsert=False)
return 'POST REQUEST add_related IS RECEIVED'
def video_info():
if request.method == 'POST':
# specific for 'try it' on /
# since it is my own api_key used for now...
if request.form.get('api_key_test') is not None:
api_key = app.config['api_key_test']
elif 'api_key' in session:
api_key = session['api_key']
else:
return render_template('explore.html',
message="""
<h4><strong class="text-danger">You can try Pytheas but to go further you will need to get an api_key from Google services
<br>Please go to <a href="./config" class="btn btn-primary btn-lg active" role="button" aria-pressed="true">Config</a> and follow guidelines</strong></h4>
""")
id_video = request.form.get('unique_id_video')
id_video = YouTube.cleaning_video(id_video)
part = ', '.join(request.form.getlist('part'))
api = YouTube(api_key=api_key)
video_result = api.get_query('videos', id=id_video, part=part)
video_result_string = json.dumps(video_result,
sort_keys=True,
indent=2,
separators=(',', ': '))
return render_template('methods/view_results.html',
result=video_result,
string=video_result_string)
return render_template('explore.html')
def playlist_info():
if request.method == 'POST':
if 'api_key' in session:
id_playlist = request.form.get('unique_id_playlist')
# remember to make same as for cleanning_ytb
if 'youtube.com/watch?v=' in id_playlist:
f = furl(id_playlist)
id_playlist = f.args['list']
part = ', '.join(request.form.getlist('part'))
api = YouTube(api_key=session['api_key'])
playlist_info = api.get_query('playlists',
id=id_playlist,
part=part)
playlist_info_string = json.dumps(playlist_info,
sort_keys=True,
indent=2,
separators=(',', ': '))
return render_template('methods/view_results.html',
result=playlist_info,
string=playlist_info_string)
else:
return render_template('explore.html', message='api key not set')
return render_template('explore.html')
def add_video(user_id, query_id, query_type):
param = request.get_json()
uid = param['query_id']
query_id = param['query_id']
query = param['query']
api_key = param['api_key']
api = YouTube(api_key=api_key)
maxResults = 50
###############################
# RESULTS FOR channel
###############################
# clean empty fields here...
param["channel_username"] = [x for x in param["channel_username"] if x]
param["channel_id"] = [x for x in param["channel_id"] if x]
if query_type == 'channel':
# looking for IDs & usernames
# username need to retrieve ID first
if param['channel_username']:
# HAVE TO PUT ALL OF THIS IN A method FUNCT()
######################################
param_query = param.copy()
rm_list = [
'query', 'query_id', 'author_id', 'channel_id',
'channel_username'
]
[param_query.pop(key) for key in rm_list]
channel_usernames = re.sub("\r", "", param['channel_username'][0])
channel_usernames = re.sub("\n", "", channel_usernames)
channel_usernames = channel_usernames.split(',')
for channel_username in channel_usernames:
param_query['forUsername'] = channel_username
find_channel_id = api.get_query('channels', **param_query)
worker.logger.debug("------> find_channel_id")
worker.logger.debug(find_channel_id)
########################################################
if find_channel_id['items']:
param['query'] = query
param['query_id'] = uid
param['channel_id'] += str(
', ' + find_channel_id['items'][0]['id'])
api.get_channel_videos(mongo_curs,
find_channel_id['items'][0]['id'],
param)
# then for ID
if param['channel_id']:
# worker.logger.debug(channel_ids)
# channel_ids = re.sub("\r", "", param['channel_id'][0])
# channel_ids = re.sub("\n", "", channel_ids)
# channel_ids = channel_ids.split(',')
for channel_id in param['channel_id']:
api.get_channel_videos(mongo_curs, channel_id, param)
# finally add metrics for query in json
count_videos = int(
mongo_curs.db.videos.find({
'query_id': query_id
}).count())
mongo_curs.db.queries.update_one(
{'query_id': query_id},
{'$set': {
'count_videos': count_videos,
'status': 'done'
}})
###############################
## RESULTS FOR searchResults ##
###############################
elif query_type == 'search':
if 'mode' in param:
# FOR NEXT DATE SEARCH INTEGRATION
# param_query = {
# 'q': param['query'],
# 'part': param['part'],
# 'relevenceLanguage': param['language'],
# 'maxResults': param['maxResults'],
# 'order': param['order'],
# 'publishedAfter' : param['publishedAfter'],
# 'publishedBefore' : param['publishedBefore'],
# }
# date_results = api.get_query(
# 'search',
# **param_query,
# )
# for each in date_results['items']:
# each.update({'query_id': query_id})
# each = YouTube.cleaning_each(each)
# mongo_curs.db.videos.insert_one(each)
# while 'nextPageToken' in date_results and len(date_results['items']) != 0:
# worker.logger.debug(date_results['items'][-1]['snippet']['publishedAt'])
# param_query['publishedAfter'] = date_results['items'][-1]['snippet']['publishedAt']
# date_results = api.get_query(
# 'search',
# **param_query,
# pageToken = date_results['nextPageToken'],
# )
# for each in date_results['items']:
# each.update({'query_id': query_id})
# each = YouTube.cleaning_each(each)
# mongo_curs.db.videos.insert_one(each)
# Parse date time from form
d_start = datetime.datetime.strptime(param['publishedAfter'],
"%Y-%m-%dT%H:%M:%SZ")
d_end = datetime.datetime.strptime(param['publishedBefore'],
"%Y-%m-%dT%H:%M:%SZ")
r_after = time.parse(param['publishedAfter'])
r_before = time.parse(param['publishedBefore'])
delta = r_before - r_after
delta_days = delta.days + 1
param_query = {
'q': param['query'],
'part': param['part'],
'maxResults': param['maxResults'],
'order': param['order']
}
if not param['language'] == 'None':
param_query['language'] = param['language']
# worker.logger.debug(param_query)
# Then iterate for each days
for n in range(delta.days + 1):
# increment one day later to get a one-day period
r_after_next = r_after + dt.timedelta(days=1)
st_point = r_after.isoformat()
ed_point = r_after_next.isoformat()
# Querying
date_results = api.get_query(
'search',
**param_query,
publishedAfter=st_point,
publishedBefore=ed_point,
)
# saving
for each in date_results['items']:
each.update({'query_id': query_id})
each = YouTube.cleaning_each(each)
mongo_curs.db.videos.insert_one(each)
# loop
while 'nextPageToken' in date_results and len(
date_results['items']) != 0:
date_results = api.get_query(
'search',
**param_query,
publishedAfter=st_point,
publishedBefore=ed_point,
pageToken=date_results['nextPageToken'])
if date_results['items']:
for each in date_results['items']:
each.update({'query_id': query_id})
each = YouTube.cleaning_each(each)
mongo_curs.db.videos.insert_one(each)
else:
break
# finally increment next after day
r_after += dt.timedelta(days=1)
else:
search_results = api.get_query('search',
q=param['query'],
part=param['part'],
relevanceLanguage=param['language'],
maxResults=param['maxResults'],
order=param['order'])
# insert videos
for each in search_results['items']:
each.update({'query_id': uid})
each = YouTube.cleaning_each(each)
mongo_curs.db.videos.insert_one(each)
# Loop and save
while 'nextPageToken' in search_results:
search_results = api.get_query(
'search',
q=param['query'],
part=param['part'],
relevanceLanguage=param['language'],
maxResults=param['maxResults'],
order=param['order'],
pageToken=search_results['nextPageToken'])
if search_results['items']:
# insert video-info
for each in search_results['items']:
each.update({'query_id': uid})
each = YouTube.cleaning_each(each)
mongo_curs.db.videos.insert_one(each)
else:
# add metrics for query in json
count_videos = int(
mongo_curs.db.videos.find({
'query_id': uid
}).count())
mongo_curs.db.queries.update_one(
{'query_id': uid},
{'$set': {
'count_videos': count_videos
}})
break
count_videos = int(
mongo_curs.db.videos.find({
'query_id': uid
}).count())
mongo_curs.db.queries.update_one(
{'query_id': uid},
{'$set': {
'count_videos': count_videos,
'status': 'done'
}})
###############################
# RESULTS FOR SET OF videosList
###############################
elif query_type == 'videos':
param['videos'] = [x.strip() for x in param['videos'][0].split(',')]
param['videos'] = [i for i in param['videos'] if i]
for each in param['videos']:
video_result = api.get_query('videos', id=each, part=param['part'])
video_result = video_result['items'][0]
video_result.update({'query_id': uid})
mongo_curs.db.videos.insert_one(video_result)
count_videos = int(
mongo_curs.db.videos.find({
'query_id': uid
}).count())
mongo_curs.db.queries.update_one(
{'query_id': uid},
{'$set': {
'count_videos': count_videos,
'status': 'done'
}})
###############################
# RESULTS FOR PLAYLISTITEM
###############################
elif query_type == 'playlist':
for playlist_id in param['playlist_id']:
param.update({'playlist_id': playlist_id})
# call request
playlist_results = api.get_playlist(mongo_curs, param)
# add metrics for query in json
count_videos = int(
mongo_curs.db.videos.find({
'query_id': query_id
}).count())
mongo_curs.db.queries.update_one(
{'query_id': query_id},
{'$set': {
'count_videos': count_videos,
'status': 'done'
}})
return 'videos added'
