def add_video(user_id, query_id, query_type):
param = request.get_json()
uid = param['query_id']
query_id = param['query_id']
query = param['query']
api_key = param['api_key']
api = YouTube(api_key=api_key)
maxResults = 50
###############################
# RESULTS FOR channel
###############################
# clean empty fields here...
param["channel_username"] = [x for x in param["channel_username"] if x]
param["channel_id"] = [x for x in param["channel_id"] if x]
if query_type == 'channel':
# looking for IDs & usernames
# username need to retrieve ID first
if param['channel_username']:
# HAVE TO PUT ALL OF THIS IN A method FUNCT()
######################################
param_query = param.copy()
rm_list = [
'query', 'query_id', 'author_id', 'channel_id',
'channel_username'
]
[param_query.pop(key) for key in rm_list]
channel_usernames = re.sub("\r", "", param['channel_username'][0])
channel_usernames = re.sub("\n", "", channel_usernames)
channel_usernames = channel_usernames.split(',')
for channel_username in channel_usernames:
param_query['forUsername'] = channel_username
find_channel_id = api.get_query('channels', **param_query)
worker.logger.debug("------> find_channel_id")
worker.logger.debug(find_channel_id)
########################################################
if find_channel_id['items']:
param['query'] = query
param['query_id'] = uid
param['channel_id'] += str(
', ' + find_channel_id['items'][0]['id'])
api.get_channel_videos(mongo_curs,
find_channel_id['items'][0]['id'],
param)
# then for ID
if param['channel_id']:
# worker.logger.debug(channel_ids)
# channel_ids = re.sub("\r", "", param['channel_id'][0])
# channel_ids = re.sub("\n", "", channel_ids)
# channel_ids = channel_ids.split(',')
for channel_id in param['channel_id']:
api.get_channel_videos(mongo_curs, channel_id, param)
# finally add metrics for query in json
count_videos = int(
mongo_curs.db.videos.find({
'query_id': query_id
}).count())
mongo_curs.db.queries.update_one(
{'query_id': query_id},
{'$set': {
'count_videos': count_videos,
'status': 'done'
}})
###############################
## RESULTS FOR searchResults ##
###############################
elif query_type == 'search':
if 'mode' in param:
# FOR NEXT DATE SEARCH INTEGRATION
# param_query = {
# 'q': param['query'],
# 'part': param['part'],
# 'relevenceLanguage': param['language'],
# 'maxResults': param['maxResults'],
# 'order': param['order'],
# 'publishedAfter' : param['publishedAfter'],
# 'publishedBefore' : param['publishedBefore'],
# }
# date_results = api.get_query(
# 'search',
# **param_query,
# )
# for each in date_results['items']:
# each.update({'query_id': query_id})
# each = YouTube.cleaning_each(each)
# mongo_curs.db.videos.insert_one(each)
# while 'nextPageToken' in date_results and len(date_results['items']) != 0:
# worker.logger.debug(date_results['items'][-1]['snippet']['publishedAt'])
# param_query['publishedAfter'] = date_results['items'][-1]['snippet']['publishedAt']
# date_results = api.get_query(
# 'search',
# **param_query,
# pageToken = date_results['nextPageToken'],
# )
# for each in date_results['items']:
# each.update({'query_id': query_id})
# each = YouTube.cleaning_each(each)
# mongo_curs.db.videos.insert_one(each)
# Parse date time from form
d_start = datetime.datetime.strptime(param['publishedAfter'],
"%Y-%m-%dT%H:%M:%SZ")
d_end = datetime.datetime.strptime(param['publishedBefore'],
"%Y-%m-%dT%H:%M:%SZ")
r_after = time.parse(param['publishedAfter'])
r_before = time.parse(param['publishedBefore'])
delta = r_before - r_after
delta_days = delta.days + 1
param_query = {
'q': param['query'],
'part': param['part'],
'maxResults': param['maxResults'],
'order': param['order']
}
if not param['language'] == 'None':
param_query['language'] = param['language']
# worker.logger.debug(param_query)
# Then iterate for each days
for n in range(delta.days + 1):
# increment one day later to get a one-day period
r_after_next = r_after + dt.timedelta(days=1)
st_point = r_after.isoformat()
ed_point = r_after_next.isoformat()
# Querying
date_results = api.get_query(
'search',
**param_query,
publishedAfter=st_point,
publishedBefore=ed_point,
)
# saving
for each in date_results['items']:
each.update({'query_id': query_id})
each = YouTube.cleaning_each(each)
mongo_curs.db.videos.insert_one(each)
# loop
while 'nextPageToken' in date_results and len(
date_results['items']) != 0:
date_results = api.get_query(
'search',
**param_query,
publishedAfter=st_point,
publishedBefore=ed_point,
pageToken=date_results['nextPageToken'])
if date_results['items']:
for each in date_results['items']:
each.update({'query_id': query_id})
each = YouTube.cleaning_each(each)
mongo_curs.db.videos.insert_one(each)
else:
break
# finally increment next after day
r_after += dt.timedelta(days=1)
else:
search_results = api.get_query('search',
q=param['query'],
part=param['part'],
relevanceLanguage=param['language'],
maxResults=param['maxResults'],
order=param['order'])
# insert videos
for each in search_results['items']:
each.update({'query_id': uid})
each = YouTube.cleaning_each(each)
mongo_curs.db.videos.insert_one(each)
# Loop and save
while 'nextPageToken' in search_results:
search_results = api.get_query(
'search',
q=param['query'],
part=param['part'],
relevanceLanguage=param['language'],
maxResults=param['maxResults'],
order=param['order'],
pageToken=search_results['nextPageToken'])
if search_results['items']:
# insert video-info
for each in search_results['items']:
each.update({'query_id': uid})
each = YouTube.cleaning_each(each)
mongo_curs.db.videos.insert_one(each)
else:
# add metrics for query in json
count_videos = int(
mongo_curs.db.videos.find({
'query_id': uid
}).count())
mongo_curs.db.queries.update_one(
{'query_id': uid},
{'$set': {
'count_videos': count_videos
}})
break
count_videos = int(
mongo_curs.db.videos.find({
'query_id': uid
}).count())
mongo_curs.db.queries.update_one(
{'query_id': uid},
{'$set': {
'count_videos': count_videos,
'status': 'done'
}})
###############################
# RESULTS FOR SET OF videosList
###############################
elif query_type == 'videos':
param['videos'] = [x.strip() for x in param['videos'][0].split(',')]
param['videos'] = [i for i in param['videos'] if i]
for each in param['videos']:
video_result = api.get_query('videos', id=each, part=param['part'])
video_result = video_result['items'][0]
video_result.update({'query_id': uid})
mongo_curs.db.videos.insert_one(video_result)
count_videos = int(
mongo_curs.db.videos.find({
'query_id': uid
}).count())
mongo_curs.db.queries.update_one(
{'query_id': uid},
{'$set': {
'count_videos': count_videos,
'status': 'done'
}})
###############################
# RESULTS FOR PLAYLISTITEM
###############################
elif query_type == 'playlist':
for playlist_id in param['playlist_id']:
param.update({'playlist_id': playlist_id})
# call request
playlist_results = api.get_playlist(mongo_curs, param)
# add metrics for query in json
count_videos = int(
mongo_curs.db.videos.find({
'query_id': query_id
}).count())
mongo_curs.db.queries.update_one(
{'query_id': query_id},
{'$set': {
'count_videos': count_videos,
'status': 'done'
}})
return 'videos added'
