def check_playlist_exisxtence(self, email):
youtube_related = Youtube()
## Get the user's watch history playlist id
mine_channel_response = youtube_related.get_mine_channel_details(email)
for item in mine_channel_response['items']:
if 'contentDetails' in item:
watch_history_playlist_id = item['contentDetails'][
'relatedPlaylists']['watchHistory']
## Search if the user already have yougroupe playlist
db_playlist_result = UserPlayList.query(
UserPlayList.email == email).get()
if db_playlist_result == None:
playlist_id = youtube_related.add_new_playlist(email)
UserPlayList(
email=email,
playlist_id=playlist_id,
watchhistory_playlist_id=watch_history_playlist_id).put()
logging.info('%s add new playlist' % email)
else:
playlist_id = db_playlist_result.playlist_id
return playlist_id
def add_videos(self, email, videos):
youtube_related = Youtube()
## Check if the playlist exists or not
playlist_id = self.check_playlist_exisxtence(email)
for video_id in videos:
youtube_related.add_video_into_playlist(email, playlist_id, video_id)
AddedVideo(email=email, video_id=video_id).put()
logging.info('%s add video %s into playlist' % (email, video_id))
def daily_check_video_status(self):
youtube_related = Youtube()
video_details = Video.query().fetch()
for video_detail in video_details:
logging.info('Daily check video status query %s' % video_detail.video_id)
response = youtube_related.query_video(CronJob.query_email, video_detail.video_id)
if response['pageInfo']['totalResults'] == 0:
video_detail.key.delete()
logging.info('Remove %s from db' % video_detail.video_id)
def daily_check_video_status(self):
youtube_related = Youtube()
video_details = Video.query().fetch()
for video_detail in video_details:
logging.info('Daily check video status query %s' %
video_detail.video_id)
response = youtube_related.query_video(CronJob.query_email,
video_detail.video_id)
if response['pageInfo']['totalResults'] == 0:
video_detail.key.delete()
logging.info('Remove %s from db' % video_detail.video_id)
def add_videos(self, email, videos):
youtube_related = Youtube()
## Check if the playlist exists or not
playlist_id = self.check_playlist_exisxtence(email)
for video_id in videos:
response = youtube_related.add_video_into_playlist(
email, playlist_id, video_id)
if response:
AddedVideo(email=email, video_id=video_id).put()
logging.info('%s add video %s into playlist' %
(email, video_id))
else:
failed_video = Video.query(Video.video_id == video_id).get()
failed_video.key.delete()
logging.info('[Error] Adding video %s failed' % video_id)
def add_group(self, email, select_channel_ids, group_names):
youtube_related = Youtube()
logging.info('[Action] %s add %s channels into %s' %
(email, len(select_channel_ids), group_names))
for select_channel_id in select_channel_ids:
for group_name in group_names.split(','):
## Add the group name into UserChannel collection
search_user_channel_result = UserChannel.query(
UserChannel.email == email,
UserChannel.channel_id == select_channel_id).get()
if search_user_channel_result.group_name == None:
search_user_channel_result.group_name = [group_name]
search_user_channel_result.put()
else:
if group_name not in search_user_channel_result.group_name:
search_user_channel_result.group_name.append(
group_name)
## Update the new group name
search_user_channel_result.put()
## Add the group name into Channel collection
search_channel_result = Channel.query(
Channel.channel_id == select_channel_id).get()
channel_details = youtube_related.get_channel_details(
channel_id=select_channel_id, email=email)
upload_playlist_id = channel_details['items'][0][
'contentDetails']['relatedPlaylists']['uploads']
## If there is no group for the channel, then add. If there is group, then append
if search_channel_result.groups == None:
search_channel_result.groups = [group_name]
search_channel_result.upload_playlist_id = upload_playlist_id
search_channel_result.put()
## Get the latest uploaded video
self.parse_latest_videos(select_channel_id)
elif group_name not in search_channel_result.groups:
search_channel_result.groups.append(group_name)
search_channel_result.upload_playlist_id = upload_playlist_id
search_channel_result.put()
logging.info('[Success] %s add %s channels into %s' %
(email, len(select_channel_ids), group_names))
def check_playlist_exisxtence(self, email):
youtube_related = Youtube()
## Get the user's watch history playlist id
mine_channel_response = youtube_related.get_mine_channel_details(email)
for item in mine_channel_response['items']:
if 'contentDetails' in item:
watch_history_playlist_id = item['contentDetails']['relatedPlaylists']['watchHistory']
## Search if the user already have yougroupe playlist
db_playlist_result = UserPlayList.query(UserPlayList.email == email).get()
if db_playlist_result == None:
playlist_id = youtube_related.add_new_playlist(email)
UserPlayList(email=email, playlist_id=playlist_id, watchhistory_playlist_id=watch_history_playlist_id).put()
logging.info('%s add new playlist' % email)
else:
playlist_id = db_playlist_result.playlist_id
return playlist_id
def add_group(self, email, select_channel_ids, group_names):
youtube_related = Youtube()
logging.info('[Action] %s add %s channels into %s' % (email, len(select_channel_ids), group_names))
for select_channel_id in select_channel_ids:
for group_name in group_names.split(','):
## Add the group name into UserChannel collection
search_user_channel_result = UserChannel.query(UserChannel.email == email,
UserChannel.channel_id == select_channel_id).get()
if search_user_channel_result.group_name == None:
search_user_channel_result.group_name = [group_name]
search_user_channel_result.put()
else:
if group_name not in search_user_channel_result.group_name:
search_user_channel_result.group_name.append(group_name)
## Update the new group name
search_user_channel_result.put()
## Add the group name into Channel collection
search_channel_result = Channel.query(Channel.channel_id == select_channel_id).get()
channel_details = youtube_related.get_channel_details(channel_id=select_channel_id, email=email)
upload_playlist_id = channel_details['items'][0]['contentDetails']['relatedPlaylists']['uploads']
## If there is no group for the channel, then add. If there is group, then append
if search_channel_result.groups == None:
search_channel_result.groups = [group_name]
search_channel_result.upload_playlist_id = upload_playlist_id
search_channel_result.put()
## Get the latest uploaded video
self.parse_latest_videos(select_channel_id)
elif group_name not in search_channel_result.groups:
search_channel_result.groups.append(group_name)
search_channel_result.upload_playlist_id = upload_playlist_id
search_channel_result.put()
logging.info('[Success] %s add %s channels into %s' % (email, len(select_channel_ids), group_names))
def get_upload_viedos(self, email, group_name):
youtube = Youtube()
upload_videos = memcache.get('%s_%s' % (email, group_name))
if upload_videos == None:
upload_videos = []
## Get the channels of the select group
group_details = UserChannel.query(
UserChannel.email == email).fetch()
for group_detail in group_details:
if group_detail.group_name == None:
continue
if group_name in group_detail.group_name:
## Use the channel id to query video to get the information of the video.
channel_upload_videos = Video.query(
Video.channel_id == group_detail.channel_id)
for channel_upload_video in channel_upload_videos:
## Checkt if the video is already in playlist
added_video_result = AddedVideo.query(
AddedVideo.email == email, AddedVideo.video_id ==
channel_upload_video.video_id).get()
if added_video_result == None:
upload_videos.append({
'title':
channel_upload_video.title,
'video_id':
channel_upload_video.video_id,
'upload_date':
channel_upload_video.upload_date,
'channel_id':
channel_upload_video.channel_id,
'thumbnail':
channel_upload_video.thumbnail
})
memcache.add(key='%s_%s' % (email, group_name),
value=upload_videos,
time=43200)
upload_videos = sorted(upload_videos,
key=lambda k: k['upload_date'],
reverse=True)
upload_videos = [
upload_videos[x:x + 4] for x in xrange(0, len(upload_videos), 4)
]
return upload_videos
def get_daily_uplaod_videos(self):
day_before = (datetime.date.today() -
datetime.timedelta(days=1)).strftime('%Y-%m-%d')
entry_ptn = re.compile('\<entry\>(.*?)\<\/entry\>', re.DOTALL)
video_id_ptn = re.compile('\<yt\:videoId\>(.*?)\<\/yt\:videoId\>')
title_ptn = re.compile('\<title\>(.*?)\<\/title\>')
upload_date_ptn = re.compile('\<published\>(.*?)\<\/published\>')
thumbnail_ptn = re.compile('media\:thumbnail url\=\"(.*?)\"')
youtube_related = Youtube()
## Get the channel with tags
channels = Channel.query(Channel.groups != None).fetch()
for channel in channels:
channel_id = channel.channel_id
logging.info('Parsing channel %s' % channel_id)
feed_url = '%s%s' % (CronJob.feed_url_prefix, channel.channel_id)
try:
page = urllib2.urlopen(feed_url)
except urllib2.HTTPError:
continue
page_source = page.read()
entries = entry_ptn.findall(page_source)
for entry in entries:
video_id = video_id_ptn.findall(entry)[0]
title = title_ptn.findall(entry)[0]
thumbnail = thumbnail_ptn.findall(entry)[0]
upload_date = upload_date_ptn.findall(entry)[0].split('T')[0]
if video_id != None and title != None and upload_date != None and upload_date == day_before:
upload_date = datetime.datetime.strptime(
upload_date, '%Y-%m-%d')
## Store new uploaded video
store_doc = Video(channel_id=channel_id,
title=title,
video_id=video_id,
upload_date=upload_date,
thumbnail=thumbnail)
store_doc.put()
logging.info('[Success] Store video %s into db' % video_id)
else:
logging.info(
'[Failed] Store video %s into db, title = %s, upload_date = %s, day_before = %s'
% (video_id, title, upload_date, day_before))
def daily_check_channel_status(self):
youtube_related = Youtube()
return
