def check_playlist_exisxtence(self, email): youtube_related = Youtube() ## Get the user's watch history playlist id mine_channel_response = youtube_related.get_mine_channel_details(email) for item in mine_channel_response['items']: if 'contentDetails' in item: watch_history_playlist_id = item['contentDetails'][ 'relatedPlaylists']['watchHistory'] ## Search if the user already have yougroupe playlist db_playlist_result = UserPlayList.query( UserPlayList.email == email).get() if db_playlist_result == None: playlist_id = youtube_related.add_new_playlist(email) UserPlayList( email=email, playlist_id=playlist_id, watchhistory_playlist_id=watch_history_playlist_id).put() logging.info('%s add new playlist' % email) else: playlist_id = db_playlist_result.playlist_id return playlist_id
def add_videos(self, email, videos): youtube_related = Youtube() ## Check if the playlist exists or not playlist_id = self.check_playlist_exisxtence(email) for video_id in videos: youtube_related.add_video_into_playlist(email, playlist_id, video_id) AddedVideo(email=email, video_id=video_id).put() logging.info('%s add video %s into playlist' % (email, video_id))
def daily_check_video_status(self): youtube_related = Youtube() video_details = Video.query().fetch() for video_detail in video_details: logging.info('Daily check video status query %s' % video_detail.video_id) response = youtube_related.query_video(CronJob.query_email, video_detail.video_id) if response['pageInfo']['totalResults'] == 0: video_detail.key.delete() logging.info('Remove %s from db' % video_detail.video_id)
def add_videos(self, email, videos): youtube_related = Youtube() ## Check if the playlist exists or not playlist_id = self.check_playlist_exisxtence(email) for video_id in videos: response = youtube_related.add_video_into_playlist( email, playlist_id, video_id) if response: AddedVideo(email=email, video_id=video_id).put() logging.info('%s add video %s into playlist' % (email, video_id)) else: failed_video = Video.query(Video.video_id == video_id).get() failed_video.key.delete() logging.info('[Error] Adding video %s failed' % video_id)
def add_group(self, email, select_channel_ids, group_names): youtube_related = Youtube() logging.info('[Action] %s add %s channels into %s' % (email, len(select_channel_ids), group_names)) for select_channel_id in select_channel_ids: for group_name in group_names.split(','): ## Add the group name into UserChannel collection search_user_channel_result = UserChannel.query( UserChannel.email == email, UserChannel.channel_id == select_channel_id).get() if search_user_channel_result.group_name == None: search_user_channel_result.group_name = [group_name] search_user_channel_result.put() else: if group_name not in search_user_channel_result.group_name: search_user_channel_result.group_name.append( group_name) ## Update the new group name search_user_channel_result.put() ## Add the group name into Channel collection search_channel_result = Channel.query( Channel.channel_id == select_channel_id).get() channel_details = youtube_related.get_channel_details( channel_id=select_channel_id, email=email) upload_playlist_id = channel_details['items'][0][ 'contentDetails']['relatedPlaylists']['uploads'] ## If there is no group for the channel, then add. If there is group, then append if search_channel_result.groups == None: search_channel_result.groups = [group_name] search_channel_result.upload_playlist_id = upload_playlist_id search_channel_result.put() ## Get the latest uploaded video self.parse_latest_videos(select_channel_id) elif group_name not in search_channel_result.groups: search_channel_result.groups.append(group_name) search_channel_result.upload_playlist_id = upload_playlist_id search_channel_result.put() logging.info('[Success] %s add %s channels into %s' % (email, len(select_channel_ids), group_names))
def check_playlist_exisxtence(self, email): youtube_related = Youtube() ## Get the user's watch history playlist id mine_channel_response = youtube_related.get_mine_channel_details(email) for item in mine_channel_response['items']: if 'contentDetails' in item: watch_history_playlist_id = item['contentDetails']['relatedPlaylists']['watchHistory'] ## Search if the user already have yougroupe playlist db_playlist_result = UserPlayList.query(UserPlayList.email == email).get() if db_playlist_result == None: playlist_id = youtube_related.add_new_playlist(email) UserPlayList(email=email, playlist_id=playlist_id, watchhistory_playlist_id=watch_history_playlist_id).put() logging.info('%s add new playlist' % email) else: playlist_id = db_playlist_result.playlist_id return playlist_id
def add_group(self, email, select_channel_ids, group_names): youtube_related = Youtube() logging.info('[Action] %s add %s channels into %s' % (email, len(select_channel_ids), group_names)) for select_channel_id in select_channel_ids: for group_name in group_names.split(','): ## Add the group name into UserChannel collection search_user_channel_result = UserChannel.query(UserChannel.email == email, UserChannel.channel_id == select_channel_id).get() if search_user_channel_result.group_name == None: search_user_channel_result.group_name = [group_name] search_user_channel_result.put() else: if group_name not in search_user_channel_result.group_name: search_user_channel_result.group_name.append(group_name) ## Update the new group name search_user_channel_result.put() ## Add the group name into Channel collection search_channel_result = Channel.query(Channel.channel_id == select_channel_id).get() channel_details = youtube_related.get_channel_details(channel_id=select_channel_id, email=email) upload_playlist_id = channel_details['items'][0]['contentDetails']['relatedPlaylists']['uploads'] ## If there is no group for the channel, then add. If there is group, then append if search_channel_result.groups == None: search_channel_result.groups = [group_name] search_channel_result.upload_playlist_id = upload_playlist_id search_channel_result.put() ## Get the latest uploaded video self.parse_latest_videos(select_channel_id) elif group_name not in search_channel_result.groups: search_channel_result.groups.append(group_name) search_channel_result.upload_playlist_id = upload_playlist_id search_channel_result.put() logging.info('[Success] %s add %s channels into %s' % (email, len(select_channel_ids), group_names))
def get_upload_viedos(self, email, group_name): youtube = Youtube() upload_videos = memcache.get('%s_%s' % (email, group_name)) if upload_videos == None: upload_videos = [] ## Get the channels of the select group group_details = UserChannel.query( UserChannel.email == email).fetch() for group_detail in group_details: if group_detail.group_name == None: continue if group_name in group_detail.group_name: ## Use the channel id to query video to get the information of the video. channel_upload_videos = Video.query( Video.channel_id == group_detail.channel_id) for channel_upload_video in channel_upload_videos: ## Checkt if the video is already in playlist added_video_result = AddedVideo.query( AddedVideo.email == email, AddedVideo.video_id == channel_upload_video.video_id).get() if added_video_result == None: upload_videos.append({ 'title': channel_upload_video.title, 'video_id': channel_upload_video.video_id, 'upload_date': channel_upload_video.upload_date, 'channel_id': channel_upload_video.channel_id, 'thumbnail': channel_upload_video.thumbnail }) memcache.add(key='%s_%s' % (email, group_name), value=upload_videos, time=43200) upload_videos = sorted(upload_videos, key=lambda k: k['upload_date'], reverse=True) upload_videos = [ upload_videos[x:x + 4] for x in xrange(0, len(upload_videos), 4) ] return upload_videos
def get_daily_uplaod_videos(self): day_before = (datetime.date.today() - datetime.timedelta(days=1)).strftime('%Y-%m-%d') entry_ptn = re.compile('\<entry\>(.*?)\<\/entry\>', re.DOTALL) video_id_ptn = re.compile('\<yt\:videoId\>(.*?)\<\/yt\:videoId\>') title_ptn = re.compile('\<title\>(.*?)\<\/title\>') upload_date_ptn = re.compile('\<published\>(.*?)\<\/published\>') thumbnail_ptn = re.compile('media\:thumbnail url\=\"(.*?)\"') youtube_related = Youtube() ## Get the channel with tags channels = Channel.query(Channel.groups != None).fetch() for channel in channels: channel_id = channel.channel_id logging.info('Parsing channel %s' % channel_id) feed_url = '%s%s' % (CronJob.feed_url_prefix, channel.channel_id) try: page = urllib2.urlopen(feed_url) except urllib2.HTTPError: continue page_source = page.read() entries = entry_ptn.findall(page_source) for entry in entries: video_id = video_id_ptn.findall(entry)[0] title = title_ptn.findall(entry)[0] thumbnail = thumbnail_ptn.findall(entry)[0] upload_date = upload_date_ptn.findall(entry)[0].split('T')[0] if video_id != None and title != None and upload_date != None and upload_date == day_before: upload_date = datetime.datetime.strptime( upload_date, '%Y-%m-%d') ## Store new uploaded video store_doc = Video(channel_id=channel_id, title=title, video_id=video_id, upload_date=upload_date, thumbnail=thumbnail) store_doc.put() logging.info('[Success] Store video %s into db' % video_id) else: logging.info( '[Failed] Store video %s into db, title = %s, upload_date = %s, day_before = %s' % (video_id, title, upload_date, day_before))
def daily_check_channel_status(self): youtube_related = Youtube() return