def add_feed_media(cls, collection_id, feed_category=None, approve_all=False): collection = Collection.get_by_id(int(collection_id)) medias = [] if feed_category: uri = Collection.YOUTUBE_CATEGORY_FEED % ((collection.region or 'US'), collection.feed_id, feed_category) else: uri = Collection.YOUTUBE_FEED % ((collection.region or 'US'), collection.feed_id) response = urlfetch.fetch(uri) if response.status_code == 200: data = simplejson.loads(response.content) or {} entries = data['feed']['entry'] if data.get('feed') else [] ids = '' publisher_map = {} for entry in entries: id = re.search('video:(.*)', entry['id']['$t']).group(1) publisher = Publisher.add(host=MediaHost.YOUTUBE, host_id=entry['author'][0]['yt$userId']['$t']) deferred.defer(Publisher.fetch_details, publisher.id, _name='publisher-' + publisher.id + '-' + str(uuid.uuid1()), _queue='youtube') ids += id + ',' publisher_map[id] = publisher if len(ids): youtube3 = get_youtube3_service() videos_response = youtube3.videos().list( id=ids, part='id,snippet,topicDetails,contentDetails,statistics' ).execute() logging.info('ADD FEED VIDEO: ' + id) medias = Media.add_from_snippet(videos_response.get('items', []), collection=collection, publisher=publisher, approve=approve_all) return medias
def add_reddit_media(cls, collection_id, reddit_id, approve_all): collection = Collection.get_by_id(int(collection_id)) response = urlfetch.fetch('http://www.reddit.com/r/%s.json?limit=50' % reddit_id) content = simplejson.loads(response.content) or {} if content.get('error'): logging.error('REDDIT ERROR: ' + str(content.get('error'))) data = content.get('data') if data: links = data.get('children') ids = [] reddit_ids = {} logging.info('FETCHING %s VIDEOS FROM REDDIT' % str(len(links))) for link in links: link_data = link.get('data') or {} if link_data.get('domain') == 'youtube.com' and not link_data.get('over_18') and link_data.get('score') > 100: url = link_data.get('url') parsed_url = urlparse.urlparse(url) host_id = urlparse.parse_qs(parsed_url.query)['v'][0] reddit_ids[host_id] = link.get('kind') + '_' + link_data.get('id') ids.append(host_id) youtube3 = get_youtube3_service() while len(ids) > 0: videos_response = youtube3.videos().list( id=','.join(ids[0:10]), part="id,snippet,topicDetails,contentDetails,statistics" ).execute() if videos_response.get('error'): logging.error(videos_response.get('error').get('error')[0]['reason']) return medias = Media.add_from_snippet(videos_response.get("items", []), collection=collection, approve=approve_all) for media in medias: if reddit_ids[media.host_id]: media.reddit_id = reddit_ids[media.host_id] media.put() ids = ids[10:]
def youtube_channel(cls, name=None, user=None, token=None, yt_channel_id=None, yt_playlist_id=None, page_token=None): name = name or 'ytchannel' channel = Channel(key_name=(yt_channel_id or yt_playlist_id), name=name, privacy=Privacy.FRIENDS, online=True, user=user, youtube=True) channel_id = (yt_channel_id or yt_playlist_id) medias = [] all = True cached_channel = memcache.get(channel_id) #if cached_channel: # return simplejson.loads(cached_channel) youtube3 = get_youtube3_service() search_response = {} if yt_channel_id: if yt_channel_id.startswith('HC'): channel_response = youtube3.channels().list( id=yt_channel_id, part='topicDetails', maxResults=1 ).execute() if len(channel_response.get('items', [])): topic_id = channel_response.get('items')[0]['topicDetails']['topicIds'][0] search_response = youtube3.search().list( topicId=topic_id, part='id,snippet', order='date', maxResults=(20 if all else 1), pageToken=page_token, fields='items,nextPageToken', type='video' ).execute() else: search_response = youtube3.search().list( channelId=yt_channel_id, part='id,snippet', order='date', maxResults=(20 if all else 1), pageToken=page_token, fields='items,nextPageToken', type='video' ).execute() elif yt_playlist_id: search_response = youtube3.playlistItems().list( playlistId=yt_playlist_id, part='id,snippet', maxResults=(20 if all else 1), pageToken=page_token, fields='items,nextPageToken' ).execute() logging.info('SEARCH RESPONSE \n \n \n') logging.info(search_response) next_page_token = search_response.get("nextPageToken") if len(search_response.get('items', [])): search_ids = '' for item in search_response.get('items', []): if item['kind'] == 'youtube#searchResult': search_ids += item['id']['videoId'] + ',' elif item['kind'] == 'youtube#playlistItem': search_ids += item['snippet']['resourceId']['videoId'] + ',' videos_response = youtube3.videos().list( id=search_ids, part="id,snippet,topicDetails,contentDetails,statistics" ).execute() medias = Media.add_from_snippet(videos_response.get("items", []), approve=True) if not all: deferred.defer(util.schedule_youtube_channel, name=name, user_id=user.id if user else None, token=token, channel_id=channel_id, yt_channel_id=yt_channel_id, yt_playlist_id=yt_playlist_id, _name='youtube-channel-' + channel_id + '-' + str(uuid.uuid1()), _queue='youtube') else: logging.error('No search results for youtube channel: ' + str(search_response)) programs = [] if user and False: programs = programming.Programming.set_user_channel_programs(user.id, channel, medias, time=datetime.datetime.now(), reset=True) else: next_time = datetime.datetime.now() for media in medias: program = Program.add_program(channel, media, time=next_time) if program: programs.append(program) next_time = next_time + datetime.timedelta(seconds=media.duration) if not all: broadcast.broadcastNewPrograms(channel, programs, new_channel=True, to_owner=False, token=token) if len(medias): deferred.defer(programming.Programming.fetch_related_tweets, medias, _name='twitter-' + channel_id + '-' + str(uuid.uuid1()), _queue='twitter', _countdown=30) channel_json = channel.toJson() channel_json['next_page_token'] = next_page_token data_json = { 'channel': channel_json, 'programs': [p.toJson() for p in programs] } memcache.set(channel_id, simplejson.dumps(data_json), time=86400) return data_json
def fetch(self, collection=None, approve_all=False): all_medias = [] if self.host == MediaHost.YOUTUBE: if not self.channel_id: yt_service = get_youtube_service() try: user_entry = yt_service.GetYouTubeUserEntry(username=self.host_id) except Exception as e: logging.error(e) return desc = user_entry.content.text desc = desc.decode('utf-8').replace("\n", r" ") if desc else None picture = urlfetch.fetch(user_entry.thumbnail.url) self.name = user_entry.title.text self.picture = db.Blob(picture.content) self.description = desc self.link = user_entry.link[0].href self.channel_id = re.search('/channel/(.*)', self.link).groups()[0] self.put() youtube3 = get_youtube3_service() publishedAfter = '1970-01-01T00:00:00Z' if constants.PUBLISHED_AFTER: publishedAfter = constants.PUBLISHED_AFTER elif self.last_fetch: publishedAfter = self.last_fetch.isoformat('T') + 'Z' ordering = ['date'] if self.last_fetch else ['date', 'rating', 'viewCount'] for order in ordering: medias = [] next_page_token = '' while next_page_token is not None: search_response = youtube3.search().list( channelId=self.channel_id, part='id', order=order, pageToken=next_page_token, publishedAfter=publishedAfter, maxResults=20 ).execute() search_ids = '' for item in search_response.get('items', []): if item['id']['kind'] == 'youtube#video': search_ids += item['id']['videoId'] + ',' videos_response = youtube3.videos().list( id=search_ids, part="id,snippet,topicDetails,contentDetails,statistics" ).execute() medias = Media.add_from_snippet(videos_response.get("items", []), collection=collection, publisher=self, enforce_category=len(collection.categories) > 0, approve=approve_all) all_medias += medias next_page_token = search_response.get('tokenPagination', {}).get('nextPageToken') self.last_fetch = datetime.datetime.now() self.put() if len(all_medias) and not approve_all: msg = emailer.Email(emailer.Message.FETCH, data={'count': len(medias)}) for uid in constants.SUPER_ADMINS: user = User.get_by_key_name(uid) msg.send(user)