def _get_atom_video_data(youtube_data, playlist=None): def get_category(categories): for category in categories: if category.scheme.endswith('categories.cat'): return category.text media = youtube_data.media video = Video( source_videoid=media.FindExtensions('videoid')[0].text, source_listid=playlist, source_username=media.credit.text, date_published=_parse_datetime(youtube_data.published.text), title=youtube_data.title.text, duration=int(media.duration.seconds) if media.duration else 0, ) video.source_category = get_category(media.category) for thumbnail in media.thumbnail: if 'time' not in thumbnail.extension_attributes: video.thumbnails.append( VideoThumbnail( url=thumbnail.url, width=thumbnail.width, height=thumbnail.height)) for restriction in media.FindExtensions('restriction'): if restriction.attributes['type'] == 'country': video.restrictions.extend( VideoRestriction( relationship=restriction.attributes['relationship'], country=country) for country in restriction.text.split()) return video
def import_google_movies(): freshold = datetime.now() - timedelta(days=app.config.get('GOOGLE_MOVIE_FRESHOLD', 120)) year_format = re.compile(' \((20\d\d)\)') for channelid, location in app.config['GOOGLE_MOVIE_LOCATIONS']: start = 0 video_ids = set() channel = Channel.query.get(channelid) existing = set(v for v, in VideoInstance.query. filter_by(channel=channelid).join(Video).values('source_videoid')) while True: url = app.config['GOOGLE_MOVIE_URL'] % (location, start) html = get_external_resource(url).read() video_ids.update(re.findall('youtube.com/watch%3Fv%3D(.{11})', html)) next = re.search('<a [^>]*start=(\d+)[^>]*><img[^>]*><br>Next</a>', html) if next: start = int(next.group(1)) time.sleep(1) # Don't get blocked by google else: break feed_ids = [('videos', id) for id in video_ids - existing] if feed_ids: playlist = batch_query(feed_ids, playlist='Googlemovietrailers/uploads') videos = [] for video in playlist.videos: year_match = year_format.search(video.title) if video.date_published > freshold and ( not year_match or int(year_match.group(1)) >= freshold.year): videos.append(video) else: app.logger.debug('Skipped import of trailer "%s" (%s)', video.title, video.date_published) added = Video.add_videos(videos, 1) channel.add_videos(videos) app.logger.info('Added %d trailers to "%s"', added, channel.title)
def _import_videos(self, form): for video in form.import_data.videos: video.rockpack_curated = True video.category = form.category.data count = Video.add_videos(form.import_data.videos, form.source.data) if not form.channel.data and not form.user.data: self._set_form_data_from_source(form) channelid = form.channel.data if channelid.startswith('_new:'): channel = Channel.create( title=channelid.split(':', 1)[1], owner=form.user.data, description=form.channel_description.data, cover=form.cover.data, category=form.category.data) self.record_action('created', channel) else: channel = Channel.query.get_or_404(channelid) channel.add_videos( form.import_data.videos, form.tags.data, category=form.category.data, date_added=form.date_added.data ) self.record_action('imported', channel, '%d videos' % count) push_config = getattr(form.import_data, 'push_config', None) if push_config and channel.id: try: subscribe(push_config.hub, push_config.topic, channel.id) except Exception, e: flash('Unable to subscribe to channel: %s' % e.message, 'error')
def get_video_data(id, fetch_all_videos=True, fetch_metadata=False): """Return video data from youtube api as playlist of one.""" data = _ooyala_feed('assets', id) video = Video( source_videoid=data['embed_code'], source_listid=None, source_username=None, date_published=_parse_datetime(data['updated_at']), title=data['name'], duration=data['duration'] / 1000, ) video.source_date_uploaded = _parse_datetime(data['created_at']) video.restricted = bool(data['time_restrictions']) update_thumbnails(video) if fetch_metadata: video.meta = _ooyala_feed('assets', id, 'metadata') video.category = video.meta.get('category', None) return Videolist(1, [video])
def _get_video_data_v3(youtube_data, playlist=None): snippet = youtube_data['snippet'] video = Video( source_videoid=youtube_data['id']['videoId'], source_listid=playlist, title=snippet['title'], # http://code.google.com/p/gdata-issues/issues/detail?id=4294 # duration=snippet['duration'], ) video.source_category = None video.source_view_count = None video.source_date_uploaded = snippet['publishedAt'] video.restricted = None for label, thumbnail in snippet['thumbnails'].items(): video.thumbnails.append( VideoThumbnail( url=thumbnail['url'], width=None, height=None)) return video
def _get_video_data(youtube_data, playlist=None): """Extract data from youtube video json record and return Video model.""" def get_category(categories): for category in categories: if category['scheme'].endswith('categories.cat'): return category['$t'] # TODO: map category media = youtube_data['media$group'] video = Video( source_videoid=media['yt$videoid']['$t'], source_listid=playlist, source_username=media['media$credit'][0]['$t'], date_published=_parse_datetime(youtube_data['published']['$t']), title=youtube_data['title']['$t'], duration=int(media['yt$duration']['seconds']) if 'yt$duration' in media else -1, ) video.source_category = get_category(media.get('media$category', [])) video.source_view_count = int(youtube_data['yt$statistics']['viewCount']) if 'yt$statistics' in youtube_data else -1 video.source_date_uploaded = media['yt$uploaded']['$t'] access_control = dict( (i['action'], i['permission'] == 'allowed') for i in youtube_data.get('yt$accessControl', [])) video.restricted = access_control.get('embed') is False if 'app$control' in youtube_data: if 'yt$incomplete' in youtube_data['app$control']: video.restricted = True else: state = youtube_data['app$control']['yt$state'] if state['name'] == 'restricted': if state['reasonCode'] == 'limitedSyndication': # see https://groups.google.com/d/msg/youtube-api-gdata/on504fCOEk0/oErUbCptWu4J video.restricted = not any(c.get('yt$format') == 5 for c in media.get('media$content', [])) else: video.restricted = True for thumbnail in media.get('media$thumbnail', []): if 'time' not in thumbnail: video.thumbnails.append( VideoThumbnail( url=thumbnail['url'], width=thumbnail['width'], height=thumbnail['height'])) for restriction in media.get('media$restriction', []): if restriction['type'] == 'country': video.restrictions.extend( VideoRestriction( relationship=restriction['relationship'], country=country) for country in restriction['$t'].split()) return video
def _format_results(self, videos, with_channels=True, with_stars=False, add_tracking=None): vlist = [] channel_list = set() IMAGE_CDN = app.config.get('IMAGE_CDN', '') BASE_URL = url_for('basews.discover') def _format_user_data(user): return dict( id=user.resource_url.lstrip('/').split('/')[1], display_name=user.display_name, resource_url=urljoin(BASE_URL, user.resource_url), avatar_thumbnail_url=urljoin(IMAGE_CDN, user.avatar) if user.avatar else '' ) for pos, v in enumerate(videos, self.paging[0]): published = v.video.date_published video = dict( id=v.id, channel=dict( id=v.channel, title=v.channel_title), title=v.title, label=v.label, date_added=_format_datetime(v.date_added), public=v.public, category='', video=dict( id=v.video.id, view_count=sum(l['view_count'] for l in v['locales'].values()), star_count=sum(l['star_count'] for l in v['locales'].values()), source=Source.id_to_label(v.video.source), source_id=v.video.source_id, source_username=v.video.source_username, source_date_uploaded=published.isoformat() if hasattr(published, 'isoformat') else published, duration=v.video.duration, description=Video.cleaned_description(v.video.description), thumbnail_url=urljoin(app.config.get('IMAGE_CDN', ''), v.video.thumbnail_url) if v.video.thumbnail_url else '', ), position=pos, child_instance_count=getattr(v, 'child_instance_count', 0) ) video['video'].update(Video.extra_meta(v.video)) if v.link_url: video['video'].update(link_url=v.link_url, link_title=v.link_title) if v.owner: video['channel']['owner'] = _format_user_data(v.owner) if v.owner and v.channel: video['channel']['resource_url'] = urljoin(BASE_URL, url_for('userws.channel_info', userid=video['channel']['owner']['id'], channelid=v.channel)) if v.original_channel_owner: video['original_channel_owner'] = _format_user_data(v.original_channel_owner) if app.config.get('DOLLY'): video.update({ "comments": { "total": getattr(v.comments, 'count', 0) } }) if v.category: video['category'] = max(v.category) if isinstance(v.category, list) else v.category if with_stars: video['recent_user_stars'] = v.get('recent_user_stars', []) if add_tracking: add_tracking(video) channel_list.add(v.channel) vlist.append(video) if with_channels and channel_list: ch = ChannelSearch(self.locale) ch.add_id(channel_list) channel_map = {c['id']: c for c in ch.channels(with_owners=True)} self.add_channels_to_videos(vlist, channel_map) return vlist
def _update_channel_videos(channel, data): playlist = youtube.parse_atom_playlist_data(data) source = Source.label_to_id('youtube') Video.add_videos(playlist.videos, source) channel.add_videos(playlist.videos)