def _get_atom_video_data(youtube_data, playlist=None):
def get_category(categories):
for category in categories:
if category.scheme.endswith('categories.cat'):
return category.text
media = youtube_data.media
video = Video(
source_videoid=media.FindExtensions('videoid')[0].text,
source_listid=playlist,
source_username=media.credit.text,
date_published=_parse_datetime(youtube_data.published.text),
title=youtube_data.title.text,
duration=int(media.duration.seconds) if media.duration else 0,
)
video.source_category = get_category(media.category)
for thumbnail in media.thumbnail:
if 'time' not in thumbnail.extension_attributes:
video.thumbnails.append(
VideoThumbnail(
url=thumbnail.url,
width=thumbnail.width,
height=thumbnail.height))
for restriction in media.FindExtensions('restriction'):
if restriction.attributes['type'] == 'country':
video.restrictions.extend(
VideoRestriction(
relationship=restriction.attributes['relationship'],
country=country) for country in restriction.text.split())
return video
def import_google_movies():
freshold = datetime.now() - timedelta(days=app.config.get('GOOGLE_MOVIE_FRESHOLD', 120))
year_format = re.compile(' \((20\d\d)\)')
for channelid, location in app.config['GOOGLE_MOVIE_LOCATIONS']:
start = 0
video_ids = set()
channel = Channel.query.get(channelid)
existing = set(v for v, in VideoInstance.query.
filter_by(channel=channelid).join(Video).values('source_videoid'))
while True:
url = app.config['GOOGLE_MOVIE_URL'] % (location, start)
html = get_external_resource(url).read()
video_ids.update(re.findall('youtube.com/watch%3Fv%3D(.{11})', html))
next = re.search('<a [^>]*start=(\d+)[^>]*><img[^>]*><br>Next</a>', html)
if next:
start = int(next.group(1))
time.sleep(1) # Don't get blocked by google
else:
break
feed_ids = [('videos', id) for id in video_ids - existing]
if feed_ids:
playlist = batch_query(feed_ids, playlist='Googlemovietrailers/uploads')
videos = []
for video in playlist.videos:
year_match = year_format.search(video.title)
if video.date_published > freshold and (
not year_match or int(year_match.group(1)) >= freshold.year):
videos.append(video)
else:
app.logger.debug('Skipped import of trailer "%s" (%s)',
video.title, video.date_published)
added = Video.add_videos(videos, 1)
channel.add_videos(videos)
app.logger.info('Added %d trailers to "%s"', added, channel.title)
def _import_videos(self, form):
for video in form.import_data.videos:
video.rockpack_curated = True
video.category = form.category.data
count = Video.add_videos(form.import_data.videos, form.source.data)
if not form.channel.data and not form.user.data:
self._set_form_data_from_source(form)
channelid = form.channel.data
if channelid.startswith('_new:'):
channel = Channel.create(
title=channelid.split(':', 1)[1],
owner=form.user.data,
description=form.channel_description.data,
cover=form.cover.data,
category=form.category.data)
self.record_action('created', channel)
else:
channel = Channel.query.get_or_404(channelid)
channel.add_videos(
form.import_data.videos,
form.tags.data,
category=form.category.data,
date_added=form.date_added.data
)
self.record_action('imported', channel, '%d videos' % count)
push_config = getattr(form.import_data, 'push_config', None)
if push_config and channel.id:
try:
subscribe(push_config.hub, push_config.topic, channel.id)
except Exception, e:
flash('Unable to subscribe to channel: %s' % e.message, 'error')
def get_video_data(id, fetch_all_videos=True, fetch_metadata=False):
"""Return video data from youtube api as playlist of one."""
data = _ooyala_feed('assets', id)
video = Video(
source_videoid=data['embed_code'],
source_listid=None,
source_username=None,
date_published=_parse_datetime(data['updated_at']),
title=data['name'],
duration=data['duration'] / 1000,
)
video.source_date_uploaded = _parse_datetime(data['created_at'])
video.restricted = bool(data['time_restrictions'])
update_thumbnails(video)
if fetch_metadata:
video.meta = _ooyala_feed('assets', id, 'metadata')
video.category = video.meta.get('category', None)
return Videolist(1, [video])
def _get_video_data_v3(youtube_data, playlist=None):
snippet = youtube_data['snippet']
video = Video(
source_videoid=youtube_data['id']['videoId'],
source_listid=playlist,
title=snippet['title'],
# http://code.google.com/p/gdata-issues/issues/detail?id=4294
# duration=snippet['duration'],
)
video.source_category = None
video.source_view_count = None
video.source_date_uploaded = snippet['publishedAt']
video.restricted = None
for label, thumbnail in snippet['thumbnails'].items():
video.thumbnails.append(
VideoThumbnail(
url=thumbnail['url'],
width=None,
height=None))
return video
def _get_video_data(youtube_data, playlist=None):
"""Extract data from youtube video json record and return Video model."""
def get_category(categories):
for category in categories:
if category['scheme'].endswith('categories.cat'):
return category['$t'] # TODO: map category
media = youtube_data['media$group']
video = Video(
source_videoid=media['yt$videoid']['$t'],
source_listid=playlist,
source_username=media['media$credit'][0]['$t'],
date_published=_parse_datetime(youtube_data['published']['$t']),
title=youtube_data['title']['$t'],
duration=int(media['yt$duration']['seconds']) if 'yt$duration' in media else -1,
)
video.source_category = get_category(media.get('media$category', []))
video.source_view_count = int(youtube_data['yt$statistics']['viewCount']) if 'yt$statistics' in youtube_data else -1
video.source_date_uploaded = media['yt$uploaded']['$t']
access_control = dict(
(i['action'], i['permission'] == 'allowed')
for i in youtube_data.get('yt$accessControl', []))
video.restricted = access_control.get('embed') is False
if 'app$control' in youtube_data:
if 'yt$incomplete' in youtube_data['app$control']:
video.restricted = True
else:
state = youtube_data['app$control']['yt$state']
if state['name'] == 'restricted':
if state['reasonCode'] == 'limitedSyndication':
# see https://groups.google.com/d/msg/youtube-api-gdata/on504fCOEk0/oErUbCptWu4J
video.restricted = not any(c.get('yt$format') == 5 for c in
media.get('media$content', []))
else:
video.restricted = True
for thumbnail in media.get('media$thumbnail', []):
if 'time' not in thumbnail:
video.thumbnails.append(
VideoThumbnail(
url=thumbnail['url'],
width=thumbnail['width'],
height=thumbnail['height']))
for restriction in media.get('media$restriction', []):
if restriction['type'] == 'country':
video.restrictions.extend(
VideoRestriction(
relationship=restriction['relationship'],
country=country) for country in restriction['$t'].split())
return video
def _format_results(self, videos, with_channels=True, with_stars=False, add_tracking=None):
vlist = []
channel_list = set()
IMAGE_CDN = app.config.get('IMAGE_CDN', '')
BASE_URL = url_for('basews.discover')
def _format_user_data(user):
return dict(
id=user.resource_url.lstrip('/').split('/')[1],
display_name=user.display_name,
resource_url=urljoin(BASE_URL, user.resource_url),
avatar_thumbnail_url=urljoin(IMAGE_CDN, user.avatar) if user.avatar else ''
)
for pos, v in enumerate(videos, self.paging[0]):
published = v.video.date_published
video = dict(
id=v.id,
channel=dict(
id=v.channel,
title=v.channel_title),
title=v.title,
label=v.label,
date_added=_format_datetime(v.date_added),
public=v.public,
category='',
video=dict(
id=v.video.id,
view_count=sum(l['view_count'] for l in v['locales'].values()),
star_count=sum(l['star_count'] for l in v['locales'].values()),
source=Source.id_to_label(v.video.source),
source_id=v.video.source_id,
source_username=v.video.source_username,
source_date_uploaded=published.isoformat() if hasattr(published, 'isoformat') else published,
duration=v.video.duration,
description=Video.cleaned_description(v.video.description),
thumbnail_url=urljoin(app.config.get('IMAGE_CDN', ''), v.video.thumbnail_url) if v.video.thumbnail_url else '',
),
position=pos,
child_instance_count=getattr(v, 'child_instance_count', 0)
)
video['video'].update(Video.extra_meta(v.video))
if v.link_url:
video['video'].update(link_url=v.link_url, link_title=v.link_title)
if v.owner:
video['channel']['owner'] = _format_user_data(v.owner)
if v.owner and v.channel:
video['channel']['resource_url'] = urljoin(BASE_URL, url_for('userws.channel_info', userid=video['channel']['owner']['id'], channelid=v.channel))
if v.original_channel_owner:
video['original_channel_owner'] = _format_user_data(v.original_channel_owner)
if app.config.get('DOLLY'):
video.update({
"comments": {
"total": getattr(v.comments, 'count', 0)
}
})
if v.category:
video['category'] = max(v.category) if isinstance(v.category, list) else v.category
if with_stars:
video['recent_user_stars'] = v.get('recent_user_stars', [])
if add_tracking:
add_tracking(video)
channel_list.add(v.channel)
vlist.append(video)
if with_channels and channel_list:
ch = ChannelSearch(self.locale)
ch.add_id(channel_list)
channel_map = {c['id']: c for c in ch.channels(with_owners=True)}
self.add_channels_to_videos(vlist, channel_map)
return vlist
def _update_channel_videos(channel, data):
playlist = youtube.parse_atom_playlist_data(data)
source = Source.label_to_id('youtube')
Video.add_videos(playlist.videos, source)
channel.add_videos(playlist.videos)