def import_google_movies():
freshold = datetime.now() - timedelta(days=app.config.get('GOOGLE_MOVIE_FRESHOLD', 120))
year_format = re.compile(' \((20\d\d)\)')
for channelid, location in app.config['GOOGLE_MOVIE_LOCATIONS']:
start = 0
video_ids = set()
channel = Channel.query.get(channelid)
existing = set(v for v, in VideoInstance.query.
filter_by(channel=channelid).join(Video).values('source_videoid'))
while True:
url = app.config['GOOGLE_MOVIE_URL'] % (location, start)
html = get_external_resource(url).read()
video_ids.update(re.findall('youtube.com/watch%3Fv%3D(.{11})', html))
next = re.search('<a [^>]*start=(\d+)[^>]*><img[^>]*><br>Next</a>', html)
if next:
start = int(next.group(1))
time.sleep(1) # Don't get blocked by google
else:
break
feed_ids = [('videos', id) for id in video_ids - existing]
if feed_ids:
playlist = batch_query(feed_ids, playlist='Googlemovietrailers/uploads')
videos = []
for video in playlist.videos:
year_match = year_format.search(video.title)
if video.date_published > freshold and (
not year_match or int(year_match.group(1)) >= freshold.year):
videos.append(video)
else:
app.logger.debug('Skipped import of trailer "%s" (%s)',
video.title, video.date_published)
added = Video.add_videos(videos, 1)
channel.add_videos(videos)
app.logger.info('Added %d trailers to "%s"', added, channel.title)
def _import_videos(self, form):
for video in form.import_data.videos:
video.rockpack_curated = True
video.category = form.category.data
count = Video.add_videos(form.import_data.videos, form.source.data)
if not form.channel.data and not form.user.data:
self._set_form_data_from_source(form)
channelid = form.channel.data
if channelid.startswith('_new:'):
channel = Channel.create(
title=channelid.split(':', 1)[1],
owner=form.user.data,
description=form.channel_description.data,
cover=form.cover.data,
category=form.category.data)
self.record_action('created', channel)
else:
channel = Channel.query.get_or_404(channelid)
channel.add_videos(
form.import_data.videos,
form.tags.data,
category=form.category.data,
date_added=form.date_added.data
)
self.record_action('imported', channel, '%d videos' % count)
push_config = getattr(form.import_data, 'push_config', None)
if push_config and channel.id:
try:
subscribe(push_config.hub, push_config.topic, channel.id)
except Exception, e:
flash('Unable to subscribe to channel: %s' % e.message, 'error')
def _update_channel_videos(channel, data):
playlist = youtube.parse_atom_playlist_data(data)
source = Source.label_to_id('youtube')
Video.add_videos(playlist.videos, source)
channel.add_videos(playlist.videos)