def import_google_movies():
freshold = datetime.now() - timedelta(days=app.config.get('GOOGLE_MOVIE_FRESHOLD', 120))
year_format = re.compile(' \((20\d\d)\)')
for channelid, location in app.config['GOOGLE_MOVIE_LOCATIONS']:
start = 0
video_ids = set()
channel = Channel.query.get(channelid)
existing = set(v for v, in VideoInstance.query.
filter_by(channel=channelid).join(Video).values('source_videoid'))
while True:
url = app.config['GOOGLE_MOVIE_URL'] % (location, start)
html = get_external_resource(url).read()
video_ids.update(re.findall('youtube.com/watch%3Fv%3D(.{11})', html))
next = re.search('<a [^>]*start=(\d+)[^>]*><img[^>]*><br>Next</a>', html)
if next:
start = int(next.group(1))
time.sleep(1) # Don't get blocked by google
else:
break
feed_ids = [('videos', id) for id in video_ids - existing]
if feed_ids:
playlist = batch_query(feed_ids, playlist='Googlemovietrailers/uploads')
videos = []
for video in playlist.videos:
year_match = year_format.search(video.title)
if video.date_published > freshold and (
not year_match or int(year_match.group(1)) >= freshold.year):
videos.append(video)
else:
app.logger.debug('Skipped import of trailer "%s" (%s)',
video.title, video.date_published)
added = Video.add_videos(videos, 1)
channel.add_videos(videos)
app.logger.info('Added %d trailers to "%s"', added, channel.title)
def _process_image_data(self, form, model=None):
for field in form:
if isinstance(field, wtf.FileField):
data = None
cfgkey = self.model.__table__.columns.get(field.name).type.cfgkey
aoi = form.data.get(field.name + '_aoi')
if aoi:
aoi = get_box_value(aoi)
if field.data:
# New image upload
data = field.data
elif aoi and model and not getattr(model, field.name + '_aoi') == aoi:
# Same image, aoi updated - need to fetch the image data
data = getattr(model, field.name).original
else:
# Allow form to be edited without replacing existing image
form._fields.pop(field.name)
if data:
try:
if not isinstance(data, FileStorage):
data = get_external_resource(data)
field.data = resize_and_upload(data, cfgkey, aoi)
except IOError, e:
# The form has already been validated at this stage but
# if we return False then BaseModelView.create_view will
# still drop through and render the errors
field.errors = [getattr(e, 'message', str(e))]
return False
def update_image_thumbnails(fieldname):
"""Re-process all images for the specified Model.field."""
model, fieldname, cfgkey = _parse_fieldname(fieldname)
for instance in model.query.filter(getattr(model, fieldname) != ''):
try:
data = get_external_resource(getattr(instance, fieldname).original)
except Exception, e:
msg = e.response.reason if hasattr(e, 'response') else e.message
logging.error('Unable to process %s: %s', getattr(instance, fieldname).path, msg)
continue
aoi = getattr(instance, '%s_aoi' % fieldname, None)
image_path = resize_and_upload(data, cfgkey, aoi)
setattr(instance, fieldname, image_path)
instance.save()
def refresh_pubsubhubbub_subscriptions(id=None):
"""Re-subscribe expired PubSubHubbub subscriptions."""
if id:
subs = [Subscription.query.get(id)]
else:
subs = Subscription.query.filter(Subscription.lease_expires < datetime.now())
for sub in subs:
try:
sub.subscribe()
except Exception, e:
if hasattr(e, 'response'):
message = re.sub('.*<h2>([^<]+).*', '\\1', e.response.text, 1, re.DOTALL)
logging.error('Failed to subscribe: %d: %d: %s', sub.id, e.response.status_code, message)
else:
logging.exception('Failed to subscribe: %d', sub.id)
else:
logging.info('Subscribed: %d', sub.id)
try:
update_channel_videos(sub.channel, get_external_resource(sub.topic).read())
except Exception:
logging.exception('Failed to update: %d', sub.id)
