def import_google_movies(): freshold = datetime.now() - timedelta(days=app.config.get('GOOGLE_MOVIE_FRESHOLD', 120)) year_format = re.compile(' \((20\d\d)\)') for channelid, location in app.config['GOOGLE_MOVIE_LOCATIONS']: start = 0 video_ids = set() channel = Channel.query.get(channelid) existing = set(v for v, in VideoInstance.query. filter_by(channel=channelid).join(Video).values('source_videoid')) while True: url = app.config['GOOGLE_MOVIE_URL'] % (location, start) html = get_external_resource(url).read() video_ids.update(re.findall('youtube.com/watch%3Fv%3D(.{11})', html)) next = re.search('<a [^>]*start=(\d+)[^>]*><img[^>]*><br>Next</a>', html) if next: start = int(next.group(1)) time.sleep(1) # Don't get blocked by google else: break feed_ids = [('videos', id) for id in video_ids - existing] if feed_ids: playlist = batch_query(feed_ids, playlist='Googlemovietrailers/uploads') videos = [] for video in playlist.videos: year_match = year_format.search(video.title) if video.date_published > freshold and ( not year_match or int(year_match.group(1)) >= freshold.year): videos.append(video) else: app.logger.debug('Skipped import of trailer "%s" (%s)', video.title, video.date_published) added = Video.add_videos(videos, 1) channel.add_videos(videos) app.logger.info('Added %d trailers to "%s"', added, channel.title)
def _process_image_data(self, form, model=None): for field in form: if isinstance(field, wtf.FileField): data = None cfgkey = self.model.__table__.columns.get(field.name).type.cfgkey aoi = form.data.get(field.name + '_aoi') if aoi: aoi = get_box_value(aoi) if field.data: # New image upload data = field.data elif aoi and model and not getattr(model, field.name + '_aoi') == aoi: # Same image, aoi updated - need to fetch the image data data = getattr(model, field.name).original else: # Allow form to be edited without replacing existing image form._fields.pop(field.name) if data: try: if not isinstance(data, FileStorage): data = get_external_resource(data) field.data = resize_and_upload(data, cfgkey, aoi) except IOError, e: # The form has already been validated at this stage but # if we return False then BaseModelView.create_view will # still drop through and render the errors field.errors = [getattr(e, 'message', str(e))] return False
def update_image_thumbnails(fieldname): """Re-process all images for the specified Model.field.""" model, fieldname, cfgkey = _parse_fieldname(fieldname) for instance in model.query.filter(getattr(model, fieldname) != ''): try: data = get_external_resource(getattr(instance, fieldname).original) except Exception, e: msg = e.response.reason if hasattr(e, 'response') else e.message logging.error('Unable to process %s: %s', getattr(instance, fieldname).path, msg) continue aoi = getattr(instance, '%s_aoi' % fieldname, None) image_path = resize_and_upload(data, cfgkey, aoi) setattr(instance, fieldname, image_path) instance.save()
def refresh_pubsubhubbub_subscriptions(id=None): """Re-subscribe expired PubSubHubbub subscriptions.""" if id: subs = [Subscription.query.get(id)] else: subs = Subscription.query.filter(Subscription.lease_expires < datetime.now()) for sub in subs: try: sub.subscribe() except Exception, e: if hasattr(e, 'response'): message = re.sub('.*<h2>([^<]+).*', '\\1', e.response.text, 1, re.DOTALL) logging.error('Failed to subscribe: %d: %d: %s', sub.id, e.response.status_code, message) else: logging.exception('Failed to subscribe: %d', sub.id) else: logging.info('Subscribed: %d', sub.id) try: update_channel_videos(sub.channel, get_external_resource(sub.topic).read()) except Exception: logging.exception('Failed to update: %d', sub.id)