def save_relations(self): missing_movies = set() for provider in self.providers: key = Person.make_key(provider=provider, item_id=getattr(self, '%s_id' % provider), person_type=self.person_type) sections = ['best_works', 'actor', 'director', 'writer', 'producer'] for section in sections: section_key = '%s_%s' % (key, section) keys = redis.smembers(section_key) if not keys: continue movie_keys = ParserFactory.group_keys_by_parser(keys) movies = getattr(self, section, []) for k, value in movie_keys.iteritems(): field = getattr(Movie, '%s_id' % k, None) if not field is None: existed = db.session.query(Movie, field).filter( field.in_(value)).all() existed_movies = set() for movie, movie_key in existed: if not movie in movies: movies.append(movie) existed_key = Movie.make_key( provider=k, item_id=movie_key) existed_movies.add(existed_key) redis.srem(section_key, existed_key) try: db.session.commit() except IntegrityError: db.session.rollback() missing_movies = missing_movies.union( set(keys).difference(existed_movies)) if not redis.exists(section_key): redis.srem(Person.not_saved_relations_key, key) redis.sadd(Person.saved_relations_key, key) return missing_movies
def save_relations(self): missing_movies = set() for provider in self.providers: key = Movie.make_key(provider=provider, id=getattr(self, '%s_id' % provider)) movie_related_key = '%s:related' % key keys = redis.smembers(movie_related_key) groups = Movie.group_keys_by_provider(keys=keys) for group in groups: provider = group['provider'] field = getattr(Movie, '%s_id' % provider, None) if field is None: continue existed = db.session.query(Movie, field).filter( field.in_(group['keys'])).all() existed_movies = set() for movie, movie_key in existed: self.related_movies = movie existed_key = Movie.make_key(provider=provider, id=movie_key) existed_movies.add(existed_key) redis.srem(movie_related_key, existed_key) try: db.session.commit() except IntegrityError: db.session.rollback() missing_movies = missing_movies.union( set(keys).difference(existed_movies)) del groups, keys movie_genres_key = '%s:genres' % key if redis.exists(movie_genres_key): genres = redis.smembers(movie_genres_key) existed = db.session.query(Genre).filter( Genre.title.in_(genres)).all() existed_genres = set() for genre in existed: self.genres.append(genre) title = str(genre.title) existed_genres.add(title) redis.srem(movie_genres_key, title) missing = set(genres).difference(existed_genres) if missing: for g in missing: genre = Genre(title=g) self.genres.append(genre) redis.srem(movie_genres_key, g) try: db.session.commit() except IntegrityError: db.session.rollback() if (not redis.exists(movie_related_key) and not redis.exists(movie_genres_key)): redis.srem(Movie.not_saved_relations_key, key) redis.sadd(Movie.saved_relations_key, key) return missing_movies