def sync_loaded_movies(logger, **kwargs): user_id = kwargs.get('user_id') pattern = re.compile(ur'^([a-zA-ZА-Яа-я\d\s:&\'-]+)\s*\(?(\d{4})?\)?$') path = instance.config.get('LOADED_MOVIE_PATH') prefix = 'Process loaded movies' logger.info('%s: start.' % prefix) for root, dirs, files in os.walk(path): for filename in files: if filename.endswith('.mkv'): name = filename[:-4] matches = pattern.findall(name) logger.info('%s: %s founded.' % (prefix, name)) try: movie = Movie.query.filter_by( year=matches[0][1]).whoosh_search(matches[0][0]).one() except NoResultFound: logger.info('%s: %s does not exists.' % (prefix, name)) redis.sadd('missind_loaded_movies', name) else: try: params = [ Watchlist.user_id == user_id, Watchlist.movie_id == movie.id ] watchlist_movie = Watchlist.query.filter(*params).one() except NoResultFound: watchlist_movie = Watchlist(user_id=user_id, movie_id=movie.id) logger.info('%s: %s add movie to watchlist' % ( prefix, name)) db.session.add(watchlist_movie) watchlist_movie.status = Watchlist.LOADED_MOVIE watchlist_movie.path = filename logger.info('%s: %s update movie in watchlist' % ( prefix, name)) try: db.session.commit() except IntegrityError: db.session.rollback() logger.info('%s: finished.' % prefix)
def save_relations(self): missing_movies = set() for provider in self.providers: key = Person.make_key(provider=provider, item_id=getattr(self, '%s_id' % provider), person_type=self.person_type) sections = ['best_works', 'actor', 'director', 'writer', 'producer'] for section in sections: section_key = '%s_%s' % (key, section) keys = redis.smembers(section_key) if not keys: continue movie_keys = ParserFactory.group_keys_by_parser(keys) movies = getattr(self, section, []) for k, value in movie_keys.iteritems(): field = getattr(Movie, '%s_id' % k, None) if not field is None: existed = db.session.query(Movie, field).filter( field.in_(value)).all() existed_movies = set() for movie, movie_key in existed: if not movie in movies: movies.append(movie) existed_key = Movie.make_key( provider=k, item_id=movie_key) existed_movies.add(existed_key) redis.srem(section_key, existed_key) try: db.session.commit() except IntegrityError: db.session.rollback() missing_movies = missing_movies.union( set(keys).difference(existed_movies)) if not redis.exists(section_key): redis.srem(Person.not_saved_relations_key, key) redis.sadd(Person.saved_relations_key, key) return missing_movies
def sync_encoded_movies(logger, **kwargs): user_id = kwargs.get('user_id') pattern = re.compile(ur'^([a-zA-ZА-Яа-я\d\s:&\'-]+) \(?(\d{4})?\)?$') path = instance.config.get('MOVIE_COLLECTION_PATH') prefix = 'Process movies in collection' logger.info('%s: start.' % prefix) for root, dirs, files in os.walk(path): name = root.replace(path, '').encode('utf-8') matches = pattern.findall(name) logger.info('%s: %s founded.' % (prefix, name)) if matches: try: movie = Movie.query.filter_by( year=matches[0][1]).whoosh_search(matches[0][0]).one() except NoResultFound: redis.sadd('missing_encoded_movies', name) else: try: params = [ Watchlist.user_id == user_id, Watchlist.movie_id == movie.id ] watchlist_movie = Watchlist.query.filter(*params).one() except NoResultFound: watchlist_movie = Watchlist(user_id=user_id, movie_id=movie.id) logger.info('%s: %s add movie to watchlist' % ( prefix, name)) db.session.add(watchlist_movie) watchlist_movie.status = Watchlist.IN_COLLECTION_MOVIE watchlist_movie.path = name logger.info('%s: %s update movie in watchlist' % ( prefix, name)) try: db.session.commit() except IntegrityError: db.session.rollback() logger.info('%s: finished.' % prefix)
def add(cls, **kwargs): key = kwargs.get('key', None) keys = kwargs.get('keys', None) provider, person_id, person_type = cls.parse_key(key=key) if not cls.is_added(key): raw = cls.fetch(key) person = Person(provider=provider, pid=raw['id'], name=raw['name'], person_type=person_type, poster=raw['poster']) try: db.session.add(person) db.session.commit() except IntegrityError: db.session.rollback() return False else: redis.sadd(cls.added_key, key) for s, movies in raw['filmography'].iteritems(): section = s if s == 'actress' or s == 'actor': section = 'actor' section_key = '%s_%s' % (key, section) for movie in movies: movie_key = Movie.make_key(provider=provider, item_id=movie['id']) redis.sadd(section_key, movie_key) Movie.push(movie_key) redis.sadd(cls.not_saved_relations_key, key) else: params = {'%s_id' % provider: person_id} try: person = db.session.query(cls).filter_by(**params).one() except NoResultFound: return ValueError('No results found') except MultipleResultsFound: return ValueError('Multiple results found') return person
def save_relations(self): missing_movies = set() for provider in self.providers: key = Movie.make_key(provider=provider, id=getattr(self, '%s_id' % provider)) movie_related_key = '%s:related' % key keys = redis.smembers(movie_related_key) groups = Movie.group_keys_by_provider(keys=keys) for group in groups: provider = group['provider'] field = getattr(Movie, '%s_id' % provider, None) if field is None: continue existed = db.session.query(Movie, field).filter( field.in_(group['keys'])).all() existed_movies = set() for movie, movie_key in existed: self.related_movies = movie existed_key = Movie.make_key(provider=provider, id=movie_key) existed_movies.add(existed_key) redis.srem(movie_related_key, existed_key) try: db.session.commit() except IntegrityError: db.session.rollback() missing_movies = missing_movies.union( set(keys).difference(existed_movies)) del groups, keys movie_genres_key = '%s:genres' % key if redis.exists(movie_genres_key): genres = redis.smembers(movie_genres_key) existed = db.session.query(Genre).filter( Genre.title.in_(genres)).all() existed_genres = set() for genre in existed: self.genres.append(genre) title = str(genre.title) existed_genres.add(title) redis.srem(movie_genres_key, title) missing = set(genres).difference(existed_genres) if missing: for g in missing: genre = Genre(title=g) self.genres.append(genre) redis.srem(movie_genres_key, g) try: db.session.commit() except IntegrityError: db.session.rollback() if (not redis.exists(movie_related_key) and not redis.exists(movie_genres_key)): redis.srem(Movie.not_saved_relations_key, key) redis.sadd(Movie.saved_relations_key, key) return missing_movies
def add(cls, **kwargs): keys = kwargs.get('keys', []) result = [] groups = cls.group_keys_by_provider(keys=keys, make_parser=True) for group in groups: parser = group['parser'] provider = group['provider'] ids = set(group['keys']) # try to find already existed movies field = getattr(cls, '%s_id' % provider) existed = db.session.query(cls, field).filter(field.in_(ids)).all() existed_ids = set() for movie, field in existed: existed_ids.add(field) result.append(movie) missing_ids = ids.difference(existed_ids) # add missing movies for movie_id in missing_ids: key = Movie.make_key(id=movie_id, provider=provider) if cls.is_added(key): continue # fetch raw movie info raw = parser.parse_movie_page(movie_id) if raw: # save movie instance to db try: movie = Movie(provider=provider, mid=raw['id'], title=raw['title'], year=str(raw['year']), rating=raw['rating'], poster=raw['poster'], title_rus=raw['title_rus'], is_series=raw['is_series'], world_premiere=raw['premiere']['world'], rus_premiere=raw['premiere']['russian'], dvd_premiere=raw['premiere']['dvd']) except KeyError: continue try: db.session.add(movie) db.session.commit() except IntegrityError: db.session.rollback() continue # if movie is TV Show, save episodes if raw['is_series']: episodes = [] for s in raw['seasons']: for e in s['episodes']: episode = { 'season_number': s['number'], 'episode_number': e['number'], 'title': e['title'], 'series_id': movie.id } if e['airdate']: episode.update({'airdate': e['airdate']}) else: episode.update({'airdate': None}) episodes.append(episode) try: db.session.execute(SeriesEpisode.__table__.insert(), episodes) db.session.commit() except IntegrityError: db.session.rollback() del episodes redis.sadd(cls.added_key, key) # add movie relations genres_key = '%s:genres' % key related_key = '%s:related' % key for genre in raw['genres']: redis.sadd(genres_key, genre) for movie_id in raw['related']: movie_key = Movie.make_key(id=movie_id, provider=provider) redis.sadd(related_key, movie_key) for person_type in ['actors', 'directors']: persons = raw.get(person_type) for p in persons: # TODO: Переписать с использованием нескольких ключей person_key = Person.make_key( provider=provider, item_id=p['id'], person_type=person_type[0]) Person.push(person_key) redis.sadd(cls.not_saved_relations_key, key) result.append(movie) return result