def sync_loaded_movies(logger, **kwargs):
user_id = kwargs.get('user_id')
pattern = re.compile(ur'^([a-zA-ZА-Яа-я\d\s:&\'-]+)\s*\(?(\d{4})?\)?$')
path = instance.config.get('LOADED_MOVIE_PATH')
prefix = 'Process loaded movies'
logger.info('%s: start.' % prefix)
for root, dirs, files in os.walk(path):
for filename in files:
if filename.endswith('.mkv'):
name = filename[:-4]
matches = pattern.findall(name)
logger.info('%s: %s founded.' % (prefix, name))
try:
movie = Movie.query.filter_by(
year=matches[0][1]).whoosh_search(matches[0][0]).one()
except NoResultFound:
logger.info('%s: %s does not exists.' % (prefix, name))
redis.sadd('missind_loaded_movies', name)
else:
try:
params = [
Watchlist.user_id == user_id,
Watchlist.movie_id == movie.id
]
watchlist_movie = Watchlist.query.filter(*params).one()
except NoResultFound:
watchlist_movie = Watchlist(user_id=user_id,
movie_id=movie.id)
logger.info('%s: %s add movie to watchlist' % (
prefix, name))
db.session.add(watchlist_movie)
watchlist_movie.status = Watchlist.LOADED_MOVIE
watchlist_movie.path = filename
logger.info('%s: %s update movie in watchlist' % (
prefix, name))
try:
db.session.commit()
except IntegrityError:
db.session.rollback()
logger.info('%s: finished.' % prefix)
def save_relations(self):
missing_movies = set()
for provider in self.providers:
key = Person.make_key(provider=provider,
item_id=getattr(self, '%s_id' % provider),
person_type=self.person_type)
sections = ['best_works', 'actor', 'director', 'writer',
'producer']
for section in sections:
section_key = '%s_%s' % (key, section)
keys = redis.smembers(section_key)
if not keys:
continue
movie_keys = ParserFactory.group_keys_by_parser(keys)
movies = getattr(self, section, [])
for k, value in movie_keys.iteritems():
field = getattr(Movie, '%s_id' % k, None)
if not field is None:
existed = db.session.query(Movie, field).filter(
field.in_(value)).all()
existed_movies = set()
for movie, movie_key in existed:
if not movie in movies:
movies.append(movie)
existed_key = Movie.make_key(
provider=k, item_id=movie_key)
existed_movies.add(existed_key)
redis.srem(section_key, existed_key)
try:
db.session.commit()
except IntegrityError:
db.session.rollback()
missing_movies = missing_movies.union(
set(keys).difference(existed_movies))
if not redis.exists(section_key):
redis.srem(Person.not_saved_relations_key, key)
redis.sadd(Person.saved_relations_key, key)
return missing_movies
def sync_encoded_movies(logger, **kwargs):
user_id = kwargs.get('user_id')
pattern = re.compile(ur'^([a-zA-ZА-Яа-я\d\s:&\'-]+) \(?(\d{4})?\)?$')
path = instance.config.get('MOVIE_COLLECTION_PATH')
prefix = 'Process movies in collection'
logger.info('%s: start.' % prefix)
for root, dirs, files in os.walk(path):
name = root.replace(path, '').encode('utf-8')
matches = pattern.findall(name)
logger.info('%s: %s founded.' % (prefix, name))
if matches:
try:
movie = Movie.query.filter_by(
year=matches[0][1]).whoosh_search(matches[0][0]).one()
except NoResultFound:
redis.sadd('missing_encoded_movies', name)
else:
try:
params = [
Watchlist.user_id == user_id,
Watchlist.movie_id == movie.id
]
watchlist_movie = Watchlist.query.filter(*params).one()
except NoResultFound:
watchlist_movie = Watchlist(user_id=user_id,
movie_id=movie.id)
logger.info('%s: %s add movie to watchlist' % (
prefix, name))
db.session.add(watchlist_movie)
watchlist_movie.status = Watchlist.IN_COLLECTION_MOVIE
watchlist_movie.path = name
logger.info('%s: %s update movie in watchlist' % (
prefix, name))
try:
db.session.commit()
except IntegrityError:
db.session.rollback()
logger.info('%s: finished.' % prefix)
def add(cls, **kwargs):
key = kwargs.get('key', None)
keys = kwargs.get('keys', None)
provider, person_id, person_type = cls.parse_key(key=key)
if not cls.is_added(key):
raw = cls.fetch(key)
person = Person(provider=provider, pid=raw['id'],
name=raw['name'], person_type=person_type,
poster=raw['poster'])
try:
db.session.add(person)
db.session.commit()
except IntegrityError:
db.session.rollback()
return False
else:
redis.sadd(cls.added_key, key)
for s, movies in raw['filmography'].iteritems():
section = s
if s == 'actress' or s == 'actor':
section = 'actor'
section_key = '%s_%s' % (key, section)
for movie in movies:
movie_key = Movie.make_key(provider=provider,
item_id=movie['id'])
redis.sadd(section_key, movie_key)
Movie.push(movie_key)
redis.sadd(cls.not_saved_relations_key, key)
else:
params = {'%s_id' % provider: person_id}
try:
person = db.session.query(cls).filter_by(**params).one()
except NoResultFound:
return ValueError('No results found')
except MultipleResultsFound:
return ValueError('Multiple results found')
return person
def save_relations(self):
missing_movies = set()
for provider in self.providers:
key = Movie.make_key(provider=provider,
id=getattr(self, '%s_id' % provider))
movie_related_key = '%s:related' % key
keys = redis.smembers(movie_related_key)
groups = Movie.group_keys_by_provider(keys=keys)
for group in groups:
provider = group['provider']
field = getattr(Movie, '%s_id' % provider, None)
if field is None:
continue
existed = db.session.query(Movie, field).filter(
field.in_(group['keys'])).all()
existed_movies = set()
for movie, movie_key in existed:
self.related_movies = movie
existed_key = Movie.make_key(provider=provider,
id=movie_key)
existed_movies.add(existed_key)
redis.srem(movie_related_key, existed_key)
try:
db.session.commit()
except IntegrityError:
db.session.rollback()
missing_movies = missing_movies.union(
set(keys).difference(existed_movies))
del groups, keys
movie_genres_key = '%s:genres' % key
if redis.exists(movie_genres_key):
genres = redis.smembers(movie_genres_key)
existed = db.session.query(Genre).filter(
Genre.title.in_(genres)).all()
existed_genres = set()
for genre in existed:
self.genres.append(genre)
title = str(genre.title)
existed_genres.add(title)
redis.srem(movie_genres_key, title)
missing = set(genres).difference(existed_genres)
if missing:
for g in missing:
genre = Genre(title=g)
self.genres.append(genre)
redis.srem(movie_genres_key, g)
try:
db.session.commit()
except IntegrityError:
db.session.rollback()
if (not redis.exists(movie_related_key) and
not redis.exists(movie_genres_key)):
redis.srem(Movie.not_saved_relations_key, key)
redis.sadd(Movie.saved_relations_key, key)
return missing_movies
def add(cls, **kwargs):
keys = kwargs.get('keys', [])
result = []
groups = cls.group_keys_by_provider(keys=keys, make_parser=True)
for group in groups:
parser = group['parser']
provider = group['provider']
ids = set(group['keys'])
# try to find already existed movies
field = getattr(cls, '%s_id' % provider)
existed = db.session.query(cls, field).filter(field.in_(ids)).all()
existed_ids = set()
for movie, field in existed:
existed_ids.add(field)
result.append(movie)
missing_ids = ids.difference(existed_ids)
# add missing movies
for movie_id in missing_ids:
key = Movie.make_key(id=movie_id, provider=provider)
if cls.is_added(key):
continue
# fetch raw movie info
raw = parser.parse_movie_page(movie_id)
if raw:
# save movie instance to db
try:
movie = Movie(provider=provider, mid=raw['id'],
title=raw['title'], year=str(raw['year']),
rating=raw['rating'], poster=raw['poster'],
title_rus=raw['title_rus'],
is_series=raw['is_series'],
world_premiere=raw['premiere']['world'],
rus_premiere=raw['premiere']['russian'],
dvd_premiere=raw['premiere']['dvd'])
except KeyError:
continue
try:
db.session.add(movie)
db.session.commit()
except IntegrityError:
db.session.rollback()
continue
# if movie is TV Show, save episodes
if raw['is_series']:
episodes = []
for s in raw['seasons']:
for e in s['episodes']:
episode = {
'season_number': s['number'],
'episode_number': e['number'],
'title': e['title'],
'series_id': movie.id
}
if e['airdate']:
episode.update({'airdate': e['airdate']})
else:
episode.update({'airdate': None})
episodes.append(episode)
try:
db.session.execute(SeriesEpisode.__table__.insert(),
episodes)
db.session.commit()
except IntegrityError:
db.session.rollback()
del episodes
redis.sadd(cls.added_key, key)
# add movie relations
genres_key = '%s:genres' % key
related_key = '%s:related' % key
for genre in raw['genres']:
redis.sadd(genres_key, genre)
for movie_id in raw['related']:
movie_key = Movie.make_key(id=movie_id, provider=provider)
redis.sadd(related_key, movie_key)
for person_type in ['actors', 'directors']:
persons = raw.get(person_type)
for p in persons:
# TODO: Переписать с использованием нескольких ключей
person_key = Person.make_key(
provider=provider, item_id=p['id'],
person_type=person_type[0])
Person.push(person_key)
redis.sadd(cls.not_saved_relations_key, key)
result.append(movie)
return result
