def save_movie(lookup, path, genres):
try:
movie = Movie.get(media_id=lookup.media_id)
except Movie.DoesNotExist:
movie = Movie.create(
media_id=lookup.media_id,
title=lookup.title,
path=path,
year=lookup.year,
)
if genres:
movie.add_genres(genres)
else:
assert lookup.cd is not None, "Multiple files for {} ({!r}, {!r}) but not cds".format(lookup.media_id, movie.path, path)
movie.path = '|'.join(sorted([movie.path, path]))
movie.save()
def get_movie_or_tv_show(path):
try:
media = TVShowEpisode.select().where(TVShowEpisode.path == path).get()
except TVShowEpisode.DoesNotExist:
media = Movie.select().where(Movie.path == path).get()
return media
def catalog_videos(database, source, max_lookups):
from aesop.processor.movie import MovieLookup
from aesop.processor.episode import AnimeLookup, TVShowLookup
model, lookup_model = {
'movies': (Movie, MovieLookup),
'tv': (TVShow, TVShowLookup),
'anime': (TVShow, AnimeLookup),
}[source.type]
log.info("Cataloguing {} videos for {}", source.type, source.path)
if model == Movie:
query = model.select(model.path)
known_paths = set(itertools.chain.from_iterable(m.path.split('|') for m in query))
else:
query = TVShowEpisode.select(TVShowEpisode.path).where(TVShowEpisode.path.startswith(source.path))
known_paths = {m.path for m in query}
known_video_types = set(Config.get('processor', 'video types', default='avi, mp4, mkv, ogm').replace(' ', '').split(','))
lookups = []
paths = []
log.debug("Known paths {}", known_paths)
removed = 0
for path in set(known_paths):
if not os.path.exists(path):
known_paths.remove(path)
log.info("{} does not exist, removing from database.", path)
if model == Movie:
Movie.delete().where(
(Movie.path == path) |
Movie.path.contains(path+'|') |
Movie.path.contains('|'+path)
).execute()
else:
ep = TVShowEpisode.select().where(TVShowEpisode.path == path).get()
show = ep.show
removed += 1
with database_proxy.transaction():
ep.delete_instance()
if not list(show.episodes):
show.delete_instance()
else:
if all([episode.watched for episode in show.episodes]):
show.watched = True
if show.is_dirty():
show.save()
start_time = time.time()
successes = 0
lookup_failures = 0
for root, dirs, files in os.walk(source.path):
for path in (os.path.join(root, p) for p in files):
if '/.AppleDouble/' in path:
log.debug('Skipping {} as it looks like an Apple double.', path)
continue
if os.path.basename(path).startswith('.'):
continue
if os.path.splitext(path)[1][1:] not in known_video_types:
continue
if '/sample/' in path.lower() or os.path.splitext(path)[0].lower().endswith('-sample'):
log.debug('Skipping {} as it looks like a sample.', path)
continue
if path in known_paths:
continue
with FingersCrossedHandler(default_handler):
try:
path_lookups = lookup_model.from_path(path)
except SkipIt as e:
log.error("Skipping path: {} {}", path, str(e))
except Exception as e:
lookup_failures += 1
exception = ''.join(traceback.format_exception(e.__class__, e, e.__traceback__))
log.error("Error retrieving information for {}: {}", path, exception)
else:
lookups.extend(path_lookups)
paths.extend([path]*len(path_lookups))
loop = asyncio.get_event_loop()
log.info("{} lookups to do", len(lookups))
for i in range(0, len(lookups), max_lookups):
chunk = lookups[i:i+max_lookups]
f = asyncio.gather(
*[asyncio.async(c, loop=loop) for c in chunk],
loop=loop, return_exceptions=True)
completed = loop.run_until_complete(f)
with database.transaction():
for path, lookup in zip(paths[i:i+max_lookups], completed):
if isinstance(lookup, SkipIt):
# logged earlier, no need to log now
lookup_failures += 1
elif isinstance(lookup, Exception):
lookup_failures += 1
exception = ''.join(traceback.format_exception(lookup.__class__, lookup, lookup.__traceback__))
log.error("Error retrieving information for {}: {}", path, exception)
else:
genres = [Genre.get_or_create(text=g) for g in lookup.genres]
if source.type == 'movies':
save_movie(lookup, path, genres)
else:
save_episode(lookup, path, genres, source.type)
successes += 1
end_time = time.time()
log.info("Took {:.2f} seconds to do {} lookups", end_time - start_time, len(lookups))
log.info("{} lookups failed", lookup_failures)
return successes, lookup_failures, removed