def read_config(self):
"""Get the config file or create it with default values."""
self.config = configparser.ConfigParser()
# Try to get config, if this doesn't work a new default config will be created
if os.path.exists(self.config_path):
try:
self.config.read(self.config_path)
return
except BaseException:
Logger.info(
'Error while parsing config file. Deleting old config')
# Default configuration
self.config['encoding'] = {
'crf': '18',
'preset': 'slow',
'audio': 'None',
'kbitrate-audio': 'None',
'threads': '4',
}
self.config['default'] = {
'min-size': '{0}'.format(1024 * 1024 * 1024 * 6),
'SQL_URI': '/var/lib/encarne/encarne.sql',
'niceness': '15',
}
self.write_config()
def add_task(self, task):
"""Schedule and manage encoding of a movie.
2. The command is added to pueue.
3. Wait for the task to finish.
4. Check if the encoding was successful.
4.1 If it wasn't successful, we delete the encoded file and mark the
old file as `encarne-failed`.
4.2 Remove the old file and move the encoded file to the proper location.
5. Repeat
"""
# Check if the current command already in the queue.
status = self.get_newest_status(task.ffmpeg_command)
# Send the command to pueue for scheduling, if it isn't in the queue yet
if status is None:
# In case a previous run failed and pueue has been resetted,
# we need to check, if the encoded file is still there.
if os.path.exists(task.temp_path):
os.remove(task.temp_path)
# Create a new pueue task
args = {
'command': [task.ffmpeg_command],
'path': task.origin_folder,
}
Logger.info(f'Add task pueue:\n {task.ffmpeg_command}')
execute_add(args, os.path.expanduser('~'))
def clean_movies(session):
"""Remove all deleted movies."""
movies = session.query(Movie).all()
for movie in movies:
# Can't find the file. Remove the movie.
path = os.path.join(movie.directory, movie.name)
if not os.path.exists(path):
Logger.info(f'Remove {path}')
session.delete(movie)
session.commit()
def check_file_size(origin, temp):
"""Compare the file size of original and re encoded file."""
origin_filesize = os.path.getsize(origin)
filesize = os.path.getsize(temp)
if origin_filesize < filesize:
Logger.info('Encoded movie is bigger than the original movie')
return False, True
else:
difference = origin_filesize - filesize
mebibyte = int(difference / 1024 / 1024)
Logger.info(
f'The new movie is {mebibyte} MIB smaller than the old one')
return True, False
def format_args(self, args):
"""Check arguments and format them to be compatible with `self.config`."""
args = {key: value for key, value in args.items() if value}
for key, value in args.items():
if key == 'directory':
self.directory = value
# Encoding
if key == 'crf':
self.config['encoding']['crf'] = str(value)
elif key == 'preset':
self.config['encoding']['preset'] = value
elif key == 'audio':
self.config['encoding']['audio'] = value
elif key == 'audio':
self.config['encoding']['kbitrate-audio'] = value
elif key == 'threads':
self.config['encoding']['threads'] = str(value)
elif key == 'size':
self.config['default']['min-size'] = str(
humanfriendly.parse_size(value))
# Default if no dir is specified
if not self.directory:
self.directory = '.'
# Get absolute path of directory
self.directory = os.path.abspath(self.directory)
Logger.info(f'Searching for files in directory {self.directory}')
# Check if path is a dir
if not os.path.isdir(self.directory):
Logger.warning('A valid directory needs to be specified')
Logger.warning(self.directory)
sys.exit(1)
def show_stats(args):
"""Print how much has already been saved by reencoding."""
session = get_session()
movies = session.query(Movie).all()
saved = 0
failed = 0
encoded = 0
for movie in movies:
# Only count movies which exist in the file system.
path = os.path.join(movie.directory, movie.name)
if not os.path.exists(path):
continue
diff = movie.original_size - movie.size
if movie.failed:
failed += 1
elif movie.encoded and diff > 0:
saved += diff
encoded += 1
saved_formatted = humanfriendly.format_size(saved)
Logger.info(f'Saved space: {saved_formatted}')
Logger.info(f'Reencoded container: {encoded}')
Logger.info(f'Failed movies: {failed}')
def get_media_duration(path):
"""Execute external mediainfo command and find the video encoding library."""
process = subprocess.run(
['mediainfo', '--Output=PBCore2', path],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
root = etree.XML(process.stdout)
try:
duration = root.find(
".//ns:instantiationDuration",
namespaces={
'ns': 'http://www.pbcore.org/PBCore/PBCoreNamespace.html'
},
).text
except IndexError:
Logger.info(f'Could not find duration for {path}')
return None
try:
duration = duration.split('.')[0].split(';')[0]
date = datetime.strptime(duration, '%H:%M:%S')
except BaseException:
try:
duration = duration.rsplit(':', 1)[0]
date = datetime.strptime(duration, '%H:%M:%S')
except BaseException:
Logger.info(f'Unknown duration: {duration}')
return None
delta = timedelta(
hours=date.hour,
minutes=date.minute,
seconds=date.second,
)
return delta
def create_tasks(self, files):
"""Filter files and check if they are already done or failed in a previous run.
Ignore previously failed movies (too big, duration differs) and already encoded movies.
Create a task with all paths and the compiled ffmpeg command.
"""
for path in files:
# Get absolute path
path = os.path.abspath(path)
mediainfo = get_media_encoding(path)
task = Task(path, self.config)
size = os.path.getsize(path)
# Get movie from db and check for already encoded or failed files.
task.movie = Movie.get_or_create(self.session, task.origin_file,
task.origin_folder, size)
if task.movie.encoded or task.movie.failed:
continue
# Already encoded
if '265' in mediainfo or '265' in path:
task.movie.encoded = True
self.session.add(task.movie)
continue
# File to small for encoding
elif size < int(self.config['default']['min-size']):
Logger.debug('File smaller than min-size: {path}')
continue
# Unknown encoding
elif mediainfo == 'unknown':
Logger.info(f'Failed to get encoding for {path}')
self.tasks.append(task)
self.session.commit()
def check_duration(origin, temp, seconds=1):
"""Check if the duration is bigger than a specific amount."""
# If the destination movie is shorter than a maximum of 1 seconds as the
# original or has no duration property in mediainfo, the task will be dropped.
origin_duration = get_media_duration(origin)
duration = get_media_duration(temp)
# If we can't get the duration the user needs to check manually.
if origin_duration is None:
Logger.info(f'Unknown time format for {origin}, compare them by hand.')
return False, False,
if duration is None:
Logger.info(f'Unknown time format for {temp}, compare them by hand.')
return False, False
diff = origin_duration - duration
THRESHOLD = 1
if math.fabs(diff.total_seconds()) > THRESHOLD:
Logger.info(f'Length differs more than {THRESHOLD} seconds.')
return False, True
return True, False
def run(self):
"""Get all known video files by recursive extension search."""
extensions = ['mkv', 'mp4', 'avi']
files = []
for extension in extensions:
found = glob.glob(os.path.join(self.directory,
f'**/*.{extension}'),
recursive=True)
files = files + found
self.create_tasks(files)
if len(self.tasks) == 0:
Logger.info('No files for encoding found.')
sys.exit(0)
else:
Logger.info(f'{len(self.tasks)} files found.')
self.receive_pueue_status()
# Add all tasks to pueue
for task in self.tasks:
self.add_task(task)
while len(self.tasks) > 0:
self.receive_pueue_status()
# Wait for all tasks
remaining_tasks = []
for task in self.tasks:
if self.is_task_done(task):
self.validate_encoded_file(task)
else:
remaining_tasks.append(task)
self.tasks = remaining_tasks
time.sleep(60)
Logger.info(
f'Successfully encoded {self.processed_files} movies. Exiting')
def get_or_create(session, name, directory, size, **kwargs):
"""Get or create a new Movie."""
movie = session.query(Movie) \
.filter(Movie.name == name) \
.filter(Movie.directory == directory) \
.filter(Movie.size == size) \
.one_or_none()
if movie:
if movie.sha1 is None:
movie.sha1 = get_sha1(os.path.join(directory, name))
if not movie:
# Delete any other movies with differing size.
# This might be necessary in case we get a new release, with a different size.
session.query(Movie) \
.filter(Movie.name == name) \
.filter(Movie.directory == directory) \
.delete()
# Found a movie with the same sha1.
# It probably moved from one directory into another
sha1 = get_sha1(os.path.join(directory, name))
movies = session.query(Movie) \
.filter(Movie.sha1 == sha1) \
.all()
if len(movies) > 0:
# Found multiple movies with the same hash. Use the first one.
if len(movies) > 1:
for movie in movies:
path = os.path.join(movie.directory, movie.name)
Logger.info(f'Found duplicate movies: {path}')
path = os.path.join(movies[0].directory, movies[0].name)
Logger.info(f'Using movie: {path}')
# Always use the first result
movie = movies[0]
# Inform user about rename or directory change
old_path = os.path.join(movie.directory, movie.name)
new_path = os.path.join(directory, name)
Logger.info(f'{name} moved in some kind of way.')
Logger.info(f'Moving from {old_path} to new path {new_path}.')
# Set attributes to new location
movie.name = name
movie.directory = directory
movie.size = size
# Create new movie
if not movie:
movie = Movie(sha1, name, directory, size, **kwargs)
session.add(movie)
session.commit()
movie = session.query(Movie) \
.filter(Movie.name == name) \
.filter(Movie.directory == directory) \
.filter(Movie.size == size) \
.one()
return movie
def validate_encoded_file(self, task):
"""Validate that the encoded file is not malformed."""
if os.path.exists(task.temp_path):
Logger.info("Pueue task completed:")
Logger.info(task.origin_file)
# Check if the duration of both movies differs.
copy, delete = check_duration(task.origin_path,
task.temp_path,
seconds=1)
# Check if the filesize of the x.265 encoded object is bigger
# than the original.
if copy:
copy, delete = check_file_size(task.origin_path,
task.temp_path)
# Only copy if checks above passed
if copy:
# Save new path, size, sha1 and mark as encoded
task.movie.sha1 = get_sha1(task.temp_path)
task.movie.size = os.path.getsize(task.temp_path)
task.movie.encoded = True
task.movie.name = os.path.basename(task.target_path)
self.session.add(task.movie)
self.session.commit()
# Get original file permissions
stat = os.stat(task.origin_path)
# Remove the old file and copy the new one to the proper directory.
os.remove(task.origin_path)
os.rename(task.temp_path, task.target_path)
# Set original file permissions on new file.
os.chmod(task.target_path, stat.st_mode)
try:
os.chown(task.target_path, stat.st_uid, stat.st_gid)
except PermissionError:
Logger.info("Failed to set ownership for {0}".format(
task.target_path))
pass
self.processed_files += 1
Logger.info("New encoded file is now in place")
elif delete:
# Mark as failed and save
task.movie.failed = True
self.session.add(task.movie)
self.session.commit()
os.remove(task.temp_path)
Logger.warning("Didn't copy new file, see message above")
else:
Logger.error("Pueue task failed in some kind of way.")