def read_config(self): """Get the config file or create it with default values.""" self.config = configparser.ConfigParser() # Try to get config, if this doesn't work a new default config will be created if os.path.exists(self.config_path): try: self.config.read(self.config_path) return except BaseException: Logger.info( 'Error while parsing config file. Deleting old config') # Default configuration self.config['encoding'] = { 'crf': '18', 'preset': 'slow', 'audio': 'None', 'kbitrate-audio': 'None', 'threads': '4', } self.config['default'] = { 'min-size': '{0}'.format(1024 * 1024 * 1024 * 6), 'SQL_URI': '/var/lib/encarne/encarne.sql', 'niceness': '15', } self.write_config()
def add_task(self, task): """Schedule and manage encoding of a movie. 2. The command is added to pueue. 3. Wait for the task to finish. 4. Check if the encoding was successful. 4.1 If it wasn't successful, we delete the encoded file and mark the old file as `encarne-failed`. 4.2 Remove the old file and move the encoded file to the proper location. 5. Repeat """ # Check if the current command already in the queue. status = self.get_newest_status(task.ffmpeg_command) # Send the command to pueue for scheduling, if it isn't in the queue yet if status is None: # In case a previous run failed and pueue has been resetted, # we need to check, if the encoded file is still there. if os.path.exists(task.temp_path): os.remove(task.temp_path) # Create a new pueue task args = { 'command': [task.ffmpeg_command], 'path': task.origin_folder, } Logger.info(f'Add task pueue:\n {task.ffmpeg_command}') execute_add(args, os.path.expanduser('~'))
def clean_movies(session): """Remove all deleted movies.""" movies = session.query(Movie).all() for movie in movies: # Can't find the file. Remove the movie. path = os.path.join(movie.directory, movie.name) if not os.path.exists(path): Logger.info(f'Remove {path}') session.delete(movie) session.commit()
def check_file_size(origin, temp): """Compare the file size of original and re encoded file.""" origin_filesize = os.path.getsize(origin) filesize = os.path.getsize(temp) if origin_filesize < filesize: Logger.info('Encoded movie is bigger than the original movie') return False, True else: difference = origin_filesize - filesize mebibyte = int(difference / 1024 / 1024) Logger.info( f'The new movie is {mebibyte} MIB smaller than the old one') return True, False
def format_args(self, args): """Check arguments and format them to be compatible with `self.config`.""" args = {key: value for key, value in args.items() if value} for key, value in args.items(): if key == 'directory': self.directory = value # Encoding if key == 'crf': self.config['encoding']['crf'] = str(value) elif key == 'preset': self.config['encoding']['preset'] = value elif key == 'audio': self.config['encoding']['audio'] = value elif key == 'audio': self.config['encoding']['kbitrate-audio'] = value elif key == 'threads': self.config['encoding']['threads'] = str(value) elif key == 'size': self.config['default']['min-size'] = str( humanfriendly.parse_size(value)) # Default if no dir is specified if not self.directory: self.directory = '.' # Get absolute path of directory self.directory = os.path.abspath(self.directory) Logger.info(f'Searching for files in directory {self.directory}') # Check if path is a dir if not os.path.isdir(self.directory): Logger.warning('A valid directory needs to be specified') Logger.warning(self.directory) sys.exit(1)
def show_stats(args): """Print how much has already been saved by reencoding.""" session = get_session() movies = session.query(Movie).all() saved = 0 failed = 0 encoded = 0 for movie in movies: # Only count movies which exist in the file system. path = os.path.join(movie.directory, movie.name) if not os.path.exists(path): continue diff = movie.original_size - movie.size if movie.failed: failed += 1 elif movie.encoded and diff > 0: saved += diff encoded += 1 saved_formatted = humanfriendly.format_size(saved) Logger.info(f'Saved space: {saved_formatted}') Logger.info(f'Reencoded container: {encoded}') Logger.info(f'Failed movies: {failed}')
def get_media_duration(path): """Execute external mediainfo command and find the video encoding library.""" process = subprocess.run( ['mediainfo', '--Output=PBCore2', path], stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) root = etree.XML(process.stdout) try: duration = root.find( ".//ns:instantiationDuration", namespaces={ 'ns': 'http://www.pbcore.org/PBCore/PBCoreNamespace.html' }, ).text except IndexError: Logger.info(f'Could not find duration for {path}') return None try: duration = duration.split('.')[0].split(';')[0] date = datetime.strptime(duration, '%H:%M:%S') except BaseException: try: duration = duration.rsplit(':', 1)[0] date = datetime.strptime(duration, '%H:%M:%S') except BaseException: Logger.info(f'Unknown duration: {duration}') return None delta = timedelta( hours=date.hour, minutes=date.minute, seconds=date.second, ) return delta
def create_tasks(self, files): """Filter files and check if they are already done or failed in a previous run. Ignore previously failed movies (too big, duration differs) and already encoded movies. Create a task with all paths and the compiled ffmpeg command. """ for path in files: # Get absolute path path = os.path.abspath(path) mediainfo = get_media_encoding(path) task = Task(path, self.config) size = os.path.getsize(path) # Get movie from db and check for already encoded or failed files. task.movie = Movie.get_or_create(self.session, task.origin_file, task.origin_folder, size) if task.movie.encoded or task.movie.failed: continue # Already encoded if '265' in mediainfo or '265' in path: task.movie.encoded = True self.session.add(task.movie) continue # File to small for encoding elif size < int(self.config['default']['min-size']): Logger.debug('File smaller than min-size: {path}') continue # Unknown encoding elif mediainfo == 'unknown': Logger.info(f'Failed to get encoding for {path}') self.tasks.append(task) self.session.commit()
def check_duration(origin, temp, seconds=1): """Check if the duration is bigger than a specific amount.""" # If the destination movie is shorter than a maximum of 1 seconds as the # original or has no duration property in mediainfo, the task will be dropped. origin_duration = get_media_duration(origin) duration = get_media_duration(temp) # If we can't get the duration the user needs to check manually. if origin_duration is None: Logger.info(f'Unknown time format for {origin}, compare them by hand.') return False, False, if duration is None: Logger.info(f'Unknown time format for {temp}, compare them by hand.') return False, False diff = origin_duration - duration THRESHOLD = 1 if math.fabs(diff.total_seconds()) > THRESHOLD: Logger.info(f'Length differs more than {THRESHOLD} seconds.') return False, True return True, False
def run(self): """Get all known video files by recursive extension search.""" extensions = ['mkv', 'mp4', 'avi'] files = [] for extension in extensions: found = glob.glob(os.path.join(self.directory, f'**/*.{extension}'), recursive=True) files = files + found self.create_tasks(files) if len(self.tasks) == 0: Logger.info('No files for encoding found.') sys.exit(0) else: Logger.info(f'{len(self.tasks)} files found.') self.receive_pueue_status() # Add all tasks to pueue for task in self.tasks: self.add_task(task) while len(self.tasks) > 0: self.receive_pueue_status() # Wait for all tasks remaining_tasks = [] for task in self.tasks: if self.is_task_done(task): self.validate_encoded_file(task) else: remaining_tasks.append(task) self.tasks = remaining_tasks time.sleep(60) Logger.info( f'Successfully encoded {self.processed_files} movies. Exiting')
def get_or_create(session, name, directory, size, **kwargs): """Get or create a new Movie.""" movie = session.query(Movie) \ .filter(Movie.name == name) \ .filter(Movie.directory == directory) \ .filter(Movie.size == size) \ .one_or_none() if movie: if movie.sha1 is None: movie.sha1 = get_sha1(os.path.join(directory, name)) if not movie: # Delete any other movies with differing size. # This might be necessary in case we get a new release, with a different size. session.query(Movie) \ .filter(Movie.name == name) \ .filter(Movie.directory == directory) \ .delete() # Found a movie with the same sha1. # It probably moved from one directory into another sha1 = get_sha1(os.path.join(directory, name)) movies = session.query(Movie) \ .filter(Movie.sha1 == sha1) \ .all() if len(movies) > 0: # Found multiple movies with the same hash. Use the first one. if len(movies) > 1: for movie in movies: path = os.path.join(movie.directory, movie.name) Logger.info(f'Found duplicate movies: {path}') path = os.path.join(movies[0].directory, movies[0].name) Logger.info(f'Using movie: {path}') # Always use the first result movie = movies[0] # Inform user about rename or directory change old_path = os.path.join(movie.directory, movie.name) new_path = os.path.join(directory, name) Logger.info(f'{name} moved in some kind of way.') Logger.info(f'Moving from {old_path} to new path {new_path}.') # Set attributes to new location movie.name = name movie.directory = directory movie.size = size # Create new movie if not movie: movie = Movie(sha1, name, directory, size, **kwargs) session.add(movie) session.commit() movie = session.query(Movie) \ .filter(Movie.name == name) \ .filter(Movie.directory == directory) \ .filter(Movie.size == size) \ .one() return movie
def validate_encoded_file(self, task): """Validate that the encoded file is not malformed.""" if os.path.exists(task.temp_path): Logger.info("Pueue task completed:") Logger.info(task.origin_file) # Check if the duration of both movies differs. copy, delete = check_duration(task.origin_path, task.temp_path, seconds=1) # Check if the filesize of the x.265 encoded object is bigger # than the original. if copy: copy, delete = check_file_size(task.origin_path, task.temp_path) # Only copy if checks above passed if copy: # Save new path, size, sha1 and mark as encoded task.movie.sha1 = get_sha1(task.temp_path) task.movie.size = os.path.getsize(task.temp_path) task.movie.encoded = True task.movie.name = os.path.basename(task.target_path) self.session.add(task.movie) self.session.commit() # Get original file permissions stat = os.stat(task.origin_path) # Remove the old file and copy the new one to the proper directory. os.remove(task.origin_path) os.rename(task.temp_path, task.target_path) # Set original file permissions on new file. os.chmod(task.target_path, stat.st_mode) try: os.chown(task.target_path, stat.st_uid, stat.st_gid) except PermissionError: Logger.info("Failed to set ownership for {0}".format( task.target_path)) pass self.processed_files += 1 Logger.info("New encoded file is now in place") elif delete: # Mark as failed and save task.movie.failed = True self.session.add(task.movie) self.session.commit() os.remove(task.temp_path) Logger.warning("Didn't copy new file, see message above") else: Logger.error("Pueue task failed in some kind of way.")