def __get_plugin_settings(self, plugin_id):
"""
Given a plugin ID , return a list of plugin settings for that plugin
:param plugin_id:
:return:
"""
settings = []
# Check plugin for settings
plugin_executor = PluginExecutor()
plugin_settings = plugin_executor.get_plugin_settings(plugin_id)
if plugin_settings:
for key in plugin_settings:
key_id = hashlib.md5(key.encode('utf8')).hexdigest()
value = plugin_settings.get(key)
var_type = "str"
if isinstance(value, bool):
var_type = "bool"
elif isinstance(value, (int, float)):
var_type = "int"
settings.append({
"key_id": key_id,
"key": key,
"value": value,
"var_type": var_type,
})
return settings
def __get_plugin_settings(self, plugin_id):
"""
Given a plugin ID , return a list of plugin settings for that plugin
:param plugin_id:
:return:
"""
settings = []
# Check plugin for settings
plugin_executor = PluginExecutor()
plugin_settings, plugin_settings_meta = plugin_executor.get_plugin_settings(
plugin_id)
if plugin_settings:
for key in plugin_settings:
form_input = {
"key_id": hashlib.md5(key.encode('utf8')).hexdigest(),
"key": key,
"value": plugin_settings.get(key),
"input_type": None,
"label": None,
"select_options": [],
}
plugin_setting_meta = plugin_settings_meta.get(key, {})
# Set input type for form
form_input['input_type'] = plugin_setting_meta.get(
'input_type', None)
if not form_input['input_type']:
form_input['input_type'] = "text"
if isinstance(form_input['value'], bool):
form_input['input_type'] = "checkbox"
# Handle unsupported input types (where they may be supported in future versions of Unmanic)
supported_input_types = [
"text",
"textarea",
"select",
"checkbox",
"browse_directory",
]
if form_input['input_type'] not in supported_input_types:
form_input['input_type'] = "text"
# Set input label text
form_input['label'] = plugin_setting_meta.get('label', None)
if not form_input['label']:
form_input['label'] = key
# Set options if form input is select
if form_input['input_type'] == 'select':
form_input['select_options'] = plugin_setting_meta.get(
'select_options', [])
if not form_input['select_options']:
# No options are given. Revert back to text input
form_input['input_type'] = 'text'
settings.append(form_input)
return settings
def __get_plugin_long_description(self, plugin_id):
"""
Given a plugin ID , return a list of lines read from the plugin's changelog
:param plugin_id:
:return:
"""
# Fetch plugin changelog
plugin_executor = PluginExecutor()
return plugin_executor.get_plugin_long_description(plugin_id)
def get_plugin_changelog(plugin_id):
"""
Given a plugin installation ID, return a list of lines read from the plugin's changelog
:param plugin_id:
:return:
"""
# Fetch plugin changelog
plugin_executor = PluginExecutor()
return plugin_executor.get_plugin_changelog(plugin_id)
def uninstall_plugins_by_db_table_id(self, plugin_table_ids: list):
"""
Remove a Plugin by it's DB table ID column.
This will also remove the Plugin directory and all it's contents.
:param plugin_table_ids:
:return:
"""
self._log("Uninstall plugins '{}'".format(plugin_table_ids),
level='debug')
# Fetch records
records_by_id = self.get_plugin_list_filtered_and_sorted(
id_list=plugin_table_ids)
# Remove each plugin from disk
for record in records_by_id:
# Unload plugin modules
try:
PluginExecutor.unload_plugin_module(record.get('plugin_id'))
except Exception as e:
self._log("Exception while unloading python module {}:".format(
record.get('plugin_id')),
message2=str(e),
level="exception")
# Remove from disk
plugin_directory = self.get_plugin_path(record.get('plugin_id'))
self._log("Removing plugin files from disk '{}'".format(
plugin_directory),
level='debug')
try:
# Delete the info file first to prevent any other process trying to read the plugin.
# Without the info file, the plugin is effectivly uninstalled
info_file = os.path.join(plugin_directory, 'info.json')
if os.path.exists(info_file):
os.remove(info_file)
# Cleanup the rest of the plugin directory
shutil.rmtree(plugin_directory)
except Exception as e:
self._log("Exception while removing directory {}:".format(
plugin_directory),
message2=str(e),
level="exception")
# Unlink from library by ID in DB
EnabledPlugins.delete().where(
EnabledPlugins.plugin_id.in_(plugin_table_ids)).execute()
# Delete by ID in DB
if not Plugins.delete().where(
Plugins.id.in_(plugin_table_ids)).execute():
return False
return True
def exec_plugin_runner(self, data, plugin_id, plugin_type):
"""
Execute a plugin runner
:param data:
:param plugin_id:
:param plugin_type:
:return:
"""
plugin_executor = PluginExecutor()
return plugin_executor.execute_plugin_runner(data, plugin_id,
plugin_type)
def get_plugin_types_with_flows():
"""
Returns a list of all available plugin types
:return:
"""
return_plugin_types = []
plugin_ex = PluginExecutor()
types_list = plugin_ex.get_all_plugin_types()
# Filter out the types without flows
for plugin_type in types_list:
if plugin_type.get('has_flow'):
return_plugin_types.append(plugin_type.get('id'))
return return_plugin_types
def update_plugin_settings(self, *args, **kwargs):
return_data = {"success": False}
# Fetch ID of plugin to get Info for
plugin_id = self.get_argument('plugin_id')
# Fetch plugin info (and settings if any)
plugin_data = self.__get_plugin_info_and_settings(plugin_id)
# If no plugin data was found for the posted plugin table ID, then return a failure response
if not plugin_data:
return return_data
# Create a dictionary of all posted arguments
post_params = {}
for k, v in self.request.arguments.items():
post_params[k] = v[0].decode("utf-8")
# Loop over all plugin settings in order to find matches in the posted params
settings_to_save = {}
for setting in plugin_data.get('settings'):
key = setting.get('key')
key_id = setting.get('key_id')
input_type = setting.get('input_type')
# Check if setting is in params
if key_id in post_params:
post_value = post_params.get(key_id, '')
# Check if value should be boolean
if input_type == 'checkbox':
post_value = True if post_value.lower(
) == 'true' else False
# Add that to our dictionary of settings to save
settings_to_save[key] = post_value
# If we found settings in the post params that need to be saved, save them...
result = False
if settings_to_save:
plugin_executor = PluginExecutor()
saved_all_settings = plugin_executor.save_plugin_settings(
plugin_data.get('plugin_id'), settings_to_save)
# If the save function was successful
if saved_all_settings:
# Update settings in plugin data that will be returned
plugin_data['settings'] = settings_to_save
result = True
self.write(json.dumps({"success": result, "plugin_info": plugin_data}))
def update_plugin_settings(plugin_id, settings, library_id=None):
"""
Updates the settings for the requested plugin_id
:param plugin_id:
:param settings:
:param library_id:
:return:
"""
# Fetch plugin info (and settings if any)
plugin_data = prepare_plugin_info_and_settings(plugin_id,
library_id=library_id)
# If no plugin data was found for the posted plugin table ID, then return a failure response
if not plugin_data:
return False
# Loop over all plugin settings in order to find matches in the posted params
settings_to_save = {}
for s in settings:
key = s.get('key')
key_id = s.get('key_id')
input_type = s.get('input_type')
# Check if setting is in params
value = s.get('value')
# Check if value should be boolean
if input_type == 'checkbox':
if isinstance(value, str):
value = True if value.lower() == 'true' else False
elif isinstance(value, int):
value = True if value > 0 else False
# Add that to our dictionary of settings to save
settings_to_save[key] = value
# If we found settings that need to be saved, save them...
if settings_to_save:
plugin_executor = PluginExecutor()
saved_all_settings = plugin_executor.save_plugin_settings(
plugin_data.get('plugin_id'),
settings_to_save,
library_id=library_id)
# If the save function was successful
if saved_all_settings:
# Update settings in plugin data that will be returned
return True
return False
def test_installed_plugins():
"""
Test all plugin runners for correct return data
:return:
"""
plugin_executor = PluginExecutor()
plugins = PluginsHandler()
order = {
"column": 'position',
"dir": 'desc',
}
plugin_results = plugins.get_plugin_list_filtered_and_sorted(order=order, start=0, length=None)
for plugin_result in plugin_results:
# plugin_runners = plugin_executor.get_plugin_runners('worker.process_item')
print("{1}Testing plugin: '{0}'{2}".format(plugin_result.get("name"), BColours.HEADER, BColours.ENDC))
plugin_id = plugin_result.get("plugin_id")
# Test Plugin runners
print(" {0}Testing runners{1}".format(BColours.SUBHEADER, BColours.ENDC))
plugin_types_in_plugin = plugin_executor.get_all_plugin_types_in_plugin(plugin_id)
if not plugin_types_in_plugin:
error = "No runners found in plugin"
print(" - {1}FAILED: {0}{2}".format(error, BColours.FAIL, BColours.ENDC))
else:
for plugin_type_in_plugin in plugin_types_in_plugin:
errors = plugin_executor.test_plugin_runner(plugin_id, plugin_type_in_plugin)
if errors:
for error in errors:
print(" - {1}FAILED: {0}{2}".format(error, BColours.FAIL, BColours.ENDC))
else:
print(" - {}PASSED{}".format(BColours.OKGREEN, BColours.ENDC))
# Test Plugin settings
print(" {0}Testing settings{1}".format(BColours.SUBHEADER, BColours.ENDC))
errors, plugin_settings = plugin_executor.test_plugin_settings(plugin_id)
if errors:
for error in errors:
print(" - {1}FAILED: {0}{2}".format(error, BColours.FAIL, BColours.ENDC))
else:
formatted_plugin_settings = json.dumps(plugin_settings, indent=1)
formatted_plugin_settings = formatted_plugin_settings.replace('\n', '\n' + ' ')
print(" - {1}Settings: {0}{2}".format(formatted_plugin_settings, BColours.RESULTS, BColours.ENDC))
print(" - {}PASSED{}".format(BColours.OKGREEN, BColours.ENDC))
print()
def reset_plugin_settings(plugin_id, library_id=None):
"""
Reset a plugin's settings back to defaults (or global config if a library ID is provided)
:param plugin_id:
:param library_id:
:return:
"""
# Fetch plugin info (and settings if any)
plugin_data = prepare_plugin_info_and_settings(plugin_id,
library_id=library_id)
# If no plugin data was found for the posted plugin table ID, then return a failure response
if not plugin_data:
return False
# Reset the plugin settings
plugin_executor = PluginExecutor()
return plugin_executor.reset_plugin_settings(plugin_data.get('plugin_id'),
library_id=library_id)
def install_plugin_by_id(self, plugin_id, repo_id=None):
"""
Find the matching plugin info for the given plugin ID.
Download the plugin if it is found and return the result.
If it is not found, return False.
:param plugin_id:
:param repo_id:
:return:
"""
plugin_list = self.get_installable_plugins_list(filter_repo_id=repo_id)
for plugin in plugin_list:
if plugin.get('plugin_id') == plugin_id:
success = self.install_plugin(plugin)
if success:
try:
plugin_directory = self.get_plugin_path(
plugin.get("plugin_id"))
result = self.write_plugin_data_to_db(
plugin, plugin_directory)
if result:
self._log(
"Installed plugin '{}'".format(plugin_id),
level="info")
# Ensure the plugin module is reloaded (if it was previously loaded)
plugin_executor = PluginExecutor()
plugin_executor.reload_plugin_module(
plugin.get('plugin_id'))
return result
except Exception as e:
self._log(
"Exception while installing plugin '{}'.".format(
plugin),
str(e),
level="exception")
return False
def __set_default_plugin_flow_priority(self, plugin_list):
from unmanic.libs.unplugins import PluginExecutor
plugin_executor = PluginExecutor()
from unmanic.libs.plugins import PluginsHandler
plugin_handler = PluginsHandler()
# Fetch current items
configured_plugin_ids = []
query = LibraryPluginFlow.select().where(
LibraryPluginFlow.library_id == self.model.id)
for flow_item in query:
configured_plugin_ids.append(flow_item.plugin_id.plugin_id)
for plugin in plugin_list:
# Ignore already configured plugins
if plugin.get('plugin_id') in configured_plugin_ids:
continue
plugin_info = plugin_handler.get_plugin_info(
plugin.get('plugin_id'))
plugin_priorities = plugin_info.get('priorities')
if plugin_priorities:
# Fetch the plugin info back from the DB
plugin_info = Plugins.select().where(
Plugins.plugin_id == plugin.get("plugin_id")).first()
# Fetch all plugin types in this plugin
plugin_types_in_plugin = plugin_executor.get_all_plugin_types_in_plugin(
plugin.get("plugin_id"))
# Loop over the plugin types in this plugin
for plugin_type in plugin_types_in_plugin:
# get the plugin runner function name for this runner
plugin_type_meta = plugin_executor.get_plugin_type_meta(
plugin_type)
runner_string = plugin_type_meta.plugin_runner()
if plugin_priorities.get(runner_string) and int(
plugin_priorities.get(runner_string, 0)) > 0:
# If the runner has a priority set and that value is greater than 0 (default that wont set anything),
# Save the priority
PluginsHandler.set_plugin_flow_position_for_single_plugin(
plugin_info, plugin_type, self.model.id,
plugin_priorities.get(runner_string))
def get_enabled_plugin_modules_by_type(self, plugin_type, library_id=None):
"""
Return a list of enabled plugin modules when given a plugin type
Runners are filtered by the given 'plugin_type' and sorted by
configured order of execution.
If no library ID is provided, this will return all installed plugins for that type.
This case should only be used for plugin runner types that are not associated with a library.
:param plugin_type:
:param library_id:
:return:
"""
# Refresh session
s = Session()
s.register_unmanic()
# First fetch all enabled plugins
order = [
{
"model": LibraryPluginFlow,
"column": 'position',
"dir": 'asc',
},
{
"column": 'name',
"dir": 'asc',
},
]
enabled_plugins = self.get_plugin_list_filtered_and_sorted(
order=order, plugin_type=plugin_type, library_id=library_id)
# Fetch all plugin modules from the given list of enabled plugins
plugin_executor = PluginExecutor()
plugin_data = plugin_executor.get_plugin_data_by_type(
enabled_plugins, plugin_type)
# Return modules
return plugin_data
def get_enabled_plugins(self, include_settings=False):
"""
Get all enabled plugins for this library
:return:
"""
# Fetch enabled plugins for this library
query = self.model.enabled_plugins.select(Plugins,
EnabledPlugins.library_id)
query = query.join(Plugins,
join_type='LEFT OUTER JOIN',
on=(EnabledPlugins.plugin_id == Plugins.id))
query = query.order_by(Plugins.name)
from unmanic.libs.unplugins import PluginExecutor
plugin_executor = PluginExecutor()
# Extract required data
enabled_plugins = []
for enabled_plugin in query.dicts():
# Check if plugin is able to be configured
has_config = False
plugin_settings, plugin_settings_meta = plugin_executor.get_plugin_settings(
enabled_plugin.get('plugin_id'), library_id=self.model.id)
if plugin_settings:
has_config = True
# Add plugin to list of enabled plugins
item = {
'plugin_id': enabled_plugin.get('plugin_id'),
'name': enabled_plugin.get('name'),
'description': enabled_plugin.get('description'),
'icon': enabled_plugin.get('icon'),
'has_config': has_config,
}
if include_settings:
item['settings'] = plugin_settings
enabled_plugins.append(item)
return enabled_plugins
def get_settings_of_all_installed_plugins(self):
all_settings = {}
# First fetch all enabled plugins
order = [
{
"column": 'name',
"dir": 'asc',
},
]
installed_plugins = self.get_plugin_list_filtered_and_sorted(
order=order)
# Fetch settings for each plugin
plugin_executor = PluginExecutor()
for plugin in installed_plugins:
plugin_settings, plugin_settings_meta = plugin_executor.get_plugin_settings(
plugin.get('plugin_id'))
all_settings[plugin.get('plugin_id')] = plugin_settings
# Return modules
return all_settings
def get_plugin_flow(self):
"""
Fetch the plugin flow for a library
:return:
"""
plugin_flow = {}
from unmanic.libs.plugins import PluginsHandler
plugin_handler = PluginsHandler()
from unmanic.libs.unplugins import PluginExecutor
plugin_ex = PluginExecutor()
for plugin_type in plugin_ex.get_all_plugin_types():
# Ignore types without flows
if not plugin_type.get('has_flow'):
continue
# Create list of plugins in this plugin type
plugin_flow[plugin_type.get('id')] = []
plugin_modules = plugin_handler.get_enabled_plugin_modules_by_type(
plugin_type.get('id'), library_id=self.model.id)
for plugin_module in plugin_modules:
plugin_flow[plugin_type.get('id')].append({
"plugin_id":
plugin_module.get("plugin_id"),
"name":
plugin_module.get("name", ""),
"author":
plugin_module.get("author", ""),
"description":
plugin_module.get("description", ""),
"version":
plugin_module.get("version", ""),
"icon":
plugin_module.get("icon", ""),
})
return plugin_flow
def get_plugin_modules_by_type(self, plugin_type):
"""
Return a list of enabled plugin modules when given a plugin type
Runners are filtered by the given 'plugin_type' and sorted by
configured order of execution.
:param plugin_type:
:return:
"""
# Refresh session
s = Session()
s.register_unmanic(s.get_installation_uuid())
# Update enabled plugins
self.ensure_session_level_for_plugins(s.level)
# First fetch all enabled plugins
order = [
{
"model": PluginFlow,
"column": 'position',
"dir": 'asc',
},
{
"column": 'name',
"dir": 'asc',
},
]
enabled_plugins = self.get_plugin_list_filtered_and_sorted(order=order, enabled=True, plugin_type=plugin_type)
# Fetch all plugin modules from the given list of enabled plugins
plugin_executor = PluginExecutor()
plugin_data = plugin_executor.get_plugin_data_by_type(enabled_plugins, plugin_type)
# Return modules
return plugin_data
def save_library_config(library_id, library_config=None, plugin_config=None):
"""
Save a complete library configuration
:param library_id:
:param library_config:
:param plugin_config:
:return:
"""
# Parse library config
if plugin_config is None:
plugin_config = {}
if library_config is None:
library_config = {}
# Check if this save requires a new library entry
if int(library_id) > 0:
# Fetch existing library by ID
new_library = False
library = Library(library_id)
else:
# Create a new library with required data
new_library = True
library = Library.create({
'name': library_config.get('name'),
'path': library_config.get('path'),
})
library_id = library.get_id()
# Update library config (if the data was given)
if library_config:
library.set_name(library_config.get('name', library.get_name()))
library.set_path(library_config.get('path', library.get_path()))
library.set_locked(library_config.get('locked', library.get_locked()))
library.set_enable_remote_only(
library_config.get('enable_remote_only',
library.get_enable_remote_only()))
library.set_enable_scanner(
library_config.get('enable_scanner', library.get_enable_scanner()))
library.set_enable_inotify(
library_config.get('enable_inotify', library.get_enable_inotify()))
library.set_priority_score(
library_config.get('priority_score', library.get_priority_score()))
library.set_tags(library_config.get('tags', library.get_tags()))
# Update enabled plugins (if the data was given)
enabled_plugins = plugin_config.get('enabled_plugins')
if enabled_plugins is not None:
# Ensure plugins are installed (install them if they are not)
repo_refreshed = False
for ep in enabled_plugins:
if not plugins.check_if_plugin_is_installed(ep.get('plugin_id')):
# Trigger plugin repo refresh if this is the first install
if not repo_refreshed:
plugins.reload_plugin_repos_data()
repo_refreshed = True
# Install the plugin
if not plugins.install_plugin_by_id(ep.get('plugin_id')):
if new_library:
library.delete()
raise Exception(
"Failed to install plugin by plugin ID '{}'".format(
ep.get('plugin_id')))
# Enable the plugins against this library
library.set_enabled_plugins(enabled_plugins)
# Import settings
plugin_executor = PluginExecutor()
for ep in enabled_plugins:
if ep.get('has_config'):
plugin_executor.save_plugin_settings(ep.get('plugin_id'),
ep.get('settings', {}),
library_id=library_id)
# Update plugin flow (if the data was given)
plugin_flow = plugin_config.get('plugin_flow')
if plugin_flow is not None:
for plugin_type in plugins.get_plugin_types_with_flows():
flow = []
for plugin_id in plugin_flow.get(plugin_type, []):
flow.append({'plugin_id': plugin_id})
plugins.save_enabled_plugin_flows_for_plugin_type(
plugin_type, library_id, flow)
# Save config
return library.save()
def test_plugin_runner(plugin_id, plugin_type, test_data=None):
plugin_executor = PluginExecutor()
return plugin_executor.test_plugin_runner(plugin_id, plugin_type, test_data)
def should_file_be_added_to_task_list(self):
"""
Test if this file needs to be added to the task list
:return:
"""
return_value = True
file_issues = []
# Init plugins
plugin_executor = PluginExecutor()
# Run task success plugins
plugin_modules = plugin_executor.get_plugin_modules_by_type('postprocessor.task_result')
if self.file_in_unmanic_ignore_lockfile():
file_issues.append({
'id': 'unmanicignore',
'message': "File found in unmanic ignore file - '{}'".format(self.path),
})
return_value = False
if not self.file_ends_in_allowed_search_extensions():
file_issues.append({
'id': 'extension',
'message': "File suffix is not in allowed search extensions - '{}'".format(self.path),
})
return_value = False
# Check if file has failed in history.
if self.file_failed_in_history():
file_issues.append({
'id': 'blacklisted',
'message': "File found already failed in history - '{}'".format(self.path),
})
return_value = False
# Check if this file is already the correct format:
if self.file_already_in_target_format():
file_issues.append({
'id': 'format',
'message': "File is already in target format - '{}'".format(self.path),
})
return_value = False
# Run tests against plugins
for plugin_module in plugin_modules:
data = {
'path': self.path,
'issues': file_issues,
'add_file_to_pending_tasks': return_value,
}
# Test return data against schema and ensure there are no errors
runner_errors = plugin_executor.test_plugin_runner(plugin_module.get('plugin_id'), 'library_management.file_test',
data)
if runner_errors:
self._log(
"Error while running library management file test '{}' on file '{}'".format(plugin_module.get('plugin_id'),
self.path), runner_errors,
level="error")
# Don't execute this runner. It failed
continue
# Run plugin and fetch return data
plugin_runner = plugin_module.get("runner")
try:
plugin_runner(data)
except Exception as e:
self._log("Exception while carrying out plugin runner on library management file test '{}'".format(
plugin_module.get('plugin_id')), message2=str(e), level="exception")
continue
pass
return return_value, file_issues
def post_process_file(self):
# Check if the job was a success
if not self.current_task.task.success:
self._log("Task was marked as failed.", level='debug')
self._log("Removing cached file",
self.current_task.task.cache_path,
level='debug')
self.remove_current_task_cache_file()
return
# Ensure file is correct format
self.current_task.task.success = self.validate_streams(
self.current_task.task.cache_path)
# Init plugins
plugin_executor = PluginExecutor()
# Read current task data
# task_data = self.current_task.get_task_data()
cache_path = self.current_task.get_cache_path()
source_data = self.current_task.get_source_data()
destination_data = self.current_task.get_destination_data()
# Move file back to original folder and remove source
file_move_processes_success = True
# Create a list for filling with destination paths
destination_files = []
if self.current_task.task.success:
# Run a postprocess file movement on the cache file for for each plugin that configures it
plugin_modules = plugin_executor.get_plugin_modules_by_type(
'postprocessor.file_move')
# Check if the source file needs to be remove by default (only if it does not match the destination file)
remove_source_file = False
if source_data['abspath'] != destination_data['abspath']:
remove_source_file = True
# Set initial data (some fields will be overwritten further down)
data = {
"source_data": None,
'remove_source_file': remove_source_file,
'copy_file': None,
"file_in": None,
"file_out": None,
}
for plugin_module in plugin_modules:
# Always set source_data to the original file's source_data
data["source_data"] = source_data
# Always set copy_file to True
data["copy_file"] = True
# Always set file in to cache path
data["file_in"] = cache_path
# Always set file out to destination data absolute path
data["file_out"] = destination_data.get('abspath')
# Test return data against schema and ensure there are no errors
errors = plugin_executor.test_plugin_runner(
plugin_module.get('plugin_id'), 'postprocessor.file_move',
data)
if errors:
self._log(
"Error while running postprocessor file movement '{}' on file '{}'"
.format(plugin_module.get('plugin_id'), cache_path),
errors,
level="error")
# Don't execute this runner. It failed
continue
# Run plugin and fetch return data
plugin_runner = plugin_module.get("runner")
try:
data = plugin_runner(data)
except Exception as e:
self._log(
"Exception while carrying out plugin runner on postprocessor file movement '{}'"
.format(plugin_module.get('plugin_id')),
message2=str(e),
level="exception")
# Do not continue with this plugin module's loop
continue
if data.get('copy_file'):
# Copy the file
self._log("Copying file {} --> {}".format(
data.get('file_in'), data.get('file_out')))
shutil.copyfile(data.get('file_in'), data.get('file_out'))
destination_files.append(data.get('file_out'))
# Run another validation on the copied file to ensure it is still correct
copy_valid = self.validate_streams(data.get('file_out'))
if not copy_valid:
# Something went wrong during that file copy
self._log(
"Copy function failed during postprocessor file movement '{}' on file '{}'"
.format(plugin_module.get('plugin_id'),
cache_path),
level='warning')
file_move_processes_success = False
# Check if the remove source flag is still True after all plugins have run. If so, we will remove the source file
if data.get('remove_source_file'):
# Only carry out a source removal if the whole postprocess was successful
if file_move_processes_success:
self._log("Removing source: {}".format(
source_data['abspath']))
os.remove(source_data['abspath'])
# If we need to keep the filename history, do that here
if self.settings.get_keep_filename_history():
dirname = os.path.dirname(source_data['abspath'])
self.keep_filename_history(
dirname, destination_data["basename"],
source_data["basename"])
else:
self._log(
"Keeping source file '{}'. Not all postprocessor file movement functions completed."
.format(source_data['abspath']),
level="warning")
if not file_move_processes_success:
self._log(
"Error while running postprocessor file movement on file '{}'. Not all postprocessor file movement functions completed."
.format(cache_path),
level="error")
else:
self._log("Encoded file failed post processing test '{}'".format(
cache_path),
level='warning')
return
# Run task success plugins
plugin_modules = plugin_executor.get_plugin_modules_by_type(
'postprocessor.task_result')
for plugin_module in plugin_modules:
data = {
"source_data": source_data,
'task_processing_success': self.current_task.task.success,
'file_move_processes_success': file_move_processes_success,
'destination_files': destination_files,
}
# Test return data against schema and ensure there are no errors
errors = plugin_executor.test_plugin_runner(
plugin_module.get('plugin_id'), 'postprocessor.task_result',
data)
if errors:
self._log(
"Error while running postprocessor task result'{}' on file '{}'"
.format(plugin_module.get('plugin_id'), cache_path),
errors,
level="error")
# Don't execute this runner. It failed
continue
# Run plugin and fetch return data
plugin_runner = plugin_module.get("runner")
try:
plugin_runner(data)
except Exception as e:
self._log(
"Exception while carrying out plugin runner on postprocessor task result '{}'"
.format(plugin_module.get('plugin_id')),
message2=str(e),
level="exception")
continue
# Cleanup cache files
task_cache_directory = os.path.dirname(cache_path)
if os.path.exists(
task_cache_directory
) and "unmanic_file_conversion" in task_cache_directory:
for f in os.listdir(task_cache_directory):
cache_file_path = os.path.join(task_cache_directory, f)
# if not f.endswith(".bak"):
# continue
self._log("Removing task cache directory file '{}'".format(
cache_file_path))
# Remove the cache file
os.remove(cache_file_path)
# Remove the directory
self._log("Removing task cache directory '{}'".format(
task_cache_directory))
os.rmdir(task_cache_directory)
def process_item(self):
# Reset the ffmpeg class when a new item is received
self.setup_ffmpeg()
abspath = self.current_task.get_source_abspath()
self._log("{} processing job - {}".format(self.name, abspath))
# # Process item in loop for the default config
# file_in = abspath
# file_out = self.current_task.task.cache_path
# data = self.convert_file(file_in, file_out)
# Then process the item for for each plugin that configures it
from unmanic.libs.unplugins import PluginExecutor
plugin_executor = PluginExecutor()
plugin_modules = plugin_executor.get_plugin_modules_by_type(
'worker.process_item')
# Process item in loop.
# First process the item for for each plugin that configures it, then run the default Unmanic configuration
task_cache_path = self.current_task.get_cache_path()
file_in = abspath
overall_success = True
current_file_out = ""
runner_count = 0
for plugin_module in plugin_modules:
runner_count += 1
# Fetch file out details
# This creates a temp file labeled "WORKING" that will be moved to the cache_path on completion
tmp_file_out = os.path.splitext(task_cache_path)
file_out = current_file_out = "{}-{}-{}{}".format(
tmp_file_out[0], "WORKING", runner_count, tmp_file_out[1])
# Fetch initial file probe
file_probe = self.ffmpeg.file_probe(file_in)
# Create args from
ffmpeg_args = self.ffmpeg.generate_ffmpeg_args(
file_probe, file_in, file_out)
data = {
"exec_ffmpeg": True,
"file_probe": file_probe,
"ffmpeg_args": ffmpeg_args,
"file_in": file_in,
"file_out": file_out,
}
# Test return data against schema and ensure there are no errors
errors = plugin_executor.test_plugin_runner(
plugin_module.get('plugin_id'), 'worker.process_item', data)
if errors:
self._log(
"Error while running worker process '{}' on file '{}'".
format(plugin_module.get('plugin_id'), abspath),
errors,
level="error")
# Dont execute this runner. It failed
continue
# Run plugin and fetch return data
plugin_runner = plugin_module.get("runner")
try:
data = plugin_runner(data)
except Exception as e:
self._log(
"Exception while carrying out plugin runner on worker process '{}'"
.format(plugin_module.get('plugin_id')),
message2=str(e),
level="exception")
# Skip this plugin module's loop
continue
self._log("Worker process '{}' file in".format(
plugin_module.get('plugin_id')),
data.get("file_in"),
level='debug')
self._log("Worker process '{}' file out".format(
plugin_module.get('plugin_id')),
data.get("file_out"),
level='debug')
# Only run the conversion process if "exec_ffmpeg" is True
if data.get("exec_ffmpeg"):
# Run conversion process
success = self.convert_file(data,
plugin_module.get('plugin_id'))
if success:
# If file conversion was successful
self._log(
"Successfully ran worker process '{}' on file '{}'".
format(plugin_module.get('plugin_id'), abspath))
# Set the file in as the file out for the next loop
file_in = file_out
else:
# If file conversion was successful
self._log(
"Error while running worker process '{}' on file '{}'".
format(plugin_module.get('plugin_id'), abspath),
level="error")
overall_success = False
else:
self._log(
"Worker process '{}' set to not run the FFMPEG command.",
level='debug')
if overall_success:
# If file conversion was successful, we will get here
self._log("Successfully converted file '{}'".format(abspath))
# Move file to original cache path
shutil.move(current_file_out, task_cache_path)
return True
self._log("Failed to convert file '{}'".format(abspath),
level='warning')
return False
def get_plugin_settings(plugin_id: str, library_id=None):
"""
Given a plugin installation ID, return a list of plugin settings for that plugin
:param plugin_id:
:param library_id:
:return:
"""
settings = []
# Check plugin for settings
plugin_executor = PluginExecutor()
plugin_settings, plugin_settings_meta = plugin_executor.get_plugin_settings(
plugin_id, library_id=library_id)
if plugin_settings:
for key in plugin_settings:
form_input = {
"key_id": hashlib.md5(key.encode('utf8')).hexdigest(),
"key": key,
"value": plugin_settings.get(key),
"input_type": None,
"label": None,
"select_options": [],
"slider_options": {},
"display": "visible",
}
plugin_setting_meta = plugin_settings_meta.get(key, {})
# Set input type for form
form_input['input_type'] = plugin_setting_meta.get(
'input_type', None)
if not form_input['input_type']:
form_input['input_type'] = "text"
if isinstance(form_input['value'], bool):
form_input['input_type'] = "checkbox"
# Handle unsupported input types (where they may be supported in future versions of Unmanic)
supported_input_types = [
"text",
"textarea",
"select",
"checkbox",
"slider",
"browse_directory",
]
if form_input['input_type'] not in supported_input_types:
form_input['input_type'] = "text"
# Set input display options
form_input['display'] = plugin_setting_meta.get(
'display', 'visible')
# Set input label text
form_input['label'] = plugin_setting_meta.get('label', None)
if not form_input['label']:
form_input['label'] = key
# Set options if form input is select
if form_input['input_type'] == 'select':
form_input['select_options'] = plugin_setting_meta.get(
'select_options', [])
if not form_input['select_options']:
# No options are given. Revert back to text input
form_input['input_type'] = 'text'
# Set options if form input is slider
if form_input['input_type'] == 'slider':
slider_options = plugin_setting_meta.get('slider_options')
if not slider_options:
# No options are given. Revert back to text input
form_input['input_type'] = 'text'
else:
form_input['slider_options'] = {
'min': slider_options.get('min', '0'),
'max': slider_options.get('max', '1'),
'step': slider_options.get('step', '1'),
'suffix': slider_options.get('suffix', ''),
}
settings.append(form_input)
return settings
def prepare_filtered_plugins(params):
"""
Returns a object of records filtered and sorted
according to the provided request.
:param params:
:return:
"""
start = params.get('start', 0)
length = params.get('length', 0)
search_value = params.get('search_value', '')
# Note that plugins can be ordered in multiple ways. So this must be a list
order = [params.get('order', {
"column": 'name',
"dir": 'desc',
})]
# Fetch Plugins
plugins = PluginsHandler()
plugin_executor = PluginExecutor()
# Get total count
records_total_count = plugins.get_total_plugin_list_count()
# Get quantity after filters (without pagination)
records_filtered_count = plugins.get_plugin_list_filtered_and_sorted(
order=order, start=0, length=0, search_value=search_value).count()
# Get filtered/sorted results
plugin_results = plugins.get_plugin_list_filtered_and_sorted(
order=order, start=start, length=length, search_value=search_value)
# Build return data
return_data = {
"recordsTotal": records_total_count,
"recordsFiltered": records_filtered_count,
"results": []
}
# Iterate over plugins and append them to the plugin data
for plugin_result in plugin_results:
# Set plugin status
plugin_status = {
"update_available": plugin_result.get('update_available'),
}
# Check if plugin is able to be configured
has_config = False
plugin_settings, plugin_settings_meta = plugin_executor.get_plugin_settings(
plugin_result.get('plugin_id'))
if plugin_settings:
has_config = True
# Set params as required in template
item = {
'id': plugin_result.get('id'),
'plugin_id': plugin_result.get('plugin_id'),
'icon': plugin_result.get('icon'),
'name': plugin_result.get('name'),
'description': plugin_result.get('description'),
'tags': plugin_result.get('tags'),
'author': plugin_result.get('author'),
'version': plugin_result.get('version'),
'status': plugin_status,
'has_config': has_config,
}
return_data["results"].append(item)
# Return results
return return_data