def on_task_start(self, task, config):
if not config:
return
files = config
if isinstance(config, basestring):
files = [config]
for name in files:
name = os.path.expanduser(name)
if not os.path.isabs(name):
name = os.path.join(task.manager.config_base, name)
include = yaml.load(io.open(name, encoding='utf-8'))
errors = process_config(include, plugin.plugin_schemas(context='task'))
if errors:
log.error('Included file %s has invalid config:' % name)
for error in errors:
log.error('[%s] %s', error.json_pointer, error.message)
task.abort('Invalid config in included file %s' % name)
log.debug('Merging %s into task %s' % (name, task.name))
# merge
try:
merge_dict_from_to(include, task.config)
except MergeException:
raise plugin.PluginError('Failed to merge include file to task %s, incompatible datatypes' % task.name)
def on_task_start(self, task, config):
if not config:
return
files = config
if isinstance(config, basestring):
files = [config]
for name in files:
name = os.path.expanduser(name)
if not os.path.isabs(name):
name = os.path.join(task.manager.config_base, name)
include = yaml.load(io.open(name, encoding='utf-8'))
errors = process_config(include,
plugin.plugin_schemas(context='task'))
if errors:
log.error('Included file %s has invalid config:' % name)
for error in errors:
log.error('[%s] %s', error.json_pointer, error.message)
task.abort('Invalid config in included file %s' % name)
log.debug('Merging %s into task %s' % (name, task.name))
# merge
try:
merge_dict_from_to(include, task.config)
except MergeException:
raise plugin.PluginError(
'Failed to merge include file to task %s, incompatible datatypes'
% task.name)
def on_process_start(self, feed, config):
if config is False: # handles 'preset: no' form to turn off preset on this feed
return
config = self.prepare_config(config)
# implements --preset NAME
if feed.manager.options.preset:
if feed.manager.options.preset not in config:
feed.enabled = False
return
# add global in except when disabled with no_global
if 'no_global' in config:
config.remove('no_global')
if 'global' in config:
config.remove('global')
elif not 'global' in config:
config.append('global')
log.trace('presets: %s' % config)
toplevel_presets = feed.manager.config.get('presets', {})
# check for indentation error (plugin as a preset)
if (feed.manager.options.test or feed.manager.options.validate) and not self.warned:
plugins = get_plugin_keywords()
for name in toplevel_presets.iterkeys():
if name in plugins:
log.warning('Plugin \'%s\' seems to be in the wrong place? You probably wanted to put it in a preset. Please fix the indentation level!' % name)
self.warned = True
# apply presets
for preset in config:
if preset not in toplevel_presets:
if preset == 'global':
continue
raise PluginError('Unable to find preset %s for feed %s' % (preset, feed.name), log)
log.debug('Merging preset %s into feed %s' % (preset, feed.name))
# We make a copy here because we need to remove
preset_config = toplevel_presets[preset]
# When there are presets within presets we remove the preset
# key from the config and append it's items to our own
if 'preset' in preset_config:
nested_presets = self.prepare_config(preset_config['preset'])
for nested_preset in nested_presets:
if nested_preset not in config:
config.append(nested_preset)
else:
log.warning('Presets contain eachother in a loop.')
# Replace preset_config with a copy without the preset key, to avoid merging errors
preset_config = dict(preset_config)
del preset_config['preset']
# merge
from flexget.utils.tools import MergeException, merge_dict_from_to
try:
merge_dict_from_to(preset_config, feed.config)
except MergeException, exc:
raise PluginError('Failed to merge preset %s to feed %s due to %s' % (preset, feed.name, exc))
def on_process_start(self, task, config):
if config is False: # handles 'preset: no' form to turn off preset on this task
return
config = self.prepare_config(config)
# add global in except when disabled with no_global
if 'no_global' in config:
config.remove('no_global')
if 'global' in config:
config.remove('global')
elif not 'global' in config:
config.append('global')
toplevel_presets = task.manager.config.get('presets', {})
# apply presets
for preset in config:
if preset not in toplevel_presets:
if preset == 'global':
continue
raise PluginError(
'Unable to find preset %s for task %s' %
(preset, task.name), log)
if toplevel_presets[preset] is None:
log.warning('Preset `%s` is empty. Nothing to merge.' % preset)
continue
log.debug('Merging preset %s into task %s' % (preset, task.name))
# We make a copy here because we need to remove
preset_config = toplevel_presets[preset]
# When there are presets within presets we remove the preset
# key from the config and append it's items to our own
if 'preset' in preset_config:
nested_presets = self.prepare_config(preset_config['preset'])
for nested_preset in nested_presets:
if nested_preset not in config:
config.append(nested_preset)
else:
log.warning('Presets contain each other in a loop.')
# Replace preset_config with a copy without the preset key, to avoid merging errors
preset_config = dict(preset_config)
del preset_config['preset']
# Merge
try:
merge_dict_from_to(preset_config, task.config)
except MergeException as exc:
raise PluginError(
'Failed to merge preset %s to task %s. Error: %s' %
(preset, task.name, exc.value))
log.trace('presets: %s' % config)
# implements --preset NAME
if task.manager.options.preset:
if task.manager.options.preset not in config:
task.enabled = False
return
def test_merge_dict_to_dict_add(self):
d1 = {'setting': {'parameter_1': ['item_1']}}
d2 = {'setting': {'parameter_2': 'item_2'}}
merge_dict_from_to(d1, d2)
assert d2 == {
'setting': {
'parameter_1': ['item_1'],
'parameter_2': 'item_2'
}
}
def on_process_start(self, task, config):
if config is False: # handles 'preset: no' form to turn off preset on this task
return
config = self.prepare_config(config)
# add global in except when disabled with no_global
if 'no_global' in config:
config.remove('no_global')
if 'global' in config:
config.remove('global')
elif not 'global' in config:
config.append('global')
toplevel_presets = task.manager.config.get('presets', {})
# apply presets
for preset in config:
if preset not in toplevel_presets:
if preset == 'global':
continue
raise PluginError('Unable to find preset %s for task %s' % (preset, task.name), log)
if toplevel_presets[preset] is None:
log.warning('Preset `%s` is empty. Nothing to merge.' % preset)
continue
log.debug('Merging preset %s into task %s' % (preset, task.name))
# We make a copy here because we need to remove
preset_config = toplevel_presets[preset]
# When there are presets within presets we remove the preset
# key from the config and append it's items to our own
if 'preset' in preset_config:
nested_presets = self.prepare_config(preset_config['preset'])
for nested_preset in nested_presets:
if nested_preset not in config:
config.append(nested_preset)
else:
log.warning('Presets contain each other in a loop.')
# Replace preset_config with a copy without the preset key, to avoid merging errors
preset_config = dict(preset_config)
del preset_config['preset']
# Merge
try:
merge_dict_from_to(preset_config, task.config)
except MergeException as exc:
raise PluginError('Failed to merge preset %s to task %s. Error: %s' %
(preset, task.name, exc.value))
log.trace('presets: %s' % config)
# implements --preset NAME
if task.manager.options.preset:
if task.manager.options.preset not in config:
task.enabled = False
return
def on_task_start(self, task, config):
if config is False: # handles 'template: no' form to turn off template on this task
return
# implements --template NAME
if task.options.template:
if not config or task.options.template not in config:
task.abort('does not use `%s` template' % task.options.template, silent=True)
config = self.prepare_config(config)
# add global in except when disabled with no_global
if 'no_global' in config:
config.remove('no_global')
if 'global' in config:
config.remove('global')
elif not 'global' in config:
config.append('global')
toplevel_templates = task.manager.config.get('templates', {})
# apply templates
for template in config:
if template not in toplevel_templates:
if template == 'global':
continue
raise plugin.PluginError('Unable to find template %s for task %s' % (template, task.name), log)
if toplevel_templates[template] is None:
log.warning('Template `%s` is empty. Nothing to merge.' % template)
continue
log.debug('Merging template %s into task %s' % (template, task.name))
# We make a copy here because we need to remove
template_config = toplevel_templates[template]
# When there are templates within templates we remove the template
# key from the config and append it's items to our own
if 'template' in template_config:
nested_templates = self.prepare_config(template_config['template'])
for nested_template in nested_templates:
if nested_template not in config:
config.append(nested_template)
else:
log.warning('Templates contain each other in a loop.')
# Replace template_config with a copy without the template key, to avoid merging errors
template_config = dict(template_config)
del template_config['template']
# Merge
try:
merge_dict_from_to(template_config, task.config)
except MergeException as exc:
raise plugin.PluginError('Failed to merge template %s to task %s. Error: %s' %
(template, task.name, exc.value))
log.trace('templates: %s' % config)
def on_task_start(self, task, config):
if config is False: # handles 'template: no' form to turn off template on this task
return
# implements --template NAME
if task.options.template:
if not config or task.options.template not in config:
task.abort('does not use `%s` template' % task.options.template, silent=True)
config = self.prepare_config(config)
# add global in except when disabled with no_global
if 'no_global' in config:
config.remove('no_global')
if 'global' in config:
config.remove('global')
elif 'global' not in config:
config.append('global')
toplevel_templates = task.manager.config.get('templates', {})
# apply templates
for template in config:
if template not in toplevel_templates:
if template == 'global':
continue
raise plugin.PluginError('Unable to find template %s for task %s' % (template, task.name), log)
if toplevel_templates[template] is None:
log.warning('Template `%s` is empty. Nothing to merge.' % template)
continue
log.debug('Merging template %s into task %s' % (template, task.name))
# We make a copy here because we need to remove
template_config = toplevel_templates[template]
# When there are templates within templates we remove the template
# key from the config and append it's items to our own
if 'template' in template_config:
nested_templates = self.prepare_config(template_config['template'])
for nested_template in nested_templates:
if nested_template not in config:
config.append(nested_template)
else:
log.warning('Templates contain each other in a loop.')
# Replace template_config with a copy without the template key, to avoid merging errors
template_config = dict(template_config)
del template_config['template']
# Merge
try:
merge_dict_from_to(template_config, task.config)
except MergeException as exc:
raise plugin.PluginError('Failed to merge template %s to task %s. Error: %s' %
(template, task.name, exc.value))
log.trace('templates: %s' % config)
def setup(manager):
if not "email" in manager.config:
return
config = prepare_config(manager.config["email"])
config["global"] = True
global task_content
task_content = {}
for task_name, task_config in manager.config["tasks"].iteritems():
task_config.setdefault("email", {})
try:
merge_dict_from_to(config, task_config["email"])
except MergeException as exc:
raise plugin.PluginError("Failed to merge email config to task %s due to %s" % (task_name, exc))
task_config.setdefault("email", config)
def setup(manager):
if not 'email' in manager.config:
return
config = prepare_config(manager.config['email'])
config['global'] = True
global task_content
task_content = {}
for task in manager.tasks.itervalues():
task.config.setdefault('email', {})
try:
merge_dict_from_to(config, task.config['email'])
except MergeException as exc:
raise PluginError('Failed to merge email config to task %s due to %s' % (task.name, exc))
task.config.setdefault('email', config)
def setup(manager, options):
if not 'email' in manager.config:
return
config = prepare_config(manager.config['email'])
config['global'] = True
global task_content
task_content = {}
for task_name, task_config in manager.config['tasks'].iteritems():
task_config.setdefault('email', {})
try:
merge_dict_from_to(config, task_config['email'])
except MergeException as exc:
raise plugin.PluginError('Failed to merge email config to task %s due to %s' % (task_name, exc))
task_config.setdefault('email', config)
def setup(manager, options):
if 'email' not in manager.config:
return
config = prepare_config(manager.config['email'])
config['global'] = True
global task_content
task_content = {}
for task_name, task_config in manager.config['tasks'].iteritems():
task_config.setdefault('email', {})
try:
merge_dict_from_to(config, task_config['email'])
except MergeException as exc:
raise plugin.PluginError('Failed to merge email config to task %s due to %s' % (task_name, exc))
task_config.setdefault('email', config)
def on_task_start(self, task, config):
if not config:
return
files = config
if isinstance(config, str):
files = [config]
for file_name in files:
file_name = os.path.expanduser(file_name)
if not os.path.isabs(file_name):
file_name = os.path.join(task.manager.config_base, file_name)
with io.open(file_name, encoding='utf-8') as inc_file:
include = yaml.load(inc_file)
inc_file.flush()
errors = process_config(include,
plugin.plugin_schemas(interface='task'))
if errors:
log.error('Included file %s has invalid config:', file_name)
for error in errors:
log.error('[%s] %s', error.json_pointer, error.message)
task.abort('Invalid config in included file %s' % file_name)
new_hash = str(get_config_hash(include))
with Session() as session:
last_hash = session.query(LastHash).filter(
LastHash.task == task.name).filter(
LastHash.file == file_name).first()
if not last_hash:
log.debug(
'no config hash detected for task %s with file %s, creating',
task.name, file_name)
last_hash = LastHash(task=task.name, file=file_name)
session.add(last_hash)
if last_hash.hash != new_hash:
log.debug(
'new hash detected, triggering config change event')
task.config_changed()
last_hash.hash = new_hash
log.debug('Merging %s into task %s', file_name, task.name)
# merge
try:
merge_dict_from_to(include, task.config)
except MergeException:
raise plugin.PluginError(
'Failed to merge include file to task %s, incompatible datatypes'
% task.name)
def setup(manager):
if not 'twitter' in manager.config:
return
try:
import tweepy
except ImportError:
raise PluginError('The Twtter plugin requires the tweepy module to be installed, please install it before using.')
global task_content
task_content = {}
print config
for task in manager.tasks.itervalues():
task.config.setdefault('twitter', {})
try:
merge_dict_from_to(config, task.config['twitter'])
except MergeException, exc:
raise PluginError('Failed to merge twitter config to task %s due to %s' % (task.name, exc))
task.config.setdefault('twitter', config)
def apply_group_options(self, config):
"""Applies group settings to each item in series group and removes settings dict."""
# Make sure config is in grouped format first
config = self.make_grouped_config(config)
for group_name in config:
if group_name == 'settings':
continue
group_series = []
# if group name is known quality, convenience create settings with that quality
if isinstance(group_name, basestring) and group_name.lower() in qualities.registry:
config['settings'].setdefault(group_name, {}).setdefault('quality', group_name)
for series in config[group_name]:
# convert into dict-form if necessary
series_settings = {}
group_settings = config['settings'].get(group_name, {})
if isinstance(series, dict):
series, series_settings = series.items()[0]
if series_settings is None:
raise Exception('Series %s has unexpected \':\'' % series)
# make sure series name is a string to accommodate for "24"
if not isinstance(series, basestring):
series = str(series)
# if series have given path instead of dict, convert it into a dict
if isinstance(series_settings, basestring):
series_settings = {'path': series_settings}
# merge group settings into this series settings
merge_dict_from_to(group_settings, series_settings)
# Convert to dict if watched is in SXXEXX format
if isinstance(series_settings.get('watched'), basestring):
season, episode = series_settings['watched'].upper().split('E')
season = season.lstrip('S')
series_settings['watched'] = {'season': int(season), 'episode': int(episode)}
# Add quality: 720p if timeframe is specified with no quality
if 'timeframe' in series_settings:
series_settings.setdefault('quality', '720p')
group_series.append({series: series_settings})
config[group_name] = group_series
del config['settings']
return config
def on_process_start(self, task):
if not 'include' in task.config:
return
import yaml
import os
files = self.get_config(task)
for name in files:
name = os.path.expanduser(name)
if not os.path.isabs(name):
name = os.path.join(task.manager.config_base, name)
include = yaml.load(file(name))
log.debug('Merging %s into task %s' % (name, task.name))
# merge
from flexget.utils.tools import MergeException, merge_dict_from_to
try:
merge_dict_from_to(include, task.config)
except MergeException:
raise PluginError('Failed to merge include file to task %s, incompatible datatypes' % (task.name))
def on_process_start(self, task, config):
if not config:
return
files = config
if isinstance(config, basestring):
files = [config]
for name in files:
name = os.path.expanduser(name)
if not os.path.isabs(name):
name = os.path.join(task.manager.config_base, name)
include = yaml.load(file(name))
if not isinstance(include, dict):
raise PluginError('Include file format is invalid: %s' % name)
log.debug('Merging %s into task %s' % (name, task.name))
# merge
from flexget.utils.tools import MergeException, merge_dict_from_to
try:
merge_dict_from_to(include, task.config)
except MergeException:
raise PluginError('Failed to merge include file to task %s, incompatible datatypes' % (task.name))
def on_process_start(self, task):
if not 'include' in task.config:
return
import yaml
import os
files = self.get_config(task)
for name in files:
name = os.path.expanduser(name)
if not os.path.isabs(name):
name = os.path.join(task.manager.config_base, name)
include = yaml.load(file(name))
log.debug('Merging %s into task %s' % (name, task.name))
# merge
from flexget.utils.tools import MergeException, merge_dict_from_to
try:
merge_dict_from_to(include, task.config)
except MergeException:
raise PluginError(
'Failed to merge include file to task %s, incompatible datatypes'
% (task.name))
def merge_config(self, new_config):
try:
merge_dict_from_to(new_config, self.config)
except MergeException as e:
raise PluginError('Failed to merge configs for task %s: %s' % (self.name, e))
def test_merge_dict_to_dict_str(self):
d1 = {'setting': {'parameter': 'item_1'}}
d2 = {'setting': {'parameter': 'item_2'}}
merge_dict_from_to(d1, d2)
assert d2 == {'setting': {'parameter': 'item_2'}}
def test_merge_str_to_list(self):
d1 = {'setting': 'string'}
d2 = {'setting': ['list']}
merge_dict_from_to(d1, d2)
assert d2 == {'setting': ['list']}
def test_merge_str_to_str(self):
d1 = {'setting': 'string_1'}
d2 = {'setting': 'string_2'}
merge_dict_from_to(d1, d2)
assert d2 == {'setting': 'string_2'}
def merge_config(self, new_config):
try:
merge_dict_from_to(new_config, self.config)
except MergeException as e:
raise PluginError('Failed to merge configs for task %s: %s' %
(self.name, e))
def test_merge_dict_to_dict_list(self):
d1 = {'setting': {'parameter': ['item_1']}}
d2 = {'setting': {'parameter': ['item_2']}}
merge_dict_from_to(d1, d2)
assert d2 == {'setting': {'parameter': ['item_2', 'item_1']}}
def test_merge_dict_to_dict_override(self):
d1 = {'setting': {'parameter': ['item_1']}}
d2 = {'setting': {'parameter': 2}}
merge_dict_from_to(d1, d2)
assert d2 == {'setting': {'parameter': 2}}
