def on_task_start(self, task, config): if not config: return files = config if isinstance(config, basestring): files = [config] for name in files: name = os.path.expanduser(name) if not os.path.isabs(name): name = os.path.join(task.manager.config_base, name) include = yaml.load(io.open(name, encoding='utf-8')) errors = process_config(include, plugin.plugin_schemas(context='task')) if errors: log.error('Included file %s has invalid config:' % name) for error in errors: log.error('[%s] %s', error.json_pointer, error.message) task.abort('Invalid config in included file %s' % name) log.debug('Merging %s into task %s' % (name, task.name)) # merge try: merge_dict_from_to(include, task.config) except MergeException: raise plugin.PluginError('Failed to merge include file to task %s, incompatible datatypes' % task.name)
def on_task_start(self, task, config): if not config: return files = config if isinstance(config, basestring): files = [config] for name in files: name = os.path.expanduser(name) if not os.path.isabs(name): name = os.path.join(task.manager.config_base, name) include = yaml.load(io.open(name, encoding='utf-8')) errors = process_config(include, plugin.plugin_schemas(context='task')) if errors: log.error('Included file %s has invalid config:' % name) for error in errors: log.error('[%s] %s', error.json_pointer, error.message) task.abort('Invalid config in included file %s' % name) log.debug('Merging %s into task %s' % (name, task.name)) # merge try: merge_dict_from_to(include, task.config) except MergeException: raise plugin.PluginError( 'Failed to merge include file to task %s, incompatible datatypes' % task.name)
def on_process_start(self, feed, config): if config is False: # handles 'preset: no' form to turn off preset on this feed return config = self.prepare_config(config) # implements --preset NAME if feed.manager.options.preset: if feed.manager.options.preset not in config: feed.enabled = False return # add global in except when disabled with no_global if 'no_global' in config: config.remove('no_global') if 'global' in config: config.remove('global') elif not 'global' in config: config.append('global') log.trace('presets: %s' % config) toplevel_presets = feed.manager.config.get('presets', {}) # check for indentation error (plugin as a preset) if (feed.manager.options.test or feed.manager.options.validate) and not self.warned: plugins = get_plugin_keywords() for name in toplevel_presets.iterkeys(): if name in plugins: log.warning('Plugin \'%s\' seems to be in the wrong place? You probably wanted to put it in a preset. Please fix the indentation level!' % name) self.warned = True # apply presets for preset in config: if preset not in toplevel_presets: if preset == 'global': continue raise PluginError('Unable to find preset %s for feed %s' % (preset, feed.name), log) log.debug('Merging preset %s into feed %s' % (preset, feed.name)) # We make a copy here because we need to remove preset_config = toplevel_presets[preset] # When there are presets within presets we remove the preset # key from the config and append it's items to our own if 'preset' in preset_config: nested_presets = self.prepare_config(preset_config['preset']) for nested_preset in nested_presets: if nested_preset not in config: config.append(nested_preset) else: log.warning('Presets contain eachother in a loop.') # Replace preset_config with a copy without the preset key, to avoid merging errors preset_config = dict(preset_config) del preset_config['preset'] # merge from flexget.utils.tools import MergeException, merge_dict_from_to try: merge_dict_from_to(preset_config, feed.config) except MergeException, exc: raise PluginError('Failed to merge preset %s to feed %s due to %s' % (preset, feed.name, exc))
def on_process_start(self, task, config): if config is False: # handles 'preset: no' form to turn off preset on this task return config = self.prepare_config(config) # add global in except when disabled with no_global if 'no_global' in config: config.remove('no_global') if 'global' in config: config.remove('global') elif not 'global' in config: config.append('global') toplevel_presets = task.manager.config.get('presets', {}) # apply presets for preset in config: if preset not in toplevel_presets: if preset == 'global': continue raise PluginError( 'Unable to find preset %s for task %s' % (preset, task.name), log) if toplevel_presets[preset] is None: log.warning('Preset `%s` is empty. Nothing to merge.' % preset) continue log.debug('Merging preset %s into task %s' % (preset, task.name)) # We make a copy here because we need to remove preset_config = toplevel_presets[preset] # When there are presets within presets we remove the preset # key from the config and append it's items to our own if 'preset' in preset_config: nested_presets = self.prepare_config(preset_config['preset']) for nested_preset in nested_presets: if nested_preset not in config: config.append(nested_preset) else: log.warning('Presets contain each other in a loop.') # Replace preset_config with a copy without the preset key, to avoid merging errors preset_config = dict(preset_config) del preset_config['preset'] # Merge try: merge_dict_from_to(preset_config, task.config) except MergeException as exc: raise PluginError( 'Failed to merge preset %s to task %s. Error: %s' % (preset, task.name, exc.value)) log.trace('presets: %s' % config) # implements --preset NAME if task.manager.options.preset: if task.manager.options.preset not in config: task.enabled = False return
def test_merge_dict_to_dict_add(self): d1 = {'setting': {'parameter_1': ['item_1']}} d2 = {'setting': {'parameter_2': 'item_2'}} merge_dict_from_to(d1, d2) assert d2 == { 'setting': { 'parameter_1': ['item_1'], 'parameter_2': 'item_2' } }
def on_process_start(self, task, config): if config is False: # handles 'preset: no' form to turn off preset on this task return config = self.prepare_config(config) # add global in except when disabled with no_global if 'no_global' in config: config.remove('no_global') if 'global' in config: config.remove('global') elif not 'global' in config: config.append('global') toplevel_presets = task.manager.config.get('presets', {}) # apply presets for preset in config: if preset not in toplevel_presets: if preset == 'global': continue raise PluginError('Unable to find preset %s for task %s' % (preset, task.name), log) if toplevel_presets[preset] is None: log.warning('Preset `%s` is empty. Nothing to merge.' % preset) continue log.debug('Merging preset %s into task %s' % (preset, task.name)) # We make a copy here because we need to remove preset_config = toplevel_presets[preset] # When there are presets within presets we remove the preset # key from the config and append it's items to our own if 'preset' in preset_config: nested_presets = self.prepare_config(preset_config['preset']) for nested_preset in nested_presets: if nested_preset not in config: config.append(nested_preset) else: log.warning('Presets contain each other in a loop.') # Replace preset_config with a copy without the preset key, to avoid merging errors preset_config = dict(preset_config) del preset_config['preset'] # Merge try: merge_dict_from_to(preset_config, task.config) except MergeException as exc: raise PluginError('Failed to merge preset %s to task %s. Error: %s' % (preset, task.name, exc.value)) log.trace('presets: %s' % config) # implements --preset NAME if task.manager.options.preset: if task.manager.options.preset not in config: task.enabled = False return
def on_task_start(self, task, config): if config is False: # handles 'template: no' form to turn off template on this task return # implements --template NAME if task.options.template: if not config or task.options.template not in config: task.abort('does not use `%s` template' % task.options.template, silent=True) config = self.prepare_config(config) # add global in except when disabled with no_global if 'no_global' in config: config.remove('no_global') if 'global' in config: config.remove('global') elif not 'global' in config: config.append('global') toplevel_templates = task.manager.config.get('templates', {}) # apply templates for template in config: if template not in toplevel_templates: if template == 'global': continue raise plugin.PluginError('Unable to find template %s for task %s' % (template, task.name), log) if toplevel_templates[template] is None: log.warning('Template `%s` is empty. Nothing to merge.' % template) continue log.debug('Merging template %s into task %s' % (template, task.name)) # We make a copy here because we need to remove template_config = toplevel_templates[template] # When there are templates within templates we remove the template # key from the config and append it's items to our own if 'template' in template_config: nested_templates = self.prepare_config(template_config['template']) for nested_template in nested_templates: if nested_template not in config: config.append(nested_template) else: log.warning('Templates contain each other in a loop.') # Replace template_config with a copy without the template key, to avoid merging errors template_config = dict(template_config) del template_config['template'] # Merge try: merge_dict_from_to(template_config, task.config) except MergeException as exc: raise plugin.PluginError('Failed to merge template %s to task %s. Error: %s' % (template, task.name, exc.value)) log.trace('templates: %s' % config)
def on_task_start(self, task, config): if config is False: # handles 'template: no' form to turn off template on this task return # implements --template NAME if task.options.template: if not config or task.options.template not in config: task.abort('does not use `%s` template' % task.options.template, silent=True) config = self.prepare_config(config) # add global in except when disabled with no_global if 'no_global' in config: config.remove('no_global') if 'global' in config: config.remove('global') elif 'global' not in config: config.append('global') toplevel_templates = task.manager.config.get('templates', {}) # apply templates for template in config: if template not in toplevel_templates: if template == 'global': continue raise plugin.PluginError('Unable to find template %s for task %s' % (template, task.name), log) if toplevel_templates[template] is None: log.warning('Template `%s` is empty. Nothing to merge.' % template) continue log.debug('Merging template %s into task %s' % (template, task.name)) # We make a copy here because we need to remove template_config = toplevel_templates[template] # When there are templates within templates we remove the template # key from the config and append it's items to our own if 'template' in template_config: nested_templates = self.prepare_config(template_config['template']) for nested_template in nested_templates: if nested_template not in config: config.append(nested_template) else: log.warning('Templates contain each other in a loop.') # Replace template_config with a copy without the template key, to avoid merging errors template_config = dict(template_config) del template_config['template'] # Merge try: merge_dict_from_to(template_config, task.config) except MergeException as exc: raise plugin.PluginError('Failed to merge template %s to task %s. Error: %s' % (template, task.name, exc.value)) log.trace('templates: %s' % config)
def setup(manager): if not "email" in manager.config: return config = prepare_config(manager.config["email"]) config["global"] = True global task_content task_content = {} for task_name, task_config in manager.config["tasks"].iteritems(): task_config.setdefault("email", {}) try: merge_dict_from_to(config, task_config["email"]) except MergeException as exc: raise plugin.PluginError("Failed to merge email config to task %s due to %s" % (task_name, exc)) task_config.setdefault("email", config)
def setup(manager): if not 'email' in manager.config: return config = prepare_config(manager.config['email']) config['global'] = True global task_content task_content = {} for task in manager.tasks.itervalues(): task.config.setdefault('email', {}) try: merge_dict_from_to(config, task.config['email']) except MergeException as exc: raise PluginError('Failed to merge email config to task %s due to %s' % (task.name, exc)) task.config.setdefault('email', config)
def setup(manager, options): if not 'email' in manager.config: return config = prepare_config(manager.config['email']) config['global'] = True global task_content task_content = {} for task_name, task_config in manager.config['tasks'].iteritems(): task_config.setdefault('email', {}) try: merge_dict_from_to(config, task_config['email']) except MergeException as exc: raise plugin.PluginError('Failed to merge email config to task %s due to %s' % (task_name, exc)) task_config.setdefault('email', config)
def setup(manager, options): if 'email' not in manager.config: return config = prepare_config(manager.config['email']) config['global'] = True global task_content task_content = {} for task_name, task_config in manager.config['tasks'].iteritems(): task_config.setdefault('email', {}) try: merge_dict_from_to(config, task_config['email']) except MergeException as exc: raise plugin.PluginError('Failed to merge email config to task %s due to %s' % (task_name, exc)) task_config.setdefault('email', config)
def on_task_start(self, task, config): if not config: return files = config if isinstance(config, str): files = [config] for file_name in files: file_name = os.path.expanduser(file_name) if not os.path.isabs(file_name): file_name = os.path.join(task.manager.config_base, file_name) with io.open(file_name, encoding='utf-8') as inc_file: include = yaml.load(inc_file) inc_file.flush() errors = process_config(include, plugin.plugin_schemas(interface='task')) if errors: log.error('Included file %s has invalid config:', file_name) for error in errors: log.error('[%s] %s', error.json_pointer, error.message) task.abort('Invalid config in included file %s' % file_name) new_hash = str(get_config_hash(include)) with Session() as session: last_hash = session.query(LastHash).filter( LastHash.task == task.name).filter( LastHash.file == file_name).first() if not last_hash: log.debug( 'no config hash detected for task %s with file %s, creating', task.name, file_name) last_hash = LastHash(task=task.name, file=file_name) session.add(last_hash) if last_hash.hash != new_hash: log.debug( 'new hash detected, triggering config change event') task.config_changed() last_hash.hash = new_hash log.debug('Merging %s into task %s', file_name, task.name) # merge try: merge_dict_from_to(include, task.config) except MergeException: raise plugin.PluginError( 'Failed to merge include file to task %s, incompatible datatypes' % task.name)
def setup(manager): if not 'twitter' in manager.config: return try: import tweepy except ImportError: raise PluginError('The Twtter plugin requires the tweepy module to be installed, please install it before using.') global task_content task_content = {} print config for task in manager.tasks.itervalues(): task.config.setdefault('twitter', {}) try: merge_dict_from_to(config, task.config['twitter']) except MergeException, exc: raise PluginError('Failed to merge twitter config to task %s due to %s' % (task.name, exc)) task.config.setdefault('twitter', config)
def apply_group_options(self, config): """Applies group settings to each item in series group and removes settings dict.""" # Make sure config is in grouped format first config = self.make_grouped_config(config) for group_name in config: if group_name == 'settings': continue group_series = [] # if group name is known quality, convenience create settings with that quality if isinstance(group_name, basestring) and group_name.lower() in qualities.registry: config['settings'].setdefault(group_name, {}).setdefault('quality', group_name) for series in config[group_name]: # convert into dict-form if necessary series_settings = {} group_settings = config['settings'].get(group_name, {}) if isinstance(series, dict): series, series_settings = series.items()[0] if series_settings is None: raise Exception('Series %s has unexpected \':\'' % series) # make sure series name is a string to accommodate for "24" if not isinstance(series, basestring): series = str(series) # if series have given path instead of dict, convert it into a dict if isinstance(series_settings, basestring): series_settings = {'path': series_settings} # merge group settings into this series settings merge_dict_from_to(group_settings, series_settings) # Convert to dict if watched is in SXXEXX format if isinstance(series_settings.get('watched'), basestring): season, episode = series_settings['watched'].upper().split('E') season = season.lstrip('S') series_settings['watched'] = {'season': int(season), 'episode': int(episode)} # Add quality: 720p if timeframe is specified with no quality if 'timeframe' in series_settings: series_settings.setdefault('quality', '720p') group_series.append({series: series_settings}) config[group_name] = group_series del config['settings'] return config
def on_process_start(self, task): if not 'include' in task.config: return import yaml import os files = self.get_config(task) for name in files: name = os.path.expanduser(name) if not os.path.isabs(name): name = os.path.join(task.manager.config_base, name) include = yaml.load(file(name)) log.debug('Merging %s into task %s' % (name, task.name)) # merge from flexget.utils.tools import MergeException, merge_dict_from_to try: merge_dict_from_to(include, task.config) except MergeException: raise PluginError('Failed to merge include file to task %s, incompatible datatypes' % (task.name))
def on_process_start(self, task, config): if not config: return files = config if isinstance(config, basestring): files = [config] for name in files: name = os.path.expanduser(name) if not os.path.isabs(name): name = os.path.join(task.manager.config_base, name) include = yaml.load(file(name)) if not isinstance(include, dict): raise PluginError('Include file format is invalid: %s' % name) log.debug('Merging %s into task %s' % (name, task.name)) # merge from flexget.utils.tools import MergeException, merge_dict_from_to try: merge_dict_from_to(include, task.config) except MergeException: raise PluginError('Failed to merge include file to task %s, incompatible datatypes' % (task.name))
def on_process_start(self, task): if not 'include' in task.config: return import yaml import os files = self.get_config(task) for name in files: name = os.path.expanduser(name) if not os.path.isabs(name): name = os.path.join(task.manager.config_base, name) include = yaml.load(file(name)) log.debug('Merging %s into task %s' % (name, task.name)) # merge from flexget.utils.tools import MergeException, merge_dict_from_to try: merge_dict_from_to(include, task.config) except MergeException: raise PluginError( 'Failed to merge include file to task %s, incompatible datatypes' % (task.name))
def merge_config(self, new_config): try: merge_dict_from_to(new_config, self.config) except MergeException as e: raise PluginError('Failed to merge configs for task %s: %s' % (self.name, e))
def test_merge_dict_to_dict_str(self): d1 = {'setting': {'parameter': 'item_1'}} d2 = {'setting': {'parameter': 'item_2'}} merge_dict_from_to(d1, d2) assert d2 == {'setting': {'parameter': 'item_2'}}
def test_merge_str_to_list(self): d1 = {'setting': 'string'} d2 = {'setting': ['list']} merge_dict_from_to(d1, d2) assert d2 == {'setting': ['list']}
def test_merge_str_to_str(self): d1 = {'setting': 'string_1'} d2 = {'setting': 'string_2'} merge_dict_from_to(d1, d2) assert d2 == {'setting': 'string_2'}
def test_merge_dict_to_dict_list(self): d1 = {'setting': {'parameter': ['item_1']}} d2 = {'setting': {'parameter': ['item_2']}} merge_dict_from_to(d1, d2) assert d2 == {'setting': {'parameter': ['item_2', 'item_1']}}
def test_merge_dict_to_dict_override(self): d1 = {'setting': {'parameter': ['item_1']}} d2 = {'setting': {'parameter': 2}} merge_dict_from_to(d1, d2) assert d2 == {'setting': {'parameter': 2}}