def check_strategy(self, strategy): ''' Return what is wrong with a strategy as a dictionary. returns: 'missing_methods': list of (stage_name,) tuples -- Stage has no specified method at all. 'non_existing_methods': list of (stage_name, method_name) tuples -- Stage has method that isn't a plugin on this system. 'non_existing_settings': list of (stage_name, method_name, settings_name) tuples -- Stage has setting that isn't one of the settings that is supposed to be specified for this method. 'missing_settings': list of (stage_name, methods_name, settings_name) tuples -- Stage does not have a setting that is supposed to be specified for this method. 'invalid_settings': list of (stage_name, methods_name, settings_name, settings_value) tuples -- Stage has setting that is not valid (i.e. out of range or wrong type) for this method. ''' result = defaultdict(list) for stage_name in stages.stages: try: method_name = strategy.methods_used[stage_name] except KeyError: result['missing_methods'].append((stage_name,)) continue try: plugin = plugin_manager.find_plugin(stage_name, method_name) except MissingPluginError: result['non_existing_methods'].append((stage_name, method_name)) continue default_settings = plugin.get_parameter_defaults() try: settings = strategy.settings[stage_name] except KeyError: # all settings are missing. for setting_name in default_settings.keys(): result['missing_settings'].append( (stage_name, method_name, setting_name)) continue result = check_plugin_settings(stage_name, method_name, settings) stage_name = 'auxiliary' for method_name, settings in strategy.auxiliary_stages.items(): try: plugin = plugin_manager.find_plugin(stage_name, method_name) except MissingPluginError: result['non_existing_methods'].append((stage_name, method_name)) continue result = check_plugin_settings(stage_name, method_name, settings) return result
def build_tasks_from_strategy(self, strategy, stage_name=None): '''Create a task for each stage of the strategy.''' tasks = [] marked_trials = self.trial_manager.marked_trials if stage_name == 'auxiliary': for plugin_name, plugin_kwargs in strategy.auxiliary_stages.items(): plugin = plugin_manager.find_plugin('auxiliary', plugin_name) if plugin.runs_with_stage == 'auxiliary': tasks.extend(build_tasks(marked_trials, plugin, stage_name, plugin_kwargs)) if stage_name is not None: if stage_name != 'auxiliary': plugin_name = strategy.methods_used[stage_name] plugin = plugin_manager.find_plugin(stage_name, plugin_name) plugin_kwargs = strategy.settings[stage_name] tasks.extend(build_tasks(marked_trials, plugin, stage_name, plugin_kwargs)) # get auxiliary stages that should run with this stage. for plugin_name, plugin_kwargs in \ strategy.auxiliary_stages.items(): plugin = plugin_manager.find_plugin('auxiliary', plugin_name) if plugin.runs_with_stage == stage_name: tasks.extend(build_tasks(marked_trials, plugin, 'auxiliary', plugin_kwargs)) else: # do for all stages for stage_name, plugin_name in strategy.methods_used.items(): plugin = plugin_manager.find_plugin(stage_name, plugin_name) plugin_kwargs = strategy.settings[stage_name] tasks.extend(build_tasks(marked_trials, plugin, stage_name, plugin_kwargs)) for plugin_name, plugin_kwargs in strategy.auxiliary_stages.items(): plugin = plugin_manager.find_plugin('auxiliary', plugin_name) tasks.extend(build_tasks(marked_trials, plugin, 'auxiliary', plugin_kwargs)) return tasks
def fix_strategy(strategy, problems_dict): ''' Return a fixed strategy given a broken strategy and the output of self.check_strategy. ''' strategy = strategy.copy() for stage_name, method_name in problems_dict['non_existing_methods']: del strategy.methods_used[stage_name] problems_dict['missing_methods'].append((stage_name,)) for value in problems_dict['missing_methods']: stage_name = value[-1] plugin = plugin_manager.get_default_plugin(stage_name) strategy.methods_used[stage_name] = plugin.name strategy.settings[stage_name] = plugin.get_parameter_defaults() for stage_name, method_name, setting_name in problems_dict[ 'non_existing_settings']: if stage_name != 'auxiliary': del strategy.settings[stage_name][setting_name] else: del strategy.auxiliary_stages[method_name][setting_name] for stage_name, method_name, setting_name in problems_dict[ 'invalid_settings']: if stage_name != 'auxiliary': del strategy.settings[stage_name][setting_name] else: del strategy.auxiliary_stages[method_name][setting_name] problems_dict['missing_settings'].append( (stage_name, method_name, setting_name)) for stage_name, method_name, setting_name in problems_dict[ 'missing_settings']: plugin = plugin_manager.find_plugin(stage_name, method_name) default_settings = plugin.get_parameter_defaults() if stage_name != 'auxiliary': strategy.settings[stage_name][setting_name] =\ default_settings[setting_name] else: strategy.auxiliary_stages[method_name][setting_name] =\ default_settings[setting_name] #TODO change name? return strategy
def task_worker(input_queue, results_queue): '''Worker process to handle task operations.''' for task_info in iter(input_queue.get, None): args = task_info['args'] kwargs = task_info['kwargs'] stage_name = task_info['plugin_info']['stage'] plugin_name = task_info['plugin_info']['name'] plugin = plugin_manager.find_plugin(stage_name, plugin_name) results_dict = {} begin_time = time.time() try: results_dict['result'] = plugin.run(*args, **kwargs) except: results_dict['result'] = None results_dict['traceback'] = traceback.format_exc() traceback.print_exc() end_time = time.time() results_dict['task_id'] = task_info['task_id'] results_dict['runtime'] = end_time - begin_time results_queue.put(results_dict)
def check_plugin_settings(stage_name, method_name, settings, result=None): if result is None: result = defaultdict(list) plugin = plugin_manager.find_plugin(stage_name, method_name) default_settings = plugin.get_parameter_defaults() for needed_setting in default_settings.keys(): if needed_setting not in settings.keys(): result['missing_settings'].append( (stage_name, method_name, needed_setting)) for existing_setting, value in settings.items(): if existing_setting not in default_settings.keys(): result['non_existing_settings'].append( (stage_name, method_name, existing_setting)) else: try: getattr(plugin, existing_setting)(value) except: result['invalid_settings'].append( (stage_name, method_name, existing_setting)) return result