def load(): global PLUGINS_LOADED if not PLUGINS_LOADED: from rally.common import opts opts.register() # NOTE(andreykurilin): `rally.plugins.common` includes deprecated # modules. As soon as they will be removed the direct import of # validators should be replaced by # # discover.import_modules_from_package("rally.plugins.common") from rally.plugins.common import validators # noqa: F401 discover.import_modules_from_package("rally.plugins.task") discover.import_modules_from_package("rally.plugins.verification") packages = discover.find_packages_by_entry_point() for package in packages: if "options" in package: opts.register_options_from_path(package["options"]) discover.import_modules_by_entry_point(_packages=packages) discover.load_plugins("/opt/rally/plugins/") discover.load_plugins(os.path.expanduser("~/.rally/plugins/")) PLUGINS_LOADED = True
def load(): global PLUGINS_LOADED if not PLUGINS_LOADED: from rally.common import opts opts.register() discover.import_modules_from_package("rally.plugins.common") try: import rally_openstack # noqa except ImportError: discover.LOG.warning( "OpenStack plugins moved to the separate package " "(see https://pypi.org/project/rally-openstack). In-tree " "OpenStack plugins will be removed from the Rally main package" " soon.") discover.import_modules_from_package("rally.plugins.openstack") discover.import_modules_from_package("rally.plugins.workload") packages = discover.find_packages_by_entry_point() for package in packages: if "options" in package: opts.register_options_from_path(package["options"]) discover.import_modules_by_entry_point(_packages=packages) discover.load_plugins("/opt/rally/plugins/") discover.load_plugins(os.path.expanduser("~/.rally/plugins/")) PLUGINS_LOADED = True
def test_schema_is_valid( self, mock_benchmark_engine__validate_config_semantic): discover.load_plugins(os.path.join(self.rally_jobs_path, "plugins")) for filename in ["rally.yaml", "rally-neutron.yaml", "rally-zaqar.yaml", "rally-designate.yaml"]: full_path = os.path.join(self.rally_jobs_path, filename) with open(full_path) as task_file: try: args_file = os.path.join( self.rally_jobs_path, filename.rsplit(".", 1)[0] + "_args.yaml") args = {} if os.path.exists(args_file): args = yaml.safe_load(open(args_file).read()) if not isinstance(args, dict): raise TypeError( "args file %s must be dict in yaml or json " "presenatation" % args_file) task = api.Task.render_template(task_file.read(), **args) task = yaml.safe_load(task) eng = engine.BenchmarkEngine(task, mock.MagicMock()) eng.validate() except Exception: print(traceback.format_exc()) self.fail("Wrong task input file: %s" % full_path)
def load(): global PLUGINS_LOADED if not PLUGINS_LOADED: from rally.common import opts opts.register() discover.import_modules_from_package("rally.deployment.engines") discover.import_modules_from_package("rally.deployment.serverprovider") discover.import_modules_from_package("rally.plugins.common") try: import rally_openstack # noqa except ImportError: discover.LOG.warning( "OpenStack plugins moved to the separate package " "(see https://pypi.python.org/pypi/rally-openstack). In-tree " "OpenStack plugins will be removed from the Rally main package" " soon.") discover.import_modules_from_package("rally.plugins.openstack") discover.import_modules_from_package("rally.plugins.workload") discover.import_modules_by_entry_point() discover.load_plugins("/opt/rally/plugins/") discover.load_plugins(os.path.expanduser("~/.rally/plugins/")) PLUGINS_LOADED = True
def test_load_plugins_fails(self, mock_isfile, mock_spec_from_file_location, mock_module_from_spec): mock_spec_from_file_location.side_effect = Exception() # test no fails if module is broken # TODO(olkonami): check exception is handled correct discover.load_plugins("/somewhere/foo.py")
def test_load_plugins_from_dir_successful(self, mock_os_walk, mock_spec_from_file_location, mock_module_from_spec, mock_isdir, mock_isfile): mock_os_walk.return_value = [ ("/somewhere", ("/subdir", ), ("plugin1.py", )), ("/somewhere/subdir", ("/subsubdir", ), ("plugin2.py", "withoutextension")), ("/somewhere/subdir/subsubdir", [], ("plugin3.py", )) ] def fake_isdir(p): return not (p.endswith(".py") or p.endswith("withoutextension")) mock_isdir.side_effect = fake_isdir test_path = "/somewhere" discover.load_plugins(test_path) expected = [ mock.call(p.rsplit("/", 1)[1][:-3], p) for p in ("/somewhere/plugin1.py", "/somewhere/subdir/plugin2.py", "/somewhere/subdir/subsubdir/plugin3.py") ] self.assertEqual(expected, mock_spec_from_file_location.call_args_list) self.assertEqual(3, len(mock_module_from_spec.mock_calls))
def test_load_plugins_from_nonexisting_and_empty_dir(self, mock_os): # test no fails for nonexisting directory mock_os.path.isdir.return_value = False discover.load_plugins("/somewhere") # test no fails for empty directory mock_os.path.isdir.return_value = True mock_os.walk.return_value = [] discover.load_plugins("/somewhere")
def test_load_plugins_from_file_successful(self, mock_spec_from_file_location, mock_module_from_spec, mock_isfile): path = "/somewhere/plugin.py" discover.load_plugins(path) mock_spec_from_file_location.assert_called_once_with("plugin", path) mock_module_from_spec.assert_called_once_with( mock_spec_from_file_location.return_value)
def test_load_plugins_from_dir_successful(self, mock_os_walk, mock_find_module, mock_load_module, mock_isdir): test_path = "/somewhere" discover.load_plugins(test_path) expected = [ mock.call("plugin1", ["/somewhere"]), mock.call("plugin2", ["/somewhere/subdir"]), mock.call("plugin3", ["/somewhere/subdir/subsubdir"]) ] self.assertEqual(expected, mock_find_module.mock_calls) self.assertEqual(3, len(mock_load_module.mock_calls))
def load(): global PLUGINS_LOADED if not PLUGINS_LOADED: discover.import_modules_from_package("rally.deployment.engines") discover.import_modules_from_package("rally.deployment.serverprovider") discover.import_modules_from_package("rally.plugins") discover.load_plugins("/opt/rally/plugins/") discover.load_plugins(os.path.expanduser("~/.rally/plugins/")) PLUGINS_LOADED = True
def load(): global PLUGINS_LOADED if not PLUGINS_LOADED: from rally.common import opts opts.register() discover.import_modules_from_package("rally.plugins.common") packages = discover.find_packages_by_entry_point() for package in packages: if "options" in package: opts.register_options_from_path(package["options"]) discover.import_modules_by_entry_point(_packages=packages) discover.load_plugins("/opt/rally/plugins/") discover.load_plugins(os.path.expanduser("~/.rally/plugins/")) PLUGINS_LOADED = True
def load(): global PLUGINS_LOADED if not PLUGINS_LOADED: discover.import_modules_from_package("rally.deployment.engines") discover.import_modules_from_package("rally.deployment.serverprovider") discover.import_modules_from_package("rally.plugins.common") try: import rally_openstack # noqa except ImportError: # print warnings when rally_openstack will be released discover.import_modules_from_package("rally.plugins.openstack") discover.import_modules_from_package("rally.plugins.workload") discover.import_modules_by_entry_point() discover.load_plugins("/opt/rally/plugins/") discover.load_plugins(os.path.expanduser("~/.rally/plugins/")) PLUGINS_LOADED = True
def test_schema_is_valid(self): discover.load_plugins(os.path.join(self.rally_jobs_path, "plugins")) files = { f for f in os.listdir(self.rally_jobs_path) if (os.path.isfile(os.path.join(self.rally_jobs_path, f)) and f.endswith(".yaml") and not f.endswith("_args.yaml")) } # TODO(andreykurilin): figure out why it fails files -= {"rally-mos.yaml", "sahara-clusters.yaml"} for filename in files: full_path = os.path.join(self.rally_jobs_path, filename) with open(full_path) as task_file: try: args_file = os.path.join( self.rally_jobs_path, filename.rsplit(".", 1)[0] + "_args.yaml") args = {} if os.path.exists(args_file): args = yaml.safe_load(open(args_file).read()) if not isinstance(args, dict): raise TypeError( "args file %s must be dict in yaml or json " "presentation" % args_file) task_inst = api._Task(api.API(skip_db_check=True)) task = task_inst.render_template( task_template=task_file.read(), **args) task = task_cfg.TaskConfig(yaml.safe_load(task)) task_obj = fakes.FakeTask({"uuid": full_path}) eng = engine.TaskEngine(task, task_obj, mock.Mock()) eng.validate(only_syntax=True) except Exception: print(traceback.format_exc()) self.fail("Wrong task input file: %s" % full_path)
def test_schema_is_valid(self): discover.load_plugins(os.path.join(self.rally_jobs_path, "plugins")) files = {f for f in os.listdir(self.rally_jobs_path) if (os.path.isfile(os.path.join(self.rally_jobs_path, f)) and f.endswith(".yaml") and not f.endswith("_args.yaml"))} # TODO(andreykurilin): figure out why it fails files -= {"rally-mos.yaml", "sahara-clusters.yaml"} for filename in files: full_path = os.path.join(self.rally_jobs_path, filename) with open(full_path) as task_file: try: args_file = os.path.join( self.rally_jobs_path, filename.rsplit(".", 1)[0] + "_args.yaml") args = {} if os.path.exists(args_file): args = yaml.safe_load(open(args_file).read()) if not isinstance(args, dict): raise TypeError( "args file %s must be dict in yaml or json " "presentation" % args_file) task_inst = api._Task(api.API(skip_db_check=True)) task = task_inst.render_template( task_template=task_file.read(), **args) task = engine.TaskConfig(yaml.safe_load(task)) task_obj = fakes.FakeTask({"uuid": full_path}) eng = engine.TaskEngine(task, task_obj, mock.Mock()) eng.validate(only_syntax=True) except Exception: print(traceback.format_exc()) self.fail("Wrong task input file: %s" % full_path)
def load(): global PLUGINS_LOADED if not PLUGINS_LOADED: from rally.common import opts opts.register() discover.import_modules_from_package("rally.deployment.engines") discover.import_modules_from_package("rally.deployment.serverprovider") discover.import_modules_from_package("rally.plugins.common") try: import rally_openstack # noqa except ImportError: # print warnings when rally_openstack will be released discover.import_modules_from_package("rally.plugins.openstack") discover.import_modules_from_package("rally.plugins.workload") discover.import_modules_by_entry_point() discover.load_plugins("/opt/rally/plugins/") discover.load_plugins(os.path.expanduser("~/.rally/plugins/")) PLUGINS_LOADED = True
def load(): global PLUGINS_LOADED if not PLUGINS_LOADED: discover.import_modules_from_package("rally.deployment.engines") discover.import_modules_from_package("rally.deployment.serverprovider") discover.import_modules_from_package("rally.plugins.common") path = [os.path.dirname(rally_ovs.__file__)] + __name__.split(".")[1:] path = os.path.join(*path) discover.load_plugins(path) discover.load_plugins("/opt/rally/plugins/") discover.load_plugins(os.path.expanduser("~/.rally/plugins/")) PLUGINS_LOADED = True
def test_load_plugins_from_file_successful(self, mock_load_source, mock_isfile): discover.load_plugins("/somewhere/plugin.py") expected = [mock.call("plugin", "/somewhere/plugin.py")] self.assertEqual(expected, mock_load_source.mock_calls)
def __init__(self, config_file=None, config_args=None, rally_endpoint=None, plugin_paths=None, skip_db_check=False): """Initialize Rally API instance :param config_file: Path to rally configuration file. If None, default path will be selected :type config_file: str :param config_args: Arguments for initialization current configuration :type config_args: list :param rally_endpoint: [Restricted]Rally endpoint connection string. :type rally_endpoint: str :param plugin_paths: Additional custom plugin locations :type plugin_paths: list :param skip_db_check: Allows to skip db revision check :type skip_db_check: bool """ if rally_endpoint: raise NotImplementedError( _LE("Sorry, but Rally-as-a-Service is " "not ready yet.")) try: config_files = ([config_file] if config_file else self._default_config_file()) CONF(config_args or [], project="rally", version=rally_version.version_string(), default_config_files=config_files) logging.setup("rally") if not CONF.get("log_config_append"): # The below two lines are to disable noise from request module. # The standard way should be we make such lots of settings on # the root rally. However current oslo codes doesn't support # such interface. So I choose to use a 'hacking' way to avoid # INFO logs from request module where user didn't give specific # log configuration. And we could remove this hacking after # oslo.log has such interface. LOG.debug( "INFO logs from urllib3 and requests module are hide.") requests_log = logging.getLogger("requests").logger requests_log.setLevel(logging.WARNING) urllib3_log = logging.getLogger("urllib3").logger urllib3_log.setLevel(logging.WARNING) LOG.debug("urllib3 insecure warnings are hidden.") for warning in ("InsecurePlatformWarning", "SNIMissingWarning", "InsecureRequestWarning"): warning_cls = getattr(urllib3.exceptions, warning, None) if warning_cls is not None: urllib3.disable_warnings(warning_cls) # NOTE(wtakase): This is for suppressing boto error logging. LOG.debug("ERROR log from boto module is hide.") boto_log = logging.getLogger("boto").logger boto_log.setLevel(logging.CRITICAL) # Set alembic log level to ERROR alembic_log = logging.getLogger("alembic").logger alembic_log.setLevel(logging.ERROR) except cfg.ConfigFilesNotFoundError as e: cfg_files = e.config_files raise exceptions.RallyException( _LE("Failed to read configuration file(s): %s") % cfg_files) # Check that db is upgraded to the latest revision if not skip_db_check: self.check_db_revision() # Load plugins plugin_paths = plugin_paths or [] if "plugin_paths" in CONF: plugin_paths.extend(CONF.get("plugin_paths") or []) for path in plugin_paths: discover.load_plugins(path) # NOTE(andreykurilin): There is no reason to auto-discover API's. We # have only 4 classes, so let's do it in good old way - hardcode them:) self._deployment = _Deployment self._task = _Task self._verifier = _Verifier self._verification = _Verification
def load_plugins(): path = os.path.dirname(rally_ovs.__file__) + "/plugins" discover.load_plugins(path)
def test_load_plugins_from_nonexisting_file(self, mock_isfile): # test no fails for nonexisting file discover.load_plugins("/somewhere/plugin.py")
def test_load_plugins_from_file_fails(self, mock_isfile): discover.load_plugins("/somewhere/plugin.py")
def run(argv, categories): parser = lambda subparsers: _add_command_parsers(categories, subparsers) category_opt = cfg.SubCommandOpt("category", title="Command categories", help="Available categories", handler=parser) CONF.register_cli_opt(category_opt) help_msg = ("Additional custom plugin locations. Multiple files or " "directories may be specified. All plugins in the specified" " directories and subdirectories will be imported. Plugins in" " /opt/rally/plugins and ~/.rally/plugins will always be " "imported.") CONF.register_cli_opt(cfg.ListOpt("plugin-paths", default=os.environ.get( "RALLY_PLUGIN_PATHS"), help=help_msg)) try: CONF(argv[1:], project="rally", version=version.version_string(), default_config_files=find_config_files(CONFIG_SEARCH_PATHS)) logging.setup("rally") if not CONF.get("log_config_append"): # The below two lines are to disable noise from request module. The # standard way should be we make such lots of settings on the root # rally. However current oslo codes doesn't support such interface. # So I choose to use a 'hacking' way to avoid INFO logs from # request module where user didn't give specific log configuration. # And we could remove this hacking after oslo.log has such # interface. LOG.debug("INFO logs from urllib3 and requests module are hide.") requests_log = logging.getLogger("requests").logger requests_log.setLevel(logging.WARNING) urllib3_log = logging.getLogger("urllib3").logger urllib3_log.setLevel(logging.WARNING) # NOTE(wtakase): This is for suppressing boto error logging. LOG.debug("ERROR log from boto module is hide.") boto_log = logging.getLogger("boto").logger boto_log.setLevel(logging.CRITICAL) except cfg.ConfigFilesNotFoundError: cfgfile = CONF.config_file[-1] if CONF.config_file else None if cfgfile and not os.access(cfgfile, os.R_OK): st = os.stat(cfgfile) print(_("Could not read %s. Re-running with sudo") % cfgfile) try: os.execvp("sudo", ["sudo", "-u", "#%s" % st.st_uid] + sys.argv) except Exception: print(_("sudo failed, continuing as if nothing happened")) print(_("Please re-run %s as root.") % argv[0]) return(2) if CONF.category.name == "version": print(version.version_string()) return(0) if CONF.category.name == "bash-completion": print(_generate_bash_completion_script()) return(0) fn = CONF.category.action_fn fn_args = [encodeutils.safe_decode(arg) for arg in CONF.category.action_args] fn_kwargs = {} for k in CONF.category.action_kwargs: v = getattr(CONF.category, "action_kwarg_" + k) if v is None: continue if isinstance(v, six.string_types): v = encodeutils.safe_decode(v) fn_kwargs[k] = v # call the action with the remaining arguments # check arguments try: validate_args(fn, *fn_args, **fn_kwargs) except MissingArgs as e: # NOTE(mikal): this isn't the most helpful error message ever. It is # long, and tells you a lot of things you probably don't want to know # if you just got a single arg wrong. print(fn.__doc__) CONF.print_help() print("Missing arguments:") for missing in e.missing: for arg in fn.args: if arg[1].get("dest", "").endswith(missing): print(" " + arg[0][0]) break return(1) try: for path in CONF.plugin_paths or []: discover.load_plugins(path) validate_deprecated_args(argv, fn) if getattr(fn, "_suppress_warnings", False): with warnings.catch_warnings(): warnings.simplefilter("ignore") ret = fn(*fn_args, **fn_kwargs) else: ret = fn(*fn_args, **fn_kwargs) return(ret) except (IOError, TypeError, ValueError, exceptions.DeploymentNotFound, exceptions.TaskNotFound, jsonschema.ValidationError) as e: if logging.is_debug(): LOG.exception(e) print(e) return 1 except sqlalchemy.exc.OperationalError as e: if logging.is_debug(): LOG.exception(e) print(e) print("Looks like Rally can't connect to its DB.") print("Make a sure that connection string in rally.conf is proper:") print(CONF.database.connection) return 1 except Exception: print(_("Command failed, please check log for more info")) raise
def test_load_plugins_fails(self, mock_os_walk, mock_os_path, mock_find_module, mock_load_module): # test no fails if module is broken # TODO(olkonami): check exception is handled correct discover.load_plugins("/somewhere")