def __call__(self, cmd, getjson=False, report_path=None, raw=False, suffix=None, extension=None, keep_old=False, write_report=True, no_logs=False): """Call rally in the shell :param cmd: rally command :param getjson: in cases, when rally prints JSON, you can catch output deserialized :param report_path: if present, rally command and its output will be written to file with passed file name :param raw: don't write command itself to report file. Only output will be written """ if not isinstance(cmd, list): cmd = cmd.split(" ") try: if no_logs or getjson: cmd = self.args + ["--log-file", "/dev/null"] + cmd with open(os.devnull, "w") as DEVNULL: output = encodeutils.safe_decode( subprocess.check_output(cmd, stderr=DEVNULL, env=self.env)) else: cmd = self.args + cmd output = encodeutils.safe_decode( subprocess.check_output(cmd, stderr=subprocess.STDOUT, env=self.env)) if getjson: return json.loads(output) return output except subprocess.CalledProcessError as e: output = e.output raise RallyCliError(cmd, e.returncode, e.output) finally: if write_report: if not report_path: report_path = self.gen_report_path(suffix=suffix, extension=extension, keep_old=keep_old) with open(report_path, "a") as rep: if not raw: rep.write("\n%s:\n" % " ".join(cmd)) rep.write("%s\n" % output)
def _parse(self): # NOTE(andreykurilin): When whole test class is marked as skipped or # failed, there is only one event with reason and status. So we should # modify all tests of test class manually. for test_id in self._unknown_entities: known_test_ids = filter( lambda t: t == test_id or t.startswith("%s." % test_id), self._tests) for t_id in known_test_ids: if self._tests[t_id]["status"] == "init": self._tests[t_id]["status"] = ( self._unknown_entities[test_id]["status"]) if self._unknown_entities[test_id].get("reason"): self._tests[t_id]["reason"] = ( self._unknown_entities[test_id]["reason"]) elif self._unknown_entities[test_id].get("traceback"): self._tests[t_id]["traceback"] = ( self._unknown_entities[test_id]["traceback"]) # decode data for test_id in self._tests: for file_name in ["traceback", "reason"]: # TODO(andreykurilin): decode fields based on mime_type if file_name in self._tests[test_id]: self._tests[test_id][file_name] = (encodeutils.safe_decode( self._tests[test_id][file_name])) self._is_parsed = True
def __init__(self, cmd, code, output): self.command = cmd self.code = code self.output = encodeutils.safe_decode(output) self.msg = "Command: %s Code: %d Output: %s\n" % (self.command, self.code, self.output)
def __init__(self, cmd, code, output): self.command = cmd self.code = code self.output = encodeutils.safe_decode(output) self.msg = "Command: %s Code: %d Output: %s\n" % (self.command, self.code, self.output)
def _parse(self): # NOTE(andreykurilin): When whole test class is marked as skipped or # failed, there is only one event with reason and status. So we should # modify all tests of test class manually. for test_id in self._unknown_entities: known_test_ids = filter(lambda t: t == test_id or t.startswith( "%s." % test_id), self._tests) for t_id in known_test_ids: if self._tests[t_id]["status"] == "init": self._tests[t_id]["status"] = ( self._unknown_entities[test_id]["status"]) if self._unknown_entities[test_id].get("reason"): self._tests[t_id]["reason"] = ( self._unknown_entities[test_id]["reason"]) elif self._unknown_entities[test_id].get("traceback"): self._tests[t_id]["traceback"] = ( self._unknown_entities[test_id]["traceback"]) # decode data for test_id in self._tests: for file_name in ["traceback", "reason"]: # TODO(andreykurilin): decode fields based on mime_type if file_name in self._tests[test_id]: self._tests[test_id][file_name] = ( encodeutils.safe_decode( self._tests[test_id][file_name])) self._is_parsed = True
def run(self, payload, algorithm, bit_length, mode): """Create and delete symmetric secret :param payload: The unecrypted data :param algorithm: the algorithm associated with the secret key :param bit_length: the big length of the secret key :param mode: the algorithm mode used with the secret key """ from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC payload = encodeutils.safe_encode(payload) salt = os.urandom(16) kdf = PBKDF2HMAC(algorithm=hashes.SHA256(), length=32, salt=salt, iterations=1000, backend=default_backend()) payload = base64.b64encode(kdf.derive(payload)) payload = encodeutils.safe_decode(payload) expire_time = (dt.datetime.utcnow() + dt.timedelta(days=5)) secret = self.admin_barbican.create_secret( expiration=expire_time.isoformat(), algorithm=algorithm, bit_length=bit_length, mode=mode, payload=payload, payload_content_type="application/octet-stream", payload_content_encoding="base64") self.admin_barbican.delete_secret(secret.secret_ref)
def test_rally_cli(self): try: subprocess.check_output(["rally"], stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: output = encodeutils.safe_decode(e.output) else: self.fail("It should ve non-zero exit code.") self.assertIn("the following arguments are required: category", output)
def __call__(self, cmd, getjson=False, report_path=None, raw=False, suffix=None, extension=None, keep_old=False, write_report=True, no_logs=False): """Call rally in the shell :param cmd: rally command :param getjson: in cases, when rally prints JSON, you can catch output deserialized :param report_path: if present, rally command and its output will be written to file with passed file name :param raw: don't write command itself to report file. Only output will be written """ if not isinstance(cmd, list): cmd = cmd.split(" ") try: if no_logs or getjson: cmd = self.args + ["--log-file", "/dev/null"] + cmd with open(os.devnull, "w") as DEVNULL: output = encodeutils.safe_decode(subprocess.check_output( cmd, stderr=DEVNULL, env=self.env)) else: cmd = self.args + cmd output = encodeutils.safe_decode(subprocess.check_output( cmd, stderr=subprocess.STDOUT, env=self.env)) if getjson: return json.loads(output) return output except subprocess.CalledProcessError as e: output = e.output raise RallyCliError(cmd, e.returncode, e.output) finally: if write_report: if not report_path: report_path = self.gen_report_path( suffix=suffix, extension=extension, keep_old=keep_old) with open(report_path, "a") as rep: if not raw: rep.write("\n%s:\n" % " ".join(cmd)) rep.write("%s\n" % output)
def check_output(*args, **kwargs): """Run command with arguments and return its output. If the exit code was non-zero it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute and output in the output attribute. The difference between check_output from subprocess package and this function: * Additional arguments: - "msg_on_err" argument. It is a message that should be written in case of error. Reduces a number of try...except blocks - "debug_output" argument(Defaults to True). Print or not output to LOG.debug * stderr is hardcoded to stdout * In case of error, prints failed command and output to LOG.error * Prints output to LOG.debug """ msg_on_err = kwargs.pop("msg_on_err", None) debug_output = kwargs.pop("debug_output", True) kwargs["stderr"] = subprocess.STDOUT try: output = subprocess.check_output(*args, **kwargs) except subprocess.CalledProcessError as exc: if msg_on_err: LOG.error(msg_on_err) LOG.error("Failed cmd: '%s'" % exc.cmd) LOG.error("Error output: '%s'" % encodeutils.safe_decode(exc.output)) raise output = encodeutils.safe_decode(output) if output and debug_output: LOG.debug("Subprocess output: '%s'" % output) return output
def check_output(*args, **kwargs): """Run command with arguments and return its output. If the exit code was non-zero it raises a CalledProcessError. The CalledProcessError object will have the return code in the returncode attribute and output in the output attribute. The difference between check_output from subprocess package and this function: * Additional arguments: - "msg_on_err" argument. It is a message that should be written in case of error. Reduces a number of try...except blocks - "debug_output" argument(Defaults to True). Print or not output to LOG.debug * stderr is hardcoded to stdout * In case of error, prints failed command and output to LOG.error * Prints output to LOG.debug """ msg_on_err = kwargs.pop("msg_on_err", None) debug_output = kwargs.pop("debug_output", True) kwargs["stderr"] = subprocess.STDOUT try: output = subprocess.check_output(*args, **kwargs) except subprocess.CalledProcessError as exc: if msg_on_err: LOG.error(msg_on_err) LOG.error("Failed cmd: '%s'" % exc.cmd) LOG.error("Error output: '%s'" % encodeutils.safe_decode(exc.output)) raise output = encodeutils.safe_decode(output) if output and debug_output: LOG.debug("Subprocess output: '%s'" % output) return output
def test_rally_cli(self): try: subprocess.check_output(["rally"], stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: output = encodeutils.safe_decode(e.output) else: self.fail("It should ve non-zero exit code.") # NOTE(andreykurilin): we should have the same errors... if six.PY2: self.assertIn("too few arguments", output) else: self.assertIn("the following arguments are required: category", output)
def run(argv, categories): if len(argv) > 1 and argv[1] in ["version", "--version"]: _print_version() return 0 parser = lambda subparsers: _add_command_parsers(categories, subparsers) category_opt = cfg.SubCommandOpt("category", title="Command categories", help="Available categories", handler=parser) CONF.register_cli_opt(category_opt) help_msg = ("Additional custom plugin locations. Multiple files or " "directories may be specified. All plugins in the specified" " directories and subdirectories will be imported. Plugins in" " /opt/rally/plugins and ~/.rally/plugins will always be " "imported.") CONF.register_cli_opt(cfg.ListOpt("plugin-paths", default=os.environ.get( "RALLY_PLUGIN_PATHS"), help=help_msg)) # NOTE(andreykurilin): this dirty hack is done to unblock the gates. # Currently, we are using oslo.config for CLI purpose (don't do this!) # and it makes the things too complicated. # To discover which CLI method can be affected by warnings and which not # (based on suppress_warnings decorator) we need to obtain a desired # CLI method. It can be done only after initialization of oslo_config # which is located in rally.api.API init method. # Initialization of rally.api.API can produce a warning (for example, # from pymysql), so suppressing of warnings later will not work in such # case (it is what actually had happened now in our CI with the latest # release of PyMySQL). # # https://bitbucket.org/zzzeek/sqlalchemy/issues/4120/mysql-5720-warns-on-tx_isolation try: import pymysql warnings.filterwarnings("ignore", category=pymysql.Warning) except ImportError: pass try: rapi = api.API(config_args=argv[1:], skip_db_check=True) except exceptions.RallyException as e: print(e) return(2) if CONF.category.name == "bash-completion": print(_generate_bash_completion_script()) return(0) fn = CONF.category.action_fn fn_args = [encodeutils.safe_decode(arg) for arg in CONF.category.action_args] # api instance always is the first argument fn_args.insert(0, rapi) fn_kwargs = {} for k in CONF.category.action_kwargs: v = getattr(CONF.category, "action_kwarg_" + k) if v is None: continue if isinstance(v, str): v = encodeutils.safe_decode(v) fn_kwargs[k] = v # call the action with the remaining arguments # check arguments try: validate_args(fn, *fn_args, **fn_kwargs) except MissingArgs as e: # NOTE(mikal): this isn't the most helpful error message ever. It is # long, and tells you a lot of things you probably don't want to know # if you just got a single arg wrong. print(fn.__doc__) CONF.print_help() print("Missing arguments:") for missing in e.missing: for arg in fn.args: if arg[1].get("dest", "").endswith(missing): print(" " + arg[0][0]) break return(1) try: validate_deprecated_args(argv, fn) # skip db check for db and plugin commands if CONF.category.name not in ("db", "plugin"): rapi.check_db_revision() if getattr(fn, "_suppress_warnings", False): with warnings.catch_warnings(): warnings.simplefilter("ignore") ret = fn(*fn_args, **fn_kwargs) else: ret = fn(*fn_args, **fn_kwargs) return ret except (IOError, TypeError, ValueError, exceptions.RallyException, jsonschema.ValidationError) as e: known_errors = (exceptions.InvalidTaskConfig, ) if logging.is_debug() and not isinstance(e, known_errors): LOG.exception("Unexpected exception in CLI") else: print(e) return getattr(e, "error_code", 1) except sqlalchemy.exc.OperationalError as e: if logging.is_debug(): LOG.exception("Something went wrong with database") print(e) print("Looks like Rally can't connect to its DB.") print("Make sure that connection string in rally.conf is proper:") print(CONF.database.connection) return 1 except Exception: print("Command failed, please check log for more info") raise
def run(argv, categories): if len(argv) > 1 and argv[1] in ["version", "--version"]: _print_version() return 0 parser = lambda subparsers: _add_command_parsers(categories, subparsers) category_opt = cfg.SubCommandOpt("category", title="Command categories", help="Available categories", handler=parser) CONF.register_cli_opt(category_opt) help_msg = ("Additional custom plugin locations. Multiple files or " "directories may be specified. All plugins in the specified" " directories and subdirectories will be imported. Plugins in" " /opt/rally/plugins and ~/.rally/plugins will always be " "imported.") CONF.register_cli_opt(cfg.ListOpt("plugin-paths", default=os.environ.get( "RALLY_PLUGIN_PATHS"), help=help_msg)) # NOTE(andreykurilin): this dirty hack is done to unblock the gates. # Currently, we are using oslo.config for CLI purpose (don't do this!) # and it makes the things too complicated. # To discover which CLI method can be affected by warnings and which not # (based on suppress_warnings decorator) we need to obtain a desired # CLI method. It can be done only after initialization of oslo_config # which is located in rally.api.API init method. # Initialization of rally.api.API can produce a warning (for example, # from pymysql), so suppressing of warnings later will not work in such # case (it is what actually had happened now in our CI with the latest # release of PyMySQL). # # https://bitbucket.org/zzzeek/sqlalchemy/issues/4120/mysql-5720-warns-on-tx_isolation try: import pymysql warnings.filterwarnings("ignore", category=pymysql.Warning) except ImportError: pass try: rapi = api.API(config_args=argv[1:], skip_db_check=True) except exceptions.RallyException as e: print(e) return(2) if CONF.category.name == "bash-completion": print(_generate_bash_completion_script()) return(0) fn = CONF.category.action_fn fn_args = [encodeutils.safe_decode(arg) for arg in CONF.category.action_args] # api instance always is the first argument fn_args.insert(0, rapi) fn_kwargs = {} for k in CONF.category.action_kwargs: v = getattr(CONF.category, "action_kwarg_" + k) if v is None: continue if isinstance(v, six.string_types): v = encodeutils.safe_decode(v) fn_kwargs[k] = v # call the action with the remaining arguments # check arguments try: validate_args(fn, *fn_args, **fn_kwargs) except MissingArgs as e: # NOTE(mikal): this isn't the most helpful error message ever. It is # long, and tells you a lot of things you probably don't want to know # if you just got a single arg wrong. print(fn.__doc__) CONF.print_help() print("Missing arguments:") for missing in e.missing: for arg in fn.args: if arg[1].get("dest", "").endswith(missing): print(" " + arg[0][0]) break return(1) try: validate_deprecated_args(argv, fn) # skip db check for db and plugin commands if CONF.category.name not in ("db", "plugin"): rapi.check_db_revision() if getattr(fn, "_suppress_warnings", False): with warnings.catch_warnings(): warnings.simplefilter("ignore") ret = fn(*fn_args, **fn_kwargs) else: ret = fn(*fn_args, **fn_kwargs) return ret except (IOError, TypeError, ValueError, exceptions.RallyException, jsonschema.ValidationError) as e: if logging.is_debug(): LOG.exception("Unexpected exception in CLI") else: print(e) return getattr(e, "error_code", 1) except sqlalchemy.exc.OperationalError as e: if logging.is_debug(): LOG.exception("Something went wrong with database") print(e) print("Looks like Rally can't connect to its DB.") print("Make sure that connection string in rally.conf is proper:") print(CONF.database.connection) return 1 except Exception: print("Command failed, please check log for more info") raise
def test_version_cli(self): output = encodeutils.safe_decode( subprocess.check_output(["rally", "version"], stderr=subprocess.STDOUT)) self.assertIn("Rally version:", output)