def __init__(self, address, client, username, password, port, remote_path, limit="", log_filename=None, verbose=False, timeout=600): """ Initialization of RemoteFile class. @param address: Address of remote host(guest) @param client: Type of transfer client @param username: Username (if required) @param password: Password (if requried) @param remote_path: Path of file which we want to edit on remote. @param limit: Speed limit of file transfer. @param log_filename: If specified, log all output to this file(SCP only) @param verbose: If True, log some stats using logging.debug (RSS only) @param timeout: The time duration (in seconds) to wait for the transfer tocomplete. """ self.address = address self.client = client self.username = username self.password = password self.port = port self.remote_path = remote_path self.limit = limit self.log_filename = log_filename self.verbose = verbose self.timeout = timeout #Get a local_path and all actions is taken on it. filename = os.path.basename(self.remote_path) #Get a local_path. tmp_dir = data_dir.get_tmp_dir() local_file = tempfile.NamedTemporaryFile(prefix=("%s_" % filename), dir=tmp_dir) self.local_path = local_file.name local_file.close() #Get a backup_path. back_dir = data_dir.get_backing_data_dir() backup_file = tempfile.NamedTemporaryFile(prefix=("%s_" % filename), dir=tmp_dir) self.backup_path = backup_file.name backup_file.close() #Get file from remote. self._pull_file() #Save a backup. shutil.copy(self.local_path, self.backup_path)
def run_tests(parser, options): """ Runs the sequence of KVM tests based on the list of dctionaries generated by the configuration system, handling dependencies. @param parser: Config parser object. @return: True, if all tests ran passed, False if any of them failed. """ debugdir = os.path.join(ROOT_PATH, 'logs', 'run-%s' % time.strftime('%Y-%m-%d-%H.%M.%S')) if not os.path.isdir(debugdir): os.makedirs(debugdir) debuglog = os.path.join(debugdir, "debug.log") configure_file_logging(debuglog) print_stdout(bcolors.HEADER + "DATA DIR: %s" % data_dir.get_backing_data_dir() + bcolors.ENDC) print_header("DEBUG LOG: %s" % debuglog) last_index = -1 for i, d in enumerate(parser.get_dicts()): if options.config is None: shortname = ".".join(d['name'].split(".")[12:]) else: shortname = ".".join(d['shortname'].split(".")) logging.info("Test %4d: %s" % (i + 1, shortname)) last_index += 1 if last_index == -1: print_stdout("No tests generated by config file %s" % parser.filename) print_stdout("Please check the file for errors (bad variable names, " "wrong indentation)") sys.exit(-1) n_tests = last_index + 1 print_header("TESTS: %s" % n_tests) status_dct = {} failed = False # Add the parameter decide if setup host env in the test case # For some special tests we only setup host in the first and last case # When we need to setup host env we need the host_setup_flag as following: # 0(00): do nothing # 1(01): setup env # 2(10): cleanup env # 3(11): setup and cleanup env index = 0 setup_flag = 1 cleanup_flag = 2 for dct in parser.get_dicts(): if options.config is None: shortname = ".".join(d['name'].split(".")[12:]) else: shortname = ".".join(d['shortname'].split(".")) if index == 0: if dct.get("host_setup_flag", None) is not None: flag = int(dct["host_setup_flag"]) dct["host_setup_flag"] = flag | setup_flag else: dct["host_setup_flag"] = setup_flag if index == last_index: if dct.get("host_setup_flag", None) is not None: flag = int(dct["host_setup_flag"]) dct["host_setup_flag"] = flag | cleanup_flag else: dct["host_setup_flag"] = cleanup_flag index += 1 # Add kvm module status dct["kvm_default"] = utils_misc.get_module_params( dct.get("sysfs_dir", "sys"), "kvm") if dct.get("skip") == "yes": continue dependencies_satisfied = True for dep in dct.get("dep"): for test_name in status_dct.keys(): if not dep in test_name: continue if not status_dct[test_name]: dependencies_satisfied = False break current_status = False if dependencies_satisfied: t = Test(dct, options) t.set_debugdir(debugdir) print_stdout("%s:" % t.tag, end=False) try: try: t_begin = time.time() t.start_file_logging() current_status = t.run_once() logging.info("PASS") t.stop_file_logging() finally: t_end = time.time() t_elapsed = t_end - t_begin except error.TestNAError, reason: logging.info("SKIP -> %s: %s", reason.__class__.__name__, reason) t.stop_file_logging() print_skip() status_dct[dct.get("name")] = False continue except error.TestWarn, reason: logging.info("WARN -> %s: %s", reason.__class__.__name__, reason) t.stop_file_logging() print_warn(t_elapsed) status_dct[dct.get("name")] = True continue except Exception, reason: exc_type, exc_value, exc_traceback = sys.exc_info() logging.error("") tb_info = traceback.format_exception(exc_type, exc_value, exc_traceback.tb_next) tb_info = "".join(tb_info) for e_line in tb_info.splitlines(): logging.error(e_line) logging.error("") logging.error("FAIL -> %s: %s", reason.__class__.__name__, reason) t.stop_file_logging() current_status = False
def create(self, params, ignore_errors=False): """ Create an image using qemu_img or dd. :param params: Dictionary containing the test parameters. :param ignore_errors: Whether to ignore errors on the image creation cmd. :note: params should contain: image_name name of the image file, without extension image_format format of the image (qcow2, raw etc) image_cluster_size (optional) cluster size for the image image_size requested size of the image (a string qemu-img can understand, such as '10G') create_with_dd use dd to create the image (raw format only) base_image(optional) the base image name when create snapshot base_format(optional) the format of base image encrypted(optional) if the image is encrypted, allowed values: on and off. Default is "off" preallocated(optional) if preallocation when create image, allowed values: off, metadata. Default is "off" :return: tuple (path to the image created, utils.CmdResult object containing the result of the creation command). """ if params.get("create_with_dd") == "yes" and self.image_format == "raw": # maps K,M,G,T => (count, bs) human = {'K': (1, 1), 'M': (1, 1024), 'G': (1024, 1024), 'T': (1024, 1048576), } if human.has_key(self.size[-1]): block_size = human[self.size[-1]][1] size = int(self.size[:-1]) * human[self.size[-1]][0] qemu_img_cmd = ("dd if=/dev/zero of=%s count=%s bs=%sK" % (self.image_filename, size, block_size)) else: qemu_img_cmd = self.image_cmd qemu_img_cmd += " create" qemu_img_cmd += " -f %s" % self.image_format image_cluster_size = params.get("image_cluster_size", None) preallocated = params.get("preallocated", "off") encrypted = params.get("encrypted", "off") image_extra_params = params.get("image_extra_params", "") has_backing_file = params.get('has_backing_file') qemu_img_cmd += " -o " if preallocated != "off": qemu_img_cmd += "preallocation=%s," % preallocated if encrypted != "off": qemu_img_cmd += "encrypted=%s," % encrypted if image_cluster_size is not None: qemu_img_cmd += "cluster_size=%s," % image_cluster_size if has_backing_file == "yes": backing_param = params.object_params("backing_file") backing_file = storage.get_image_filename(backing_param, self.root_dir) backing_fmt = backing_param.get("image_format") qemu_img_cmd += "backing_file=%s," % backing_file qemu_img_cmd += "backing_fmt=%s," % backing_fmt if image_extra_params: qemu_img_cmd += "%s," % image_extra_params qemu_img_cmd = qemu_img_cmd.rstrip(" -o") qemu_img_cmd = qemu_img_cmd.rstrip(",") if self.base_tag: qemu_img_cmd += " -b %s" % self.base_image_filename if self.base_format: qemu_img_cmd += " -F %s" % self.base_format qemu_img_cmd += " %s" % self.image_filename qemu_img_cmd += " %s" % self.size if (params.get("image_backend", "filesystem") == "filesystem"): image_dirname = os.path.dirname(self.image_filename) if image_dirname and not os.path.isdir(image_dirname): e_msg = ("Parent directory of the image file %s does " "not exist" % self.image_filename) logging.error(e_msg) logging.error("This usually means a serious setup error.") logging.error("Please verify if your data dir contains the " "expected directory structure") logging.error("Backing data dir: %s", data_dir.get_backing_data_dir()) logging.error("Directory structure:") for root, _, _ in os.walk(data_dir.get_backing_data_dir()): logging.error(root) logging.warning("We'll try to proceed by creating the dir. " "Other errors may ensue") os.makedirs(image_dirname) msg = "Create image by command: %s" % qemu_img_cmd error.context(msg, logging.info) cmd_result = utils.run(qemu_img_cmd, verbose=False, ignore_status=True) if cmd_result.exit_status != 0 and not ignore_errors: raise error.TestError("Failed to create image %s" % self.image_filename) return self.image_filename, cmd_result
debugdir = os.path.join(logdir, debugbase) latestdir = os.path.join(logdir, "latest") if not os.path.isdir(debugdir): os.makedirs(debugdir) try: os.unlink(latestdir) except OSError, detail: pass os.symlink(debugbase, latestdir) debuglog = os.path.join(debugdir, "debug.log") loglevel = options.vt_log_level configure_file_logging(debuglog, loglevel) print_stdout(bcolors.HEADER + "DATA DIR: %s" % data_dir.get_backing_data_dir() + bcolors.ENDC) print_header("DEBUG LOG: %s" % debuglog) last_index = -1 logging.info("Starting test job at %s", test_start_time) logging.info("") logging.info(version.get_pretty_version_info()) logging.info("") cleanup_env(parser, options) d = parser.get_dicts().next()
def run_tests(parser, options): """ Runs the sequence of KVM tests based on the list of dctionaries generated by the configuration system, handling dependencies. @param parser: Config parser object. @return: True, if all tests ran passed, False if any of them failed. """ debugdir = os.path.join(data_dir.get_root_dir(), 'logs', 'run-%s' % time.strftime('%Y-%m-%d-%H.%M.%S')) if not os.path.isdir(debugdir): os.makedirs(debugdir) debuglog = os.path.join(debugdir, "debug.log") configure_file_logging(debuglog) print_stdout(bcolors.HEADER + "DATA DIR: %s" % data_dir.get_backing_data_dir() + bcolors.ENDC) print_header("DEBUG LOG: %s" % debuglog) last_index = -1 logging.info("Starting test job at %s", time.strftime('%Y-%m-%d %H:%M:%S')) logging.info("") logging.debug("Cleaning up previous job tmp files") d = parser.get_dicts().next() env_filename = os.path.join(data_dir.get_root_dir(), options.type, d.get("env", "env")) env = utils_env.Env(env_filename, Test.env_version) env.destroy() try: address_pool_files = glob.glob("/tmp/address_pool*") for address_pool_file in address_pool_files: os.remove(address_pool_file) aexpect_tmp = "/tmp/aexpect_spawn/" if os.path.isdir(aexpect_tmp): shutil.rmtree("/tmp/aexpect_spawn/") except (IOError, OSError): pass logging.debug("") if options.restore_image_between_tests: logging.debug("Creating first backup of guest image") qemu_img = storage.QemuImg(d, data_dir.get_data_dir(), "image") qemu_img.backup_image(d, data_dir.get_data_dir(), 'backup', True) logging.debug("") tag_index = get_tag_index(options, d) for line in get_cartesian_parser_details(parser).splitlines(): logging.info(line) logging.info("Defined test set:") for i, d in enumerate(parser.get_dicts()): shortname = get_tag(d, tag_index) logging.info("Test %4d: %s", i + 1, shortname) last_index += 1 if last_index == -1: print_stdout("No tests generated by config file %s" % parser.filename) print_stdout("Please check the file for errors (bad variable names, " "wrong indentation)") sys.exit(-1) logging.info("") n_tests = last_index + 1 n_tests_failed = 0 n_tests_skipped = 0 print_header("TESTS: %s" % n_tests) status_dct = {} failed = False # Add the parameter decide if setup host env in the test case # For some special tests we only setup host in the first and last case # When we need to setup host env we need the host_setup_flag as following: # 0(00): do nothing # 1(01): setup env # 2(10): cleanup env # 3(11): setup and cleanup env index = 0 setup_flag = 1 cleanup_flag = 2 job_start_time = time.time() for dct in parser.get_dicts(): shortname = get_tag(dct, tag_index) if index == 0: if dct.get("host_setup_flag", None) is not None: flag = int(dct["host_setup_flag"]) dct["host_setup_flag"] = flag | setup_flag else: dct["host_setup_flag"] = setup_flag if index == last_index: if dct.get("host_setup_flag", None) is not None: flag = int(dct["host_setup_flag"]) dct["host_setup_flag"] = flag | cleanup_flag else: dct["host_setup_flag"] = cleanup_flag index += 1 # Add kvm module status dct["kvm_default"] = utils_misc.get_module_params( dct.get("sysfs_dir", "sys"), "kvm") if dct.get("skip") == "yes": continue dependencies_satisfied = True for dep in dct.get("dep"): for test_name in status_dct.keys(): if not dep in test_name: continue if not status_dct[test_name]: dependencies_satisfied = False break current_status = False if dependencies_satisfied: t = Test(dct, options) t.set_debugdir(debugdir) pretty_index = "(%d/%d)" % (index, n_tests) print_stdout("%s %s:" % (pretty_index, t.tag), end=False) try: try: t_begin = time.time() t.start_file_logging() current_status = t.run_once() logging.info("PASS %s", t.tag) logging.info("") t.stop_file_logging() finally: t_end = time.time() t_elapsed = t_end - t_begin except error.TestError, reason: n_tests_failed += 1 logging.info("ERROR %s -> %s: %s", t.tag, reason.__class__.__name__, reason) logging.info("") t.stop_file_logging() print_error(t_elapsed) status_dct[dct.get("name")] = False continue except error.TestNAError, reason: n_tests_skipped += 1 logging.info("SKIP %s -> %s: %s", t.tag, reason.__class__.__name__, reason) logging.info("") t.stop_file_logging() print_skip() status_dct[dct.get("name")] = False continue except error.TestWarn, reason: logging.info("WARN %s -> %s: %s", t.tag, reason.__class__.__name__, reason) logging.info("") t.stop_file_logging() print_warn(t_elapsed) status_dct[dct.get("name")] = True continue
def run_tests(parser, options): """ Runs the sequence of KVM tests based on the list of dctionaries generated by the configuration system, handling dependencies. @param parser: Config parser object. @return: True, if all tests ran passed, False if any of them failed. """ debugdir = os.path.join(data_dir.get_root_dir(), 'logs', 'run-%s' % time.strftime('%Y-%m-%d-%H.%M.%S')) if not os.path.isdir(debugdir): os.makedirs(debugdir) debuglog = os.path.join(debugdir, "debug.log") configure_file_logging(debuglog) print_stdout(bcolors.HEADER + "DATA DIR: %s" % data_dir.get_backing_data_dir() + bcolors.ENDC) print_header("DEBUG LOG: %s" % debuglog) last_index = -1 logging.info("Starting test job at %s" % time.strftime('%Y-%m-%d %H:%M:%S')) logging.info("") logging.debug("Options received from the command line:") utils_misc.display_attributes(options) logging.debug("") logging.debug("Cleaning up previous job tmp files") d = parser.get_dicts().next() env_filename = os.path.join(data_dir.get_root_dir(), options.type, d.get("env", "env")) env = utils_env.Env(env_filename, Test.env_version) env.destroy() try: address_pool_files = glob.glob("/tmp/address_pool*") for address_pool_file in address_pool_files: os.remove(address_pool_file) aexpect_tmp = "/tmp/aexpect_spawn/" if os.path.isdir(aexpect_tmp): shutil.rmtree("/tmp/aexpect_spawn/") except (IOError, OSError): pass logging.debug("") if options.restore_image_between_tests: logging.debug("Creating first backup of guest image") qemu_img = storage.QemuImg(d, data_dir.get_data_dir(), "image") qemu_img.backup_image(d, data_dir.get_data_dir(), 'backup', True) logging.debug("") if options.type == 'qemu': logging.info("We're running the qemu test with:") logging.info("qemu binary: %s" % d.get('qemu_binary')) logging.info("qemu img binary: %s" % d.get('qemu_img_binary')) logging.info("qemu io binary: %s" % d.get('qemu_io_binary')) logging.info("") logging.info("Defined test set:") for i, d in enumerate(parser.get_dicts()): if options.config is None and options.type in TEST_TYPES_STRIP_NAMES: shortname = ".".join(d['name'].split(".")[12:]) else: shortname = ".".join(d['shortname'].split(".")) logging.info("Test %4d: %s" % (i + 1, shortname)) last_index += 1 if last_index == -1: print_stdout("No tests generated by config file %s" % parser.filename) print_stdout("Please check the file for errors (bad variable names, " "wrong indentation)") sys.exit(-1) logging.info("") n_tests = last_index + 1 print_header("TESTS: %s" % n_tests) status_dct = {} failed = False # Add the parameter decide if setup host env in the test case # For some special tests we only setup host in the first and last case # When we need to setup host env we need the host_setup_flag as following: # 0(00): do nothing # 1(01): setup env # 2(10): cleanup env # 3(11): setup and cleanup env index = 0 setup_flag = 1 cleanup_flag = 2 for dct in parser.get_dicts(): if options.config is None and options.type in TEST_TYPES_STRIP_NAMES: shortname = ".".join(d['name'].split(".")[12:]) else: shortname = ".".join(d['shortname'].split(".")) if index == 0: if dct.get("host_setup_flag", None) is not None: flag = int(dct["host_setup_flag"]) dct["host_setup_flag"] = flag | setup_flag else: dct["host_setup_flag"] = setup_flag if index == last_index: if dct.get("host_setup_flag", None) is not None: flag = int(dct["host_setup_flag"]) dct["host_setup_flag"] = flag | cleanup_flag else: dct["host_setup_flag"] = cleanup_flag index += 1 # Add kvm module status dct["kvm_default"] = utils_misc.get_module_params( dct.get("sysfs_dir", "sys"), "kvm") if dct.get("skip") == "yes": continue dependencies_satisfied = True for dep in dct.get("dep"): for test_name in status_dct.keys(): if not dep in test_name: continue if not status_dct[test_name]: dependencies_satisfied = False break current_status = False if dependencies_satisfied: t = Test(dct, options) t.set_debugdir(debugdir) pretty_index = "(%d/%d)" % (index, n_tests) print_stdout("%s %s:" % (pretty_index, t.tag), end=False) try: try: t_begin = time.time() t.start_file_logging() current_status = t.run_once() logging.info("PASS %s" % t.tag) logging.info("") t.stop_file_logging() finally: t_end = time.time() t_elapsed = t_end - t_begin except error.TestNAError, reason: logging.info("SKIP %s -> %s: %s", t.tag, reason.__class__.__name__, reason) logging.info("") t.stop_file_logging() print_skip() status_dct[dct.get("name")] = False continue except error.TestWarn, reason: logging.info("WARN %s -> %s: %s", t.tag, reason.__class__.__name__, reason) logging.info("") t.stop_file_logging() print_warn(t_elapsed) status_dct[dct.get("name")] = True continue except Exception, reason: exc_type, exc_value, exc_traceback = sys.exc_info() logging.error("") tb_info = traceback.format_exception(exc_type, exc_value, exc_traceback.tb_next) tb_info = "".join(tb_info) for e_line in tb_info.splitlines(): logging.error(e_line) logging.error("") logging.error("FAIL %s -> %s: %s", t.tag, reason.__class__.__name__, reason) logging.info("") t.stop_file_logging() current_status = False
debugbase = "run-%s" % test_start_time debugdir = os.path.join(logdir, debugbase) latestdir = os.path.join(logdir, "latest") if not os.path.isdir(debugdir): os.makedirs(debugdir) try: os.unlink(latestdir) except OSError, detail: pass os.symlink(debugbase, latestdir) debuglog = os.path.join(debugdir, "debug.log") loglevel = options.log_level configure_file_logging(debuglog, loglevel) print_stdout(bcolors.HEADER + "DATA DIR: %s" % data_dir.get_backing_data_dir() + bcolors.ENDC) print_header("DEBUG LOG: %s" % debuglog) last_index = -1 logging.info("Starting test job at %s", test_start_time) logging.info("") logging.info(version.get_pretty_version_info()) logging.info("") cleanup_env(parser, options) d = parser.get_dicts().next()
def run_tests(parser, options): """ Runs the sequence of KVM tests based on the list of dctionaries generated by the configuration system, handling dependencies. @param parser: Config parser object. @return: True, if all tests ran passed, False if any of them failed. """ debugdir = os.path.join(data_dir.get_root_dir(), 'logs', 'run-%s' % time.strftime('%Y-%m-%d-%H.%M.%S')) if not os.path.isdir(debugdir): os.makedirs(debugdir) debuglog = os.path.join(debugdir, "debug.log") configure_file_logging(debuglog) print_stdout(bcolors.HEADER + "DATA DIR: %s" % data_dir.get_backing_data_dir() + bcolors.ENDC) print_header("DEBUG LOG: %s" % debuglog) last_index = -1 for i, d in enumerate(parser.get_dicts()): if options.config is None: shortname = ".".join(d['name'].split(".")[12:]) else: shortname = ".".join(d['shortname'].split(".")) logging.info("Test %4d: %s" % (i + 1, shortname)) last_index += 1 if last_index == -1: print_stdout("No tests generated by config file %s" % parser.filename) print_stdout("Please check the file for errors (bad variable names, " "wrong indentation)") sys.exit(-1) # Clean environment file d = parser.get_dicts().next() env_filename = os.path.join(data_dir.get_root_dir(), options.type, d.get("env", "env")) env = utils_env.Env(env_filename, Test.env_version) env.destroy() n_tests = last_index + 1 print_header("TESTS: %s" % n_tests) status_dct = {} failed = False # Add the parameter decide if setup host env in the test case # For some special tests we only setup host in the first and last case # When we need to setup host env we need the host_setup_flag as following: # 0(00): do nothing # 1(01): setup env # 2(10): cleanup env # 3(11): setup and cleanup env index = 0 setup_flag = 1 cleanup_flag = 2 for dct in parser.get_dicts(): if options.config is None: shortname = ".".join(d['name'].split(".")[12:]) else: shortname = ".".join(d['shortname'].split(".")) if index == 0: if dct.get("host_setup_flag", None) is not None: flag = int(dct["host_setup_flag"]) dct["host_setup_flag"] = flag | setup_flag else: dct["host_setup_flag"] = setup_flag if index == last_index: if dct.get("host_setup_flag", None) is not None: flag = int(dct["host_setup_flag"]) dct["host_setup_flag"] = flag | cleanup_flag else: dct["host_setup_flag"] = cleanup_flag index += 1 # Add kvm module status dct["kvm_default"] = utils_misc.get_module_params( dct.get("sysfs_dir", "sys"), "kvm") if dct.get("skip") == "yes": continue dependencies_satisfied = True for dep in dct.get("dep"): for test_name in status_dct.keys(): if not dep in test_name: continue if not status_dct[test_name]: dependencies_satisfied = False break current_status = False if dependencies_satisfied: t = Test(dct, options) t.set_debugdir(debugdir) pretty_index = "(%d/%d)" % (index, n_tests) print_stdout("%s %s:" % (pretty_index, t.tag), end=False) try: try: t_begin = time.time() t.start_file_logging() current_status = t.run_once() logging.info("PASS") t.stop_file_logging() finally: t_end = time.time() t_elapsed = t_end - t_begin except error.TestNAError, reason: logging.info("SKIP -> %s: %s", reason.__class__.__name__, reason) t.stop_file_logging() print_skip() status_dct[dct.get("name")] = False continue except error.TestWarn, reason: logging.info("WARN -> %s: %s", reason.__class__.__name__, reason) t.stop_file_logging() print_warn(t_elapsed) status_dct[dct.get("name")] = True continue except Exception, reason: exc_type, exc_value, exc_traceback = sys.exc_info() logging.error("") tb_info = traceback.format_exception(exc_type, exc_value, exc_traceback.tb_next) tb_info = "".join(tb_info) for e_line in tb_info.splitlines(): logging.error(e_line) logging.error("") logging.error("FAIL -> %s: %s", reason.__class__.__name__, reason) t.stop_file_logging() current_status = False