def _proc_cmd(self): """Command to start child process.""" from testplan.common.utils.path import fix_home_prefix cmd = [ sys.executable, fix_home_prefix(self._child_path()), "--index", self.cfg.index, "--address", self.transport.address, "--testplan", os.path.join(os.path.dirname(testplan.__file__), ".."), "--type", "process_worker", "--log-level", TESTPLAN_LOGGER.getEffectiveLevel(), "--sys-path-file", self._write_syspath(), ] if os.environ.get(testplan.TESTPLAN_DEPENDENCIES_PATH): cmd.extend([ "--testplan-deps", fix_home_prefix( os.environ[testplan.TESTPLAN_DEPENDENCIES_PATH]), ]) return cmd
def schedule_tests_to_pool(name, pool, **pool_cfg): pool_name = pool.__name__ # Enable debug: # from testplan.logger import DEBUG # TESTPLAN_LOGGER.setLevel(DEBUG) plan = Testplan(name=name, parse_cmdline=False) pool = pool(name=pool_name, **pool_cfg) plan.add_resource(pool) this_file = fix_home_prefix(os.path.dirname(os.path.abspath(__file__))) if pool_cfg.get('workspace'): # Remote pool ws = pool_cfg['workspace'] wd = fix_home_prefix(os.getcwd()) common = os.path.commonprefix([pool_cfg['workspace'], wd]) # Relative path tp the same file but from the remote workspace. path = '/'.join( os.path.join(os.path.relpath(common, wd), os.path.relpath(this_file, ws)).split(os.sep)) else: path = this_file uids = [] for idx in range(1, 10): uids.append( plan.schedule(target='get_mtest', module='func_pool_base_tasks', path=path, kwargs=dict(name=idx), resource=pool_name)) with log_propagation_disabled(TESTPLAN_LOGGER): res = plan.run() assert res.run is True assert res.success is True assert plan.report.passed is True assert plan.report.status == Status.PASSED assert plan.report.counts.passed == 9 assert plan.report.counts.error == plan.report.counts.skipped == \ plan.report.counts.failed == plan.report.counts.incomplete == 0 names = sorted(['MTest{}'.format(x) for x in range(1, 10)]) assert sorted([entry.name for entry in plan.report.entries]) == names assert isinstance(plan.report.serialize(), dict) for idx in range(1, 10): name = 'MTest{}'.format(idx) assert plan.result.test_results[uids[idx - 1]].report.name == name # All tasks scheduled once for uid in pool.task_assign_cnt: assert pool.task_assign_cnt[uid] == 1
def _proc_cmd(self): """Command to start child process.""" from testplan.common.utils.path import fix_home_prefix cmd = [sys.executable, fix_home_prefix(self._child_path()), '--index', self.cfg.index, '--address', self.transport.address, '--testplan', os.path.join(os.path.dirname(testplan.__file__), '..'), '--type', 'process_worker', '--log-level', TESTPLAN_LOGGER.getEffectiveLevel()] if os.environ.get(testplan.TESTPLAN_DEPENDENCIES_PATH): cmd.extend( ['--testplan-deps', fix_home_prefix( os.environ[testplan.TESTPLAN_DEPENDENCIES_PATH])]) return cmd
def _build_dependencies(self, extra_deps): """ Build a list of directories to reload code from and a tree of dependencies. :param extra_deps: Modules to register as extra dependencies to reload, despite not being directly imported by __main__. :type extra_deps: ``Iterable[ModuleType]`` """ main_module_file = sys.modules['__main__'].__file__ if not main_module_file: raise RuntimeError( "Can only use interactive reloader when the __main__ module " "is a file.") reload_dir = os.path.realpath(os.path.dirname(main_module_file)) reload_dirs = {path_utils.fix_home_prefix(reload_dir)} # Add extra reload source directories if required. if extra_deps: reload_dirs = reload_dirs.union( self._extra_reload_dirs(extra_deps)) dep_graph, watched_modules = self._build_dep_graph( main_module_file, reload_dirs) return reload_dirs, dep_graph, watched_modules
def _get_reload_dirs(self, extra_deps=None): reload_dirs = [] if _has_file(sys.modules['__main__']): main_module_file = self._module_filepath( sys.modules['__main__'].__file__) reload_dir = os.path.realpath(os.path.dirname(main_module_file)) else: reload_dir = os.path.realpath(pwd()) reload_dirs.append(fix_home_prefix(reload_dir)) # Add extra reload source directories. for mod in extra_deps: module_file = self._module_filepath(mod.__file__) reload_dir = os.path.realpath(os.path.dirname(module_file)) reload_dirs.append(fix_home_prefix(reload_dir)) return reload_dirs
def _check_workspace(self): """ Check if workspace is available on remote host :return: True if exsit, false otherwise """ if self.cfg.remote_workspace: # User defined the remote workspace to be used # will raise if check fail if 0 == self._execute_cmd_remote( cmd=filepath_exist_cmd( fix_home_prefix(self.cfg.remote_workspace)), label="workspace availability check (1)", check=True, ): return True if 0 == self._execute_cmd_remote( cmd=filepath_exist_cmd(self._workspace_paths.local), label="workspace availability check (2)", check=False, ): # local workspace accessible on remote return True return False
def _prepare_remote(self): """Transfer local data to remote host.""" self._child_paths.local = self._child_path() self._workspace_paths.local = fix_home_prefix(self.cfg.workspace) if self.cfg.copy_workspace_check: cmd = self.cfg.copy_workspace_check(self.cfg.ssh_cmd, self.cfg.index, self._workspace_paths.local) self._should_transfer_workspace = self._execute_cmd( cmd, label='copy workspace check', check=False) != 0 self._define_remote_dirs() self._create_remote_dirs() self._copy_child_script() self._copy_dependencies_module() self._copy_workspace() self._working_dirs.local = pwd() self._working_dirs.remote = self._remote_working_dir self.logger.debug('Remote working path = %s', self._working_dirs.remote) self._push_files() self.setup_metadata.setup_script = self.cfg.setup_script self.setup_metadata.env = self.cfg.env self.setup_metadata.workspace_paths = self._workspace_paths
def _copy_workspace(self): """Copy the local workspace to remote host.""" self._workspace_paths.remote = '{}/{}'.format( self._remote_testplan_path, self._workspace_paths.local.split(os.sep)[-1]) if self.cfg.remote_workspace: # User defined the remote workspace to be used. # Make a soft link instead of copying workspace. self._execute_cmd(self.cfg.ssh_cmd( self.cfg.index, ' '.join( self.cfg.link_cmd(path=fix_home_prefix( self.cfg.remote_workspace), link=self._workspace_paths.remote))), label='linking to remote workspace (1).') elif self._should_transfer_workspace is True: # Workspace should be copied to remote. self._transfer_data(source=self._workspace_paths.local, target=self._remote_testplan_path, remote_target=True, exclude=self.cfg.workspace_exclude) # Mark that workspace pushed is safe to delete. Not some NFS. self.setup_metadata.workspace_pushed = True else: # Make a soft link instead of copying workspace. self._execute_cmd(self.cfg.ssh_cmd( self.cfg.index, ' '.join( self.cfg.link_cmd(path=self._workspace_paths.local, link=self._workspace_paths.remote))), label='linking to remote workspace (2).')
def _define_remote_dirs(self): """Define mandatory directories in remote host.""" self._remote_plan_runpath = self.cfg.remote_runpath or ( f"/var/tmp/{getpass.getuser()}/testplan/{self._get_plan().cfg.name}" if IS_WIN else self._get_plan().runpath) self._workspace_paths.local = fix_home_prefix( os.path.abspath(self.cfg.workspace)) self._workspace_paths.remote = "/".join( [self._remote_plan_runpath, "fetched_workspace"]) self._testplan_import_path.remote = "/".join( [self._remote_plan_runpath, "testplan_lib"]) self._remote_runid_file = os.path.join(self._remote_plan_runpath, self._get_plan().runid_filename) self._remote_resource_runpath = rebase_path( self.runpath, self._get_plan().runpath, self._remote_plan_runpath, ) self.logger.debug("Remote runpath = %s", self._remote_resource_runpath) self._working_dirs.local = pwd() self._working_dirs.remote = self._remote_working_dir() self.logger.debug("Remote working path = %s", self._working_dirs.remote)
def _copy_workspace(self): """Make the local workspace available on remote host.""" self._workspace_paths.local = fix_home_prefix(self.cfg.workspace) self._workspace_paths.remote = '{}/{}'.format( self._remote_testplan_path, self._workspace_paths.local.split(os.sep)[-1]) if self.cfg.remote_workspace: # User defined the remote workspace to be used # Make a soft link and return execute_cmd(self.cfg.ssh_cmd( self.ssh_cfg, ' '.join( self.cfg.link_cmd(path=fix_home_prefix( self.cfg.remote_workspace), link=self._workspace_paths.remote))), label='linking to remote workspace (1).', logger=self.logger) return copy = True # flag to make a copy of workspace to remote if self.cfg.copy_workspace_check: cmd = self.cfg.copy_workspace_check(self.cfg.ssh_cmd, self.ssh_cfg, self._workspace_paths.local) copy = execute_cmd(cmd, label='workspace availability check', check=False, logger=self.logger) != 0 if copy: # Workspace should be copied to remote. self._transfer_data(source=self._workspace_paths.local, target=self._remote_testplan_path, remote_target=True, exclude=self.cfg.workspace_exclude) # Mark that workspace pushed is safe to delete. Not some NFS. self.setup_metadata.workspace_pushed = True else: # Make a soft link instead of copying workspace. execute_cmd(self.cfg.ssh_cmd( self.ssh_cfg, ' '.join( self.cfg.link_cmd(path=self._workspace_paths.local, link=self._workspace_paths.remote))), label='linking to remote workspace (2).', logger=self.logger)
def schedule_tests_to_pool(plan, pool, schedule_path=None, **pool_cfg): pool_name = pool.__name__ # Enable debug: # from testplan.common.utils.logger import DEBUG # TESTPLAN_LOGGER.setLevel(DEBUG) pool = pool(name=pool_name, **pool_cfg) plan.add_resource(pool) if schedule_path is None: schedule_path = fix_home_prefix( os.path.dirname(os.path.abspath(__file__)) ) uids = [] for idx in range(1, 10): uids.append( plan.schedule( target="get_mtest", module="func_pool_base_tasks", path=schedule_path, kwargs=dict(name=idx), resource=pool_name, ) ) with log_propagation_disabled(TESTPLAN_LOGGER): res = plan.run() assert res.run is True assert res.success is True assert plan.report.passed is True assert plan.report.status == Status.PASSED # 1 testcase * 9 iterations assert plan.report.counter == {"passed": 9, "total": 9, "failed": 0} names = sorted(["MTest{}".format(x) for x in range(1, 10)]) assert sorted([entry.name for entry in plan.report.entries]) == names assert isinstance(plan.report.serialize(), dict) for idx in range(1, 10): name = "MTest{}".format(idx) assert plan.result.test_results[uids[idx - 1]].report.name == name # All tasks assigned once for uid in pool._task_retries_cnt: assert pool._task_retries_cnt[uid] == 0 assert pool.added_item(uid).reassign_cnt == 0
def _module_filepath(module): """ :param module: Module object - either a module itself of its modulefinder proxy. :type module: ``Union[module, modulefinder.module]`` :return: the normalised filepath to a module, or None if it has no __file__ attribute. :rtype: ``Optional[str]`` """ if not _has_file(module): return None ret_path = path_utils.fix_home_prefix(os.path.abspath(module.__file__)) if ret_path.endswith('c'): return ret_path[:-1] return ret_path
def schedule_tests_to_pool(name, pool, schedule_path=None, **pool_cfg): pool_name = pool.__name__ # Enable debug: # from testplan.common.utils.logger import DEBUG # TESTPLAN_LOGGER.setLevel(DEBUG) plan = Testplan( name=name, parse_cmdline=False, ) pool = pool(name=pool_name, **pool_cfg) plan.add_resource(pool) if schedule_path is None: schedule_path = fix_home_prefix( os.path.dirname(os.path.abspath(__file__))) uids = [] for idx in range(1, 10): uids.append(plan.schedule(target='get_mtest', module='func_pool_base_tasks', path=schedule_path, kwargs=dict(name=idx), resource=pool_name)) with log_propagation_disabled(TESTPLAN_LOGGER): res = plan.run() assert res.run is True assert res.success is True assert plan.report.passed is True assert plan.report.status == Status.PASSED assert plan.report.counts.passed == 9 # 1 testcase * 9 iterations assert (plan.report.counts.error == plan.report.counts.skipped == plan.report.counts.failed == plan.report.counts.incomplete == 0) names = sorted(['MTest{}'.format(x) for x in range(1, 10)]) assert sorted([entry.name for entry in plan.report.entries]) == names assert isinstance(plan.report.serialize(), dict) for idx in range(1, 10): name = 'MTest{}'.format(idx) assert plan.result.test_results[uids[idx-1]].report.name == name # All tasks scheduled once for uid in pool.task_assign_cnt: assert pool.task_assign_cnt[uid] == 1
def _extra_reload_dirs(self, deps): """ Build and return a set of reload directories used by extra dependencies. :param deps: Extra modules to add as dependencies of __main__. :type deps ``Iterable[ModuleType]`` :return: Reload directories of extra dependencies :rtype: ``set[str]`` """ for dep in deps: self.logger.debug( "Adding extra dependent %s: %s", "path" if isinstance(dep, str) else "module", dep, ) reload_dirs = set() for dep in deps: if isinstance(dep, str): dirpath = path_utils.fix_home_prefix(os.path.abspath(dep)) # Add it to `sys.path` for reloading if dirpath not in sys.path: sys.path.append(dirpath) reload_dirs.add(dirpath) else: filepath = _module_filepath(dep) dirpath = os.path.dirname(filepath) if filepath else None # Find the path where module or package can be imported if _has_package(dep): package = sys.modules.get(dep.__package__.split(".")[0]) if package is not None: filepath = _module_filepath(package) if filepath: dirpath = os.path.dirname( os.path.dirname(filepath)) if dirpath: # Even though this module has been imported, its directory # may have been removed from `sys.path`. In that case it # needs to be added back so the module can be reloaded. if dirpath not in sys.path: sys.path.append(dirpath) reload_dirs.add(dirpath) return reload_dirs
def schedule_tests_to_pool(plan, pool, schedule_path=None, **pool_cfg): pool_name = pool.__name__ pool = pool(name=pool_name, **pool_cfg) plan.add_resource(pool) if schedule_path is None: schedule_path = fix_home_prefix( os.path.dirname(os.path.abspath(__file__))) uids = [] for idx in range(1, 10): uids.append( plan.schedule( target="get_mtest", module="func_pool_base_tasks", path=schedule_path, kwargs=dict(name=idx), resource=pool_name, )) res = plan.run() assert res.run is True assert res.success is True assert plan.report.passed is True assert plan.report.status == Status.PASSED # 2 testcase * 9 iterations assert plan.report.counter == {"passed": 18, "total": 18, "failed": 0} names = sorted(["MTest{}".format(x) for x in range(1, 10)]) assert sorted([entry.name for entry in plan.report.entries]) == names assert isinstance(plan.report.serialize(), dict) for idx in range(1, 10): name = "MTest{}".format(idx) assert plan.result.test_results[uids[idx - 1]].report.name == name # check attachment exists in local assert os.path.exists( plan.report.entries[0].entries[0].entries[1].entries[0]["source_path"]) # All tasks assigned once for uid in pool._task_retries_cnt: assert pool._task_retries_cnt[uid] == 0 assert pool.added_item(uid).reassign_cnt == 0
def _remote_sys_path(self): sys_path = [self._testplan_import_path.remote] for path in sys.path: path = fix_home_prefix(path) if is_subdir(path, self._workspace_paths.local): path = rebase_path( path, self._workspace_paths.local, self._workspace_paths.remote, ) sys_path.append(path) return sys_path
def _proc_cmd(self): """Command to start child process.""" from testplan.common.utils.path import fix_home_prefix cmd = [ sys.executable, fix_home_prefix(self._child_path()), "--index", self.cfg.index, "--address", self.transport.address, "--type", "process_worker", "--log-level", TESTPLAN_LOGGER.getEffectiveLevel(), "--sys-path-file", self._syspath_file, ] return cmd
def _copy_workspace(self): """Make the local workspace available on remote host.""" if self.cfg.remote_workspace: # User defined the remote workspace to be used # Make a soft link and return self._execute_cmd_remote( cmd=link_cmd( path=fix_home_prefix(self.cfg.remote_workspace), link=self._workspace_paths.remote, ), label="linking to remote workspace (1).", ) return if 0 == self._execute_cmd_remote( cmd=filepath_exist_cmd(self._workspace_paths.local), label="workspace availability check", check=False, ): # exists on remote, make symlink self._execute_cmd_remote( cmd=link_cmd( path=self._workspace_paths.local, link=self._workspace_paths.remote, ), label="linking to remote workspace (2).", ) else: # copy to remote self._transfer_data( # join with "" to add trailing "/" to source # this will copy everything under local import path to to testplan_lib source=os.path.join(self._workspace_paths.local, ""), target=self._workspace_paths.remote, remote_target=True, exclude=self.cfg.workspace_exclude, )
def _prepare_workspace(self, exist_on_remote): """Make workspace available on remote host.""" if self.cfg.remote_workspace: self.logger.info( "User has specified workspace path on remote host, " "pointing runpath/fetched_workspace to it") # Make a soft link and return self._execute_cmd_remote( cmd=link_cmd( path=fix_home_prefix(self.cfg.remote_workspace), link=self._workspace_paths.remote, ), label="linking to remote workspace (1).", ) return if exist_on_remote: self.logger.info( "Local workspace path is accessible on %s, " "pointing runpath/fetched_workspace to it", self.ssh_cfg["host"], ) self._execute_cmd_remote( cmd=link_cmd( path=self._workspace_paths.local, link=self._workspace_paths.remote, ), label="linking to remote workspace (2).", ) else: # copy to remote self.logger.info( "Local workspace path is inaccessible on %s, " "Copying it to remote runpath/fetched_workspace", self.ssh_cfg["host"], ) self._transfer_data( # join with "" to add trailing "/" to source # this will copy everything under local import path to to testplan_lib source=os.path.join(self._workspace_paths.local, ""), target=self._workspace_paths.remote, remote_target=True, exclude=self.cfg.workspace_exclude, ) self.logger.info( "Creating symlink to imitate local workspace path on %s, " "pointing to runpath/fetched_workspace", self.ssh_cfg["host"], ) self._execute_cmd_remote( cmd=mkdir_cmd(os.path.dirname(self._workspace_paths.local)), label="imitate local workspace path on remote - mkdir", check=False, # just best effort ) self._execute_cmd_remote( cmd=link_cmd( path=self._workspace_paths.remote, link=self._workspace_paths.local, ), label="imitate local workspace path on remote - ln", check=False, # just best effort )
def _module_filepath(filepath): ret_path = fix_home_prefix(os.path.abspath(filepath)) if ret_path.endswith('c'): return ret_path[:-1] return ret_path