def _proc_cmd(self):
"""Command to start child process."""
from testplan.common.utils.path import fix_home_prefix
cmd = [
sys.executable,
fix_home_prefix(self._child_path()),
"--index",
self.cfg.index,
"--address",
self.transport.address,
"--testplan",
os.path.join(os.path.dirname(testplan.__file__), ".."),
"--type",
"process_worker",
"--log-level",
TESTPLAN_LOGGER.getEffectiveLevel(),
"--sys-path-file",
self._write_syspath(),
]
if os.environ.get(testplan.TESTPLAN_DEPENDENCIES_PATH):
cmd.extend([
"--testplan-deps",
fix_home_prefix(
os.environ[testplan.TESTPLAN_DEPENDENCIES_PATH]),
])
return cmd
def schedule_tests_to_pool(name, pool, **pool_cfg):
pool_name = pool.__name__
# Enable debug:
# from testplan.logger import DEBUG
# TESTPLAN_LOGGER.setLevel(DEBUG)
plan = Testplan(name=name, parse_cmdline=False)
pool = pool(name=pool_name, **pool_cfg)
plan.add_resource(pool)
this_file = fix_home_prefix(os.path.dirname(os.path.abspath(__file__)))
if pool_cfg.get('workspace'): # Remote pool
ws = pool_cfg['workspace']
wd = fix_home_prefix(os.getcwd())
common = os.path.commonprefix([pool_cfg['workspace'], wd])
# Relative path tp the same file but from the remote workspace.
path = '/'.join(
os.path.join(os.path.relpath(common, wd),
os.path.relpath(this_file, ws)).split(os.sep))
else:
path = this_file
uids = []
for idx in range(1, 10):
uids.append(
plan.schedule(target='get_mtest',
module='func_pool_base_tasks',
path=path,
kwargs=dict(name=idx),
resource=pool_name))
with log_propagation_disabled(TESTPLAN_LOGGER):
res = plan.run()
assert res.run is True
assert res.success is True
assert plan.report.passed is True
assert plan.report.status == Status.PASSED
assert plan.report.counts.passed == 9
assert plan.report.counts.error == plan.report.counts.skipped == \
plan.report.counts.failed == plan.report.counts.incomplete == 0
names = sorted(['MTest{}'.format(x) for x in range(1, 10)])
assert sorted([entry.name for entry in plan.report.entries]) == names
assert isinstance(plan.report.serialize(), dict)
for idx in range(1, 10):
name = 'MTest{}'.format(idx)
assert plan.result.test_results[uids[idx - 1]].report.name == name
# All tasks scheduled once
for uid in pool.task_assign_cnt:
assert pool.task_assign_cnt[uid] == 1
def _proc_cmd(self):
"""Command to start child process."""
from testplan.common.utils.path import fix_home_prefix
cmd = [sys.executable, fix_home_prefix(self._child_path()),
'--index', self.cfg.index,
'--address', self.transport.address,
'--testplan', os.path.join(os.path.dirname(testplan.__file__),
'..'),
'--type', 'process_worker',
'--log-level', TESTPLAN_LOGGER.getEffectiveLevel()]
if os.environ.get(testplan.TESTPLAN_DEPENDENCIES_PATH):
cmd.extend(
['--testplan-deps', fix_home_prefix(
os.environ[testplan.TESTPLAN_DEPENDENCIES_PATH])])
return cmd
def _build_dependencies(self, extra_deps):
"""
Build a list of directories to reload code from and a tree of
dependencies.
:param extra_deps: Modules to register as extra dependencies to reload,
despite not being directly imported by __main__.
:type extra_deps: ``Iterable[ModuleType]``
"""
main_module_file = sys.modules['__main__'].__file__
if not main_module_file:
raise RuntimeError(
"Can only use interactive reloader when the __main__ module "
"is a file.")
reload_dir = os.path.realpath(os.path.dirname(main_module_file))
reload_dirs = {path_utils.fix_home_prefix(reload_dir)}
# Add extra reload source directories if required.
if extra_deps:
reload_dirs = reload_dirs.union(
self._extra_reload_dirs(extra_deps))
dep_graph, watched_modules = self._build_dep_graph(
main_module_file, reload_dirs)
return reload_dirs, dep_graph, watched_modules
def _get_reload_dirs(self, extra_deps=None):
reload_dirs = []
if _has_file(sys.modules['__main__']):
main_module_file = self._module_filepath(
sys.modules['__main__'].__file__)
reload_dir = os.path.realpath(os.path.dirname(main_module_file))
else:
reload_dir = os.path.realpath(pwd())
reload_dirs.append(fix_home_prefix(reload_dir))
# Add extra reload source directories.
for mod in extra_deps:
module_file = self._module_filepath(mod.__file__)
reload_dir = os.path.realpath(os.path.dirname(module_file))
reload_dirs.append(fix_home_prefix(reload_dir))
return reload_dirs
def _check_workspace(self):
"""
Check if workspace is available on remote host
:return: True if exsit, false otherwise
"""
if self.cfg.remote_workspace:
# User defined the remote workspace to be used
# will raise if check fail
if 0 == self._execute_cmd_remote(
cmd=filepath_exist_cmd(
fix_home_prefix(self.cfg.remote_workspace)),
label="workspace availability check (1)",
check=True,
):
return True
if 0 == self._execute_cmd_remote(
cmd=filepath_exist_cmd(self._workspace_paths.local),
label="workspace availability check (2)",
check=False,
):
# local workspace accessible on remote
return True
return False
def _prepare_remote(self):
"""Transfer local data to remote host."""
self._child_paths.local = self._child_path()
self._workspace_paths.local = fix_home_prefix(self.cfg.workspace)
if self.cfg.copy_workspace_check:
cmd = self.cfg.copy_workspace_check(self.cfg.ssh_cmd,
self.cfg.index,
self._workspace_paths.local)
self._should_transfer_workspace = self._execute_cmd(
cmd, label='copy workspace check', check=False) != 0
self._define_remote_dirs()
self._create_remote_dirs()
self._copy_child_script()
self._copy_dependencies_module()
self._copy_workspace()
self._working_dirs.local = pwd()
self._working_dirs.remote = self._remote_working_dir
self.logger.debug('Remote working path = %s',
self._working_dirs.remote)
self._push_files()
self.setup_metadata.setup_script = self.cfg.setup_script
self.setup_metadata.env = self.cfg.env
self.setup_metadata.workspace_paths = self._workspace_paths
def _copy_workspace(self):
"""Copy the local workspace to remote host."""
self._workspace_paths.remote = '{}/{}'.format(
self._remote_testplan_path,
self._workspace_paths.local.split(os.sep)[-1])
if self.cfg.remote_workspace:
# User defined the remote workspace to be used.
# Make a soft link instead of copying workspace.
self._execute_cmd(self.cfg.ssh_cmd(
self.cfg.index, ' '.join(
self.cfg.link_cmd(path=fix_home_prefix(
self.cfg.remote_workspace),
link=self._workspace_paths.remote))),
label='linking to remote workspace (1).')
elif self._should_transfer_workspace is True:
# Workspace should be copied to remote.
self._transfer_data(source=self._workspace_paths.local,
target=self._remote_testplan_path,
remote_target=True,
exclude=self.cfg.workspace_exclude)
# Mark that workspace pushed is safe to delete. Not some NFS.
self.setup_metadata.workspace_pushed = True
else:
# Make a soft link instead of copying workspace.
self._execute_cmd(self.cfg.ssh_cmd(
self.cfg.index, ' '.join(
self.cfg.link_cmd(path=self._workspace_paths.local,
link=self._workspace_paths.remote))),
label='linking to remote workspace (2).')
def _define_remote_dirs(self):
"""Define mandatory directories in remote host."""
self._remote_plan_runpath = self.cfg.remote_runpath or (
f"/var/tmp/{getpass.getuser()}/testplan/{self._get_plan().cfg.name}"
if IS_WIN else self._get_plan().runpath)
self._workspace_paths.local = fix_home_prefix(
os.path.abspath(self.cfg.workspace))
self._workspace_paths.remote = "/".join(
[self._remote_plan_runpath, "fetched_workspace"])
self._testplan_import_path.remote = "/".join(
[self._remote_plan_runpath, "testplan_lib"])
self._remote_runid_file = os.path.join(self._remote_plan_runpath,
self._get_plan().runid_filename)
self._remote_resource_runpath = rebase_path(
self.runpath,
self._get_plan().runpath,
self._remote_plan_runpath,
)
self.logger.debug("Remote runpath = %s", self._remote_resource_runpath)
self._working_dirs.local = pwd()
self._working_dirs.remote = self._remote_working_dir()
self.logger.debug("Remote working path = %s",
self._working_dirs.remote)
def _copy_workspace(self):
"""Make the local workspace available on remote host."""
self._workspace_paths.local = fix_home_prefix(self.cfg.workspace)
self._workspace_paths.remote = '{}/{}'.format(
self._remote_testplan_path,
self._workspace_paths.local.split(os.sep)[-1])
if self.cfg.remote_workspace:
# User defined the remote workspace to be used
# Make a soft link and return
execute_cmd(self.cfg.ssh_cmd(
self.ssh_cfg, ' '.join(
self.cfg.link_cmd(path=fix_home_prefix(
self.cfg.remote_workspace),
link=self._workspace_paths.remote))),
label='linking to remote workspace (1).',
logger=self.logger)
return
copy = True # flag to make a copy of workspace to remote
if self.cfg.copy_workspace_check:
cmd = self.cfg.copy_workspace_check(self.cfg.ssh_cmd, self.ssh_cfg,
self._workspace_paths.local)
copy = execute_cmd(cmd,
label='workspace availability check',
check=False,
logger=self.logger) != 0
if copy:
# Workspace should be copied to remote.
self._transfer_data(source=self._workspace_paths.local,
target=self._remote_testplan_path,
remote_target=True,
exclude=self.cfg.workspace_exclude)
# Mark that workspace pushed is safe to delete. Not some NFS.
self.setup_metadata.workspace_pushed = True
else:
# Make a soft link instead of copying workspace.
execute_cmd(self.cfg.ssh_cmd(
self.ssh_cfg, ' '.join(
self.cfg.link_cmd(path=self._workspace_paths.local,
link=self._workspace_paths.remote))),
label='linking to remote workspace (2).',
logger=self.logger)
def schedule_tests_to_pool(plan, pool, schedule_path=None, **pool_cfg):
pool_name = pool.__name__
# Enable debug:
# from testplan.common.utils.logger import DEBUG
# TESTPLAN_LOGGER.setLevel(DEBUG)
pool = pool(name=pool_name, **pool_cfg)
plan.add_resource(pool)
if schedule_path is None:
schedule_path = fix_home_prefix(
os.path.dirname(os.path.abspath(__file__))
)
uids = []
for idx in range(1, 10):
uids.append(
plan.schedule(
target="get_mtest",
module="func_pool_base_tasks",
path=schedule_path,
kwargs=dict(name=idx),
resource=pool_name,
)
)
with log_propagation_disabled(TESTPLAN_LOGGER):
res = plan.run()
assert res.run is True
assert res.success is True
assert plan.report.passed is True
assert plan.report.status == Status.PASSED
# 1 testcase * 9 iterations
assert plan.report.counter == {"passed": 9, "total": 9, "failed": 0}
names = sorted(["MTest{}".format(x) for x in range(1, 10)])
assert sorted([entry.name for entry in plan.report.entries]) == names
assert isinstance(plan.report.serialize(), dict)
for idx in range(1, 10):
name = "MTest{}".format(idx)
assert plan.result.test_results[uids[idx - 1]].report.name == name
# All tasks assigned once
for uid in pool._task_retries_cnt:
assert pool._task_retries_cnt[uid] == 0
assert pool.added_item(uid).reassign_cnt == 0
def _module_filepath(module):
"""
:param module: Module object - either a module itself of its modulefinder
proxy.
:type module: ``Union[module, modulefinder.module]``
:return: the normalised filepath to a module, or None if it has no __file__
attribute.
:rtype: ``Optional[str]``
"""
if not _has_file(module):
return None
ret_path = path_utils.fix_home_prefix(os.path.abspath(module.__file__))
if ret_path.endswith('c'):
return ret_path[:-1]
return ret_path
def schedule_tests_to_pool(name, pool, schedule_path=None, **pool_cfg):
pool_name = pool.__name__
# Enable debug:
# from testplan.common.utils.logger import DEBUG
# TESTPLAN_LOGGER.setLevel(DEBUG)
plan = Testplan(
name=name,
parse_cmdline=False,
)
pool = pool(name=pool_name, **pool_cfg)
plan.add_resource(pool)
if schedule_path is None:
schedule_path = fix_home_prefix(
os.path.dirname(os.path.abspath(__file__)))
uids = []
for idx in range(1, 10):
uids.append(plan.schedule(target='get_mtest',
module='func_pool_base_tasks',
path=schedule_path, kwargs=dict(name=idx),
resource=pool_name))
with log_propagation_disabled(TESTPLAN_LOGGER):
res = plan.run()
assert res.run is True
assert res.success is True
assert plan.report.passed is True
assert plan.report.status == Status.PASSED
assert plan.report.counts.passed == 9 # 1 testcase * 9 iterations
assert (plan.report.counts.error == plan.report.counts.skipped ==
plan.report.counts.failed == plan.report.counts.incomplete == 0)
names = sorted(['MTest{}'.format(x) for x in range(1, 10)])
assert sorted([entry.name for entry in plan.report.entries]) == names
assert isinstance(plan.report.serialize(), dict)
for idx in range(1, 10):
name = 'MTest{}'.format(idx)
assert plan.result.test_results[uids[idx-1]].report.name == name
# All tasks scheduled once
for uid in pool.task_assign_cnt:
assert pool.task_assign_cnt[uid] == 1
def _extra_reload_dirs(self, deps):
"""
Build and return a set of reload directories used by extra
dependencies.
:param deps: Extra modules to add as dependencies of __main__.
:type deps ``Iterable[ModuleType]``
:return: Reload directories of extra dependencies
:rtype: ``set[str]``
"""
for dep in deps:
self.logger.debug(
"Adding extra dependent %s: %s",
"path" if isinstance(dep, str) else "module",
dep,
)
reload_dirs = set()
for dep in deps:
if isinstance(dep, str):
dirpath = path_utils.fix_home_prefix(os.path.abspath(dep))
# Add it to `sys.path` for reloading
if dirpath not in sys.path:
sys.path.append(dirpath)
reload_dirs.add(dirpath)
else:
filepath = _module_filepath(dep)
dirpath = os.path.dirname(filepath) if filepath else None
# Find the path where module or package can be imported
if _has_package(dep):
package = sys.modules.get(dep.__package__.split(".")[0])
if package is not None:
filepath = _module_filepath(package)
if filepath:
dirpath = os.path.dirname(
os.path.dirname(filepath))
if dirpath:
# Even though this module has been imported, its directory
# may have been removed from `sys.path`. In that case it
# needs to be added back so the module can be reloaded.
if dirpath not in sys.path:
sys.path.append(dirpath)
reload_dirs.add(dirpath)
return reload_dirs
def schedule_tests_to_pool(plan, pool, schedule_path=None, **pool_cfg):
pool_name = pool.__name__
pool = pool(name=pool_name, **pool_cfg)
plan.add_resource(pool)
if schedule_path is None:
schedule_path = fix_home_prefix(
os.path.dirname(os.path.abspath(__file__)))
uids = []
for idx in range(1, 10):
uids.append(
plan.schedule(
target="get_mtest",
module="func_pool_base_tasks",
path=schedule_path,
kwargs=dict(name=idx),
resource=pool_name,
))
res = plan.run()
assert res.run is True
assert res.success is True
assert plan.report.passed is True
assert plan.report.status == Status.PASSED
# 2 testcase * 9 iterations
assert plan.report.counter == {"passed": 18, "total": 18, "failed": 0}
names = sorted(["MTest{}".format(x) for x in range(1, 10)])
assert sorted([entry.name for entry in plan.report.entries]) == names
assert isinstance(plan.report.serialize(), dict)
for idx in range(1, 10):
name = "MTest{}".format(idx)
assert plan.result.test_results[uids[idx - 1]].report.name == name
# check attachment exists in local
assert os.path.exists(
plan.report.entries[0].entries[0].entries[1].entries[0]["source_path"])
# All tasks assigned once
for uid in pool._task_retries_cnt:
assert pool._task_retries_cnt[uid] == 0
assert pool.added_item(uid).reassign_cnt == 0
def _remote_sys_path(self):
sys_path = [self._testplan_import_path.remote]
for path in sys.path:
path = fix_home_prefix(path)
if is_subdir(path, self._workspace_paths.local):
path = rebase_path(
path,
self._workspace_paths.local,
self._workspace_paths.remote,
)
sys_path.append(path)
return sys_path
def _proc_cmd(self):
"""Command to start child process."""
from testplan.common.utils.path import fix_home_prefix
cmd = [
sys.executable,
fix_home_prefix(self._child_path()),
"--index",
self.cfg.index,
"--address",
self.transport.address,
"--type",
"process_worker",
"--log-level",
TESTPLAN_LOGGER.getEffectiveLevel(),
"--sys-path-file",
self._syspath_file,
]
return cmd
def _copy_workspace(self):
"""Make the local workspace available on remote host."""
if self.cfg.remote_workspace:
# User defined the remote workspace to be used
# Make a soft link and return
self._execute_cmd_remote(
cmd=link_cmd(
path=fix_home_prefix(self.cfg.remote_workspace),
link=self._workspace_paths.remote,
),
label="linking to remote workspace (1).",
)
return
if 0 == self._execute_cmd_remote(
cmd=filepath_exist_cmd(self._workspace_paths.local),
label="workspace availability check",
check=False,
):
# exists on remote, make symlink
self._execute_cmd_remote(
cmd=link_cmd(
path=self._workspace_paths.local,
link=self._workspace_paths.remote,
),
label="linking to remote workspace (2).",
)
else:
# copy to remote
self._transfer_data(
# join with "" to add trailing "/" to source
# this will copy everything under local import path to to testplan_lib
source=os.path.join(self._workspace_paths.local, ""),
target=self._workspace_paths.remote,
remote_target=True,
exclude=self.cfg.workspace_exclude,
)
def _prepare_workspace(self, exist_on_remote):
"""Make workspace available on remote host."""
if self.cfg.remote_workspace:
self.logger.info(
"User has specified workspace path on remote host, "
"pointing runpath/fetched_workspace to it")
# Make a soft link and return
self._execute_cmd_remote(
cmd=link_cmd(
path=fix_home_prefix(self.cfg.remote_workspace),
link=self._workspace_paths.remote,
),
label="linking to remote workspace (1).",
)
return
if exist_on_remote:
self.logger.info(
"Local workspace path is accessible on %s, "
"pointing runpath/fetched_workspace to it",
self.ssh_cfg["host"],
)
self._execute_cmd_remote(
cmd=link_cmd(
path=self._workspace_paths.local,
link=self._workspace_paths.remote,
),
label="linking to remote workspace (2).",
)
else:
# copy to remote
self.logger.info(
"Local workspace path is inaccessible on %s, "
"Copying it to remote runpath/fetched_workspace",
self.ssh_cfg["host"],
)
self._transfer_data(
# join with "" to add trailing "/" to source
# this will copy everything under local import path to to testplan_lib
source=os.path.join(self._workspace_paths.local, ""),
target=self._workspace_paths.remote,
remote_target=True,
exclude=self.cfg.workspace_exclude,
)
self.logger.info(
"Creating symlink to imitate local workspace path on %s, "
"pointing to runpath/fetched_workspace",
self.ssh_cfg["host"],
)
self._execute_cmd_remote(
cmd=mkdir_cmd(os.path.dirname(self._workspace_paths.local)),
label="imitate local workspace path on remote - mkdir",
check=False, # just best effort
)
self._execute_cmd_remote(
cmd=link_cmd(
path=self._workspace_paths.remote,
link=self._workspace_paths.local,
),
label="imitate local workspace path on remote - ln",
check=False, # just best effort
)
def _module_filepath(filepath):
ret_path = fix_home_prefix(os.path.abspath(filepath))
if ret_path.endswith('c'):
return ret_path[:-1]
return ret_path
