def _build_push_dests(self, push_sources): """ When the destination paths have not been explicitly specified, build them automatically. If an absolute path is given on Linux, we will push to the same path on remote. If on windows or a relative path is given on Linux, we will push to the save relative path in respect to working directory. """ push_locations = [] if IS_WIN: for entry in push_sources: if isinstance(entry, str): src = entry dst = rebase_path( src, self._working_dirs.local, self._working_dirs.remote, ) else: src, dst = entry if not os.path.isabs(dst): dst = rebase_path( dst, self._working_dirs.local, self._working_dirs.remote, ) push_locations.append((src, dst)) else: for entry in push_sources: if isinstance(entry, str): src = entry if os.path.isabs(src): dst = src else: dst = os.path.join(self._working_dirs.remote, src) else: src, dst = entry if not os.path.isabs(dst): dst = os.path.join( self._working_dirs.remote, os.path.relpath(dst, self._working_dirs.local), ) push_locations.append((src, dst)) return push_locations
def workspace(): """ Sets up the workspace to use for testing the remote resource. We will copy the tests subdir to workspace, and remote resource logic will copy the workspace to remote host. :return: paths to the workspace """ # Set up the workspace in a temporary directory. workspace = tempfile.mkdtemp() # Copy the tests dir. "tests" is a generic name so we don't want to # rely "import tests" here - instead navigate up to find the tests dir. script_dir = os.path.dirname(__file__) orig_tests_dir = script_dir while os.path.basename(orig_tests_dir) != "tests": orig_tests_dir = os.path.abspath( os.path.join(orig_tests_dir, os.pardir)) shutil.copytree(orig_tests_dir, os.path.join(workspace, "tests")) working_dir = rebase_path(script_dir, orig_tests_dir, os.path.join(workspace, "tests")) orig_dir = os.getcwd() os.chdir(working_dir) yield workspace os.chdir(orig_dir) shutil.rmtree(workspace)
def test_fetch_results(remote_resource, push_dir): log_file = "/".join( [remote_resource._remote_resource_runpath, "remote.log"]) remote_resource._execute_cmd_remote( cmd=f"/bin/touch {log_file}", label="create log file", ) remote_resource._fetch_results() log_file_local = rebase_path( log_file, remote_resource._remote_plan_runpath, remote_resource.parent.runpath, ) assert os.path.exists(log_file_local) assert os.path.exists( os.path.join( remote_resource.runpath, "pulled_files", os.path.basename(push_dir), "file1", )) assert not os.path.exists( os.path.join( remote_resource.runpath, "pulled_files", os.path.basename(push_dir), "file2", ))
def _define_remote_dirs(self): """Define mandatory directories in remote host.""" self._remote_plan_runpath = self.cfg.remote_runpath or ( f"/var/tmp/{getpass.getuser()}/testplan/{self._get_plan().cfg.name}" if IS_WIN else self._get_plan().runpath) self._workspace_paths.local = fix_home_prefix( os.path.abspath(self.cfg.workspace)) self._workspace_paths.remote = "/".join( [self._remote_plan_runpath, "fetched_workspace"]) self._testplan_import_path.remote = "/".join( [self._remote_plan_runpath, "testplan_lib"]) self._remote_runid_file = os.path.join(self._remote_plan_runpath, self._get_plan().runid_filename) self._remote_resource_runpath = rebase_path( self.runpath, self._get_plan().runpath, self._remote_plan_runpath, ) self.logger.debug("Remote runpath = %s", self._remote_resource_runpath) self._working_dirs.local = pwd() self._working_dirs.remote = self._remote_working_dir() self.logger.debug("Remote working path = %s", self._working_dirs.remote)
def test_prepare_remote(remote_resource, workspace, push_dir): remote_resource.make_runpath_dirs() remote_resource._prepare_remote() assert ( remote_resource._remote_plan_runpath == remote_resource.parent.runpath ) assert remote_resource._remote_resource_runpath == remote_resource.runpath assert remote_resource._remote_runid_file == "/".join( [ remote_resource._remote_plan_runpath, remote_resource.parent.runid_filename, ] ) assert remote_resource._workspace_paths.remote == "/".join( [remote_resource._remote_plan_runpath, "fetched_workspace"] ) assert remote_resource._working_dirs.remote == rebase_path( os.getcwd(), workspace, remote_resource._workspace_paths.remote, ) for remote_path in [ remote_resource._remote_runid_file, "/".join([remote_resource._workspace_paths.remote, "tests", "unit"]), "/".join([push_dir, "file1"]), "/".join([push_dir, "file1_ln"]), "/".join([remote_resource._working_dirs.remote, "file1"]), "/".join([workspace, "file2"]), ]: assert 0 == remote_resource._execute_cmd_remote( cmd=filepath_exist_cmd(remote_path), check=False, ) for remote_path in [ "/".join( [remote_resource._workspace_paths.remote, "tests", "functional"] ), "/".join([push_dir, "file3"]), ]: assert 0 != remote_resource._execute_cmd_remote( cmd=filepath_exist_cmd(remote_path), check=False, ) # for now, these setting are used by child.py rather than remote_resource assert remote_resource.setup_metadata.env == { "LOCAL_USER": getpass.getuser() } assert remote_resource.setup_metadata.setup_script == ["remote_setup.py"] assert remote_resource.setup_metadata.push_dirs == [push_dir] assert remote_resource.setup_metadata.push_files == [ "/".join([remote_resource._working_dirs.remote, "file1"]), "/".join([workspace, "file2"]), ] remote_resource._clean_remote()
def _set_child_script(self): """Specify the remote worker executable file.""" self._child_paths.local = self._child_path() self._child_paths.remote = rebase_path( self._child_paths.local, self._testplan_import_path.local, self._testplan_import_path.remote, )
def rebase_path(self, local, remote): """adapt task's path for remote execution if necessary""" if os.path.isabs(self._path): self._rebased_path = rebase_path( self._path, local, remote, )
def rebase_attachment(self, result): """Rebase the path of attachment from remote to local""" if result: for attachment in result.report.attachments: attachment.source_path = rebase_path( attachment.source_path, self._remote_plan_runpath, self._get_plan().runpath, )
def _remote_sys_path(self): sys_path = [self._testplan_import_path.remote] for path in sys.path: path = fix_home_prefix(path) if is_subdir(path, self._workspace_paths.local): path = rebase_path( path, self._workspace_paths.local, self._workspace_paths.remote, ) sys_path.append(path) return sys_path
def _remote_working_dir(self): """Choose a working directory to use on the remote host.""" if not is_subdir(self._working_dirs.local, self._workspace_paths.local): raise RuntimeError( "Current working dir is not within the workspace.\n" "Workspace = {ws}\n" "Working dir = {cwd}".format( ws=self._workspace_paths.local, cwd=self._working_dirs.local, )) # Current working directory is within the workspace - use the same # path relative to the remote workspace. return rebase_path( self._working_dirs.local, self._workspace_paths.local, self._workspace_paths.remote, )
def _handle_taskresults(self, worker, request, response): """Handle a TaskResults message from a worker.""" def task_should_rerun(): if not self.cfg.allow_task_rerun: return False if not task_result.task: return False if task_result.task.rerun == 0: return False result = task_result.result if ( task_result.status and result and result.run and result.report.passed ): return False if task_result.task.reassign_cnt >= task_result.task.rerun: self.logger.test_info( "Will not rerun %(input)s again as it already " "reached max rerun limit %(reruns)d", { "input": self._input[uid], "reruns": task_result.task.rerun, }, ) return False return True worker.respond(response.make(Message.Ack)) for task_result in request.data: uid = task_result.task.uid() worker.assigned.remove(uid) self._workers_last_result.setdefault(worker, time.time()) self.logger.test_info( "De-assign {} from {}".format(task_result.task, worker) ) if task_result.result and isinstance(worker, RemoteResource): for attachment in task_result.result.report.attachments: attachment.source_path = rebase_path( attachment.source_path, worker._remote_plan_runpath, worker._get_plan().runpath, ) if task_should_rerun(): self.logger.test_info( "Will rerun %(task)s for max %(rerun)d more times", { "task": task_result.task, "rerun": task_result.task.rerun - task_result.task.reassign_cnt, }, ) self.unassigned.put(task_result.task.priority, uid) self._task_retries_cnt[uid] = 0 self._input[uid].reassign_cnt += 1 # Will rerun task, but still need to retain the result self._append_temporary_task_result(task_result) continue self._print_test_result(task_result) self._results[uid] = task_result self.ongoing.remove(uid)