def __init__(self, session, path, python, requirements_manager): # type: (Session, PathLike, float, RequirementsManager) -> None """ :param python: base python version to use (e.g python3.6) :param path: path of env """ self.session = session self.python = python self.source = None self.requirements_manager = requirements_manager self.path = path self.extra_channels = self.session.config.get( 'agent.package_manager.conda_channels', []) self.pip = CondaPip( session=self.session, source=self.source, python=self.python, requirements_manager=self.requirements_manager, path=self.path, ) self.conda = (find_executable("conda") or Argv( select_for_platform(windows="where", linux="which"), "conda").get_output(shell=True).strip()) try: output = Argv(self.conda, "--version").get_output() except subprocess.CalledProcessError as ex: raise CommandFailedError( "Unable to determine conda version: {ex}, output={ex.output}". format(ex=ex)) self.conda_version = self.get_conda_version(output) if Version(self.conda_version, partial=True) < self.MINIMUM_VERSION: raise CommandFailedError( "conda version '{}' is smaller than minimum supported conda version '{}'" .format(self.conda_version, self.MINIMUM_VERSION))
def __init__(self, session, path, python, requirements_manager, execution_info=None, **kwargs): # type: (Session, PathLike, float, RequirementsManager, ExecutionInfo, Any) -> None """ :param python: base python version to use (e.g python3.6) :param path: path of env """ self.session = session self.python = python self.source = None self.requirements_manager = requirements_manager self.path = path self.env_read_only = False self.extra_channels = self.session.config.get( 'agent.package_manager.conda_channels', []) self.conda_env_as_base_docker = \ self.session.config.get('agent.package_manager.conda_env_as_base_docker', None) or \ bool(ENV_CONDA_ENV_PACKAGE.get()) if ENV_CONDA_ENV_PACKAGE.get(): self.conda_pre_build_env_path = ENV_CONDA_ENV_PACKAGE.get() else: self.conda_pre_build_env_path = execution_info.docker_cmd if execution_info else None self.pip = CondaPip( session=self.session, source=self.source, python=self.python, requirements_manager=self.requirements_manager, path=self.path, ) try: self.conda = (find_executable("conda") or Argv( select_for_platform(windows="where", linux="which"), "conda").get_output(shell=select_for_platform( windows=True, linux=False)).strip()) except Exception: raise ValueError("ERROR: package manager \"conda\" selected, " "but \'conda\' executable could not be located") try: output = Argv(self.conda, "--version").get_output(stderr=subprocess.STDOUT) except subprocess.CalledProcessError as ex: raise CommandFailedError( "Unable to determine conda version: {ex}, output={ex.output}". format(ex=ex)) self.conda_version = self.get_conda_version(output) if SimpleVersion.compare_versions(self.conda_version, '<', self.MINIMUM_VERSION): raise CommandFailedError( "conda version '{}' is smaller than minimum supported conda version '{}'" .format(self.conda_version, self.MINIMUM_VERSION))
def clone_repository_cached(session, execution, destination): # type: (Session, ExecutionInfo, Path) -> Tuple[VCS, RepoInfo] """ Clone a remote repository. :param execution: execution info :param destination: directory to clone to (in which a directory for the repository will be created) :param session: program session :return: repository information :raises: CommandFailedError if git/hg is not installed """ repo_url = execution.repository # type: str parsed_url = furl(repo_url) no_password_url = parsed_url.copy().remove(password=True).url clone_folder_name = Path(str(furl(repo_url).path)).name # type: str clone_folder = Path(destination) / clone_folder_name cached_repo_path = ( Path(session.config["agent.vcs_cache.path"]).expanduser() / "{}.{}".format(clone_folder_name, md5(ensure_binary(repo_url)).hexdigest()) / clone_folder_name) # type: Path vcs = VcsFactory.create(session, execution_info=execution, location=cached_repo_path) if not find_executable(vcs.executable_name): raise CommandFailedError(vcs.executable_not_found_error_help()) if session.config["agent.vcs_cache.enabled"] and cached_repo_path.exists(): print('Using cached repository in "{}"'.format(cached_repo_path)) else: print("cloning: {}".format(no_password_url)) rm_tree(cached_repo_path) # We clone the entire repository, not a specific branch vcs.clone() # branch=execution.branch) vcs.pull() rm_tree(destination) shutil.copytree(Text(cached_repo_path), Text(clone_folder)) if not clone_folder.is_dir(): raise CommandFailedError( "copying of repository failed: from {} to {}".format( cached_repo_path, clone_folder)) # checkout in the newly copy destination vcs.location = Text(clone_folder) vcs.checkout() repo_info = vcs.get_repository_copy_info(clone_folder) # make sure we have no user/pass in the returned repository structure repo_info = attr.evolve(repo_info, url=no_password_url) return vcs, repo_info
def check_directory_path(path, check_whitespace_in_path=True): message = 'Could not create directory "{}": {}' if not is_windows_platform() and check_whitespace_in_path: match = re.search(r'\s', path) if match: raise CommandFailedError( 'directories may not contain whitespace (char: {!r}, position: {})'.format(match.group(0), match.endpos)) try: Path(os.path.expandvars(path)).expanduser().mkdir(parents=True, exist_ok=True) except OSError as e: raise CommandFailedError(message.format(path, e.strerror)) except Exception as e: raise CommandFailedError(message.format(path, e))
def from_task(cls, task_info): # type: (...) -> ExecutionInfo """ extract ExecutionInfo tuple from task parameters """ if not task_info.script: raise CommandFailedError( "can not run task without script information") execution = cls.from_dict(task_info.script.to_dict()) if not execution.entry_point: log.warning("notice: `script.entry_point` is empty") if not execution.working_dir: entry_point, _, working_dir = execution.entry_point.partition(":") execution.entry_point = entry_point execution.working_dir = working_dir or "" return execution
def get_conda_version(output): match = re.search(r"(\d+\.){0,2}\d+", output) if not match: raise CommandFailedError("Unidentified conda version string:", output) return match.group(0)