예제 #1
0
 def __init__(self, session, path, python, requirements_manager):
     # type: (Session, PathLike, float, RequirementsManager) -> None
     """
     :param python: base python version to use (e.g python3.6)
     :param path: path of env
     """
     self.session = session
     self.python = python
     self.source = None
     self.requirements_manager = requirements_manager
     self.path = path
     self.extra_channels = self.session.config.get(
         'agent.package_manager.conda_channels', [])
     self.pip = CondaPip(
         session=self.session,
         source=self.source,
         python=self.python,
         requirements_manager=self.requirements_manager,
         path=self.path,
     )
     self.conda = (find_executable("conda") or Argv(
         select_for_platform(windows="where", linux="which"),
         "conda").get_output(shell=True).strip())
     try:
         output = Argv(self.conda, "--version").get_output()
     except subprocess.CalledProcessError as ex:
         raise CommandFailedError(
             "Unable to determine conda version: {ex}, output={ex.output}".
             format(ex=ex))
     self.conda_version = self.get_conda_version(output)
     if Version(self.conda_version, partial=True) < self.MINIMUM_VERSION:
         raise CommandFailedError(
             "conda version '{}' is smaller than minimum supported conda version '{}'"
             .format(self.conda_version, self.MINIMUM_VERSION))
예제 #2
0
 def __init__(self,
              session,
              path,
              python,
              requirements_manager,
              execution_info=None,
              **kwargs):
     # type: (Session, PathLike, float, RequirementsManager, ExecutionInfo, Any) -> None
     """
     :param python: base python version to use (e.g python3.6)
     :param path: path of env
     """
     self.session = session
     self.python = python
     self.source = None
     self.requirements_manager = requirements_manager
     self.path = path
     self.env_read_only = False
     self.extra_channels = self.session.config.get(
         'agent.package_manager.conda_channels', [])
     self.conda_env_as_base_docker = \
         self.session.config.get('agent.package_manager.conda_env_as_base_docker', None) or \
         bool(ENV_CONDA_ENV_PACKAGE.get())
     if ENV_CONDA_ENV_PACKAGE.get():
         self.conda_pre_build_env_path = ENV_CONDA_ENV_PACKAGE.get()
     else:
         self.conda_pre_build_env_path = execution_info.docker_cmd if execution_info else None
     self.pip = CondaPip(
         session=self.session,
         source=self.source,
         python=self.python,
         requirements_manager=self.requirements_manager,
         path=self.path,
     )
     try:
         self.conda = (find_executable("conda") or Argv(
             select_for_platform(windows="where", linux="which"),
             "conda").get_output(shell=select_for_platform(
                 windows=True, linux=False)).strip())
     except Exception:
         raise ValueError("ERROR: package manager \"conda\" selected, "
                          "but \'conda\' executable could not be located")
     try:
         output = Argv(self.conda,
                       "--version").get_output(stderr=subprocess.STDOUT)
     except subprocess.CalledProcessError as ex:
         raise CommandFailedError(
             "Unable to determine conda version: {ex}, output={ex.output}".
             format(ex=ex))
     self.conda_version = self.get_conda_version(output)
     if SimpleVersion.compare_versions(self.conda_version, '<',
                                       self.MINIMUM_VERSION):
         raise CommandFailedError(
             "conda version '{}' is smaller than minimum supported conda version '{}'"
             .format(self.conda_version, self.MINIMUM_VERSION))
예제 #3
0
def clone_repository_cached(session, execution, destination):
    # type: (Session, ExecutionInfo, Path) -> Tuple[VCS, RepoInfo]
    """
    Clone a remote repository.
    :param execution: execution info
    :param destination: directory to clone to (in which a directory for the repository will be created)
    :param session: program session
    :return: repository information
    :raises: CommandFailedError if git/hg is not installed
    """
    repo_url = execution.repository  # type: str
    parsed_url = furl(repo_url)
    no_password_url = parsed_url.copy().remove(password=True).url

    clone_folder_name = Path(str(furl(repo_url).path)).name  # type: str
    clone_folder = Path(destination) / clone_folder_name
    cached_repo_path = (
        Path(session.config["agent.vcs_cache.path"]).expanduser() /
        "{}.{}".format(clone_folder_name,
                       md5(ensure_binary(repo_url)).hexdigest()) /
        clone_folder_name)  # type: Path

    vcs = VcsFactory.create(session,
                            execution_info=execution,
                            location=cached_repo_path)
    if not find_executable(vcs.executable_name):
        raise CommandFailedError(vcs.executable_not_found_error_help())

    if session.config["agent.vcs_cache.enabled"] and cached_repo_path.exists():
        print('Using cached repository in "{}"'.format(cached_repo_path))
    else:
        print("cloning: {}".format(no_password_url))
        rm_tree(cached_repo_path)
        # We clone the entire repository, not a specific branch
        vcs.clone()  # branch=execution.branch)

    vcs.pull()
    rm_tree(destination)
    shutil.copytree(Text(cached_repo_path), Text(clone_folder))
    if not clone_folder.is_dir():
        raise CommandFailedError(
            "copying of repository failed: from {} to {}".format(
                cached_repo_path, clone_folder))

    # checkout in the newly copy destination
    vcs.location = Text(clone_folder)
    vcs.checkout()

    repo_info = vcs.get_repository_copy_info(clone_folder)

    # make sure we have no user/pass in the returned repository structure
    repo_info = attr.evolve(repo_info, url=no_password_url)

    return vcs, repo_info
예제 #4
0
def check_directory_path(path, check_whitespace_in_path=True):
    message = 'Could not create directory "{}": {}'
    if not is_windows_platform() and check_whitespace_in_path:
        match = re.search(r'\s', path)
        if match:
            raise CommandFailedError(
                'directories may not contain whitespace (char: {!r}, position: {})'.format(match.group(0),
                                                                                           match.endpos))
    try:
        Path(os.path.expandvars(path)).expanduser().mkdir(parents=True, exist_ok=True)
    except OSError as e:
        raise CommandFailedError(message.format(path, e.strerror))
    except Exception as e:
        raise CommandFailedError(message.format(path, e))
예제 #5
0
    def from_task(cls, task_info):
        # type: (...) -> ExecutionInfo
        """
        extract ExecutionInfo tuple from task parameters
        """
        if not task_info.script:
            raise CommandFailedError(
                "can not run task without script information")
        execution = cls.from_dict(task_info.script.to_dict())
        if not execution.entry_point:
            log.warning("notice: `script.entry_point` is empty")
        if not execution.working_dir:
            entry_point, _, working_dir = execution.entry_point.partition(":")
            execution.entry_point = entry_point
            execution.working_dir = working_dir or ""

        return execution
예제 #6
0
 def get_conda_version(output):
     match = re.search(r"(\d+\.){0,2}\d+", output)
     if not match:
         raise CommandFailedError("Unidentified conda version string:", output)
     return match.group(0)