コード例 #1
0
 def __init__(self,
              session,
              path,
              python,
              requirements_manager,
              execution_info=None,
              **kwargs):
     # type: (Session, PathLike, float, RequirementsManager, ExecutionInfo, Any) -> None
     """
     :param python: base python version to use (e.g python3.6)
     :param path: path of env
     """
     super(CondaAPI, self).__init__()
     self.session = session
     self.python = python
     self.source = None
     self.requirements_manager = requirements_manager
     self.path = path
     self.env_read_only = False
     self.extra_channels = self.session.config.get(
         'agent.package_manager.conda_channels', [])
     self.conda_env_as_base_docker = \
         self.session.config.get('agent.package_manager.conda_env_as_base_docker', None) or \
         bool(ENV_CONDA_ENV_PACKAGE.get())
     if ENV_CONDA_ENV_PACKAGE.get():
         self.conda_pre_build_env_path = ENV_CONDA_ENV_PACKAGE.get()
     else:
         self.conda_pre_build_env_path = execution_info.docker_cmd if execution_info else None
     self.pip = CondaPip(
         session=self.session,
         source=self.source,
         python=self.python,
         requirements_manager=self.requirements_manager,
         path=self.path,
     )
     try:
         self.conda = (find_executable("conda") or Argv(
             select_for_platform(windows="where", linux="which"),
             "conda").get_output(shell=select_for_platform(
                 windows=True, linux=False)).strip())
     except Exception:
         raise ValueError("ERROR: package manager \"conda\" selected, "
                          "but \'conda\' executable could not be located")
     try:
         output = Argv(self.conda,
                       "--version").get_output(stderr=subprocess.STDOUT)
     except subprocess.CalledProcessError as ex:
         raise CommandFailedError(
             "Unable to determine conda version: {ex}, output={ex.output}".
             format(ex=ex))
     self.conda_version = self.get_conda_version(output)
     if SimpleVersion.compare_versions(self.conda_version, '<',
                                       self.MINIMUM_VERSION):
         raise CommandFailedError(
             "conda version '{}' is smaller than minimum supported conda version '{}'"
             .format(self.conda_version, self.MINIMUM_VERSION))
コード例 #2
0
def check_directory_path(path, check_whitespace_in_path=True):
    message = 'Could not create directory "{}": {}'
    if not is_windows_platform() and check_whitespace_in_path:
        match = re.search(r'\s', path)
        if match:
            raise CommandFailedError(
                'directories may not contain whitespace (char: {!r}, position: {})'.format(match.group(0),
                                                                                           match.endpos))
    try:
        Path(os.path.expandvars(path)).expanduser().mkdir(parents=True, exist_ok=True)
    except OSError as e:
        raise CommandFailedError(message.format(path, e.strerror))
    except Exception as e:
        raise CommandFailedError(message.format(path, e))
コード例 #3
0
    def from_task(cls, task_info):
        # type: (...) -> ExecutionInfo
        """
        extract ExecutionInfo tuple from task parameters
        """
        if not task_info.script:
            raise CommandFailedError("can not run task without script information")
        execution = cls.from_dict(task_info.script.to_dict())
        if not execution.entry_point:
            log.warning("notice: `script.entry_point` is empty")
        if not execution.working_dir:
            entry_point, _, working_dir = execution.entry_point.partition(":")
            execution.entry_point = entry_point
            execution.working_dir = working_dir or ""

        # noinspection PyBroadException
        try:
            execution.docker_cmd = task_info.execution.docker_cmd
        except Exception:
            pass

        return execution
コード例 #4
0
def clone_repository_cached(session, execution, destination):
    # type: (Session, ExecutionInfo, Path) -> Tuple[VCS, RepoInfo]
    """
    Clone a remote repository.
    :param execution: execution info
    :param destination: directory to clone to (in which a directory for the repository will be created)
    :param session: program session
    :return: repository information
    :raises: CommandFailedError if git/hg is not installed
    """
    repo_url = execution.repository  # type: str
    parsed_url = furl(repo_url)
    no_password_url = parsed_url.copy().remove(password=True).url

    clone_folder_name = Path(str(furl(repo_url).path)).name  # type: str
    clone_folder = Path(destination) / clone_folder_name

    standalone_mode = session.config.get("agent.standalone_mode", False)
    if standalone_mode:
        cached_repo_path = clone_folder
    else:
        cached_repo_path = (
            Path(session.config["agent.vcs_cache.path"]).expanduser() /
            "{}.{}".format(clone_folder_name,
                           md5(ensure_binary(repo_url)).hexdigest()) /
            clone_folder_name)  # type: Path

    vcs = VcsFactory.create(session,
                            execution_info=execution,
                            location=cached_repo_path)
    if not find_executable(vcs.executable_name):
        raise CommandFailedError(vcs.executable_not_found_error_help())

    if not standalone_mode:
        if session.config[
                "agent.vcs_cache.enabled"] and cached_repo_path.exists():
            print('Using cached repository in "{}"'.format(cached_repo_path))

        else:
            print("cloning: {}".format(no_password_url))
            rm_tree(cached_repo_path)
            # We clone the entire repository, not a specific branch
            vcs.clone()  # branch=execution.branch)

        vcs.pull()
        rm_tree(destination)
        shutil.copytree(Text(cached_repo_path), Text(clone_folder))
        if not clone_folder.is_dir():
            raise CommandFailedError(
                "copying of repository failed: from {} to {}".format(
                    cached_repo_path, clone_folder))

    # checkout in the newly copy destination
    vcs.location = Text(clone_folder)
    vcs.checkout()

    repo_info = vcs.get_repository_copy_info(clone_folder)

    # make sure we have no user/pass in the returned repository structure
    repo_info = attr.evolve(repo_info, url=no_password_url)

    return vcs, repo_info
コード例 #5
0
 def get_conda_version(output):
     match = re.search(r"(\d+\.){0,2}\d+", output)
     if not match:
         raise CommandFailedError("Unidentified conda version string:",
                                  output)
     return match.group(0)
コード例 #6
0
def clone_repository_cached(session, execution, destination):
    # type: (Session, ExecutionInfo, Path) -> Tuple[VCS, RepoInfo]
    """
    Clone a remote repository.
    :param execution: execution info
    :param destination: directory to clone to (in which a directory for the repository will be created)
    :param session: program session
    :return: repository information
    :raises: CommandFailedError if git/hg is not installed
    """
    # mock lock
    repo_lock = Lock()
    repo_lock_timeout_sec = 300
    repo_url = execution.repository  # type: str
    parsed_url = furl(repo_url)
    no_password_url = parsed_url.copy().remove(password=True).url

    clone_folder_name = Path(str(furl(repo_url).path)).name  # type: str
    clone_folder = Path(destination) / clone_folder_name

    standalone_mode = session.config.get("agent.standalone_mode", False)
    if standalone_mode:
        cached_repo_path = clone_folder
    else:
        vcs_cache_path = Path(
            session.config["agent.vcs_cache.path"]).expanduser()
        repo_hash = md5(ensure_binary(repo_url)).hexdigest()
        # create lock
        repo_lock = FileLock(filename=(vcs_cache_path /
                                       '{}.lock'.format(repo_hash)).as_posix())
        # noinspection PyBroadException
        try:
            repo_lock.acquire(timeout=repo_lock_timeout_sec)
        except BaseException:
            print(
                'Could not lock cache folder "{}" (timeout {} sec), using temp vcs cache.'
                .format(clone_folder_name, repo_lock_timeout_sec))
            repo_hash = '{}_{}'.format(repo_hash,
                                       str(random()).replace('.', ''))
            # use mock lock for the context
            repo_lock = Lock()
        # select vcs cache folder
        cached_repo_path = vcs_cache_path / "{}.{}".format(
            clone_folder_name, repo_hash) / clone_folder_name

    with repo_lock:
        vcs = VcsFactory.create(session,
                                execution_info=execution,
                                location=cached_repo_path)
        if not find_executable(vcs.executable_name):
            raise CommandFailedError(vcs.executable_not_found_error_help())

        if not standalone_mode:
            if session.config[
                    "agent.vcs_cache.enabled"] and cached_repo_path.exists():
                print(
                    'Using cached repository in "{}"'.format(cached_repo_path))

            else:
                print("cloning: {}".format(no_password_url))
                rm_tree(cached_repo_path)
                # We clone the entire repository, not a specific branch
                vcs.clone()  # branch=execution.branch)

            vcs.pull()
            rm_tree(destination)
            shutil.copytree(Text(cached_repo_path), Text(clone_folder))
            if not clone_folder.is_dir():
                raise CommandFailedError(
                    "copying of repository failed: from {} to {}".format(
                        cached_repo_path, clone_folder))

    # checkout in the newly copy destination
    vcs.location = Text(clone_folder)
    vcs.checkout()

    repo_info = vcs.get_repository_copy_info(clone_folder)

    # make sure we have no user/pass in the returned repository structure
    repo_info = attr.evolve(repo_info, url=no_password_url)

    return vcs, repo_info