def __init__(self, session, path, python, requirements_manager):
# type: (Session, PathLike, float, RequirementsManager) -> None
"""
:param python: base python version to use (e.g python3.6)
:param path: path of env
"""
self.session = session
self.python = python
self.source = None
self.requirements_manager = requirements_manager
self.path = path
self.extra_channels = self.session.config.get(
'agent.package_manager.conda_channels', [])
self.pip = CondaPip(
session=self.session,
source=self.source,
python=self.python,
requirements_manager=self.requirements_manager,
path=self.path,
)
self.conda = (find_executable("conda") or Argv(
select_for_platform(windows="where", linux="which"),
"conda").get_output(shell=True).strip())
try:
output = Argv(self.conda, "--version").get_output()
except subprocess.CalledProcessError as ex:
raise CommandFailedError(
"Unable to determine conda version: {ex}, output={ex.output}".
format(ex=ex))
self.conda_version = self.get_conda_version(output)
if Version(self.conda_version, partial=True) < self.MINIMUM_VERSION:
raise CommandFailedError(
"conda version '{}' is smaller than minimum supported conda version '{}'"
.format(self.conda_version, self.MINIMUM_VERSION))
class Hg(VCS):
executable_name = "hg"
main_branch = "default"
checkout_flags = ("--clean", )
patch_base = ("import", "--no-commit")
def executable_not_found_error_help(self):
return 'Cannot find "{}" executable. {}'.format(
self.executable_name,
select_for_platform(
linux="You can install it by running: sudo apt-get install {}".
format(self.executable_name),
windows="You can download it here: {}".format(
"https://www.mercurial-scm.org/wiki/Download"),
),
)
def pull(self):
self.call("pull",
self.url_with_auth,
cwd=self.location,
*(("-r", self.revision) if self.revision else ()))
info_commands = dict(
url=Argv(executable_name, "paths", "--verbose"),
branch=Argv(executable_name, "--debug", "id", "-b"),
commit=Argv(executable_name, "--debug", "id", "-i"),
root=Argv(executable_name, "root"),
)
def _run_command(self, command, raw=False, **kwargs):
# type: (Iterable[Text], bool, Any) -> Union[Dict, Text]
"""
Run a conda command, returning JSON output.
The command is prepended with 'conda' and run with JSON output flags.
:param command: command to run
:param raw: return text output and don't change command
:param kwargs: kwargs for Argv.get_output()
:return: JSON output or text output
"""
command = Argv(*command) # type: Executable
if not raw:
command = (self.conda, ) + command + ("--quiet", "--json")
try:
print('Executing Conda: {}'.format(command.serialize()))
result = command.get_output(stdin=DEVNULL, **kwargs)
except Exception as e:
if raw:
raise
result = e.output if hasattr(e, 'output') else ''
if raw:
return result
result = json.loads(result) if result else {}
if result.get('success', False):
print('Pass')
elif result.get('error'):
print('Conda error: {}'.format(result.get('error')))
return result
def __init__(self,
session,
path,
python,
requirements_manager,
execution_info=None,
**kwargs):
# type: (Session, PathLike, float, RequirementsManager, ExecutionInfo, Any) -> None
"""
:param python: base python version to use (e.g python3.6)
:param path: path of env
"""
self.session = session
self.python = python
self.source = None
self.requirements_manager = requirements_manager
self.path = path
self.env_read_only = False
self.extra_channels = self.session.config.get(
'agent.package_manager.conda_channels', [])
self.conda_env_as_base_docker = \
self.session.config.get('agent.package_manager.conda_env_as_base_docker', None) or \
bool(ENV_CONDA_ENV_PACKAGE.get())
if ENV_CONDA_ENV_PACKAGE.get():
self.conda_pre_build_env_path = ENV_CONDA_ENV_PACKAGE.get()
else:
self.conda_pre_build_env_path = execution_info.docker_cmd if execution_info else None
self.pip = CondaPip(
session=self.session,
source=self.source,
python=self.python,
requirements_manager=self.requirements_manager,
path=self.path,
)
try:
self.conda = (find_executable("conda") or Argv(
select_for_platform(windows="where", linux="which"),
"conda").get_output(shell=select_for_platform(
windows=True, linux=False)).strip())
except Exception:
raise ValueError("ERROR: package manager \"conda\" selected, "
"but \'conda\' executable could not be located")
try:
output = Argv(self.conda,
"--version").get_output(stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as ex:
raise CommandFailedError(
"Unable to determine conda version: {ex}, output={ex.output}".
format(ex=ex))
self.conda_version = self.get_conda_version(output)
if SimpleVersion.compare_versions(self.conda_version, '<',
self.MINIMUM_VERSION):
raise CommandFailedError(
"conda version '{}' is smaller than minimum supported conda version '{}'"
.format(self.conda_version, self.MINIMUM_VERSION))
class Git(VCS):
executable_name = "git"
main_branch = "master"
clone_flags = ("--quiet", "--recursive")
checkout_flags = ("--force", )
COMMAND_ENV = {
# do not prompt for password
"GIT_TERMINAL_PROMPT": "0",
# do not prompt for ssh key passphrase
"GIT_SSH_COMMAND": "ssh -oBatchMode=yes",
}
@staticmethod
def remote_branch_name(branch):
return "origin/{}".format(branch)
def executable_not_found_error_help(self):
return 'Cannot find "{}" executable. {}'.format(
self.executable_name,
select_for_platform(
linux="You can install it by running: sudo apt-get install {}".
format(self.executable_name),
windows="You can download it here: {}".format(
"https://gitforwindows.org/"),
),
)
def pull(self):
self.call("fetch", "--all", "--recurse-submodules", cwd=self.location)
def checkout(self): # type: () -> None
"""
Checkout repository at specified revision
"""
self.call("checkout",
self.revision,
*self.checkout_flags,
cwd=self.location)
try:
self.call("submodule", "update", "--recursive", cwd=self.location)
except:
pass
info_commands = dict(
url=Argv(executable_name, "ls-remote", "--get-url", "origin"),
branch=Argv(executable_name, "rev-parse", "--abbrev-ref", "HEAD"),
commit=Argv(executable_name, "rev-parse", "HEAD"),
root=Argv(executable_name, "rev-parse", "--show-toplevel"),
)
patch_base = ("apply", )
def create(self):
"""
Create a new environment
"""
output = Argv(
self.conda,
"create",
"--yes",
"--mkdir",
"--prefix",
self.path,
"python={}".format(self.python),
).get_output(stderr=DEVNULL)
match = re.search(
r"\W*(.*activate) ({})".format(re.escape(str(self.path))), output
)
self.source = self.pip.source = (
tuple(match.group(1).split()) + (match.group(2),)
if match
else ("activate", self.path)
)
conda_env = Path(self.conda).parent.parent / 'etc' / 'profile.d' / 'conda.sh'
if conda_env.is_file() and not is_windows_platform():
self.source = self.pip.source = CommandSequence(('source', conda_env.as_posix()), self.source)
# install cuda toolkit
try:
cuda_version = float(int(self.session.config['agent.cuda_version'])) / 10.0
if cuda_version > 0:
self._install('cudatoolkit={:.1f}'.format(cuda_version))
except Exception:
pass
return self
def install_flags(self):
"""
Configurable package installation creation arguments
"""
return super(VirtualenvPip, self).install_flags() + Argv.conditional_flag(
self.session.config["agent.package_manager.force_upgrade"], "--upgrade"
)
def run_with_env(self, command, output=False, **kwargs):
if not self.source:
return super(CondaPip, self).run_with_env(command, output=output, **kwargs)
command = CommandSequence(self.source, Argv("pip", *command))
return (command.get_output if output else command.check_call)(
stdin=DEVNULL, **kwargs
)
def get_pip_version(cls, package):
output = Argv(
'pip', 'search', package,
*(chain.from_iterable(
('-i', x) for x in cls._pip_extra_index_url))).get_output()
# ad-hoc pattern to duplicate the behavior of the old code
return re.search(r'{} \((\d+\.\d+\.[^.]+)'.format(package),
output).group(1)
def create_flags(self):
"""
Configurable environment creation arguments
"""
return Argv.conditional_flag(
self.session.config["agent.package_manager.system_site_packages"],
"--system-site-packages",
)
def run(self, *args, **kwargs):
func = kwargs.pop("func", Argv.get_output)
kwargs.setdefault("stdin", DEVNULL)
kwargs['env'] = deepcopy(os.environ)
if 'VIRTUAL_ENV' in kwargs['env'] or 'CONDA_PREFIX' in kwargs['env']:
kwargs['env'].pop('VIRTUAL_ENV', None)
kwargs['env'].pop('CONDA_PREFIX', None)
kwargs['env'].pop('PYTHONPATH', None)
if hasattr(sys, "real_prefix") and hasattr(sys, "base_prefix"):
path = ':'+kwargs['env']['PATH']
path = path.replace(':'+sys.base_prefix, ':'+sys.real_prefix, 1)
kwargs['env']['PATH'] = path
if check_if_command_exists("poetry"):
argv = Argv("poetry", *args)
else:
argv = Argv(self._python, "-m", "poetry", *args)
self.log.debug("running: %s", argv)
return func(argv, **kwargs)
def _check_script_validity(self, path):
"""
Make sure script in ``path`` is a valid python script
:param path:
:return:
"""
result = Argv(self.bin, path, "--version").call(
stdout=DEVNULL, stderr=DEVNULL, stdin=DEVNULL
)
return result == 0
def _run_command(self, command, raw=False, **kwargs):
# type: (Iterable[Text], bool, Any) -> Union[Dict, Text]
"""
Run a conda command, returning JSON output.
The command is prepended with 'conda' and run with JSON output flags.
:param command: command to run
:param raw: return text output and don't change command
:param kwargs: kwargs for Argv.get_output()
:return: JSON output or text output
"""
def escape_ansi(line):
ansi_escape = re.compile(
r'(?:\x1B[@-_]|[\x80-\x9F])[0-?]*[ -/]*[@-~]')
return ansi_escape.sub('', line)
command = Argv(*command) # type: Executable
if not raw:
command = (self.conda, ) + command + ("--quiet", "--json")
try:
print('Executing Conda: {}'.format(command.serialize()))
result = command.get_output(stdin=DEVNULL, **kwargs)
if self.session.debug_mode:
print(result)
except Exception as e:
result = e.output if hasattr(e, 'output') else ''
if self.session.debug_mode:
print(result)
if raw:
raise
if raw:
return result
result = json.loads(escape_ansi(result)) if result else {}
if result.get('success', False):
print('Pass')
elif result.get('error'):
print('Conda error: {}'.format(result.get('error')))
return result
def get_stderr(self, *argv, **kwargs):
"""
Execute argv without stdout/stdin in <cwd> and get stderr output.
Remove stdin so git/hg can't ask for passwords.
``kwargs`` can override all arguments passed to subprocess.
"""
process = self._call_subprocess(
subprocess.Popen, argv,
**dict(kwargs, stderr=subprocess.PIPE, stdout=None))
_, stderr = process.communicate()
code = process.poll()
if code == COMMAND_SUCCESS:
return stderr
with Argv.normalize_exception(censor_password=True):
raise subprocess.CalledProcessError(returncode=code,
cmd=argv,
output=stderr)
def install_packages(self, *packages):
first_install = (
Argv(
self.python,
six.text_type(self.script_path),
"venv=",
self.path,
"install=",
)
+ packages
)
later_install = first_install + (
"pip-command=",
"pip-faster",
"install",
"--upgrade", # no --prune
)
self._choose_install(first_install, later_install)
def install_from_file(self, path):
first_install = (
Argv(
self.python,
six.text_type(self.script_path),
"venv=",
"-p",
self.python,
self.path,
)
+ self.create_flags()
+ ("install=", "-r", path)
+ self.install_flags()
)
later_install = first_install + (
"pip-command=",
"pip-faster",
"install",
"--upgrade", # no --prune
)
self._choose_install(first_install, later_install)
def command(self, *args):
return Argv(*args, log=self.get_logger(Argv.__module__))
def get_cuda_version(config): # type: (ConfigTree) -> (Text, Text)
# we assume os.environ already updated the config['agent.cuda_version'] & config['agent.cudnn_version']
cuda_version = config['agent.cuda_version']
cudnn_version = config['agent.cudnn_version']
if cuda_version and cudnn_version:
return normalize_cuda_version(
cuda_version), normalize_cuda_version(cudnn_version)
if not cuda_version and is_windows_platform():
try:
cuda_vers = [
int(k.replace('CUDA_PATH_V', '').replace('_', ''))
for k in os.environ.keys() if k.startswith('CUDA_PATH_V')
]
cuda_vers = max(cuda_vers)
if cuda_vers > 40:
cuda_version = cuda_vers
except:
pass
if not cuda_version:
try:
try:
nvcc = 'nvcc.exe' if is_windows_platform() else 'nvcc'
if is_windows_platform() and 'CUDA_PATH' in os.environ:
nvcc = os.path.join(os.environ['CUDA_PATH'], nvcc)
output = Argv(nvcc, '--version').get_output()
except OSError:
raise CudaNotFound('nvcc not found')
match = re.search(r'release (.{3})', output).group(1)
cuda_version = Text(int(float(match) * 10))
except:
pass
if not cuda_version:
try:
try:
output = Argv('nvidia-smi', ).get_output()
except OSError:
raise CudaNotFound('nvcc not found')
match = re.search(r'CUDA Version: ([0-9]+).([0-9]+)', output)
match = match.group(1) + '.' + match.group(2)
cuda_version = Text(int(float(match) * 10))
except:
pass
if not cudnn_version:
try:
cuda_lib = which('nvcc')
if is_windows_platform:
cudnn_h = path.sep.join(
cuda_lib.split(path.sep)[:-2] + ['include', 'cudnn.h'])
else:
cudnn_h = path.join(
path.sep,
*(cuda_lib.split(path.sep)[:-2] +
['include', 'cudnn.h']))
cudnn_major, cudnn_minor = None, None
try:
include_file = open(cudnn_h)
except OSError:
raise CudaNotFound('Could not read cudnn.h')
with include_file:
for line in include_file:
if 'CUDNN_MAJOR' in line:
cudnn_major = line.split()[-1]
if 'CUDNN_MINOR' in line:
cudnn_minor = line.split()[-1]
if cudnn_major and cudnn_minor:
break
cudnn_version = cudnn_major + (cudnn_minor or '0')
except:
pass
return (normalize_cuda_version(cuda_version or 0),
normalize_cuda_version(cudnn_version or 0))
def get_python_command(self, extra):
if check_if_command_exists("poetry"):
return Argv("poetry", "run", "python", *extra)
else:
return Argv(self.config._python, "-m", "poetry", "run", "python", *extra)
def _make_command(self, command):
return Argv(self.bin, '-m', 'pip', *command)
def create(self):
"""
Create a new environment
"""
if self.conda_env_as_base_docker and self.conda_pre_build_env_path:
if Path(self.conda_pre_build_env_path).is_dir():
print("Using pre-existing Conda environment from {}".format(
self.conda_pre_build_env_path))
self.path = Path(self.conda_pre_build_env_path)
self.source = ("conda", "activate", self.path.as_posix())
self.pip = CondaPip(
session=self.session,
source=self.source,
python=self.python,
requirements_manager=self.requirements_manager,
path=self.path,
)
conda_env = Path(
self.conda
).parent.parent / 'etc' / 'profile.d' / 'conda.sh'
self.source = self.pip.source = CommandSequence(
('source', conda_env.as_posix()), self.source)
self.env_read_only = True
return self
elif Path(self.conda_pre_build_env_path).is_file():
print("Restoring Conda environment from {}".format(
self.conda_pre_build_env_path))
tar_path = find_executable("tar")
self.path.mkdir(parents=True, exist_ok=True)
output = Argv(
tar_path,
"-xzf",
self.conda_pre_build_env_path,
"-C",
self.path,
).get_output()
self.source = self.pip.source = ("conda", "activate",
self.path.as_posix())
conda_env = Path(
self.conda
).parent.parent / 'etc' / 'profile.d' / 'conda.sh'
self.source = self.pip.source = CommandSequence(
('source', conda_env.as_posix()), self.source)
# unpack cleanup
print("Fixing prefix in Conda environment {}".format(
self.path))
CommandSequence(('source', conda_env.as_posix()),
((self.path / 'bin' /
'conda-unpack').as_posix(), )).get_output()
return self
else:
raise ValueError(
"Could not restore Conda environment, cannot find {}".
format(self.conda_pre_build_env_path))
output = Argv(
self.conda,
"create",
"--yes",
"--mkdir",
"--prefix",
self.path,
"python={}".format(self.python),
).get_output(stderr=DEVNULL)
match = re.search(
r"\W*(.*activate) ({})".format(re.escape(str(self.path))), output)
self.source = self.pip.source = (tuple(match.group(1).split()) +
(match.group(2), ) if match else
("conda", "activate",
self.path.as_posix()))
conda_env = Path(
self.conda).parent.parent / 'etc' / 'profile.d' / 'conda.sh'
if conda_env.is_file() and not is_windows_platform():
self.source = self.pip.source = CommandSequence(
('source', conda_env.as_posix()), self.source)
# install cuda toolkit
# noinspection PyBroadException
try:
cuda_version = float(int(
self.session.config['agent.cuda_version'])) / 10.0
if cuda_version > 0:
self._install('cudatoolkit={:.1f}'.format(cuda_version))
except Exception:
pass
return self
def _get_vcs_command(self, argv):
# type: (Iterable[PathLike]) -> Argv
return Argv(self.executable_name, *argv)
def get_python_command(self, extra=()):
# type: (...) -> Executable
return Argv(self.bin, *extra)
def _make_command(self, command):
return Argv(self.bin, '-m', 'pip', '--disable-pip-version-check', *command)
def run_task(task):
return Argv("trains_agent", "--debug", "worker", "execute", "--id", task.id)
