def install_cmake_package(self, requirement_specifier, cmake_options, ignore_index):
"""
Build and install cmake package into cache & copy it to env.
"""
pkg_cache_dir = os.path.abspath(
os.path.join(self.cache, "%s-%s" % (requirement_specifier.name, requirement_specifier.version))
)
def in_cache():
return os.path.isdir(pkg_cache_dir)
if not in_cache() and not ignore_index:
cwd = os.getcwd()
archive = None
archive_name = None
try:
archive = self.download(requirement_specifier.name, requirement_specifier.version)
archive_name = unpack(archive)
logging.info("Building %s" % requirement_specifier.name)
os.chdir(archive_name)
env = os.environ.copy()
env["PKG_CONFIG_PATH"] = ",".join(self.search_pkg_config_locations())
retcode = run_shell(
["cmake", ".", "-DCMAKE_INSTALL_PREFIX=%s" % pkg_cache_dir] + cmake_options,
env=env,
verbose=self.settings["verbosity"] >= 1,
)
if retcode != 0:
raise RequirementException("%s configure failed" % requirement_specifier.name)
retcode = run_shell(["make", "-j4"], verbose=self.settings["verbosity"] >= 1)
if retcode != 0:
raise RequirementException("%s build failed" % requirement_specifier.name)
retcode = run_shell(["make", "install"], verbose=self.settings["verbosity"] >= 1)
if retcode != 0:
raise RequirementException('%s "make install" failed' % requirement_specifier.name)
finally:
safe_remove(archive)
safe_remove(archive_name)
os.chdir(cwd)
pkg_install_dir = os.path.join(
self.env, "lib/%s-%s" % (requirement_specifier.name, requirement_specifier.version)
)
if in_cache():
# install gazebo somewhere into venv
if os.path.exists(pkg_install_dir):
shutil.rmtree(pkg_install_dir)
shutil.copytree(pkg_cache_dir, pkg_install_dir)
else:
raise RequirementException("can't find gazebo-%s in robustus cache" % requirement_specifier.version)
return pkg_install_dir
def _perrepo(self, cmd_str):
verbose = self.settings["verbosity"] > 0
run_shell(
'cd "$(git rev-parse --show-toplevel)" && . "%s" && %s' % (self._activate_path(), cmd_str),
shell=True,
verbose=verbose,
)
for d in os.listdir(os.path.join(self.env, "src")):
full_path = os.path.join(self.env, "src", d)
if os.path.isdir(full_path):
logging.info("Running command in %s" % full_path)
run_shell(
'cd "%s" && . "%s" && %s' % (full_path, self._activate_path(), cmd_str), shell=True, verbose=verbose
)
def install_satisfactory_requirement_from_remote(self, requirement_specifier):
"""
If wheel for satisfactory requirement found on remote, install it.
:param requirement_specifier: specifies package namd and package version string
:return: True if package installed by function (according to pip return code);
False otherwise.
"""
logging.info("Attempting to install package from remote wheel")
for find_link in self.settings["find_links"]:
find_links_url = (find_link + "/python-wheels/index.html",) # TEMPORARY.
dtemp_path = tempfile.mkdtemp()
return_code = run_shell(
[
self.pip_executable,
"install",
"--download-cache=%s" % dtemp_path,
"--no-index",
"--use-wheel",
"--find-links=%s" % find_links_url,
requirement_specifier.freeze(),
],
verbose=self.settings["verbosity"] >= 2,
)
if return_code == 0:
# The following downloads the wheels of the requirment (and those of
# dependencies) into a pip download cache and moves (renames) the downloaded
# wheels into the local Robustus cache. Regarding the need for this see "Wheels
# for Dependencies" "http://lucumr.pocoo.org/2014/1/27/python-on-wheels/".
for file_path in glob.glob(os.path.join(dtemp_path, "http*.whl")):
if os.path.isfile(file_path):
file_name = os.path.basename(file_path)
file_name_new = file_name.rpartition("%2F")[-1]
file_path_new = os.path.join(self.cache, file_name_new)
shutil.move(file_path, file_path_new) # NOTE: Allow overwrites.
safe_remove(dtemp_path)
return True
else:
logging.info(
"pip failed to install requirement %s from remote wheels cache %s."
% (requirement_specifier.freeze(), find_links_url)
)
safe_remove(dtemp_path)
return False
def _pip_install_requirement(self, requirement_specifier):
command = " ".join([self.pip_executable, "install", requirement_specifier.freeze()])
logging.info("Got url-based requirement. " "Fall back to pip shell command:%s" % (command,))
ret_code = run_shell(command, shell=True, verbose=self.settings["verbosity"] >= 1)
return ret_code
def install_through_wheeling(self, requirement_specifier, rob_file, ignore_index):
"""
Check if package cache already contains package of specified version, if so install it.
Otherwise make a wheel and put it into cache.
Hope manual check for requirements file won't be necessary, waiting for pip 1.5 https://github.com/pypa/pip/issues/855
:param package: package name
:param version: package version string
:return: None
"""
# If wheelhouse doesn't contain necessary requirement attempt to install from remote wheel archive or make a wheel.
installed = False
if self.find_satisfactory_requirement(requirement_specifier) is None:
# Pip does not download the wheels of dependencies unless it installs.
installed = False
if not self.settings["no_remote_cache"]:
installed = self.install_satisfactory_requirement_from_remote(requirement_specifier)
if not installed:
logging.info("Wheel not found, downloading package")
return_code = run_shell(
[self.pip_executable, "install", "--download", self.cache, requirement_specifier.freeze()],
verbose=self.settings["verbosity"] >= 2,
)
if return_code != 0:
raise RequirementException("pip failed to download requirement %s" % requirement_specifier.freeze())
logging.info("Done")
logging.info("Building wheel")
wheel_cmd = [
self.pip_executable,
"wheel",
"--no-index",
"--find-links=%s" % self.cache,
"--wheel-dir=%s" % self.cache,
requirement_specifier.freeze(),
]
# we probably sometimes will want to see build log
for i in xrange(self.settings["verbosity"]):
wheel_cmd.append("-v")
return_code = run_shell(wheel_cmd, verbose=self.settings["verbosity"] >= 1)
if return_code != 0:
raise RequirementException(
"pip failed to build wheel for requirement %s" % requirement_specifier.freeze()
)
logging.info("Done")
if not installed:
# install from new prebuilt wheel
logging.info("Installing package from wheel")
return_code = run_shell(
[
self.pip_executable,
"install",
"--no-index",
"--use-wheel",
"--find-links=%s" % self.cache,
requirement_specifier.freeze(),
],
verbose=self.settings["verbosity"] >= 2,
)
if return_code != 0:
raise RequirementException(
"pip failed to install requirement %s from wheels cache %s."
% (requirement_specifier.freeze(), self.cache)
)
def env(args):
"""
Create robustus environment.
@param args: command line arguments
"""
settings = dict()
settings = Robustus._override_settings(settings, args)
# create virtualenv
python_executable = os.path.abspath(os.path.join(args.env, "bin/python"))
if os.path.isfile(python_executable):
logging.info("Found virtualenv in " + args.env)
else:
logging.info("Creating virtualenv")
virtualenv_args = ["virtualenv", args.env, "--prompt", args.prompt]
if args.python is not None:
virtualenv_args += ["--python", args.python]
if args.system_site_packages:
virtualenv_args += ["--system-site-packages"]
run_shell(virtualenv_args, settings["verbosity"] >= 1)
pip_executable = os.path.abspath(os.path.join(args.env, "bin/pip"))
if not os.path.isfile(pip_executable):
raise RobustusException("failed to create virtualenv, pip not found")
easy_install_executable = os.path.abspath(os.path.join(args.env, "bin/easy_install"))
if not os.path.isfile(easy_install_executable):
raise RobustusException("failed to create virtualenv, easy_install not found")
# http://wheel.readthedocs.org/en/latest/
# wheel is binary packager for python/pip
# we store all packages in binary wheel somewhere on the PC to avoid recompilation of packages
# wheel needs pip>=1.4, setuptools>=0.8 and wheel packages for wheeling
run_shell([pip_executable, "install", "pip==1.4.1", "--upgrade"], settings["verbosity"] >= 1)
# some sloppy maintained packages (like ROS) require outdated distribute for installation
# and we need to install it before setuptools
run_shell([pip_executable, "install", "distribute==0.7.3"], settings["verbosity"] >= 1)
run_shell([pip_executable, "install", "setuptools==1.1.6", "--upgrade"], settings["verbosity"] >= 1)
run_shell([pip_executable, "install", "wheel==0.22.0", "--upgrade"], settings["verbosity"] >= 1)
# linking BLAS and LAPACK libraries
if os.path.isfile("/usr/lib64/libblas.so.3"):
logging.info("Linking CentOS libblas to venv")
blas_so = os.path.join(args.env, "lib64/libblas.so")
ln("/usr/lib64/libblas.so.3", blas_so, True)
os.environ["BLAS"] = os.path.join(args.env, "lib64")
elif os.path.isfile("/usr/lib/libblas.so"):
logging.info("Linking Ubuntu libblas to venv")
blas_so = os.path.join(args.env, "lib/libblas.so")
ln("/usr/lib/libblas.so", blas_so, True)
os.environ["BLAS"] = os.path.join(args.env, "lib")
if os.path.isfile("/usr/lib64/liblapack.so.3"):
logging.info("Linking CentOS liblapack to venv")
lapack_so = os.path.join(args.env, "lib64/liblapack.so")
ln("/usr/lib64/liblapack.so.3", lapack_so, True)
os.environ["LAPACK"] = os.path.join(args.env, "lib64")
elif os.path.isfile("/usr/lib/liblapack.so"):
logging.info("Linking Ubuntu liblapack to venv")
lapack_so = os.path.join(args.env, "lib/liblapack.so")
ln("/usr/lib/liblapack.so", lapack_so, True)
os.environ["LAPACK"] = os.path.join(args.env, "lib")
# readline must be come before everything else
run_shell([easy_install_executable, "-q", "readline==6.2.2"], settings["verbosity"] >= 1)
# compose settings file
logging.info("Write .robustus config file")
settings = Robustus._override_settings(Robustus.default_settings, args)
with open(os.path.join(args.env, Robustus.settings_file_path), "w") as file:
file.write(str(settings))
# Install Robustus in the Python virtual environment if its "setup.py" is available.
# If Robustus has already been installed in the virtual environment, running "setup.py"
# should be harmless. This is required to pass the "test_robustus" test.
cwd = os.getcwd()
script_dir = os.path.dirname(os.path.realpath(__file__))
setup_dir = os.path.abspath(os.path.join(script_dir, os.path.pardir))
logging.info("python_executable = %s" % python_executable)
logging.info("script_dir = %s" % script_dir)
logging.info("setup_dir = %s" % setup_dir)
os.chdir(setup_dir)
if os.path.exists("setup.py"):
run_shell([python_executable, "setup.py", "install"], settings["verbosity"] >= 1)
else:
logging.warn("Cannot find setup.py in %s. Continuing..." % setup_dir)
os.chdir(cwd)
logging.info("Robustus initialized environment with cache located at %s" % settings["cache"])
