def install_cmake_package(self, requirement_specifier, cmake_options, ignore_index): """ Build and install cmake package into cache & copy it to env. """ pkg_cache_dir = os.path.abspath( os.path.join(self.cache, "%s-%s" % (requirement_specifier.name, requirement_specifier.version)) ) def in_cache(): return os.path.isdir(pkg_cache_dir) if not in_cache() and not ignore_index: cwd = os.getcwd() archive = None archive_name = None try: archive = self.download(requirement_specifier.name, requirement_specifier.version) archive_name = unpack(archive) logging.info("Building %s" % requirement_specifier.name) os.chdir(archive_name) env = os.environ.copy() env["PKG_CONFIG_PATH"] = ",".join(self.search_pkg_config_locations()) retcode = run_shell( ["cmake", ".", "-DCMAKE_INSTALL_PREFIX=%s" % pkg_cache_dir] + cmake_options, env=env, verbose=self.settings["verbosity"] >= 1, ) if retcode != 0: raise RequirementException("%s configure failed" % requirement_specifier.name) retcode = run_shell(["make", "-j4"], verbose=self.settings["verbosity"] >= 1) if retcode != 0: raise RequirementException("%s build failed" % requirement_specifier.name) retcode = run_shell(["make", "install"], verbose=self.settings["verbosity"] >= 1) if retcode != 0: raise RequirementException('%s "make install" failed' % requirement_specifier.name) finally: safe_remove(archive) safe_remove(archive_name) os.chdir(cwd) pkg_install_dir = os.path.join( self.env, "lib/%s-%s" % (requirement_specifier.name, requirement_specifier.version) ) if in_cache(): # install gazebo somewhere into venv if os.path.exists(pkg_install_dir): shutil.rmtree(pkg_install_dir) shutil.copytree(pkg_cache_dir, pkg_install_dir) else: raise RequirementException("can't find gazebo-%s in robustus cache" % requirement_specifier.version) return pkg_install_dir
def _perrepo(self, cmd_str): verbose = self.settings["verbosity"] > 0 run_shell( 'cd "$(git rev-parse --show-toplevel)" && . "%s" && %s' % (self._activate_path(), cmd_str), shell=True, verbose=verbose, ) for d in os.listdir(os.path.join(self.env, "src")): full_path = os.path.join(self.env, "src", d) if os.path.isdir(full_path): logging.info("Running command in %s" % full_path) run_shell( 'cd "%s" && . "%s" && %s' % (full_path, self._activate_path(), cmd_str), shell=True, verbose=verbose )
def install_satisfactory_requirement_from_remote(self, requirement_specifier): """ If wheel for satisfactory requirement found on remote, install it. :param requirement_specifier: specifies package namd and package version string :return: True if package installed by function (according to pip return code); False otherwise. """ logging.info("Attempting to install package from remote wheel") for find_link in self.settings["find_links"]: find_links_url = (find_link + "/python-wheels/index.html",) # TEMPORARY. dtemp_path = tempfile.mkdtemp() return_code = run_shell( [ self.pip_executable, "install", "--download-cache=%s" % dtemp_path, "--no-index", "--use-wheel", "--find-links=%s" % find_links_url, requirement_specifier.freeze(), ], verbose=self.settings["verbosity"] >= 2, ) if return_code == 0: # The following downloads the wheels of the requirment (and those of # dependencies) into a pip download cache and moves (renames) the downloaded # wheels into the local Robustus cache. Regarding the need for this see "Wheels # for Dependencies" "http://lucumr.pocoo.org/2014/1/27/python-on-wheels/". for file_path in glob.glob(os.path.join(dtemp_path, "http*.whl")): if os.path.isfile(file_path): file_name = os.path.basename(file_path) file_name_new = file_name.rpartition("%2F")[-1] file_path_new = os.path.join(self.cache, file_name_new) shutil.move(file_path, file_path_new) # NOTE: Allow overwrites. safe_remove(dtemp_path) return True else: logging.info( "pip failed to install requirement %s from remote wheels cache %s." % (requirement_specifier.freeze(), find_links_url) ) safe_remove(dtemp_path) return False
def _pip_install_requirement(self, requirement_specifier): command = " ".join([self.pip_executable, "install", requirement_specifier.freeze()]) logging.info("Got url-based requirement. " "Fall back to pip shell command:%s" % (command,)) ret_code = run_shell(command, shell=True, verbose=self.settings["verbosity"] >= 1) return ret_code
def install_through_wheeling(self, requirement_specifier, rob_file, ignore_index): """ Check if package cache already contains package of specified version, if so install it. Otherwise make a wheel and put it into cache. Hope manual check for requirements file won't be necessary, waiting for pip 1.5 https://github.com/pypa/pip/issues/855 :param package: package name :param version: package version string :return: None """ # If wheelhouse doesn't contain necessary requirement attempt to install from remote wheel archive or make a wheel. installed = False if self.find_satisfactory_requirement(requirement_specifier) is None: # Pip does not download the wheels of dependencies unless it installs. installed = False if not self.settings["no_remote_cache"]: installed = self.install_satisfactory_requirement_from_remote(requirement_specifier) if not installed: logging.info("Wheel not found, downloading package") return_code = run_shell( [self.pip_executable, "install", "--download", self.cache, requirement_specifier.freeze()], verbose=self.settings["verbosity"] >= 2, ) if return_code != 0: raise RequirementException("pip failed to download requirement %s" % requirement_specifier.freeze()) logging.info("Done") logging.info("Building wheel") wheel_cmd = [ self.pip_executable, "wheel", "--no-index", "--find-links=%s" % self.cache, "--wheel-dir=%s" % self.cache, requirement_specifier.freeze(), ] # we probably sometimes will want to see build log for i in xrange(self.settings["verbosity"]): wheel_cmd.append("-v") return_code = run_shell(wheel_cmd, verbose=self.settings["verbosity"] >= 1) if return_code != 0: raise RequirementException( "pip failed to build wheel for requirement %s" % requirement_specifier.freeze() ) logging.info("Done") if not installed: # install from new prebuilt wheel logging.info("Installing package from wheel") return_code = run_shell( [ self.pip_executable, "install", "--no-index", "--use-wheel", "--find-links=%s" % self.cache, requirement_specifier.freeze(), ], verbose=self.settings["verbosity"] >= 2, ) if return_code != 0: raise RequirementException( "pip failed to install requirement %s from wheels cache %s." % (requirement_specifier.freeze(), self.cache) )
def env(args): """ Create robustus environment. @param args: command line arguments """ settings = dict() settings = Robustus._override_settings(settings, args) # create virtualenv python_executable = os.path.abspath(os.path.join(args.env, "bin/python")) if os.path.isfile(python_executable): logging.info("Found virtualenv in " + args.env) else: logging.info("Creating virtualenv") virtualenv_args = ["virtualenv", args.env, "--prompt", args.prompt] if args.python is not None: virtualenv_args += ["--python", args.python] if args.system_site_packages: virtualenv_args += ["--system-site-packages"] run_shell(virtualenv_args, settings["verbosity"] >= 1) pip_executable = os.path.abspath(os.path.join(args.env, "bin/pip")) if not os.path.isfile(pip_executable): raise RobustusException("failed to create virtualenv, pip not found") easy_install_executable = os.path.abspath(os.path.join(args.env, "bin/easy_install")) if not os.path.isfile(easy_install_executable): raise RobustusException("failed to create virtualenv, easy_install not found") # http://wheel.readthedocs.org/en/latest/ # wheel is binary packager for python/pip # we store all packages in binary wheel somewhere on the PC to avoid recompilation of packages # wheel needs pip>=1.4, setuptools>=0.8 and wheel packages for wheeling run_shell([pip_executable, "install", "pip==1.4.1", "--upgrade"], settings["verbosity"] >= 1) # some sloppy maintained packages (like ROS) require outdated distribute for installation # and we need to install it before setuptools run_shell([pip_executable, "install", "distribute==0.7.3"], settings["verbosity"] >= 1) run_shell([pip_executable, "install", "setuptools==1.1.6", "--upgrade"], settings["verbosity"] >= 1) run_shell([pip_executable, "install", "wheel==0.22.0", "--upgrade"], settings["verbosity"] >= 1) # linking BLAS and LAPACK libraries if os.path.isfile("/usr/lib64/libblas.so.3"): logging.info("Linking CentOS libblas to venv") blas_so = os.path.join(args.env, "lib64/libblas.so") ln("/usr/lib64/libblas.so.3", blas_so, True) os.environ["BLAS"] = os.path.join(args.env, "lib64") elif os.path.isfile("/usr/lib/libblas.so"): logging.info("Linking Ubuntu libblas to venv") blas_so = os.path.join(args.env, "lib/libblas.so") ln("/usr/lib/libblas.so", blas_so, True) os.environ["BLAS"] = os.path.join(args.env, "lib") if os.path.isfile("/usr/lib64/liblapack.so.3"): logging.info("Linking CentOS liblapack to venv") lapack_so = os.path.join(args.env, "lib64/liblapack.so") ln("/usr/lib64/liblapack.so.3", lapack_so, True) os.environ["LAPACK"] = os.path.join(args.env, "lib64") elif os.path.isfile("/usr/lib/liblapack.so"): logging.info("Linking Ubuntu liblapack to venv") lapack_so = os.path.join(args.env, "lib/liblapack.so") ln("/usr/lib/liblapack.so", lapack_so, True) os.environ["LAPACK"] = os.path.join(args.env, "lib") # readline must be come before everything else run_shell([easy_install_executable, "-q", "readline==6.2.2"], settings["verbosity"] >= 1) # compose settings file logging.info("Write .robustus config file") settings = Robustus._override_settings(Robustus.default_settings, args) with open(os.path.join(args.env, Robustus.settings_file_path), "w") as file: file.write(str(settings)) # Install Robustus in the Python virtual environment if its "setup.py" is available. # If Robustus has already been installed in the virtual environment, running "setup.py" # should be harmless. This is required to pass the "test_robustus" test. cwd = os.getcwd() script_dir = os.path.dirname(os.path.realpath(__file__)) setup_dir = os.path.abspath(os.path.join(script_dir, os.path.pardir)) logging.info("python_executable = %s" % python_executable) logging.info("script_dir = %s" % script_dir) logging.info("setup_dir = %s" % setup_dir) os.chdir(setup_dir) if os.path.exists("setup.py"): run_shell([python_executable, "setup.py", "install"], settings["verbosity"] >= 1) else: logging.warn("Cannot find setup.py in %s. Continuing..." % setup_dir) os.chdir(cwd) logging.info("Robustus initialized environment with cache located at %s" % settings["cache"])