def pipInstall(cfg, pkg): """PIP installation command.""" platform = system() log = cfg["devDir"] + "/autoproj/bob/logs/os_deps.txt" path = cfg["devDir"] + "/autoproj/bob/logs" if not os.path.isdir(path): execute.makeDir(path) with open(log, "a") as f: f.write("@pip " + pkg + "\n") if platform == "Windows": if not os.popen('which pip').read(): out, err, r = execute.do([ "pacman", "--noconfirm", "-S", "mingw-w64-x86_64-python2-pip" ]) if len(out) > 0: return cmd = ["pip", "install", "-U", "--noinput", pkg] else: cmd = ["yes", "|", "pip", "install", "-U", pkg] print(" ".join(cmd)) out, err, r = execute.do(cmd) if len(out) > 0: return
def install(cfg, pkg): """Standard system package manager installation command.""" platform = system() if pkg == "": return if pkg == "cmake": if os.popen('which cmake').read(): return elif pkg == "pkg-config": if os.popen('which pkg-config').read(): return log = cfg["devDir"] + "/autoproj/bob/logs/os_deps.txt" path = cfg["devDir"] + "/autoproj/bob/logs" if not os.path.isdir(path): execute.makeDir(path) with open(log, "a") as f: f.write(pkg + "\n") if platform == "Windows": out, err, r = execute.do(["pacman", "-Qq", "mingw-w64-x86_64-" + pkg]) if len(out) > 0: return elif os.system('pkg-config --exists ' + pkg) == 0: return if platform == "Windows": if pkg == "cython": execute.do( ["pacman", "--noconfirm", "-S", "mingw-w64-x86_64-cython2"]) execute.do(["pacman", "--noconfirm", "-S", "mingw-w64-x86_64-" + pkg]) elif platform == "Darwin": pkgstr = '" ' + pkg + ' "' out, err, r = execute.do(['port', 'installed', '|', 'grep', pkgstr]) if len(out) > len(pkg): return print(c.BOLD + "Installing os dependency: " + pkg + c.END, end="") execute.do(["sudo", "port", "install", pkg]) else: out, err, r = execute.do(['dpkg', '-l', pkg]) if len(err) > 5: print(c.BOLD + "Installing os dependency: " + pkg + c.END, end="") arrPkg = pkg.split() for p in arrPkg: os.system("sudo apt-get install -y " + p) else: for line in out.split(b"\n"): arrLine = line.split() if len(arrLine) > 2 and arrLine[1] == pkg: if arrLine[0] != "ii": print(c.BOLD + "Installing os dependency: " + pkg + c.END, end="") arrPkg = pkg.split() for p in arrPkg: os.system("sudo apt-get install -y " + p) break
def installPackage(cfg, p, cmake_options=[]): if p in cfg["ignorePackages"] or "orogen" in p: return path = cfg["devDir"] + "/" + p if not os.path.isdir(cfg["devDir"] + "/" + p): cfg["errors"].append("install: " + p + " path not found") return if not os.path.isfile(cfg["devDir"] + "/" + p + "/CMakeLists.txt"): print(p + c.WARNING + " skip \"no cmake package\"" + c.END) stdout.flush() return if cfg["rebuild"]: execute.do(["rm", "-rf", path + "/build"]) start = datetime.datetime.now() if os.path.isdir(path + "/build"): cmd = ["cmake", path] else: execute.makeDir(path + "/build") #cmd = ["cmake", "..", "-DCMAKE_INSTALL_PREFIX="+cfg["devDir"]+"/install", "-DCMAKE_BUILD_TYPE=DEBUG", "-Wno-dev"] cmake = "cmake_" + cfg["defBuildType"] cmd = [cmake] + cmake_options if system() == "Windows": cmd = ["bash"] + cmd out, err, r = execute.do(cmd, cfg, None, path + "/build", p.replace("/", "_") + "_configure.txt") if r != 0: print(p + c.ERROR + " configure error" + c.END) stdout.flush() cfg["errors"].append("configure: " + p) return print(p + c.WARNING + " configured" + c.END) stdout.flush() end = datetime.datetime.now() diff1 = end - start start = end out, err, r = execute.do( ["make", "install", "-j", str(cfg["numCores"]), "-C", path + "/build"], cfg, None, None, p.replace("/", "_") + "_build.txt") if r != 0: print(p + c.ERROR + " build error" + c.END) cfg["errors"].append("build: " + p) return end = datetime.datetime.now() diff2 = end - start print(p + c.WARNING + " installed" + c.END) cfg["profiling"].append( [p, { "configure time": str(diff1) }, { "compile time": str(diff2) }]) cfg["installed"].append(p)
def setupCfg(cfg): # todo: handle this files differently path = cfg["pyScriptDir"] # load server information if not already done if not "server" in cfg: with open(path + "/server.yml") as f: cfg["server"] = yaml.load(f) # load the package information if not already done path = cfg["devDir"] + "/autoproj/bob" if not os.path.isdir(path): execute.makeDir(path) if not "packages" in cfg and os.path.isfile(path + "/packages.yml"): with open(path + "/packages.yml") as f: cfg["packages"] = yaml.load(f)
def fetch_rbdl(cfg): path = cfg["devDir"] + "/external" print(c.BOLD + "Fetching " + "external/rbdl ... " + c.END, end="") sys.stdout.flush cwd = os.getcwd() execute.makeDir(path) os.chdir(path) if not os.path.isfile(path + "/rbdl/CMakeLists.txt"): if os.path.isdir(path + "/rbdl"): execute.do(["rm", "-rf", "rbdl"]) execute.do(["hg", "clone", "https://bitbucket.org/rbdl/rbdl"]) if not os.path.isfile("rbdl/CMakeLists.txt"): cfg["errors"].append("fetch: external/rbdl") os.chdir(cwd) return True
def fetch_minizip(cfg): path = cfg["devDir"]+"/external" print c.BOLD+"Fetching "+"external/minizip ... "+c.END, sys.stdout.flush cwd = os.getcwd() execute.makeDir(path) os.chdir(path) if not os.path.isfile(path+"/unzip101e.zip"): if os.path.isdir(path+"/minizip"): uninstall_minizip(cfg) execute.do(["wget", "-q", "http://www.winimage.com/zLibDll/unzip101e.zip"]) execute.do(["unzip", "unzip101e.zip", "-d", "minizip"]) if not os.path.isfile("minizip/minizip.c"): cfg["errors"].append("fetch: external/minizip") os.chdir(cwd) patch_minizip(cfg) return True
def fetch_sisl(cfg): path = cfg["devDir"]+"/external" print c.BOLD+"Fetching "+"external/sisl ... "+c.END, sys.stdout.flush cwd = os.getcwd() execute.makeDir(path) os.chdir(path) if not os.path.isfile(path+"/sisl-4.5.0.tar.gz"): if os.path.isdir(path+"/sisl"): execute.do(["rm", "-rf", "sisl"]) execute.do(["wget", "-q", "http://www.sintef.no/upload/IKT/9011/geometri/sisl/sisl-4.5.0.tar.gz"]) execute.do(["tar", "-xzf", "sisl-4.5.0.tar.gz"]) execute.do(["mv", "sisl-4.5.0", "sisl"]) if not os.path.isfile("sisl/CMakeLists.txt"): cfg["errors"].append("fetch: external/sisl") os.chdir(cwd) patch_sisl(cfg) return True
def fetch_ode(cfg): path = cfg["devDir"]+"/simulation" print c.BOLD+"Fetching "+"external/ode ... "+c.END, sys.stdout.flush cwd = os.getcwd() execute.makeDir(path) os.chdir(path) if not os.path.isfile(path+"/ode-0.12.tar.gz"): if os.path.isdir(path+"/ode"): uninstall_ode(cfg) execute.do(["wget", "-q", "http://sourceforge.net/projects/opende/files/ODE/0.12/ode-0.12.tar.gz"]) execute.do(["tar", "-xzf", "ode-0.12.tar.gz"]) patch_ode(cfg) execute.do(["mv", "ode-0.12", "ode"]) if not os.path.isfile("ode/ode.pc.in"): cfg["errors"].append("fetch: simulation/ode") os.chdir(cwd) cfg["installed"].append("simulation/ode") return True
def fetch_ode_16(cfg): path = cfg["devDir"] + "/simulation" print(c.BOLD + "Fetching " + "external/ode ... " + c.END, end="") sys.stdout.flush() cwd = os.getcwd() execute.makeDir(path) os.chdir(path) if not os.path.isfile(path + "/ode-0.16.tar.gz"): if os.path.isdir(path + "/ode"): uninstall_ode(cfg) execute.do([ "wget", "-q", "https://bitbucket.org/odedevs/ode/downloads/ode-0.16.tar.gz", ]) execute.do(["tar", "-xzf", "ode-0.16.tar.gz"]) patch_ode_16(cfg) execute.do(["mv", "ode-0.16", "ode"]) if not os.path.isfile("ode/ode.pc.in"): cfg["errors"].append("fetch: simulation/ode") os.chdir(cwd) cfg["installed"].append("simulation/ode") return True
def setupEnv(cfg, update=False): global os prefix = cfg["devDir"] + "/install" if system() == "Windows": if prefix[1] == ':': prefix = prefix.replace(prefix[:2], "/"+prefix[0]) prefix_bin = prefix + "/bin" prefix_lib = prefix + "/lib" prefix_pkg = prefix_lib + "/pkgconfig" prefix_config = prefix + "/configuration" platform = system() # create env.sh p = subprocess.Popen("which cmake_debug", stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) out, err = p.communicate() cmakeDebugPath = out.strip() if len(cmakeDebugPath) > 0: # check if path is correct expectPath = cfg["devDir"]+"/install/bin/cmake_debug" if platform == "Windows": c.printWarning("cmake_debug path check is not working on Windows currently (please always ensure that you only sourced the env.sh in your current dev folder!") else : if cmakeDebugPath != expectPath: c.printError('"cmake_debug" found in wrong folder.') c.printError('Found: '+cmakeDebugPath) c.printError('Expected: '+expectPath) c.printError('Maybe you already sourced an "env.sh" from a different "dev" folder?') return if not update: return if not update: if os.path.isfile(cfg["devDir"]+"/env.sh"): source(cfg["devDir"]+"/env.sh") p = subprocess.Popen("which autoproj", stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) out, err = p.communicate() aPath = out.strip() if len(aPath) > 0: with open(cfg["devDir"]+"/bobenv.sh", "w") as f: f.write("#! /bin/sh\n") f.write(". env.sh\n") f.write('export MARS_SCRIPT_DIR="'+cfg["pyScriptDir"]+'"\n') f.write("alias bob='${MARS_SCRIPT_DIR}/pybob.py'\n") f.write("alias bob-bootstrap='${MARS_SCRIPT_DIR}/pybob.py bootstrap'\n") f.write("alias bob-install='${MARS_SCRIPT_DIR}/pybob.py install'\n") f.write("alias bob-rebuild='${MARS_SCRIPT_DIR}/pybob.py rebuild'\n") f.write("alias bob-build='${MARS_SCRIPT_DIR}/pybob.py'\n") f.write("alias bob-diff='${MARS_SCRIPT_DIR}/pybob.py diff'\n") f.write("alias bob-list='${MARS_SCRIPT_DIR}/pybob.py list'\n") f.write("alias bob-fetch='${MARS_SCRIPT_DIR}/pybob.py fetch'\n") f.write("alias bob-show-log='${MARS_SCRIPT_DIR}/pybob.py show-log'\n") f.write(". ${MARS_SCRIPT_DIR}/auto_complete.sh\n") else: with open(cfg["devDir"]+"/env.sh", "w") as f: f.write("#! /bin/sh\n") f.write('export AUTOPROJ_CURRENT_ROOT="'+cfg["devDir"]+'"\n') f.write('export MARS_SCRIPT_DIR="'+cfg["pyScriptDir"]+'"\n') f.write('export PATH="$PATH:'+prefix_bin+'"\n') if platform == "Darwin": f.write('export DYLD_LIBRARY_PATH="'+prefix_lib+':$DYLD_LIBRARY_PATH"\n') elif platform == "Linux": f.write('export LD_LIBRARY_PATH="'+prefix_lib+':$DYLD_LIBRARY_PATH"\n') else: f.write('export PATH="'+prefix_lib+':$PATH"\n') f.write('export ROCK_CONFIGURATION_PATH="'+prefix_config+'"\n') f.write('export PYTHONPATH="'+prefix_lib+'/python2.7/site-packages:$PYTHONPATH"\n') # todo: handle python path f.write('if [ x${PKG_CONFIG_PATH} = "x" ]; then\n') f.write(' export PKG_CONFIG_PATH="'+prefix_pkg+'"\n') f.write('else\n') f.write(' export PKG_CONFIG_PATH="'+prefix_pkg+':$PKG_CONFIG_PATH"\n') f.write('fi\n') f.write("alias bob='${MARS_SCRIPT_DIR}/pybob.py'\n") f.write("alias bob-bootstrap='${MARS_SCRIPT_DIR}/pybob.py bootstrap'\n") f.write("alias bob-install='${MARS_SCRIPT_DIR}/pybob.py install'\n") f.write("alias bob-rebuild='${MARS_SCRIPT_DIR}/pybob.py rebuild'\n") f.write("alias bob-build='${MARS_SCRIPT_DIR}/pybob.py'\n") f.write("alias bob-diff='${MARS_SCRIPT_DIR}/pybob.py diff'\n") f.write("alias bob-list='${MARS_SCRIPT_DIR}/pybob.py list'\n") f.write("alias bob-fetch='${MARS_SCRIPT_DIR}/pybob.py fetch'\n") f.write("alias bob-show-log='${MARS_SCRIPT_DIR}/pybob.py show-log'\n") f.write(". ${MARS_SCRIPT_DIR}/auto_complete.sh\n") execute.makeDir(cfg["devDir"]+"/install/bin") with open(cfg["devDir"]+"/install/bin/cmake_debug", "w") as f: f.write("#!/bin/bash\n") options = "-DROCK_TEST_ENABLED=OFF" if not "autoprojEnv" in cfg or not cfg["autoprojEnv"]: options += " -DBINDINGS_RUBY=OFF " if platform == "Windows": f.write("cmake .. "+options+"-DCMAKE_INSTALL_PREFIX="+cfg["devDir"]+"/install -DCMAKE_BUILD_TYPE=DEBUG -G \"MSYS Makefiles\" $@\n") else: f.write("cmake .. "+options+"-DCMAKE_INSTALL_PREFIX="+cfg["devDir"]+"/install -DCMAKE_BUILD_TYPE=DEBUG $@\n") with open(cfg["devDir"]+"/install/bin/cmake_release", "w") as f: f.write("#!/bin/bash\n") if platform == "Windows": f.write("cmake .. "+options+"-DCMAKE_INSTALL_PREFIX="+cfg["devDir"]+"/install -DCMAKE_BUILD_TYPE=RELEASE -G \"MSYS Makefiles\" $@\n") else: f.write("cmake .. "+options+"-DCMAKE_INSTALL_PREFIX="+cfg["devDir"]+"/install -DCMAKE_BUILD_TYPE=RELEASE $@\n") cmd = ["chmod", "+x", cfg["devDir"]+"/install/bin/cmake_debug"] execute.simpleExecute(cmd) cmd = ["chmod", "+x", cfg["devDir"]+"/install/bin/cmake_release"] execute.simpleExecute(cmd) source(cfg["devDir"]+"/env.sh")
def updatePackageSets(cfg): # the server configuration are handled in the init.rb for autoproj setupCfg(cfg) path = cfg["devDir"] + "/autoproj/" execute.makeDir(path + "remotes") execute.makeDir(cfg["devDir"] + "/.autoproj/remotes") cloned = [] deps = [] with open(path + "manifest") as f: manifest = yaml.load(f) for packageSet in manifest["package_sets"]: key, value = list(packageSet.items())[0] realPath = cfg[ "devDir"] + "/.autoproj/remotes/" + key + "__" + value.strip( ).replace("/", "_").replace("-", "_") + "_git" if not os.path.isdir(realPath): if key == "url": clonePackageSet(cfg, value.strip(), realPath, path, cloned, deps) else: clonePackageSet(cfg, cfg["server"][key] + value.strip() + ".git", realPath, path, cloned, deps) # update remotes that are not actually cloned for d in os.listdir(path + "remotes"): if os.path.isdir(path + "remotes/" + d): if d not in cloned: if cfg["update"]: c.printNormal(" Updating: " + d) out, err, r = execute.do( ["git", "-C", path + "remotes/" + d, "pull"]) if r != 0: cfg["errors"].append("update: " + d) c.printError("\ncan't update package set \"" + d + "\":\n" + err) if d not in cloned: with open(path + "remotes/" + d + "/source.yml") as f: info = yaml.load(f) if "imports" in info and info["imports"]: for i in info["imports"]: key, value = list(i.items())[0] realPath = cfg[ "devDir"] + "/.autoproj/remotes/" + key + "__" + value.strip( ).replace("/", "_").replace("-", "_") + "_git" if i not in deps and not os.path.isdir(realPath): deps.append(i) # now handle deps while len(deps) > 0: packageSet = deps.pop(0) key, value = list(packageSet.items())[0] realPath = cfg[ "devDir"] + "/.autoproj/remotes/" + key + "__" + value.strip( ).replace("/", "_").replace("-", "_") + "_git" clonePackageSet(cfg, cfg["server"][key] + value.strip() + ".git", realPath, path, cloned, deps) # last step: write all packages int a file to speed up pybob usage packages, wildcards = listPackages(cfg) pDict = {} with open(path + "/bob/packages.txt", "wb") as f: for p in packages: if len(p[1]) > 0: if sys.version_info.major <= 2: f.write(p[1] + "\n") else: f.write(bytes(p[1] + "\n", "utf-8")) pDict[p[1]] = p[0] else: if sys.version_info.major <= 2: f.write(p[0] + "\n") else: f.write(bytes(p[0] + "\n", "utf-8")) pDict[p[0]] = p[0] for p in wildcards: if len(p[1]) > 0: pDict[p[1]] = p[0] else: pDict[p[0]] = p[0] with open(path + "/bob/packages.yml", "w") as f: yaml.dump(pDict, f)
def setupEnv(cfg, update=False): global os prefix = cfg["devDir"] + "/install" if system() == "Windows": if prefix[1] == ':': prefix = prefix.replace(prefix[:2], "/"+prefix[0]) prefix_bin = prefix + "/bin" prefix_lib = prefix + "/lib" prefix_pkg = prefix_lib + "/pkgconfig" pythonpath = prefix_lib + "/python%d.%d/site-packages" % (sys.version_info.major, sys.version_info.minor) platform = system() if platform == "Windows": # todo: make this more generic pythonpath = "/mingw64/lib/python2.7:/mingw64/lib/python2.7/plat-win32:/mingw64/lib/python2.7/lib-tk:/mingw64/lib/python2.7/lib-dynload:/mingw64/lib/python2.7/site-packages:"+pythonpath elif platform == "Linux": prefix_lib += ":" + prefix + "/lib/x86_64-linux-gnu" prefix_pkg += ":" + prefix + "/lib/x86_64-linux-gnu/pkgconfig" prefix_config = prefix + "/configuration" # create env.sh p = subprocess.Popen("which cmake_debug", stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) out, err = p.communicate() cmakeDebugPath = out.strip() if len(cmakeDebugPath) > 0: # check if path is correct expectPath = cfg["devDir"]+"/install/bin/cmake_debug" if platform == "Windows": c.printWarning("cmake_debug path check is not working on Windows currently (please always ensure that you only sourced the env.sh in your current dev folder!") else : if cmakeDebugPath.decode("utf-8") != expectPath: c.printError('"cmake_debug" found in wrong folder.') c.printError('Found: ' + cmakeDebugPath.decode("utf-8")) c.printError('Expected: ' + expectPath) c.printError('Maybe you already sourced an "env.sh" from a different "dev" folder?') return if not update: return if not update: if os.path.isfile(cfg["devDir"]+"/env.sh"): source(cfg["devDir"]+"/env.sh") p = subprocess.Popen("which autoproj", stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) out, err = p.communicate() aPath = out.strip() if len(aPath) > 0: with open(cfg["devDir"]+"/bobenv.sh", "w") as f: f.write("#! /bin/bash\n") f.write(". env.sh\n") f.write('export MARS_SCRIPT_DIR="'+cfg["pyScriptDir"]+'"\n') _make_pybob_aliases(f) else: with open(cfg["devDir"]+"/env.sh", "w") as f: f.write("#! /bin/bash\n") f.write('export AUTOPROJ_CURRENT_ROOT="'+cfg["devDir"]+'"\n') f.write('if [ x${CMAKE_PREFIX_PATH} = "x" ]; then\n') f.write(' export CMAKE_PREFIX_PATH="'+cfg["devDir"]+'/install"\n') f.write('else\n') f.write(' export CMAKE_PREFIX_PATH="'+cfg["devDir"]+'/install:$CMAKE_PREFIX_PATH"\n') f.write('fi\n') f.write('export MARS_SCRIPT_DIR="'+cfg["pyScriptDir"]+'"\n') f.write('export PATH="$PATH:'+prefix_bin+'"\n') if platform == "Darwin": f.write('export DYLD_LIBRARY_PATH="'+prefix_lib+':$DYLD_LIBRARY_PATH"\n') f.write('export MYLD_LIBRARY_PATH="$DYLD_LIBRARY_PATH"\n') elif platform == "Linux": f.write('export LD_LIBRARY_PATH="'+prefix_lib+':$LD_LIBRARY_PATH"\n') f.write('export CXXFLAGS="-std=c++11"\n') else: f.write('export PATH="'+prefix_lib+':$PATH"\n') f.write('export ROCK_CONFIGURATION_PATH="'+prefix_config+'"\n') f.write('export PYTHONPATH="' + pythonpath + ':$PYTHONPATH"\n') f.write('if [ x${PKG_CONFIG_PATH} = "x" ]; then\n') f.write(' export PKG_CONFIG_PATH="'+prefix_pkg+'"\n') f.write('else\n') f.write(' export PKG_CONFIG_PATH="'+prefix_pkg+':$PKG_CONFIG_PATH"\n') f.write('fi\n') _make_pybob_aliases(f) execute.makeDir(cfg["devDir"]+"/install/bin") if len(aPath) == 0: with open(cfg["devDir"]+"/install/bin/amake", "w") as f: f.write("#!/bin/bash\n") f.write("${AUTOPROJ_CURRENT_ROOT}/pybob/pybob.py install $@\n") cmd = ["chmod", "+x", cfg["devDir"]+"/install/bin/amake"] execute.simpleExecute(cmd) with open(cfg["devDir"]+"/install/bin/cmake_debug", "w") as f: f.write("#!/bin/bash\n") options = "-DROCK_TEST_ENABLED=OFF" if not "autoprojEnv" in cfg or not cfg["autoprojEnv"]: options += " -DBINDINGS_RUBY=OFF " if platform == "Windows": f.write("cmake .. "+options+"-DCMAKE_INSTALL_PREFIX=$AUTOPROJ_CURRENT_ROOT/install -DCMAKE_BUILD_TYPE=DEBUG -G \"MSYS Makefiles\" $@\n") else: f.write("cmake .. "+options+"-DCMAKE_INSTALL_PREFIX=$AUTOPROJ_CURRENT_ROOT/install -DCMAKE_BUILD_TYPE=DEBUG $@\n") with open(cfg["devDir"]+"/install/bin/cmake_release", "w") as f: f.write("#!/bin/bash\n") if platform == "Windows": f.write("cmake .. "+options+"-DCMAKE_INSTALL_PREFIX=$AUTOPROJ_CURRENT_ROOT/install -DCMAKE_BUILD_TYPE=RELEASE -G \"MSYS Makefiles\" $@\n") else: f.write("cmake .. "+options+"-DCMAKE_INSTALL_PREFIX=$AUTOPROJ_CURRENT_ROOT/install -DCMAKE_BUILD_TYPE=RELEASE $@\n") cmd = ["chmod", "+x", cfg["devDir"]+"/install/bin/cmake_debug"] execute.simpleExecute(cmd) cmd = ["chmod", "+x", cfg["devDir"]+"/install/bin/cmake_release"] execute.simpleExecute(cmd) source(cfg["devDir"]+"/env.sh")