def show_log_():
packageList = []
package = sys.argv[2].replace("/", "_")
for file in os.listdir(os.path.join(cfg["devDir"], "autoproj/bob/logs")):
if file[-13:] == "configure.txt" and package in file:
packageList.append(file[:-14])
for package in packageList:
logFile = cfg[
"devDir"] + "/autoproj/bob/logs/" + package + "_configure.txt"
c.printWarning("configure log:")
with open(logFile) as f:
for l in f:
if "error" in l:
c.printError(l.strip())
else:
c.printNormal(l.strip())
logFile = cfg["devDir"] + "/autoproj/bob/logs/" + package + "_build.txt"
c.printWarning("build log:")
with open(logFile) as f:
for l in f:
if "error" in l:
c.printError(l.strip())
else:
c.printNormal(l.strip())
def fetchBuildconf(cfg):
if os.path.isdir(cfg["devDir"] + "/autoproj"):
if cfg["update"]:
c.printNormal(" Update buildconf.")
out, err, r = execute.do(
["git", "-C", cfg["devDir"] + "/autoproj", "pull"])
if r != 0:
cfg["errors"].append("update: buildconf")
c.printError("\ncan't update buildconf:\n" + err)
else:
address = cfg["buildconfAddress"]
if len(address) == 0:
c.printError("no address given")
return
branch = cfg["buildconfBranch"]
c.printNormal(" Fetching \"" + address + branch + "\" into " +
cfg["devDir"] + "/autoproj")
command = [
"git", "clone", "-o", "autobuild", address,
cfg["devDir"] + "/autoproj"
]
if len(branch) > 0:
command.append("-b")
command.append(branch)
execute.do(command)
def clonePackageSet(cfg, git, realPath, path, cloned, deps):
# clone in tmp folder
c.printNormal(" Fetching: " + git)
out, err, r = execute.do(
["git", "clone", "-o", "autobuild", git, realPath])
if not os.path.isdir(realPath + "/.git"):
c.printNormal(out)
c.printError(err)
cfg["errors"].append("clone: " + git)
return
# get the name of the remote
with open(realPath + "/source.yml") as f:
info = yaml.load(f)
#os.system("rm -rf "+path+"remotes/"+info["name"])
os.system("ln -s " + realPath + " " + path + "remotes/" + info["name"])
if "imports" in info and info["imports"]:
for i in info["imports"]:
key, value = list(i.items())[0]
realPath = cfg[
"devDir"] + "/.autoproj/remotes/" + key + "__" + value.strip(
).replace("/", "_").replace("-", "_") + "_git"
if i not in deps and not os.path.isdir(realPath):
deps.append(i)
# store the info which package sets we have cloned already
cloned.append(info["name"])
def show_log_():
package = sys.argv[2]
logFile = cfg["devDir"] + "/autoproj/bob/logs/"+package.replace("/", "_")+"_configure.txt"
c.printWarning("configure log:")
with open(logFile) as f:
for l in f:
if "error" in l:
c.printError(l.strip())
else:
c.printNormal(l.strip())
logFile = cfg["devDir"] + "/autoproj/bob/logs/"+package.replace("/", "_")+"_build.txt"
c.printWarning("build log:")
with open(logFile) as f:
for l in f:
if "error" in l:
c.printError(l.strip())
else:
c.printNormal(l.strip())
def updatePackageSets(cfg):
# the server configuration are handled in the init.rb for autoproj
setupCfg(cfg)
path = cfg["devDir"] + "/autoproj/"
execute.makeDir(path + "remotes")
execute.makeDir(cfg["devDir"] + "/.autoproj/remotes")
cloned = []
deps = []
with open(path + "manifest") as f:
manifest = yaml.load(f)
for packageSet in manifest["package_sets"]:
key, value = list(packageSet.items())[0]
realPath = cfg[
"devDir"] + "/.autoproj/remotes/" + key + "__" + value.strip(
).replace("/", "_").replace("-", "_") + "_git"
if not os.path.isdir(realPath):
if key == "url":
clonePackageSet(cfg, value.strip(), realPath, path, cloned,
deps)
else:
clonePackageSet(cfg,
cfg["server"][key] + value.strip() + ".git",
realPath, path, cloned, deps)
# update remotes that are not actually cloned
for d in os.listdir(path + "remotes"):
if os.path.isdir(path + "remotes/" + d):
if d not in cloned:
if cfg["update"]:
c.printNormal(" Updating: " + d)
out, err, r = execute.do(
["git", "-C", path + "remotes/" + d, "pull"])
if r != 0:
cfg["errors"].append("update: " + d)
c.printError("\ncan't update package set \"" + d +
"\":\n" + err)
if d not in cloned:
with open(path + "remotes/" + d + "/source.yml") as f:
info = yaml.load(f)
if "imports" in info and info["imports"]:
for i in info["imports"]:
key, value = list(i.items())[0]
realPath = cfg[
"devDir"] + "/.autoproj/remotes/" + key + "__" + value.strip(
).replace("/", "_").replace("-", "_") + "_git"
if i not in deps and not os.path.isdir(realPath):
deps.append(i)
# now handle deps
while len(deps) > 0:
packageSet = deps.pop(0)
key, value = list(packageSet.items())[0]
realPath = cfg[
"devDir"] + "/.autoproj/remotes/" + key + "__" + value.strip(
).replace("/", "_").replace("-", "_") + "_git"
clonePackageSet(cfg, cfg["server"][key] + value.strip() + ".git",
realPath, path, cloned, deps)
# last step: write all packages int a file to speed up pybob usage
packages, wildcards = listPackages(cfg)
pDict = {}
with open(path + "/bob/packages.txt", "wb") as f:
for p in packages:
if len(p[1]) > 0:
if sys.version_info.major <= 2:
f.write(p[1] + "\n")
else:
f.write(bytes(p[1] + "\n", "utf-8"))
pDict[p[1]] = p[0]
else:
if sys.version_info.major <= 2:
f.write(p[0] + "\n")
else:
f.write(bytes(p[0] + "\n", "utf-8"))
pDict[p[0]] = p[0]
for p in wildcards:
if len(p[1]) > 0:
pDict[p[1]] = p[0]
else:
pDict[p[0]] = p[0]
with open(path + "/bob/packages.yml", "w") as f:
yaml.dump(pDict, f)
def envsh_():
global cfg
env.setupEnv(cfg, True)
c.printNormal(" Recreated env.sh.")
def install_():
global cfg
layout_packages = []
cfg["update"] = False
if len(sys.argv) < 3:
buildconf.fetchPackages(cfg, layout_packages)
else:
buildconf.fetchPackage(cfg, sys.argv[2], layout_packages)
deps = []
checked = []
if cfg["checkDeps"]:
for p in layout_packages:
bob_package.getDeps(cfg, p, deps, checked)
#print deps
toInstall = []
for d in deps[::-1]:
if d not in toInstall:
toInstall.append(d)
for p in layout_packages:
if p not in toInstall:
toInstall.append(p)
while len(toInstall) > 0:
threads = []
jobs = []
iList = list(toInstall)
toInstall = []
for p in iList:
wait = False
#c.printWarning(str(cfg["deps"][p]))
if p in cfg["deps"]:
for d in cfg["deps"][p]:
if d in iList:
wait = True
break
if not wait:
jobs.append(p)
if p in cfg["overrides"] and "install" in cfg["overrides"][p]:
if cfg["multiprocessing"]:
threads.append(Thread(target=cfg["overrides"][p]["install"], args=(cfg,)))
else:
c.printNormal("Install: "+p)
le = len(cfg["errors"])
cfg["overrides"][p]["install"](cfg)
if len(cfg["errors"]) <= le:
c.printWarning("done")
else:
c.printError("error")
elif p in cfg["osdeps"]:
# os deps are installed in fetch phase
continue
else:
if cfg["multiprocessing"]:
threads.append(Thread(target=bob_package.installPackage, args=(cfg, p)))
else:
c.printNormal("Install: "+p)
le = len(cfg["errors"])
bob_package.installPackage(cfg, p)
if len(cfg["errors"]) <= le:
c.printWarning("done")
else:
c.printError("error")
else:
toInstall.append(p)
if cfg["multiprocessing"]:
c.printBold("Install: "+str(jobs))
le = len(cfg["errors"])
for t in threads:
t.start()
for t in threads:
t.join()
if len(cfg["errors"]) > le:
foo = ""
print
printNormal(" The following commands are available:\n "),
c.printBold(", ".join(commands))
printNormal('\n Once you have the env.sh sourced, most commands\n can also be used with "mars_command" to have\n autocompletion (e.g. mars_install)\n')
env.setupEnv(cfg, False)
if __name__ == "__main__":
if len(sys.argv) < 2 or sys.argv[1] not in commands:
print c.printBold("Please specify an action. Your options are:\n" +
", ".join(commands) + "\n")
exit(0)
if "-n" in sys.argv:
cfg["checkDeps"] = False
globals()[sys.argv[1].replace("-", "_")+"_"]()
printErrors()
if len(cfg["profiling"]) > 0:
with open(cfg["devDir"]+"/autoproj/bob/profiling.yml", "w") as f:
yaml.dump(cfg["profiling"], f, default_flow_style=False)
if len(cfg["depsInverse"]) > 0:
with open(cfg["devDir"]+"/autoproj/bob/depsInverse.yml", "w") as f:
yaml.dump(cfg["depsInverse"], f, default_flow_style=False)
c.printBold("Installed packages: ")
c.printNormal(cfg["installed"])
diff = datetime.datetime.now() - start
print "Time: "+str(diff)
def install_():
global cfg
layout_packages = []
cfg["update"] = False
filterArgs = ["-n", "-k"]
if len(sys.argv) < 3 or sys.argv[2] in filterArgs:
# search path upwards for a manifest.xml
# if not found build manifest from buildconf
pathToCheck = os.getcwd()
found = False
done = False
while not done:
if os.path.isfile(pathToCheck + "/manifest.xml"):
found = True
done = True
elif os.path.exists(pathToCheck + "/autoproj"):
# found dev root
done = True
else:
arrPath = pathToCheck.split("/")
if len(arrPath) == 1:
done = True
else:
pathToCheck = "/".join(arrPath[:-1])
if found:
layout_packages.append(os.path.relpath(pathToCheck, cfg["devDir"]))
else:
buildconf.fetchPackages(cfg, layout_packages)
else:
pathToCheck = os.path.join(os.getcwd(), sys.argv[2])
if os.path.isfile(pathToCheck + "/manifest.xml"):
layout_packages.append(os.path.relpath(pathToCheck, cfg["devDir"]))
else:
buildconf.fetchPackage(cfg, sys.argv[2], layout_packages)
deps = []
checked = []
if cfg["checkDeps"]:
for p in layout_packages:
bob_package.getDeps(cfg, p, deps, checked)
toInstall = []
for d in deps[::-1]:
if d not in toInstall:
toInstall.append(d)
for p in layout_packages:
if p not in toInstall:
toInstall.append(p)
iList = []
while len(toInstall) > 0:
threads = []
jobs = []
oldList = iList
iList = list(toInstall)
if oldList == iList:
# detect unresolved deps loop
for p in oldList:
c.printError("detect dependency cycle:\n " + str(p))
c.printWarning(" deps:")
if p in cfg["deps"]:
for d in cfg["deps"][p]:
if d in iList:
c.printWarning(" - " + str(d))
exit(-1)
toInstall = []
for p in iList:
wait = False
if p in cfg["deps"]:
for d in cfg["deps"][p]:
if d in iList:
wait = True
break
if not wait:
jobs.append(p)
if p in cfg["overrides"] and "install" in cfg["overrides"][p]:
if cfg["multiprocessing"]:
threads.append(
Thread(target=cfg["overrides"][p]["install"],
args=(cfg, )))
else:
c.printNormal("Install: " + p)
le = len(cfg["errors"])
cfg["overrides"][p]["install"](cfg)
if len(cfg["errors"]) <= le:
c.printWarning("done")
else:
c.printError("error")
elif p in cfg["osdeps"]:
# os deps are installed in fetch phase
continue
else:
if cfg["multiprocessing"]:
threads.append(
Thread(target=bob_package.installPackage,
args=(cfg, p)))
else:
c.printNormal("Install: " + p)
le = len(cfg["errors"])
bob_package.installPackage(cfg, p)
if len(cfg["errors"]) <= le:
c.printWarning("done")
else:
c.printError("error")
else:
toInstall.append(p)
if cfg["multiprocessing"]:
c.printBold("Install: " + str(jobs))
le = len(cfg["errors"])
for t in threads:
t.start()
for t in threads:
t.join()
if len(cfg["errors"]) > le:
foo = ""