def run(self):
ui.debug("Calling:", " ".join(self.cmd))
try:
self.process = subprocess.Popen(
self.cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=self.cwd, env=self.env
)
except Exception, e:
self.exception = e
return
def clone_project(worktree, url, src=None, branch=None, remote="origin",
skip_if_exists=False):
""" Add a project to a worktree given its url.
If src is not given, it will be guessed from the url
If skip_if_exists is False, an error message will be
raised if the project already exists
"""
should_add = True
if not src:
src = url.split("/")[-1].replace(".git", "")
if os.path.isabs(src):
src = os.path.relpath(worktree.root, src)
src = qibuild.sh.to_posix_path(src)
project = worktree.get_project(src, raises=False)
if project:
if not skip_if_exists:
mess = "Could not add project from %s in %s\n" % (url, src)
mess += "This path is already registered for worktree in %s\n" % worktree.root
raise Exception(mess)
else:
if os.path.exists(project.path):
ui.debug("Found project in %s, skipping" % src)
return
# Some one erase the project manually without telling qiworktree
should_add = False
path = os.path.join(worktree.root, src)
path = qibuild.sh.to_native_path(path)
if os.path.exists(path):
if skip_if_exists:
if qisrc.git.is_submodule(path):
ui.warning("erasing submodule: ", path)
qibuild.sh.rm(path)
else:
ui.debug("Adding project in %s", src)
worktree.add_project(src)
return
else:
mess = "Could not add project from %s in %s\n" % (url, src)
mess += "This path already exists\n"
raise Exception(mess)
ui.info(ui.green, "Git clone: %s -> %s" % (url, path))
dirname = os.path.dirname(path)
qibuild.sh.mkdir(dirname, recursive=True)
git = qisrc.git.Git(path)
if branch:
git.clone(url, "-b", branch, "-o", remote)
else:
git.clone(url, "-o", remote)
if should_add:
worktree.add_project(path)
def toc_open(worktree_root, args=None, qibuild_cfg=None):
""" Creates a :py:class:`Toc` object.
:param worktree: The worktree to be used. (see :py:class:`qisrc.worktree.WorkTree`)
:param args: an ``argparse.NameSpace`` object containing
the arguments passed from the comand line.
:param qibuild_cfg: A (:py:class:`qibuild.config.QiBuildConfig` instance) to use.
If None, we built a new instance to store in ``toc.config``
You should always use this function to call Toc methods from
a qibuild :term:`action`.
It takes care of all the options you specify from command line,
and calls Toc constructor accordingly (see :py:meth:`Toc.__init__`)
"""
# Not that args can come from:
# - a worktree parser
# - a toc parser
# - a build parser
# (hence all the hasattr...)
# ...
# or simply not given :)
config = None
if hasattr(args, 'config'):
config = args.config
build_type = None
if hasattr(args, 'build_type'):
build_type = args.build_type
cmake_flags = list()
if hasattr(args,'cmake_flags'):
cmake_flags = args.cmake_flags
cmake_generator = None
if hasattr(args, 'cmake_generator'):
cmake_generator = args.cmake_generator
worktree = qisrc.worktree.open_worktree(worktree_root)
toc = Toc(worktree,
config=config,
build_type=build_type,
cmake_flags=cmake_flags,
cmake_generator=cmake_generator,
qibuild_cfg=qibuild_cfg)
(active_projects, single) = _projects_from_args(toc, args)
toc.active_projects = active_projects
ui.debug("active projects: %s", ".".join(toc.active_projects))
ui.debug("single: %s", str(single))
toc.solve_deps = (not single)
return toc
def main():
ui.info(ui.red, "This is a an error message\n",
ui.reset, "And here are the details")
ui.error("could not build")
ui.warning("-j ignored for this generator")
ui.info("building foo")
ui.debug("debug message")
ui.info(ui.brown, "this is brown")
ui.info(ui.bold, ui.brown, "this is bold brown")
ui.info(ui.red, "red is dead")
ui.info(ui.darkred, "darkred is really dead")
ui.info(ui.yellow, "this is yellow")
def _dump_arguments(name, args):
""" Dump an argparser namespace to log """
output = ""
max_len = 0
for k in args.__dict__.keys():
if len(k) > max_len:
max_len = len(k)
for k, v in args.__dict__.iteritems():
pad = " " * (max_len - len(k))
output += " %s%s = %s\n" % (str(k), pad, str(v))
if output[-1] == "\n":
output = output[:-1]
ui.debug("[%s] arguments:\n%s", name, output)
def _projects_from_args(toc, args):
"""
Cases handled:
- nothing specified: get the project from the cwd
- args.single: do not resolve dependencies
- args.all: return all projects
Returns a tuple (project_names, single):
project_names: the actual list for project
single: user specified --single
"""
toc_p_names = [p.name for p in toc.projects]
if hasattr(args, "all") and args.all:
# Pretend the user has asked for all the known projects
ui.debug("select: All projects have been selected")
return (toc_p_names, False)
if hasattr(args, "projects"):
if args.projects:
ui.debug("select: projects list from arguments: %s", ",".join(args.projects))
return ([args.projects, args.single])
else:
from_cwd = None
try:
from_cwd = qibuild.project.project_from_cwd()
except:
pass
if from_cwd:
ui.debug("select: projects from cwd: %s", from_cwd)
return ([from_cwd], args.single)
else:
ui.debug("select: default to all projects")
return (toc_p_names, args.single)
else:
return (list(), False)
def get_sdk_dirs(self, project_name):
""" Return a list of sdk needed to build a project.
Iterate through the dependencies.
When it is a package (pre-compiled), add the path of
the package, when it is a project, add the path to the "sdk" dir
under the build directory of the project.
If a name is both in source and in package, use the package
(saves compile time), unless user asked explicitely for a list
of projects
"""
dirs = list()
known_project_names = [p.name for p in self.projects]
if project_name not in known_project_names:
raise TocException("%s is not a buildable project" % project_name)
# Here do not honor self.solve_deps or the software won't compile :)
dep_solver = DependenciesSolver(projects=self.projects, packages=self.packages,
active_projects=self.active_projects)
(r_project_names, _package_names, not_found) = dep_solver.solve([project_name])
# Nothing to do with with the packages:
# SDK dirs from toolchain are managed by the toolchain file in
# self.toolchain
if not_found:
# FIXME: right now there are tons of case where you could have missing
# projects. (using a cross-toolchain, or an Aldebaran SDK)
# Put this back later.
# ui.warning("Could not find projects", ", ".join(not_found))
pass
# Remove self from the list:
r_project_names.remove(project_name)
for project_name in r_project_names:
project = self.get_project(project_name)
dirs.append(project.get_sdk_dir())
ui.debug("sdk_dirs for", project_name, ":", " ".join(dirs))
return dirs
def run_action(module_name, args=None, forward_args=None):
"""
Run an action using its module path and a list of arguments
If forward_args is given, it must be an argparse.Namespace object.
This namespace will be merged with args before being
passed to the do() method of module_name.
"""
if not args:
args = list()
ui.debug("running", module_name, " ".join(args))
action_name = module_name.split(".")[-1]
package_name = ".".join(module_name.split(".")[:-1])
try:
_tmp = __import__(package_name, globals(), locals(), [action_name])
except ImportError, err:
raise InvalidAction(module_name, str(err))
def open_worktree(worktree=None):
"""
Open a qi worktree.
:return: a valid :py:class:`WorkTree` instance.
If worktree is None, guess it from the current working dir.
"""
if not worktree:
worktree = guess_worktree()
if worktree is None:
raise NotInWorktree()
if not os.path.exists(worktree):
mess = "Cannot open a worktree from %s\n" % worktree
mess += "This path does not exist"
raise Exception(mess)
res = WorkTree(worktree)
ui.debug("Opening worktree in", worktree)
return res
def execute(self):
"""Execute the command, and return a generator for iterating over
the output written to the standard output and error streams.
"""
def reader(pipe, pipe_name, queue):
"To be called in a thread"
while pipe and not pipe.closed:
line = pipe.readline()
if line == "":
break
queue.put((pipe_name, line))
if not pipe.closed:
pipe.close()
ui.debug("Calling:", " ".join(self.cmd))
process = subprocess.Popen(self.cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.cwd, env=self.env)
queue = Queue.Queue()
pipe_out = threading.Thread(target=reader, args=(process.stdout, "stdout", queue))
pipe_err = threading.Thread(target=reader, args=(process.stderr, "stderr", queue))
pipe_out.start()
pipe_err.start()
while True:
if process.poll() is not None and self.returncode is None:
self.returncode = process.returncode
try:
name, line = queue.get(block=True, timeout=0.01)
if name == "stderr":
yield (None, line)
else:
yield (line, None)
except Queue.Empty:
if self.returncode is not None:
break
pipe_out.join()
pipe_err.join()
def install_project_runtime(self, project, destdir, num_jobs=1):
"""Install runtime component of a project to a destdir """
runtime_components = [
"binary",
"data",
"conf",
"lib",
"python",
"doc"
]
for component in runtime_components:
self.build_env["DESTDIR"] = destdir
cmake_args = list()
cmake_args += ["-DCOMPONENT=%s" % component]
cmake_args += ["-P", "cmake_install.cmake", "--"]
cmake_args += num_jobs_to_args(num_jobs, self.cmake_generator)
ui.debug("Installing", component)
qibuild.command.call(["cmake"] + cmake_args,
cwd=project.build_directory,
env=self.build_env,
)
def __init__(self, in_dir, out_dir="build-doc"):
self.worktree = qisrc.worktree.open_worktree(in_dir)
self.in_dir = in_dir
self.out_dir = out_dir
# Set during configure_all()
self.opts = dict()
self.templates_path = None
self.sphinxdocs = dict()
self.doxydocs = dict()
# Will fill up self.templates_path, self.sphinxdocs and self.doxydocs
self._load_doc_projects()
self.deps_tree = self.get_deps_tree()
ui.debug("QiDocBuilder dep tree: ", pprint.pformat(self.deps_tree))
if not self.templates_path:
mess = "Could not find any template repo\n"
mess += "Please make sure that one of the qiproject.xml looks like:\n"
mess += '<qiproject template_repo="yes" />\n'
raise Exception(mess)
self.doxytags_path = os.path.join(self.out_dir, "doxytags")
def run_tests(project, build_env, pattern=None, verbose=False, slow=False, dry_run=False, valgrind=False):
""" Called by :py:meth:`qibuild.toc.Toc.test_project`
:param test_name: If given, only run this test
Always write some XML files in build-<config>/test-results
(even if they were no tests to run at all)
:return: a boolean to indicate if test was sucessful
"""
build_dir = project.build_directory
results_dir = os.path.join(project.build_directory, "test-results")
all_tests = parse_ctest_test_files(build_dir)
tests = list()
slow_tests = list()
if pattern:
tests = [x for x in all_tests if re.search(pattern, x[0])]
if not tests:
mess = "No tests matching %s\n" % pattern
mess += "Known tests are:\n"
for x in all_tests:
mess += " * " + x[0] + "\n"
raise Exception(mess)
else:
for test in all_tests:
(name, cmd_, properties) = test
cost = properties.get("COST")
if not slow and cost and float(cost) > 50:
ui.debug("Skipping test", name, "because cost",
"(%s)"% cost, "is greater than 50")
slow_tests.append(name)
continue
tests.append(test)
if not tests:
# Create a fake test result to keep CI jobs happy:
fake_test_res = TestResult("compilation")
fake_test_res.ok = True
xml_out = os.path.join(results_dir, "compilation.xml")
write_xml(xml_out, fake_test_res)
ui.warning("No tests found for project", project.name)
return
if dry_run:
ui.info(ui.green, "List of tests for", project.name)
for (test_name, _, _) in tests:
ui.info(ui.green, " * ", ui.reset, test_name)
return
ui.info(ui.green, "Testing", project.name, "...")
ok = True
fail_tests = list()
for (i, test) in enumerate(tests):
(test_name, cmd, properties) = test
ui.info(ui.green, " * ", ui.reset, ui.bold,
"(%2i/%2i)" % (i+1, len(tests)),
ui.blue, test_name.ljust(25), end="")
if verbose:
print
sys.stdout.flush()
test_res = run_test(build_dir, test_name, cmd, properties, build_env,
valgrind=valgrind, verbose=verbose)
if test_res.ok:
ui.info(ui.green, "[OK]")
else:
ok = False
ui.info(ui.red, "[FAIL]")
if not verbose:
print test_res.out
fail_tests.append(test_name)
xml_out = os.path.join(results_dir, test_name + ".xml")
if not os.path.exists(xml_out):
write_xml(xml_out, test_res)
if ok:
ui.info("Ran %i tests" % len(tests))
if slow_tests and not slow:
ui.info("Note: %i" % len(slow_tests),
"slow tests did not run, use --slow to run them")
ui.info("All pass. Congrats!")
return True
ui.error("Ran %i tests, %i failures" %
(len(tests), len(fail_tests)))
for fail_test in fail_tests:
ui.info(ui.bold, " -", ui.blue, fail_test)
return False
def install_project(self, project, destdir, prefix="/",
runtime=False, num_jobs=1,
split_debug=False):
""" Install the project
:param project: project name.
:param destdir: destination. Note that when using `qibuild install`,
we will first call `cmake` to make sure `CMAKE_INSTALL_PREFIX` is
``/``. But this function simply calls ``cmake --target install``
in the simple case.
:param runtime: Whether to install the project as a runtime
package or not.
(see :ref:`cmake-install` section for the details)
:package split_debug: split the debug symbols out of the binaries
useful for `qibuild deploy`
"""
build_dir = project.build_directory
# DESTDIR=/tmp/foo and CMAKE_PREFIX="/usr/local" means
# dest = /tmp/foo/usr/local
destdir = qibuild.sh.to_native_path(destdir)
self.build_env["DESTDIR"] = destdir
# Must make sure prefix is not seen as an absolute path here:
dest = os.path.join(destdir, prefix[1:])
dest = qibuild.sh.to_native_path(dest)
cmake_cache = os.path.join(build_dir, "CMakeCache.txt")
if not os.path.exists(cmake_cache):
mess = """Could not install project {project.name}
It appears the project has not been configured.
({cmake_cache} does not exist)
Try configuring and building the project first.
"""
mess = mess.format(config=self.active_config,
project=project, cmake_cache=cmake_cache)
raise Exception(mess)
cprefix = qibuild.cmake.get_cached_var(build_dir, "CMAKE_INSTALL_PREFIX")
if cprefix != prefix:
qibuild.cmake.cmake(project.directory, project.build_directory,
['-DCMAKE_INSTALL_PREFIX=%s' % prefix],
clean_first=False,
env=self.build_env)
else:
mess = "Skipping configuration of project %s\n" % project.name
mess += "CMAKE_INSTALL_PREFIX is already correct"
ui.debug(mess)
if not self.using_visual_studio and not self.cmake_generator == "Xcode":
self.build_project(project, target="preinstall", num_jobs=num_jobs,
fix_shared_libs=False)
if runtime:
self.install_project_runtime(project, destdir, num_jobs=num_jobs)
else:
self.build_project(project, target="install", fix_shared_libs=False)
if split_debug:
if self.using_visual_studio:
raise Exception("split debug not supported on Visual Studio")
objcopy = qibuild.cmake.get_cached_var(project.build_directory, "CMAKE_OBJCOPY")
if objcopy is None:
objcopy = qibuild.command.find_program("objcopy", env=self.build_env)
if not objcopy:
mess = """\
Could not split debug symbols from binaries for project {project.name}.
Could not find objcopy executable.\
"""
qibuild.ui.warning(mess.format(project=project))
else:
qibuild.gdb.split_debug(destdir, objcopy=objcopy)
def call(cmd, cwd=None, env=None, ignore_ret_code=False, quiet=None):
""" Execute a command line.
If ignore_ret_code is False:
raise CommandFailedException if returncode is None.
Else:
simply returns the returncode of the process
Note: first arg of the cmd is assumed to be something
inside %PATH%. (or in env[PATH] if env is not None)
Note: the shell= argument of the subprocess.Popen
call will always be False.
can raise:
* CommandFailedException if ignore_ret_code is False
and returncode is non zero
* NotInPath if first arg of cmd is not in %PATH%
* And a normal exception if cwd is given and is not
an existing directory.
"""
exe_full_path = find_program(cmd[0], env=env)
if not exe_full_path:
raise NotInPath(cmd[0], env=env)
cmd[0] = exe_full_path
if cwd:
if not os.path.exists(cwd):
# We know we are likely to have a problem on windows here,
# so always raise.
raise Exception("Trying to to run %s in non-existing %s" % (" ".join(cmd), cwd))
ui.debug("Calling:", " ".join(cmd))
ring_buffer = RingBuffer(300)
returncode = 0
if quiet:
quiet_command = quiet
else:
quiet_command = CONFIG.get("quiet", False)
# This code won't work on windows with python < 2.7,
# so quiet will be ignored
if sys.platform.startswith("win") and sys.version_info < (2, 7):
quiet_command = False
if not quiet_command:
returncode = subprocess.call(cmd, env=env, cwd=cwd)
else:
cmdline = CommandLine(cmd, cwd=cwd, env=env)
for (out, err) in cmdline.execute():
if out is not None:
ring_buffer.append(out)
if err is not None:
ring_buffer.append(err)
returncode = cmdline.returncode
if ignore_ret_code:
return returncode
if returncode != 0:
if quiet_command:
lines = ring_buffer.get()
for line in lines:
sys.stdout.write(line)
sys.stdout.flush()
# Raise correct exception
raise CommandFailedException(cmd, returncode, cwd)
