def process(input_dir, output_dir, **kwargs):
""" Process """
if not os.path.isdir(input_dir):
if os.path.exists(input_dir):
raise Exception("%s is not a directory" % input_dir)
else:
raise Exception("%s does not exist" % input_dir)
if qisys.sh.is_path_inside(output_dir, input_dir):
raise Exception("output directory is inside input directory")
ui.info(ui.green, "Generating code in", output_dir)
for root, directories, filenames in os.walk(input_dir):
rel_root = os.path.relpath(root, input_dir)
if rel_root == ".":
rel_root = ""
for directory in directories:
input_name = os.path.join(rel_root, directory)
output_name = process_string(input_name, **kwargs)
to_make = os.path.join(output_dir, output_name)
qisys.sh.mkdir(to_make, recursive=True)
for filename in filenames:
input_name = os.path.join(rel_root, filename)
output_name = process_string(input_name, **kwargs)
output_path = os.path.join(output_dir, output_name)
input_path = os.path.join(input_dir, input_name)
qisys.sh.install(input_path, output_path, quiet=True)
process_file(output_path, **kwargs)
ui.info("*", output_name)
def do(args):
"""Main entry point."""
git_worktree = qisrc.parsers.get_git_worktree(args)
ui.info(ui.green, "Current worktree:", ui.reset, ui.bold, git_worktree.root)
snapshot_path = args.snapshot_path
qisrc.snapshot.generate_snapshot(git_worktree, snapshot_path,
deprecated_format=args.deprecated_format)
def do(args):
""" Main entry point """
build_worktree = qibuild.parsers.get_build_worktree(args)
projects = qibuild.parsers.get_build_projects(build_worktree, args, solve_deps=False)
for project in projects:
path = qimvn.package.package(project, pom_path=args.pom, skip_test=args.skip_test)
ui.info(path)
def do(args):
""" Main entry point. """
cmake_builder = qibuild.parsers.get_cmake_builder(args)
deps_solver = cmake_builder.deps_solver
packages = deps_solver.get_dep_packages(cmake_builder.projects,
["build", "runtime", "test"])
projects = deps_solver.get_dep_projects(cmake_builder.projects,
["build", "runtime", "test"])
oss = args.oss
def should_add_license(license_):
""" Should Add Licence """
if license_ is None:
return False
if license_ == "proprietary" and oss:
return False
return True
res = collections.OrderedDict()
for package in packages:
license_name = package.license
if should_add_license(license_name):
res[package.name] = package.license
for project in projects:
license_name = project.license
if should_add_license(license_name):
res[project.name] = project.license
if args.json:
print(json.dumps(res, indent=2))
else:
for name, license_ in res.items():
ui.info(name, license_)
return res
def do(args):
"""Main entry point"""
# Convert 'helper' options into cmake flags
if not args.cmake_flags:
args.cmake_flags = list()
if args.effective_cplusplus:
args.cmake_flags.append("QI_EFFECTIVE_CPP=ON")
if args.werror:
args.cmake_flags.append("QI_WERROR=ON")
if args.coverage:
args.cmake_flags.append("QI_WITH_COVERAGE=ON")
# args.debug_info has 3 values: None (not set at all), True, False
if args.debug_info is True:
args.cmake_flags.append("QI_WITH_DEBUG_INFO=ON")
if args.debug_info is False:
args.cmake_flags.append("QI_WITH_DEBUG_INFO=OFF")
if args.force_32_bits:
args.cmake_flags.append("QI_FORCE_32_BITS=ON")
cmake_builder = qibuild.parsers.get_cmake_builder(args)
if args.debug_trycompile:
ui.info(ui.green, "Using cmake --debug-trycompile")
if args.trace_cmake:
ui.info(ui.green, "Tracing CMake execution")
cmake_builder.configure(clean_first=args.clean_first,
debug_trycompile=args.debug_trycompile,
trace_cmake=args.trace_cmake,
profiling=args.profiling,
summarize_options=args.summarize_options)
def generate_examples_zips(self):
for example_src in self.examples:
example_path = os.path.join(self.source_dir, example_src)
zip_path = os.path.join(self.source_dir, example_src + ".zip")
if not qisys.sh.up_to_date(zip_path, example_path):
ui.info("Generating", zip_path)
qisys.archive.compress(example_path, algo="zip", quiet=True)
def split_debug(self, destdir, file_list):
""" Split debug symbols after install """
if self.using_visual_studio:
raise Exception("split debug not supported on Visual Studio")
ui.info(ui.green, "Splitting debug symbols from binaries ...")
tool_paths = dict()
for name in ["objcopy", "objdump"]:
tool_path = qibuild.cmake.get_binutil(name,
build_dir=self.build_directory,
env=self.build_env)
tool_paths[name] = tool_path
missing = [x for x in tool_paths if not tool_paths[x]]
if missing:
mess = """\
Could not split debug symbols from binaries for project {name}.
The following tools were not found: {missing}\
"""
mess = mess.format(name=self.name, missing = ", ".join(missing))
ui.warning(mess)
return
for filename in file_list:
full_path = os.path.join(destdir, filename[1:]) # remove starting /
if qibuild.gdb.is_elf(full_path):
qibuild.gdb.split_debug(full_path, **tool_paths)
def do(args):
""""Create a new project """
try:
worktree = qisys.parsers.get_worktree(args)
except qisys.worktree.NotInWorkTree:
worktree = None
project_name = os.path.basename(args.project_name)
output_dir = args.output_dir
if not output_dir:
output_dir = qisrc.templates.attached_lower(project_name)
output_dir = os.path.join(os.getcwd(), output_dir)
if os.path.exists(output_dir):
raise Exception("%s already exists" % output_dir)
template_path = args.template_path
if not template_path:
template_path = os.path.join(qisrc.QISRC_ROOT_DIR, "templates", "project")
qisrc.templates.process(template_path, output_dir, project_name=project_name)
if args.git:
qisys.command.call(["git", "init"], cwd=output_dir)
with open(os.path.join(output_dir, ".gitignore"), "w") as fp:
fp.write("build-*\n")
qisys.command.call(["git" , "add" , "."], cwd=output_dir)
qisys.command.call(["git" , "commit" , "-m" , "initial commit"], cwd=output_dir)
ui.info(ui.green, "New project initialized in", ui.bold, output_dir)
if worktree:
worktree.add_project(output_dir)
return worktree.get_project(output_dir)
def do(args):
""" Import a binary package into a toolchain
- Convert the binary package into a qiBuild package
- Add the qiBuild package to the cache
- Add the qiBuild package from cache to toolchain
"""
name = args.name
package_path = args.package_path
converted_package_path = convert_package(package_path, name,
interactive=args.interactive)
toolchain = qitoolchain.parsers.get_toolchain(args)
tc_packages_path = qitoolchain.toolchain.get_default_packages_path(toolchain.name)
message = """
Importing '{1}' in the toolchain {0} ...
""".format(toolchain.name, package_path)
qisys.ui.info(message)
# installation of the qiBuild package
package_dest = os.path.join(tc_packages_path, name)
qisys.sh.rm(package_dest)
with qisys.sh.TempDir() as tmp:
extracted = qisys.archive.extract(converted_package_path, tmp, quiet=True,
strict_mode=False)
qisys.sh.install(extracted, package_dest, quiet=True)
qibuild_package = qitoolchain.qipackage.QiPackage(name, path=package_dest)
toolchain.add_package(qibuild_package)
ui.info("done")
def push(project_path, branch, review=True, dry_run=False, reviewers=None):
""" Push the changes for review.
Unless review is False, in this case, simply update
the remote gerrit branch
:param reviewers: A list of reviewers to invite to review
"""
git = qisrc.git.Git(project_path)
review_remote = git.get_config("review.remote")
args = list()
if dry_run:
args.append("--dry-run")
if not review_remote:
# Repository not configured for code review:
# we just follow the normal 'git push' behavior
git.push(*args)
return
args.append(review_remote)
if review:
ui.info('Pushing code to gerrit for review.')
args.append("%s:refs/for/%s" % (branch, branch))
if reviewers:
reviewers = guess_emails(git, reviewers)
receive_pack = "git receive-pack"
for reviewer in reviewers:
receive_pack += " --reviewer=%s" % reviewer
args = ["--receive-pack=%s" % receive_pack] + args
else:
args.append("%s:%s" % (branch, branch))
git.push(*args)
def generate_mo_file(self, locale):
""" Generate .mo file for the given locale """
ui.info(ui.green, "Generating translation for", ui.reset,
ui.bold, locale)
input_file = self.get_po_file(locale)
if not os.path.exists(input_file):
ui.error("No .po found for locale: ", locale, "\n",
"(looked in %s)" % input_file, "\n",
"Did you run qilinguist update?")
return None
output_file = os.path.join(self.mo_path, locale, "LC_MESSAGES",
self.domain + ".mo")
to_make = os.path.dirname(output_file)
qisys.sh.mkdir(to_make, recursive=True)
cmd = ["msgfmt", "--check", "--statistics"]
# required by libqi:
conf_file = os.path.join(self.mo_path, ".confintl")
with open(conf_file, "w") as fp:
fp.write("# THIS FILE IS AUTOGENERATED\n"
"# Do not delete or modify it\n"
"# This file is used to find translation dictionaries\n")
cmd.extend(["--output-file", output_file])
cmd.extend(["--directory", self.po_path])
cmd.append(input_file)
process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
__out, err = process.communicate()
ui.info(err.strip())
if "untranslated" in err:
return False, "Some untranslated messages were found"
if process.returncode != 0:
return False, "msgfmt failed"
return True, ""
def configure_qtcreator(qibuild_cfg):
""" Configure QtCreator
"""
ide = qibuild.config.IDE()
ide.name = "QtCreator"
build_env = qibuild.config.get_build_env()
qtcreator_path = qisys.command.find_program("qtcreator", env=build_env)
if qtcreator_path:
ui.info(ui.green, "::", ui.reset, "Found QtCreator:", qtcreator_path)
mess = "Do you want to use qtcreator from %s?\n" % qtcreator_path
mess += "Answer 'no' if you installed qtcreator from Nokia's installer"
answer = qisys.interact.ask_yes_no(mess, default=True)
if not answer:
qtcreator_path = None
else:
ui.warning("QtCreator not found")
if not qtcreator_path:
qtcreator_path = qisys.interact.ask_program(
"Please enter full qtcreator path")
if not qtcreator_path:
ui.warning("Not adding config for QtCreator",
"qibuild open will not work", sep="\n")
return
ide.path = qtcreator_path
qibuild_cfg.add_ide(ide)
def check_parent_project(toc, project_name, project_path):
""" Check if the qibuild project was not found because
there was a missing
<project src= ... /> in the parent qiproject.xml file
"""
parent_proj = get_parent_project(toc, project_path)
if not parent_proj:
return
parent_qiproj = os.path.join(parent_proj.path, "qiproject.xml")
if not os.path.exists(parent_qiproj):
return
question = "Add the path to project %s to its parent qiproject.xml"
question = question % (project_name)
answer = qisys.interact.ask_yes_no(question, default=True)
if answer:
ui.info("Patching", parent_qiproj)
tree = qixml.read(parent_qiproj)
child_src = os.path.relpath(project_path, parent_proj.path)
child_src = qisys.sh.to_posix_path(child_src)
to_add = qixml.etree.Element("project")
to_add.set("src", child_src)
tree.getroot().append(to_add)
qixml.write(tree, parent_qiproj)
toc.projects = list()
toc.worktree.load()
toc.update_projects()
def summary(self):
""" Display the tests results.
Called at the end of self.run()
Sets ``self.ok``
"""
if not self.tests:
self.ok = False
return
num_tests = len(self.results)
failures = [x for x in self.results.values() if x.ok is False]
num_failed = len(failures)
message = "Ran %i tests in %is" % (num_tests, self.elapsed_time)
ui.info(message)
self.ok = (not failures) and not self._interrupted
if self.ok:
ui.info(ui.green, "All pass. Congrats!")
return
if num_failed != 0:
ui.error(num_failed, "failures")
if failures:
max_len = max(len(x.test["name"]) for x in failures)
for i, failure in enumerate(failures):
ui.info_count(i, num_failed,
ui.blue, failure.test["name"].ljust(max_len + 2),
ui.reset, *failure.message)
def do(args):
""" Main entry point """
force_rm = args.force_remove
tc = qitoolchain.get_toolchain(args.name)
ui.info(ui.green, "Removing toolchain", ui.blue, tc.name)
tc.remove(force_remove=force_rm)
ui.info(ui.green, "done")
def ask_choice(choices, input_text):
"""Ask the user to choose from a list of choices
"""
ui.info(ui.green, "::", ui.reset, input_text)
for i, choice in enumerate(choices):
if i == 0:
choice += " \t(default)"
ui.info(" ", ui.blue, str(i+1), ui.reset, choice)
keep_asking = True
res = None
while keep_asking:
try:
answer = read_input()
except KeyboardInterrupt:
break
if not answer:
return choices[0]
try:
index = int(answer)
except ValueError:
print "Please enter number"
continue
if index not in range(1, len(choices)+1):
print "%i is out of range" % index
continue
res = choices[index-1]
keep_asking = False
return res
def do(args):
pkg_path = args.pkg_path
output_path = args.output_path
if not output_path:
output_path = os.getcwd()
# Extract the manifest to a tempfile to
# parse it
archive = zipfile.ZipFile(pkg_path)
name = None
version = None
pkg_name = None
with qisys.sh.TempDir() as tmp:
for name in archive.namelist():
if name == "manifest.xml":
archive.extract("manifest.xml", path=tmp)
manifest_xml_path = os.path.join(tmp, "manifest.xml")
pkg_name = qipkg.builder.pkg_name(manifest_xml_path)
break
if pkg_name is not None:
to_make = os.path.join(output_path, os.path.basename(pkg_name))
output_path = os.path.join(output_path, pkg_name)
qisys.sh.mkdir(to_make, recursive=True)
archive.close()
if pkg_path.endswith(".mpkg"):
basename = os.path.basename(pkg_path)
name, _ = os.path.splitext(basename)
output_path = os.path.join(output_path, name)
qisys.archive.extract(pkg_path, output_path, algo="zip", strict_mode=False)
ui.info(ui.green, "Package extracted to", ui.reset,
ui.bold, output_path)
def test_empty_end():
out = io.BytesIO()
ui.info("[skipped] ", end="", fp=out)
ui.info("Your branch has diverged", fp=out)
actual = out.getvalue()
expected = "[skipped] Your branch has diverged\n"
assert actual == expected
def handle_extensions(venv_path, python_worktree, build_worktree):
"""
Check if there is a build project matching the given source,
and add the correct path to the virtualenv.
"""
extensions_projects = list()
build_projects = build_worktree.build_projects
for project in python_worktree.python_projects:
parent_project = qisys.parsers.find_parent_project(build_projects,
project.path)
if parent_project:
extensions_projects.append(parent_project)
if extensions_projects:
ui.info()
ui.info(ui.blue, "::", ui.reset, "Registering C++ extensions")
to_write = ""
for i, project in enumerate(extensions_projects):
ui.info_count(i, len(extensions_projects),
ui.blue, project.name)
qi_pth_src = os.path.join(project.sdk_directory, "qi.pth")
if os.path.exists(qi_pth_src):
with open(qi_pth_src, "r") as fp:
to_write += fp.read()
if not to_write.endswith("\n"):
to_write += "\n"
lib_path = virtualenv.path_locations(venv_path)[1]
qi_pth_dest = os.path.join(venv_path, lib_path, "site-packages/qi.pth")
with open(qi_pth_dest, "a") as fp:
fp.write(to_write)
def get_ide(qibuild_cfg):
"""Return an IDE to use."""
known_ides = qibuild_cfg.ides.values()
ide_names = qibuild_cfg.ides.keys()
if not known_ides:
ui.warning("No IDE configured yet")
ui.info("Tips: use `qibuild config --wizard` to configure an IDE")
return None
# Remove the one that are not supported:
supported_ides = [x for x in known_ides if x.name in SUPPORTED_IDES]
if len(supported_ides) == 1:
return supported_ides[0]
if not supported_ides:
mess = "Found those IDEs in configuration: %s\n" % ", ".join(ide_names)
mess += "But `qibuild open` only supports: %s\n" % ", ".join(SUPPORTED_IDES)
raise Exception(mess)
# User chose a specific config and an IDE matches this config
if qibuild_cfg.ide:
return qibuild_cfg.ide
supported_names = [x.name for x in supported_ides]
# Several IDEs, ask the user to choose
ide_name = qisys.interact.ask_choice(supported_names,
"Please choose an IDE to use")
if not ide_name:
return None
return qibuild_cfg.ides[ide_name]
def push_projects(git_projects, dry_run=False):
""" Push Projects """
if not git_projects:
return
ui.info(ui.green, "Pushing ", len(git_projects), "projects")
for i, git_project in enumerate(git_projects):
default_branch = git_project.default_branch.name
remote_branch = git_project.default_branch.remote_branch
ui.info_count(i, len(git_projects), git_project.src)
git = qisrc.git.Git(git_project.path)
if git_project.review:
push_remote = git_project.review_remote
else:
push_remote = git_project.default_remote
remote_ref = "%s/%s" % (push_remote.name, remote_branch)
display_changes(git, default_branch, remote_ref)
answer = qisys.interact.ask_yes_no("OK to push?", default=False)
if not answer:
return
to_push = "%s:%s" % (default_branch, remote_branch)
push_args = [push_remote.name, to_push]
push_args.append("--force")
if dry_run:
push_args.append("--dry-run")
rc, out = git.push(*push_args, raises=False)
if rc == 0:
ui.info(out)
else:
ui.error(out)
def test_convert_to_strings_unicode():
""" Test Convert to Strings Unicode """
out = io.BytesIO()
ui.info("élément", ["Jérôme", "プログラミング"], fp=out)
actual = out.getvalue().decode("utf-8")
expected = "élément ['Jérôme', 'プログラミング']\n"
assert actual == expected
def configure_projects(self, projects=None):
""" Configure the given projects so that the actual git config matches
the one coming from the manifest :
Configure default remotes, default branches and code review, then save config
To be called _after_ sync()
"""
if projects is None:
projects = self.git_worktree.get_git_projects()
if not projects:
return
to_configure = list()
srcs = {project.src: project for project in projects}
for repo in self.new_repos:
if repo.src in srcs.keys():
to_configure.append(repo)
if not to_configure:
return
ui.info(ui.green, ":: Setup git projects ...")
max_src = max(len(x.src) for x in to_configure)
n = len(to_configure)
for i, repo in enumerate(to_configure):
ui.info_count(i, n, ui.white, "Setup", ui.reset,
ui.blue, repo.src.ljust(max_src), end="\r")
git_project = srcs[repo.src]
git_project.read_remote_config(repo)
git_project.apply_config()
ui.info(" " * (max_src + 19), end="\r")
self.git_worktree.save_git_config()
def test_custom_sep():
""" Test Custom Sep """
out = io.BytesIO()
ui.info("foo", "bar", sep="\n", fp=out)
actual = out.getvalue().decode("utf-8")
expected = "foo\nbar\n"
assert actual == expected
def test_convert_to_strings():
""" Test Convert to Strings """
out = io.BytesIO()
ui.info("mylist", ["a", "b", "c"], fp=out)
actual = out.getvalue().decode("utf-8")
expected = "mylist ['a', 'b', 'c']\n"
assert actual == expected
def test_do_not_add_space_after_newline():
""" Test Do Not Add Space After New Line """
out = io.BytesIO()
ui.info("foo\n", "bar", fp=out)
actual = out.getvalue().decode("utf-8")
expected = "foo\nbar\n"
assert actual == expected
def test_insert_spaces():
""" Test Insert Space """
out = io.BytesIO()
ui.info("foo:", "bar", fp=out)
actual = out.getvalue().decode("utf-8")
expected = "foo: bar\n"
assert actual == expected
def test_several_newlines():
""" Test Several New Lines """
out = io.BytesIO()
ui.info("foo\n", "bar\n", "baz", fp=out)
actual = out.getvalue().decode("utf-8")
expected = "foo\nbar\nbaz\n"
assert actual == expected
def move_repo(self, repo, new_src, force=False):
""" Move a project in the worktree (same remote url, different
src)
"""
project = self.get_git_project(repo.src)
if not project:
return
ui.info("* moving", ui.blue, project.src,
ui.reset, "to", ui.blue, new_src)
new_path = os.path.join(self.worktree.root, new_src)
new_path = qisys.sh.to_native_path(new_path)
if os.path.exists(new_path):
if force:
qisys.sh.rm(new_path)
else:
ui.error(new_path, "already exists")
ui.error("If you are sure there is nothing valuable here, "
"remove this directory and try again")
return
new_base_dir = os.path.dirname(new_path)
try:
qisys.sh.mkdir(new_base_dir, recursive=True)
os.rename(project.path, new_path)
except Exception as e:
ui.error("Error when moving", project.src, "to", new_path,
"\n", e , "\n",
"Repository left in", project.src)
return
self.worktree.move_project(project.src, new_src)
project.src = new_src
self.save_project_config(project)
return True
def make_package(self, pml_builder, output=None):
stage_path = pml_builder.stage_path
if not output:
name = pkg_name(self.manifest_xml) + ".pkg"
output = os.path.join(os.getcwd(), name)
archive = zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED)
# Add the manifest
manifest_xml = pml_builder.manifest_xml
archive.write(manifest_xml, "manifest.xml")
# Add everything from the staged path
pml_builder.install(pml_builder.stage_path)
stage_path = pml_builder.stage_path
for root,_, filenames in os.walk(stage_path):
for filename in filenames:
full_path = os.path.join(root, filename)
rel_path = os.path.relpath(full_path, stage_path)
ui.info(ui.green, "adding", ui.reset, ui.bold, rel_path)
archive.write(full_path, rel_path)
archive.close()
ui.info(ui.green, "Package generated in",
ui.reset, ui.bold, output)
return output
def run(self, num_jobs=1, repeat_until_fail=0):
""" Run all the tests """
if repeat_until_fail == 0:
self._run_once(num_jobs)
return self.ok
ui.info(ui.blue, "::", ui.reset, "Running tests until they fail")
num_runs = 0
while num_runs < repeat_until_fail:
ui.info(ui.bold, "Test run #%i" % (num_runs + 1))
self._run_once(num_jobs)
ui.info()
if self.ok:
num_runs += 1
else:
break
return self.ok
def do_sync(git_project):
""" Do Sync """
if reset:
(status, out) = git_project.reset()
else:
(status, out) = git_project.sync(rebase_devel=args.rebase_devel)
with lock:
ui.info_count(i[0], len(git_projects), ui.blue,
git_project.src.ljust(max_src))
if status is None:
ui.info(git_project.src, ui.brown, " [skipped]")
skipped.append((git_project.src, out))
if status is False:
ui.info(git_project.src, ui.red, " [failed]")
failed.append((git_project.src, out))
if out:
ui.info(ui.indent(out + "\n\n", num=2))
i[0] += 1
def _do_package(cmake_builder, destdir, build_type="Release"):
""" Helper function. On linux and mac this is only called
once.
On Windows this is called twice, both in debug and release
This is because usually debug and release version of a library
are incompatible on Windows.
"""
cmake_builder.build_config.build_type = build_type
cmake_builder.dep_types = ["build"]
ui.info(ui.blue, "::", ui.reset, ui.bold, "Configuring ... (%s)" % build_type)
cmake_builder.configure()
ui.info(ui.blue, "::", ui.reset, ui.bold, "Building ... (%s)" % build_type)
cmake_builder.build()
cmake_builder.dep_types = list()
ui.info(ui.blue, "::", ui.reset, ui.bold, "Installing ... (%s)" % build_type)
cmake_builder.install(destdir)
def _install_package(url, pkg_name, pkg_path):
import qi
app = qi.Application()
session = qi.Session()
session.connect("tcp://%s:9559" % (url.host))
package_manager = session.service("PackageManager")
ui.info(ui.blue, "::",
ui.reset, ui.bold,
"Removing previous installation of the package")
try:
package_manager.removePkg(pkg_name)
except:
pass
ui.info(ui.blue, "::",
ui.reset, ui.bold,
"Installing package")
ret = package_manager.install(
"/home/%s/%s" % (url.user, os.path.basename(pkg_path)))
ui.info("PackageManager returned:", ret)
def do(args):
"""Main method."""
git_worktree = qisrc.parsers.get_git_worktree(args)
git_projects = qisrc.parsers.get_git_projects(git_worktree, args,
default_all=True,
use_build_deps=True)
if not git_projects:
qisrc.worktree.on_no_matching_projects(git_worktree, groups=args.groups)
return
num_projs = len(git_projects)
max_len = max(len(p.src) for p in git_projects)
state_projects = list()
for (i, git_project) in enumerate(git_projects, start = 1):
if sys.stdout.isatty():
src = git_project.src
to_write = "Checking (%d/%d) " % (i, num_projs)
to_write += src.ljust(max_len)
sys.stdout.write(to_write + "\r")
sys.stdout.flush()
state_project = qisrc.status.check_state(git_project, args.untracked_files)
state_projects.append(state_project)
if sys.stdout.isatty():
ui.info("Checking (%d/%d):" % (num_projs, num_projs), "done",
" " * max_len)
dirty = [x for x in state_projects if not x.sync_and_clean]
ui.info("\n", ui.brown, "Dirty projects", len(dirty), "/", num_projs)
projects_to_display = dirty if args.short else state_projects
for git_project in projects_to_display:
qisrc.status.print_state(git_project, max_len)
max_len = max(max_len, len("Project"))
qisrc.status.print_incorrect_projs(state_projects, max_len)
qisrc.status.print_not_on_a_branch(state_projects)
if not args.untracked_files:
ui.info("Tips: use -u to show untracked files")
def checkout(self, branch, force=False):
""" Called by ``qisrc checkout``
For each project, checkout the branch if it is different than
the default branch of the manifest.
"""
ui.info(ui.green, ":: Checkout projects ...")
errors = list()
manifest_xml = os.path.join(self._syncer.manifest_repo, "manifest.xml")
manifest = qisrc.manifest.Manifest(manifest_xml)
to_checkout = list()
for project in self.git_projects:
if project.default_branch is None:
continue
branch_name = project.default_branch.name
git = qisrc.git.Git(project.path)
if git.get_current_branch() != branch_name:
to_checkout.append(project)
n = len(to_checkout)
if n == 0:
ui.info(ui.green, "Nothing to checkout")
return True
max_src = max([len(x.src) for x in to_checkout])
for i, project in enumerate(to_checkout):
ui.info_count(i, n, ui.bold, "Checkout",
ui.reset, ui.blue, project.src.ljust(max_src), end="\r")
if project.default_branch is None:
continue
branch_name = project.default_branch.name
remote_name = project.default_remote.name
git = qisrc.git.Git(project.path)
ok, err = git.safe_checkout(branch_name, remote_name, force=force)
if not ok:
errors.append((project.src, err))
if not errors:
return True
ui.error("Failed to checkout some projects")
for (project, error) in errors:
ui.info(project, ":", error)
return False
def update(self, feed):
""" Update a toolchain given a feed """
feed_parser = qitoolchain.feed.ToolchainFeedParser()
feed_parser.parse(feed)
remote_packages = feed_parser.get_packages()
local_packages = self.packages.values()
to_add = list()
to_remove = list()
svn_packages = [x for x in remote_packages
if isinstance(x, qitoolchain.svn_package.SvnPackage)]
other_packages = [x for x in remote_packages if x not in svn_packages]
if svn_packages:
ui.info(ui.green, "Updating svn packages")
for i, svn_package in enumerate(svn_packages):
ui.info_count(i, len(svn_packages), ui.blue, svn_package.name)
self.handle_svn_package(svn_package)
self.packages[svn_package.name] = svn_package
for remote_package in other_packages:
if remote_package in local_packages:
continue
to_add.append(remote_package)
for local_package in local_packages:
if local_package not in remote_packages:
to_remove.append(local_package)
if to_remove:
ui.info(ui.red, "Removing packages")
for i, package in enumerate(to_remove):
ui.info_count(i, len(to_remove), ui.blue, package.name)
self.remove_package(package.name)
if to_add:
ui.info(ui.green, "Adding packages")
for i, package in enumerate(to_add):
ui.info_count(i, len(to_add), ui.blue, package.name)
self.handle_package(package, feed)
self.packages[package.name] = package
self.save()
def get_build_worktree(args, verbose=True):
""" Get a build worktree to use from a argparse.Namespace
object
"""
worktree = qisys.parsers.get_worktree(args)
build_worktree = qibuild.worktree.BuildWorkTree(worktree)
if verbose:
ui.info(ui.green, "Current build worktree:", ui.reset, ui.bold, build_worktree.root)
build_config = get_build_config(build_worktree, args)
build_worktree.build_config = build_config
if verbose:
if build_config.toolchain:
ui.info(ui.green, "Using toolchain:", ui.blue, build_config.toolchain.name)
for profile in build_config.profiles:
ui.info(ui.green, "Using profile:", ui.blue, profile)
return build_worktree
def install(self,
destination,
install_tc_packages=False,
python_minify=False):
""" Install every project to the given destination """
qisys.sh.mkdir(destination, recursive=True)
# Copy the manifest
qisys.sh.install(self.manifest_xml,
os.path.join(destination, "manifest.xml"))
# Use every available builder to install
build_config = None
for builder in self.builders:
desc = desc_from_builder(builder)
if builder.projects:
ui.info(ui.bold, "-> Adding %s ..." % desc)
if isinstance(builder, qibuild.cmake_builder.CMakeBuilder):
builder.dep_types = ["runtime"]
build_config = builder.build_config
builder.install(destination,
components=["runtime"],
install_tc_packages=install_tc_packages)
else:
builder.install(destination)
# Install self.pml_extra_files
if self.pml_extra_files:
ui.info(ui.bold, "-> Adding extra files ...")
for src in self.pml_extra_files:
full_src = os.path.join(self.base_dir, src)
rel_src = os.path.relpath(full_src, self.base_dir)
full_dest = os.path.join(destination, rel_src)
qisys.sh.install(full_src, full_dest)
# Minify Python Files if requested
if python_minify is True:
extension = os.path.splitext(rel_src)[1].strip().lower()
if extension == ".py":
minify_python(full_dest)
# Generate and install translations
ui.info(ui.bold, "-> Generating translations ...")
pml_translator = qilinguist.pml_translator.PMLTranslator(self.pml_path)
pml_translator.release(build_config=build_config)
pml_translator.install(destination)
def _install_package(url, pkg_name, pkg_path):
""" Install Package """
# TODO: This will need NAOqi authentication
try:
import qi
session = qi.Session()
session.connect("tcp://%s:9559" % (url.host))
package_manager = session.service("PackageManager")
ui.info(ui.blue, "::", ui.reset, ui.bold,
"Removing previous installation of the package")
try:
package_manager.removePkg(pkg_name)
except Exception:
pass
ui.info(ui.blue, "::", ui.reset, ui.bold, "Installing package")
ret = package_manager.install("/home/%s/%s" %
(url.user, os.path.basename(pkg_path)))
ui.info("PackageManager returned:", ret)
except ImportError:
ui.error(
"Unable to install pkg, please install qi from pip and retry.")
def rebase_projects(git_projects, upstream_projects, branch):
""" Rebase Projects """
ui.info(ui.green, "Computing list of forked projects ...")
rebased_projects = list()
errors = list()
forked_projects = get_forked_projects(git_projects, upstream_projects, branch)
if not forked_projects:
ui.info(ui.green, "Nothing to rebase")
return list(), list()
ui.info(ui.green, "Rebasing forked projects ...")
max_src = max(len(x.src) for x in forked_projects)
for i, git_project in enumerate(forked_projects):
ui.info_count(i, len(forked_projects),
git_project.src.ljust(max_src + 2), end="")
upstream_project = upstream_projects[git_project.src]
status = rebase_project(git_project, upstream_project)
if status is True:
rebased_projects.append(git_project)
if status is False:
errors.append(git_project)
return rebased_projects, errors
def configure_local_settings(build_worktree):
""" Configure local settings for this worktree
"""
print
worktree_root = build_worktree.root
ui.info(ui.green, "::", ui.reset, "Found a worktree in", worktree_root)
qibuild_cfg = build_worktree.qibuild_cfg
answer = qisys.interact.ask_yes_no(
"Do you want to configure settings for this worktree?", default=False)
if not answer:
return
tc_names = qitoolchain.get_tc_names()
if tc_names:
ui.info(ui.green, "::", ui.reset, "Found the following toolchains: ",
", ".join(tc_names))
answer = qisys.interact.ask_yes_no(
"Use one of these toolchains by default", default=True)
if answer:
default = qisys.interact.ask_choice(
tc_names, "Choose a toolchain to use by default")
if default:
qibuild_cfg.local.defaults.config = default
qibuild_cfg.write_local_config(build_worktree.qibuild_xml)
answer = qisys.interact.ask_yes_no(
"Do you want to use a unique build dir?"
" (mandatory when using Eclipse)",
default=False)
build_dir = None
if answer:
build_dir = qisys.interact.ask_string("Path to a build directory")
build_dir = os.path.expanduser(build_dir)
full_path = os.path.join(worktree_root, build_dir)
ui.info(ui.green, "::", ui.reset, "Will use", full_path,
"as a root for all build directories")
qibuild_cfg.local.build.build_dir = build_dir
qibuild_cfg.write_local_config(build_worktree.qibuild_xml)
def setup_project(project):
"""
Setup a project for code review:
If there is :py:class:`.Remote` configured for code review,
using the ssh protocol, use it to fetch the gerrit ``commit-msg`` hook
"""
remote = project.review_remote
server = remote.server
username = remote.username
ssh_port = remote.port
# Install the hook
commit_hook = os.path.join(project.path, ".git", "hooks", "commit-msg")
if os.path.exists(commit_hook):
return True
ui.info("Configuring", ui.blue, project.src,
ui.reset, "for code review ... ", end="")
if remote.protocol == "ssh":
ok, out = fetch_gerrit_hook_ssh(project.path, username, server, port=ssh_port)
if not ok:
ui.info("\n", out, ui.red, "[FAILED]")
return False
# FIXME: make it work with http too?
ui.info(ui.green, "[OK]")
return True
def update(self, feed, branch=None, name=None):
"""
Update a toolchain given a feed
``feed`` can be:
* a path
* an url
* a git url (in this case branch and name cannot be None,
and ``feeds/<name>.xml`` must exist on the given branch)
"""
feed_parser = qitoolchain.feed.ToolchainFeedParser(self.name)
feed_parser.parse(feed, branch=branch, name=name)
self.build_target = feed_parser.target
ui.debug("Update target in database from feedParser",
self.build_target)
remote_packages = feed_parser.get_packages()
local_packages = self.packages.values()
to_add = list()
to_remove = list()
to_update = list()
svn_packages = [
x for x in remote_packages
if isinstance(x, qitoolchain.svn_package.SvnPackage)
]
other_packages = [x for x in remote_packages if x not in svn_packages]
for remote_package in other_packages:
if remote_package.name in (x.name for x in local_packages):
continue
to_add.append(remote_package)
for local_package in local_packages:
if local_package.name not in (x.name for x in remote_packages):
to_remove.append(local_package)
remote_names = [x.name for x in remote_packages]
for local_package in local_packages:
if local_package not in remote_packages and local_package.name in remote_names:
remote_package = [
x for x in remote_packages if x.name == local_package.name
][0]
to_update.append(remote_package)
# remove svn packages from the list of packages to update
to_update = [
x for x in to_update
if not isinstance(x, qitoolchain.svn_package.SvnPackage)
]
if to_update:
ui.info(ui.red, "Updating packages")
for i, package in enumerate(to_update):
remote_package = [
x for x in remote_packages if x.name == package.name
][0]
local_package = [
x for x in local_packages if x.name == package.name
][0]
ui.info_count(i, len(to_update), ui.blue, package.name, "from",
local_package.version, "to", remote_package.version)
self.remove_package(package.name)
self.handle_package(package, feed)
self.add_package(package)
if to_remove:
ui.info(ui.red, "Removing packages")
for i, package in enumerate(to_remove):
ui.info_count(i, len(to_remove), ui.blue, package.name)
self.remove_package(package.name)
if to_add:
ui.info(ui.green, "Adding packages")
for i, package in enumerate(to_add):
ui.info_count(i, len(to_add), ui.blue, package.name)
self.handle_package(package, feed)
self.add_package(package)
if svn_packages:
ui.info(ui.green, "Updating svn packages")
for i, svn_package in enumerate(svn_packages):
ui.info_count(i, len(svn_packages), ui.blue, svn_package.name)
self.handle_svn_package(svn_package)
self.add_package(svn_package)
ui.info(ui.green, "Done")
self.save()
def install(self, dest_dir, *args, **kwargs):
""" Install the projects and the packages to the dest_dir """
installed = list()
projects = self.deps_solver.get_dep_projects(self.projects,
self.dep_types)
packages = self.deps_solver.get_dep_packages(self.projects,
self.dep_types)
if "install_tc_packages" in kwargs:
install_tc_packages = kwargs["install_tc_packages"]
del kwargs["install_tc_packages"]
if not install_tc_packages:
packages = list()
# Compute the real path where to install the packages:
prefix = kwargs.get("prefix", "/")
prefix = prefix[1:]
real_dest = os.path.join(dest_dir, prefix)
components = kwargs.get("components")
build_type = "Release"
if projects:
ui.info(ui.green, "the following projects")
for project in projects:
ui.info(ui.green, " *", ui.blue, project.name)
if packages:
ui.info(ui.green, "and the following packages")
for package in packages:
ui.info(ui.green, " *", ui.blue, package.name)
ui.info(ui.green, "will be installed to", ui.blue, real_dest)
runtime_only = self.dep_types == ["runtime"]
if runtime_only:
ui.info(ui.green, "(runtime components only)")
build_type = projects[0].build_type
release = build_type == "Release"
if packages:
ui.info(ui.green, ":: ", "installing packages")
for i, package in enumerate(packages):
ui.info_count(i,
len(packages),
ui.green,
"Installing",
ui.blue,
package.name,
update_title=True)
files = package.install(real_dest,
components=components,
release=release)
installed.extend(files)
# Remove qitest.json so that we don't append tests twice
# when running qibuild install --with-tests twice
qitest_json = os.path.join(dest_dir, "qitest.json")
qisys.sh.rm(qitest_json)
if projects:
ui.info(ui.green, ":: ", "installing projects")
for i, project in enumerate(projects):
ui.info_count(i,
len(projects),
ui.green,
"Installing",
ui.blue,
project.name,
update_title=True)
files = project.install(dest_dir, **kwargs)
installed.extend(files)
return installed
def deploy(self,
url,
split_debug=False,
with_tests=False,
install_tc_packages=True):
""" Deploy the project and the packages it depends to a remote url """
# Deploy packages: install all of them in the same temp dir, then
# deploy this temp dir to the target
deploy_name = self.build_config.build_directory(prefix="deploy")
deploy_dir = os.path.join(self.build_worktree.root, ".qi", deploy_name)
if not os.path.isdir(deploy_dir):
qisys.sh.mkdir(deploy_dir)
deploy_manifest = os.path.join(deploy_dir, "deploy_manifest.txt")
if os.path.exists(deploy_manifest):
qisys.sh.rm(deploy_manifest)
to_deploy = list()
components = ["runtime"]
if with_tests:
components.append("test")
# Remove qitest.json so that we don't append tests twice
# when running `qibuild deploy --with-tests` twice
qitest_json = os.path.join(deploy_dir, "qitest.json")
qisys.sh.rm(qitest_json)
if install_tc_packages:
dep_packages = self.deps_solver.get_dep_packages(
self.projects, self.dep_types)
else:
dep_packages = list()
dep_projects = self.deps_solver.get_dep_projects(
self.projects, self.dep_types)
ui.info(ui.green, "The following projects")
for project in sorted(dep_projects, key=operator.attrgetter("name")):
ui.info(ui.green, " *", ui.reset, ui.blue, project.name)
if dep_packages:
ui.info(ui.green, "and the following packages")
for package in sorted(dep_packages,
key=operator.attrgetter("name")):
ui.info(ui.green, " *", ui.reset, ui.blue, package.name)
ui.info(ui.green, "will be deployed to", ui.blue, url.as_string)
if dep_packages:
ui.info(ui.green, ":: ", "Deploying packages")
for i, package in enumerate(dep_packages):
ui.info_count(i,
len(dep_packages),
ui.green,
"Deploying package",
ui.blue,
package.name,
ui.green,
"to",
ui.blue,
url.as_string,
update_title=True)
# Install package in local deploy dir
files = package.install(deploy_dir, components=components)
to_deploy.extend(files)
ui.info(ui.green, ":: ", "Deploying projects")
# Deploy projects: install them inside a 'deploy' dir in the worktree
# root, then deploy this dir to the target
for (i, project) in enumerate(dep_projects):
ui.info_count(i,
len(dep_projects),
ui.green,
"Deploying project",
ui.blue,
project.name,
ui.green,
"to",
ui.blue,
url.as_string,
update_title=True)
if with_tests:
to_deploy.append("qitest.json")
# Install project in local deploy dir
installed = project.install(deploy_dir,
components=components,
split_debug=split_debug)
to_deploy.extend(installed)
# Add debugging scripts
for project in self.projects:
scripts = qibuild.deploy.generate_debug_scripts(
self, deploy_dir, project.name, url)
if scripts:
to_deploy.extend(scripts)
# Write the list of files to be deployed
with open(deploy_manifest, "a") as f:
# sort and remove duplicates:
to_deploy = list(set(to_deploy))
to_deploy.sort()
f.write("\n".join(to_deploy))
ui.info(ui.green,
"::",
"Syncing to url",
ui.reset,
ui.bold,
url.as_string,
update_title=True)
qisys.remote.deploy(deploy_dir, url, filelist=deploy_manifest)
def check_local_branch(git_project):
""" Check Local Branch """
git = qisrc.git.Git(git_project.path)
rc, out = git.fetch(raises=False)
if rc != 0:
ui.info(ui.red, "[FAILED]")
ui.info(ui.red, "git fetch failed:\n" + out)
return False
current_branch = git.get_current_branch()
local_branch = git_project.default_branch.name
if current_branch != local_branch:
ui.info(ui.brown, "[skipped] ", end="")
ui.info("On %s, should be on %s" % (current_branch, local_branch))
return False
remote_branch = git_project.default_branch.remote_branch
remote_name = git_project.default_remote.name
remote_ref = "%s/%s" % (remote_name, remote_branch)
status = qisrc.git.get_status(git, local_branch, remote_ref)
if status != "no-diff":
ui.info(ui.brown, "[skipped] ", end="")
if status == "ahead":
ui.info("You have changes not pushed yet")
elif status == "behind":
ui.info("Your branch is not up to date")
elif status == "diverged":
ui.info("Your branch has diverged")
return False
return True
def fix_root_cmake(cmakelists, project_name, dry_run=True): # pylint: disable=too-many-branches
""" Fix the root CMakeLists.txt file
If not found, create a new one
If include(qibuild.cmake) is found, replace by find_package(qibuild)
If include(boostrap.cmake) is found, replace by find_package(qibuild)
If no find_package(qibuild) is found, add the line next to the
first project() line
"""
template = """# CMake file for {project_name}
cmake_minimum_required(VERSION 2.8)
project({project_name})
find_package(qibuild)
# qi_create_lib(...)
# qi_create_bin(...)
"""
template = template.format(project_name=project_name)
if not os.path.exists(cmakelists):
if not dry_run:
with open(cmakelists, "w") as fp:
fp.write(template)
return
with open(cmakelists, "r") as fp:
old_lines = fp.readlines()
new_lines = list()
# Replace old include() by new find_package
seen_find_package_qibuild = False
for line in old_lines:
match = re.match(r"\s*find_package\s*\(\s*qibuild\s*\)", line)
if match:
seen_find_package_qibuild = True
match = re.match(r"\s*include\s*\(.*/?bootstrap.cmake.*", line)
if match:
if not seen_find_package_qibuild:
new_lines.append('find_package(qibuild)\n')
new_lines.append('include(qibuild/compat/compat)\n')
seen_find_package_qibuild = True
else:
match = re.match(r"\s*include\s*\(.*/?qibuild.cmake.*", line)
if match:
if not seen_find_package_qibuild:
new_lines.append('find_package(qibuild)\n')
seen_find_package_qibuild = True
else:
new_lines.append(line)
# Add find_package(qibuild) after project() if it is not there
if not seen_find_package_qibuild:
tmp_lines = new_lines[:]
new_lines = list()
for line in tmp_lines:
new_lines.append(line)
regexp = re.compile(r'^\s*project\s*\((.*)\)', re.IGNORECASE)
if re.match(regexp, line):
new_lines.append('find_package(qibuild)\n')
if dry_run:
ui.info("Would patch", cmakelists)
# Print a nice diff
for line in difflib.unified_diff(old_lines, new_lines):
sys.stdout.write(line)
return
with open(cmakelists, "w") as fp:
ui.info("Patching", cmakelists)
fp.writelines(new_lines)
def rebase_project(git_project, upstream_project):
""" Rebase Project """
ok = check_local_branch(git_project)
if not ok:
return False
git = qisrc.git.Git(git_project.path)
local_branch = git_project.default_branch.name
upstream_branch = upstream_project.default_branch.name
upstream_ref = "%s/%s" % (upstream_project.default_remote.name, upstream_branch)
status = qisrc.git.get_status(git, local_branch, upstream_ref)
if status == "ahead":
ui.info(ui.green, "[OK]", ui.reset, "already rebased")
return None
if status == "no-diff":
ui.info(ui.green, "[OK]", ui.reset, "no diff")
return None
if status == "behind":
rc, out = git.merge(upstream_ref, raises=False)
if rc != 0:
ui.info(ui.red, "[FAILED]")
ui.info(ui.red, "git merge failed\n" + out)
return False
ui.info(ui.green, "[OK]", ui.reset, "fast-forwarded")
return True
git.call("tag", "-f", "before-rebase", raises=False) # suppress output
rc, out = git.call("rebase", upstream_ref, raises=False)
if rc == 0:
ui.info(ui.green, "[OK]", ui.reset, "rebased")
return True
ui.info(ui.red, "[FAILED]", ui.reset, "there was some conflicts")
git.call("rebase", "--abort", raises=False)
git.call("tag", "-d", "before-rebase", raises=False) # suppress output
return False
def do(args):
directory = args.directory
name = args.name
res = qibuild.cmake.modules.generate_cmake_module(directory, name)
ui.info(ui.green, "CMake module generated in", ui.reset, ui.bold, res)
def get_doc_worktree(args):
worktree = qisys.parsers.get_worktree(args)
doc_worktree = DocWorkTree(worktree)
ui.info(ui.green, "Current doc worktree:", ui.reset,
ui.bold, doc_worktree.root)
return doc_worktree
def do(args):
"""Main entry point."""
build_worktree = qibuild.parsers.get_build_worktree(args)
projects = qibuild.parsers.get_build_projects(build_worktree,
args,
solve_deps=True)
if args.remove_known_configs and args.remove_unknown_configs:
clean_selection = "all_configs"
elif args.remove_known_configs and not args.remove_unknown_configs:
clean_selection = "known_configs"
elif not args.remove_known_configs and args.remove_unknown_configs:
clean_selection = "unknown_configs"
else:
clean_selection = "given_config"
bdirs = {'known_configs': list(), 'unknown_configs': list()}
all_configs = clean_selection != "given_config"
for project in projects:
bdirs_ = project.get_build_dirs(all_configs=all_configs)
for cat in bdirs_.keys():
bdirs[cat].extend(bdirs_[cat])
if clean_selection in ["given_config", "all_configs", "known_configs"]:
bdir_count = len(bdirs['known_configs'])
if bdir_count == 0:
ui.info(ui.green, "No build directory to clean")
elif not args.force:
ui.info(ui.green, "Build directories that will be removed",
ui.reset, ui.bold, "(use -f to apply):")
for i, bdir in enumerate(bdirs['known_configs']):
message = list()
if args.force:
message.extend([ui.green, "Cleaning", ui.reset, bdir])
# delete the build directory
qisys.sh.rm(bdir)
else:
message.append(bdir)
ui.info_count(i, bdir_count, *message)
if clean_selection in ["all_configs", "unknown_configs"]:
bdir_count = len(bdirs['unknown_configs'])
if bdir_count == 0:
ui.info(
ui.green,
"No build directory matching unknown configuration to clean")
elif not args.force:
ui.info(
ui.green,
"Build directories matching unknown configuration that may be removed",
ui.reset, ui.bold, "(interactive mode, use -f to apply):")
# remove uncertain build directories, by configuration name, so sort them
sorted_bdirs = {}
for bdir in bdirs['unknown_configs']:
# all build directory names should be prefixed with "build-", so strip it
config_name = os.path.basename(bdir)[6:]
if not config_name in sorted_bdirs:
sorted_bdirs[config_name] = []
sorted_bdirs[config_name].append(bdir)
for c, sbdirs in sorted_bdirs.items():
question = "Remove build directories matching the '%s' configuration?" % c
answer = qisys.interact.ask_yes_no(question, default=False)
if not answer:
continue
bdir_count = len(sbdirs)
for i, bdir in enumerate(sbdirs, start=1):
to_print = [
ui.green, "*", ui.reset,
"(%i/%i)" % (i, bdir_count)
]
if args.force:
to_print.extend([ui.green, "Cleaning", ui.reset, bdir])
# delete the build directory
qisys.sh.rm(bdir)
else:
to_print.extend([ui.reset, bdir])
ui.info(*to_print)
def do(args):
python_worktree = qipy.parsers.get_python_worktree(args)
ui.info(ui.green, "python projects in:", ui.blue, python_worktree.root)
collector = PythonTestCollector(python_worktree)
collector.collect()
def cmake(source_dir,
build_dir,
cmake_args,
env=None,
clean_first=True,
profiling=False,
debug_trycompile=False,
trace_cmake=False,
summarize_options=False):
"""
Call cmake with from a build dir for a source dir.
cmake_args are added on the command line.
:param env: defines the environment used when calling ``cmake``
``os.environ`` will remain unchanged
:param clean_first: Clean the cmake cache
:param summarize_options: Whether to call :py:func:`display_options` at the end
For qibuild/CMake hackers:
:param profiling: Profile CMake executions
:param debug_trycompile: Call ``cmake`` with ``--debug-trycompile``
:param trace_cmake: Call ``cmake`` with ``--trace`` The results
will be written in <build>/cmake.log
"""
if not os.path.exists(source_dir):
raise Exception("source dir: %s does not exist, aborting")
if not os.path.exists(build_dir):
mess = "Could not find build directory: %s \n" % build_dir
raise Exception(mess)
# Always remove CMakeCache
if clean_first:
cache = os.path.join(build_dir, "CMakeCache.txt")
qisys.sh.rm(cache)
if debug_trycompile:
cmake_args.append("--debug-trycompile")
if profiling or trace_cmake:
cmake_args.append("--trace")
# Check that no one has made an in-source build
in_source_cache = os.path.join(source_dir, "CMakeCache.txt")
if os.path.exists(in_source_cache):
mess = "You have run CMake from your sources\n"
mess += "CMakeCache.txt found here: %s\n" % in_source_cache
mess += "Please clean your sources and try again\n"
raise Exception(mess)
# Check that the root CMakeLists file is correct
root_cmake = os.path.join(source_dir, "CMakeLists.txt")
check_root_cmake_list(root_cmake)
# Add path to source to the list of args, and set buildir for
# the current working dir.
cmake_args += [source_dir]
if not profiling and not trace_cmake:
qisys.command.call(["cmake"] + cmake_args, cwd=build_dir, env=env)
if summarize_options:
display_options(build_dir)
return
cmake_log = os.path.join(build_dir, "cmake.log")
fp = open(cmake_log, "w")
if profiling:
ui.info(ui.green, "Running cmake for profiling ...")
if trace_cmake:
ui.info(ui.green, "Running cmake with --trace ...")
ui.debug("Running cmake " + " ".join(cmake_args))
retcode = subprocess.call(["cmake"] + cmake_args,
cwd=build_dir,
env=env,
stdout=fp,
stderr=fp)
fp.close()
if retcode != 0:
mess = "CMake failed"
if retcode < 0:
mess += " (%s)" % qisys.command.str_from_signal(-retcode)
ui.error(mess)
ui.info(ui.green, "CMake trace saved in", ui.reset, ui.bold, cmake_log)
if not profiling:
return
qibuild_dir = get_cmake_qibuild_dir()
ui.info(ui.green, "Analyzing cmake logs ...")
profiling_res = qibuild.cmake.profiling.parse_cmake_log(
cmake_log, qibuild_dir)
outdir = os.path.join(build_dir, "profile")
qibuild.cmake.profiling.gen_annotations(profiling_res, outdir, qibuild_dir)
ui.info(ui.green, "Annotations generated in", outdir)
def remove_repo(self, project):
""" Remove a project from the worktree """
ui.info(ui.green, "Removing", project.src)
# not sure when to use from_disk here ...
if project in self.worktree.projects:
self.worktree.remove_project(project.src)
def main():
ui.info(ui.red, "This is a an error message\n", ui.reset,
"And here are the details")
ui.error("could not build")
ui.warning("-j ignored for this generator")
ui.info("building foo")
ui.debug("debug message")
ui.info(ui.brown, "this is brown")
ui.info(ui.bold, ui.brown, "this is bold brown")
ui.info(ui.red, "red is dead")
ui.info(ui.darkred, "darkred is really dead")
ui.info(ui.yellow, "this is yellow")
def configure(workspace, build_config, build_type="Release"):
build_dir = os.path.join(workspace, "build-%s" % build_type)
qisys.sh.mkdir(build_dir, recursive=True)
platform = get_platform(build_config)
toolchain = qitoolchain.get_toolchain(build_config.toolchain)
toolchain.update()
ui.info(str(toolchain))
toolchain_path = toolchain.db.db_path
cmake_args = list()
cmake_args.append("-Wno-dev")
cmake_args.append("-DCMAKE_BUILD_TYPE=%s" % build_type)
# Find Boost in toolchain
boost_path = toolchain.get_package("boost", raises=True).path
boost_root = qisys.sh.to_posix_path(boost_path)
cmake_args.append("-DBOOST_ROOT=%s" % boost_root)
# our boost package assumes boost libs are shared:
cmake_args.append("-DBoost_USE_STATIC_LIBS=OFF")
# Find Eigen3 in toolchain
eigen_path = toolchain.get_package("eigen3", raises=True).path
eigen_root = qisys.sh.to_posix_path(eigen_path)
cmake_args.append("-DEIGEN_ROOT=%s" % eigen_root)
# Find console_bridge in toolchain
console_bridge_path = toolchain.get_package("console_bridge",
raises=True).path
console_bridge_dir = os.path.join(console_bridge_path, "share", "cmake",
"console_bridge")
console_bridge_dir = qisys.sh.to_posix_path(console_bridge_dir)
cmake_args.append("-Dconsole_bridge_DIR=%s" % console_bridge_dir)
# Need an absolute, non-empty path for CMAKE_INSTALL_PREFIX
cmake_args.append("-DCMAKE_INSTALL_PREFIX=/prefix")
# Don't bother building tests:
cmake_args.append("-DCATKIN_ENABLE_TESTING=OFF")
if platform == "linux":
cmake_include_path = list()
cmake_lib_path = list()
# Need to find lz4 and bz2 (deps of rosbag) in toolchain:
for dep in ["bzip2", "lz4"]:
package_path = toolchain.get_package(dep, raises=True).path
include_path = os.path.join(package_path, "include")
lib_path = os.path.join(package_path, "lib")
cmake_include_path.append(include_path)
cmake_lib_path.append(lib_path)
cmake_args.append("-DCMAKE_INCLUDE_PATH=%s" %
";".join(cmake_include_path))
cmake_args.append("-DCMAKE_LIBRARY_PATH=%s" % ";".join(cmake_lib_path))
cmake_args.append("-DCMAKE_CXX_FLAGS=-std=gnu++11")
if platform == "windows":
cmake_args.append("-DCMAKE_DEBUG_POSTFIX=_d")
cmake_flags = (
# our boost package assumes boost libs are shared:
("BOOST_ALL_DYN_LINK", "1"),
# ros/time.h uses #ifdef WIN32 instead of #ifdef _WIN32 ...
("WIN32", "1"),
)
cmake_flags_arg = "-DCMAKE_CXX_FLAGS="
for key, value in cmake_flags:
cmake_flags_arg += "/D%s=%s " % (key, value)
# re-add some flags
# (they get removed when we set CMAKE_CXX_FLAGS
# for some reason)
# /EHsc is required for exceptions, and
# /MD for linking with the 'dynamic' runtime
cmake_flags_arg += "/EHsc /MD"
cmake_args.append(cmake_flags_arg)
if platform != "linux":
# Ninja may not work on linux ...
cmake_args.append("-GNinja")
cmake_args.append("../src")
build_env = get_build_env(build_config)
qisys.command.call(["cmake"] + cmake_args, env=build_env, cwd=build_dir)
def do(args):
"""Main entry point"""
standalone = args.standalone
breakpad = args.breakpad
cmake_builder = qibuild.parsers.get_cmake_builder(args)
if breakpad:
cmake_builder.build_config.build_type = "RelWithDebInfo"
projects = cmake_builder.projects
if len(projects) != 1:
raise Exception("This action can only work on one project")
project = projects[0]
archive_name = project.name
version = project.version
if not version:
project.version = "0.1"
build_dir_name = os.path.basename(project.build_directory)
archive_suffix = build_dir_name.replace("build-", "")
archive_name += "-" + version
archive_name += "-" + archive_suffix
package_dir = os.path.join(cmake_builder.build_worktree.root, "package")
destdir = os.path.join(package_dir, archive_name)
# Clean the destdir just in case the package was already generated
qisys.sh.rm(destdir)
build_type = cmake_builder.build_config.build_type
# Also build in debug on windows when building a package for a toolchain
if sys.platform.startswith("win") and not standalone:
_do_package(cmake_builder,
destdir,
build_type="Debug",
standalone=False)
_do_package(cmake_builder,
destdir,
build_type=build_type,
standalone=standalone)
package_xml_path = os.path.join(destdir, "package.xml")
project.gen_package_xml(package_xml_path)
if breakpad:
symbols_archive_name = archive_name + "-symbols.zip"
symbols_archive = os.path.join(package_dir, symbols_archive_name)
with qisys.sh.TempDir() as tmp:
pool_dir = os.path.join(tmp, "symbols")
qibuild.breakpad.dump_symbols_from_directory(destdir, pool_dir)
qisys.archive.compress(pool_dir, flat=True, output=symbols_archive)
ui.info(ui.blue, "::", ui.reset, ui.bold, "Compressing package ...")
flat = not standalone
archive = qisys.archive.compress(destdir,
algo="zip",
quiet=True,
flat=flat,
output=destdir + ".zip")
# Clean up after ourselves
qisys.sh.rm(destdir)
ui.info(ui.green, "Package generated in", ui.reset, ui.bold, archive)
if breakpad:
ui.info(ui.green, "Symbols package generated in", ui.reset, ui.bold,
symbols_archive)
return archive, symbols_archive
else:
return archive
def build(self,
build_type=None,
language=None,
spellcheck=False,
werror=False,
pdb=False):
""" Run sphinx.main() with the correct arguments """
try:
import sphinx
except ImportError, e:
ui.error(e, "skipping build")
return
if self.prebuild_script:
ui.info(ui.green, "Running pre-build script:", ui.white,
self.prebuild_script)
cmd = [sys.executable, self.prebuild_script]
qisys.command.call(cmd, cwd=self.path)
ui.info()
self.generate_examples_zips()
if self.translated and language and language != "en" \
and language not in self.linguas:
raise UnknownLingua(self, language)
if self.translated:
self.intl_build(language)
qisys.sh.mkdir(self.html_dir, recursive=True)
spell_dir = os.path.join(self.build_dir, "spellcheck")
qisys.sh.mkdir(spell_dir, recursive=True)
def test_empty_end():
ui.info("[skipped]", end="")
ui.info("Your branch has diverged")
