def do(args):
"""Main entry point"""
if args.nightly:
ui.warning("--slow option has no effect\n",
"Use `qibuild configure -DQI_WITH_NIGHTLY_TESTS=ON` instead")
if args.nightmare:
os.environ["GTEST_REPEAT"] = "20"
os.environ["GTEST_SHUFFLE"] = "yes"
if args.list:
build_worktree = qibuild.parsers.get_build_worktree(args)
project = qibuild.parsers.get_one_build_project(build_worktree, args)
json = os.path.join(project.build_directory, "qitest.json")
qisys.script.run_action("qitest.actions.list", [json])
return
cmake_builder = qibuild.parsers.get_cmake_builder(args)
deps_solver = cmake_builder.deps_solver
if args.use_deps:
dep_types = ["build"]
else:
dep_types = list()
projects = deps_solver.get_dep_projects(cmake_builder.projects,
dep_types)
res = True
for project in projects:
if args.build_first:
project.build()
res = project.run_tests(**vars(args))
if not res:
sys.exit(1)
def do(args):
""" Main method """
worktree = qisys.worktree.open_worktree(args.worktree)
if not worktree.projects:
on_empty_worktree(worktree)
regex = args.pattern
if args.pattern:
regex = re.compile(regex)
mess = [ui.green, "Projects in :", ui.reset, ui.bold, worktree.root]
if args.pattern:
mess.extend([ui.green, "matching", args.pattern])
ui.info(*mess)
to_remove = list()
for project in worktree.projects:
if not regex or regex.search(project.src):
ui.info(ui.green, " *", ui.blue, project.src)
if not os.path.exists(project.path):
to_remove.append(project.src)
if not to_remove:
return
mess = "The following projects:\n"
for rm in to_remove:
mess += " * " + rm + "\n"
mess += "are registered in the worktree, but their paths no longer exists"
ui.warning(mess)
answer = qisys.interact.ask_yes_no("Do you want to remove them", default=True)
if not answer:
return
for rm in to_remove:
ui.info(ui.green, "Removing", rm)
worktree.remove_project(rm)
def is_submodule(path):
""" Tell if the given path is a submodule
"""
if not os.path.isdir(path):
return False
# Two cases:
# * submodule not initialized -> path will be an empty dir
# * submodule initialized -> path/.git will be a file
# looking like:
# gitdir: ../../.git/modules/bar
contents = os.listdir(path)
if contents:
dot_git = os.path.join(path, ".git")
if os.path.isdir(dot_git):
return False
parent_repo_root = get_repo_root(os.path.dirname(path))
else:
parent_repo_root = get_repo_root(path)
parent_git = Git(parent_repo_root)
(retcode, out) = parent_git.submodule(raises=False)
if retcode == 0:
if not out:
return False
else:
lines = out.splitlines()
submodules = [x.split()[1] for x in lines]
rel_path = os.path.relpath(path, parent_repo_root)
return rel_path in submodules
else:
ui.warning("git submodules configuration is broken for",
parent_repo_root, "!",
"\nError was: ", ui.reset, "\n", " " + out)
return True
def read_remote_manifest(self, manifest_xml=None, warn_if_missing_group=True):
""" Read the manifest file in .qi/manifests/<name>/manifest.xml
using the settings in .qi/manifest.xml (to know the name and the groups
to use)
"""
if not manifest_xml:
manifest_xml = os.path.join(self.manifest_repo, "manifest.xml")
remote_manifest = qisrc.manifest.Manifest(manifest_xml, review=self.manifest.review)
groups = self.manifest.groups
# if self.manifest.groups is empty but there is a default
# group in the manifest, we need to set self.manifest.groups
# so that subsequent calls to qisrc add-group, remove-group
# work
if self.manifest.groups is None:
default_group = remote_manifest.groups.default_group
if default_group:
self.manifest.groups = [default_group.name]
# Maybe some groups were removed from the manifest.
# Only consider those which exist
if groups is None:
groups_to_use = None
else:
groups_to_use = list()
for group in groups:
if group in remote_manifest.groups.group_names:
groups_to_use.append(group)
else:
if warn_if_missing_group:
ui.warning("Group %s not found in the manifest" % group)
repos = remote_manifest.get_repos(groups=groups_to_use)
return repos
def num_jobs_to_args(num_jobs, cmake_generator):
""" Convert a number of jobs to a list of cmake args
>>> num_jobs_to_args(3, "Unix Makefiles")
["-j", "3"]
>>> num_jobs_to_args(3, "NMake Makefiles"
Error: -j is not supported for NMake, use Jom
>>> num_jobs_to_args(3, "Visual Studio")
Warning: -j is ignored for Visual Studio
"""
if num_jobs == 1:
return list()
if "Unix Makefiles" in cmake_generator:
return ["-j", str(num_jobs)]
if cmake_generator == "NMake Makefiles":
mess = "-j is not supported for %s\n" % cmake_generator
mess += "On windows, you can use Jom instead to compile "
mess += "with multiple processors"
raise Exception(mess)
if "Visual Studio" in cmake_generator or \
cmake_generator == "Xcode" or \
"JOM" in cmake_generator:
ui.warning("-j is ignored when used with", cmake_generator)
return list()
ui.warning("cannot parse -j into a cmake option for generator: %s" % cmake_generator)
return list()
def do(args):
""" Main entry point """
toolchain = qitoolchain.get_toolchain(args.name)
svn_packages = list()
for package in toolchain.packages:
svn_dir = os.path.join(package.path, ".svn")
if os.path.exists(svn_dir):
svn_packages.append(package)
not_clean = list()
for i, svn_package in enumerate(svn_packages, start=1):
to_write = "Checking (%d/%d) " % (i, len(svn_packages))
sys.stdout.write(to_write + "\r")
sys.stdout.flush()
svn = qisrc.svn.Svn(svn_package.path)
_rc, out = svn.call("status", raises=False)
if out:
not_clean.append((svn_package.name, out))
if not not_clean:
ui.info("\n", ui.green, "All OK")
sys.exit(0)
ui.warning("Some svn packages are not clean")
for name, message in not_clean:
ui.info(ui.green, "*", ui.reset, ui.blue, name)
ui.info(message)
sys.exit(1)
def get_git_projects(self, groups=None):
""" Get the git projects matching a given group """
if not groups:
return self.git_projects
res = set()
git_project_names = dict()
group_names = groups
for git_project in self.git_projects:
git_project_names[git_project.name] = git_project
projects = list()
groups = qisrc.groups.get_groups(self.worktree)
for group_name in group_names:
warn_for_group = True
project_names = groups.projects(group_name)
for project_name in project_names:
git_project = git_project_names.get(project_name)
if git_project:
res.add(git_project)
else:
if warn_for_group:
ui.warning("Group", group_name, "is not currently in use\n",
"Please use `qisrc add-group %s`" % group_name)
warn_for_group = False
res = list(res)
res.sort(key=operator.attrgetter("src"))
return res
def parse_num_jobs(self, num_jobs, cmake_generator=None):
""" Convert a number of jobs to a list of cmake args. """
if not cmake_generator:
cmake_generator = \
qibuild.cmake.get_cached_var(self.build_directory, "CMAKE_GENERATOR")
if num_jobs is None:
# By default, use the "good" number just like Ninja
if "Visual Studio" in cmake_generator:
return ["/maxcpucount"]
return list()
if "Unix Makefiles" in cmake_generator or \
"Ninja" in cmake_generator:
return ["-j", str(num_jobs)]
if cmake_generator == "NMake Makefiles":
mess = "-j is not supported for %s\n" % cmake_generator
mess += "On Windows, you can use Jom or Ninja instead to compile "
mess += "with multiple processors"
raise Exception(mess)
if cmake_generator == "Xcode" or "JOM" in cmake_generator:
ui.warning("-j is ignored when used with", cmake_generator)
return list()
if "Visual Studio" in cmake_generator:
return ["/maxcpucount:%i" % num_jobs]
ui.warning("Unknown generator: %s, ignoring -j option" % cmake_generator)
return list()
def read_remote_config(self, repo, quiet=False):
""" Apply the configuration read from the "repo" setting
of a remote manifest.
Called by WorkTreeSyncer
"""
previous_default = None
if self.default_remote:
previous_default = self.default_remote.name
self.name = repo.project
self.remotes = list()
for remote in repo.remotes:
self.configure_remote(remote)
if repo.default_branch and repo.default_remote:
self.configure_branch(repo.default_branch, tracks=repo.default_remote.name,
remote_branch=repo.default_branch, default=True,
quiet=quiet)
new_default = self.default_remote.name
if previous_default is not None and previous_default != new_default:
if not quiet:
ui.warning("Default remote changed", previous_default, "->",
new_default)
self.review = False
if repo.review:
ok = qisrc.review.setup_project(self)
if ok:
self.review = True
def split_debug(self, destdir, file_list):
""" Split debug symbols after install """
if self.using_visual_studio:
raise Exception("split debug not supported on Visual Studio")
ui.info(ui.green, "Splitting debug symbols from binaries ...")
tool_paths = dict()
for name in ["objcopy", "objdump"]:
tool_path = qibuild.cmake.get_binutil(name,
build_dir=self.build_directory,
env=self.build_env)
tool_paths[name] = tool_path
missing = [x for x in tool_paths if not tool_paths[x]]
if missing:
mess = """\
Could not split debug symbols from binaries for project {name}.
The following tools were not found: {missing}\
"""
mess = mess.format(name=self.name, missing = ", ".join(missing))
ui.warning(mess)
return
for filename in file_list:
full_path = os.path.join(destdir, filename[1:]) # remove starting /
if qibuild.gdb.is_elf(full_path):
qibuild.gdb.split_debug(full_path, **tool_paths)
def configure(self, **kwargs):
""" Create a correct conf.py in self.build_dir """
rel_paths = kwargs.get("rel_paths", False)
in_conf_py = os.path.join(self.source_dir, "conf.in.py")
should_use_template = False
if os.path.exists(in_conf_py):
should_use_template = True
else:
in_conf_py = os.path.join(self.source_dir, "conf.py")
if not os.path.exists(in_conf_py):
ui.error("Could not find a conf.py or a conf.in.py in", self.source_dir)
return
with open(in_conf_py) as fp:
conf = fp.read()
if should_use_template:
if self.template_project:
from_template = self.template_project.sphinx_conf
from_template = from_template.format(**kwargs)
conf = from_template + conf
else:
ui.warning("Found a conf.in.py but no template project found "
"in the worktree")
from_conf = dict()
try:
# quick hack if conf.in.py used __file__
from_conf["__file__"] = in_conf_py
exec(conf, from_conf)
conf = conf.replace("__file__", 'r"%s"' % in_conf_py)
except Exception, e:
ui.error("Could not read", in_conf_py, "\n", e)
return
def ask_gerrit_username(server, ssh_port=29418):
"""
Run a wizard to try to configure gerrit access
If that fails, ask the user for its username
If that fails, give up and suggest upload the public key
"""
ui.info(ui.green, "Configuring gerrit ssh access ...")
username = os.environ.get("GERRIT_USER", qisys.sh.username())
if not username:
username = qisys.interact.ask_string("Please enter your username")
if not username:
return None
ui.info("Checking gerrit connection with %s@%s:%i" %
(username, server, ssh_port))
if check_gerrit_connection(username, server, ssh_port=ssh_port):
ui.info("Success")
return username
ui.warning("Could not connect to ssh using username", username)
try_other = qisys.interact.ask_yes_no("Do you want to try with another username?")
if not try_other:
return None
username = qisys.interact.ask_string("Please enter your username")
if not username:
return None
if check_gerrit_connection(username, server, ssh_port=ssh_port):
return username
return None
def handle_pure_python(venv_path, python_worktree, env=None):
""" Add the paths of all python projects to the virtualenv """
lib_path = virtualenv.path_locations(venv_path)[1]
qi_pth_dest = os.path.join(venv_path, lib_path, "site-packages/qi.pth")
res = True
with open(qi_pth_dest, "w") as fp:
fp.write("")
for i, project in enumerate(python_worktree.python_projects):
ui.info_count(i, len(python_worktree.python_projects),
ui.blue, project.name)
if project.setup_with_distutils:
cmd = [python_worktree.pip, "install"]
if not ui.CONFIG["verbose"]:
cmd.append("--quiet")
cmd.extend(["--editable", "."])
rc = qisys.command.call(cmd, cwd=project.path, ignore_ret_code=True,
env=env)
if rc != 0:
ui.warning("Failed to run pip install on", project.src)
res = False
else:
ui.debug("Adding python path for project", project.name, ":\n",
project.python_path)
for path in project.python_path:
fp.write(path + "\n")
return res
def do(args):
"""Main entry point
"""
feed = args.feed
tc_name = args.name
if tc_name:
toolchain = qitoolchain.get_toolchain(tc_name)
if not feed:
feed = toolchain.feed_url
if not feed:
mess = "Could not find feed for toolchain %s\n" % tc_name
mess += "Please check configuration or " \
"specifiy a feed on the command line\n"
raise Exception(mess)
toolchain.update(feed)
else:
tc_names = qitoolchain.get_tc_names()
for i, tc_name in enumerate(tc_names, start=1):
toolchain = qitoolchain.toolchain.Toolchain(tc_name)
tc_feed = toolchain.feed_url
if not tc_feed:
ui.warning("No feed found for %s, skipping" % tc_name)
continue
ui.info(ui.green, "*", ui.reset, "(%i/%i)" % (i, len(tc_names)),
ui.green, "Updating", ui.blue, tc_name)
toolchain.update(tc_feed)
def get_ide(qibuild_cfg):
"""Return an IDE to use."""
known_ides = qibuild_cfg.ides.values()
ide_names = qibuild_cfg.ides.keys()
if not known_ides:
ui.warning("No IDE configured yet")
ui.info("Tips: use `qibuild config --wizard` to configure an IDE")
return None
# Remove the one that are not supported:
supported_ides = [x for x in known_ides if x.name in SUPPORTED_IDES]
if len(supported_ides) == 1:
return supported_ides[0]
if not supported_ides:
mess = "Found those IDEs in configuration: %s\n" % ", ".join(ide_names)
mess += "But `qibuild open` only supports: %s\n" % ", ".join(SUPPORTED_IDES)
raise Exception(mess)
# User chose a specific config and an IDE matches this config
if qibuild_cfg.ide:
return qibuild_cfg.ide
supported_names = [x.name for x in supported_ides]
# Several IDEs, ask the user to choose
ide_name = qisys.interact.ask_choice(supported_names,
"Please choose an IDE to use")
if not ide_name:
return None
return qibuild_cfg.ides[ide_name]
def do(args):
""" Main entry point """
git_worktree = qisrc.parsers.get_git_worktree(args)
git_projects = qisrc.parsers.get_git_projects(git_worktree, args)
for git_project in git_projects:
maintainers = qisrc.maintainers.get(git_project)
if not maintainers:
mess = """\
The project in {src} has no maintainer.
Please edit {qiproject_xml} to silence this warning
"""
ui.warning(mess.format(src=git_project.src, qiproject_xml=git_project.qiproject_xml), end="")
reviewers = [x["email"] for x in maintainers]
reviewers.extend(args.reviewers or list())
git = qisrc.git.Git(git_project.path)
current_branch = git.get_current_branch()
if not current_branch:
ui.error("Not currently on any branch")
sys.exit(2)
if git_project.review:
qisrc.review.push(
git_project,
current_branch,
bypass_review=(not args.review),
dry_run=args.dry_run,
reviewers=reviewers,
topic=args.topic,
)
else:
if args.dry_run:
git.push("-n")
else:
git.push()
def do(args):
"""Main entry point
"""
if "--name" in sys.argv:
ui.warning("--name is deprecated, use --feed-name instead")
feed = args.feed
# Normalize feed path:
if feed and os.path.exists(feed):
feed = qisys.sh.to_native_path(feed)
tc_name = args.name
# Validate the name: must be a valid filename:
bad_chars = r'<>:"/\|?*'
for bad_char in bad_chars:
if bad_char in tc_name:
mess = "Invalid toolchain name: '%s'\n" % tc_name
mess += "A valid toolchain name should not contain any "
mess += "of the following chars:\n"
mess += " ".join(bad_chars)
raise Exception(mess)
build_worktree = None
if tc_name in qitoolchain.get_tc_names():
toolchain = qitoolchain.Toolchain(tc_name)
ui.info(tc_name, "already exists,",
"updating without removing")
toolchain = qitoolchain.Toolchain(tc_name)
if feed:
toolchain.update(feed, branch=args.branch, name=args.feed_name)
return toolchain
def do(args):
"""Main entry point
"""
feed = args.feed
tc_name = args.name
dry_run = args.dry_run
if tc_name:
toolchain = qitoolchain.get_toolchain(tc_name)
if not feed:
feed = qitoolchain.toolchain.get_tc_feed(tc_name)
if not feed:
mess = "Could not find feed for toolchain %s\n" % tc_name
mess += "Pleas check configuration or specifiy a feed on the command line\n"
raise Exception(mess)
ui.info(ui.green, "Updating toolchain", tc_name, "with", feed)
toolchain.parse_feed(feed, dry_run=dry_run)
else:
tc_names = qitoolchain.get_tc_names()
for i, tc_name in enumerate(tc_names, start=1):
tc_feed = qitoolchain.toolchain.get_tc_feed(tc_name)
ui.info(ui.green, "*", ui.reset, "(%i/%i)" % (i, len(tc_names)), ui.green, "Updating", ui.blue, tc_name)
if not tc_feed:
ui.warning("No feed found for %s, skipping" % tc_name)
continue
ui.info(ui.green, "Reading", tc_feed)
toolchain = qitoolchain.Toolchain(tc_name)
toolchain.parse_feed(tc_feed, dry_run=dry_run)
def no_project_args_on_root(worktree):
""" Called when user ran a qisys.command at the top
of a worktree.
"""
wt_root = worktree.root
qiproj_xml = os.path.join(wt_root, "qiproject.xml")
if os.path.exists(qiproj_xml):
mess = """ Found a qiproject.xml at the root of the worktree
(in {qiproj_xml})
This is not recommended. You should create your worktree at
the parent directory instead
Please remove {dot_qi} and run:
cd {parent}
qibuild init
to do so
"""
parent = os.path.join(wt_root, "..")
parent = os.path.abspath(parent)
mess = mess.format(qiproj_xml=qiproj_xml, wt_root=wt_root,
parent=parent,
dot_qi=os.path.join(wt_root, ".qi"))
ui.warning(mess, end="")
else:
mess = """No project specified
Please specify the name or the path of a project
or go to the subdirectory of a project
Tip: use `qibuild list' to get the list of known projects"""
raise Exception(mess)
def do(args):
test_runners = qitest.parsers.get_test_runners(args)
# rule to check for tests which doesn't follow naming convention
expr = re.compile("^test_.*")
warn_name_count = 0
warn_type_count = 0
for test_runner in test_runners:
ui.info("Tests in ", test_runner.project.sdk_directory)
for i, test in enumerate(test_runner.tests):
n = len(test_runner.tests)
name = test["name"]
name_ok = re.match(expr, name)
type_ok = (test.get("pytest") or test.get("gtest"))
if name_ok and type_ok:
ui.info_count(i, n, test["name"])
else:
message = ""
if not name_ok:
warn_name_count += 1
message += "(invalid name) "
if not type_ok:
warn_type_count += 1
message += "(no type)"
ui.info_count(i, n, name, ui.brown, message)
if warn_name_count:
msg = "%i on %i tests do not respect naming convention" % (warn_name_count, len(test_runner.tests))
ui.warning(msg)
if warn_type_count:
msg = "%i on %i tests do not have any type" % (warn_type_count, len(test_runner.tests))
ui.warning(msg)
def configure_qtcreator(qibuild_cfg):
""" Configure QtCreator
"""
ide = qibuild.config.IDE()
ide.name = "QtCreator"
build_env = qibuild.config.get_build_env()
qtcreator_path = qisys.command.find_program("qtcreator", env=build_env)
if qtcreator_path:
ui.info(ui.green, "::", ui.reset, "Found QtCreator:", qtcreator_path)
mess = "Do you want to use qtcreator from %s?\n" % qtcreator_path
mess += "Answer 'no' if you installed qtcreator from Nokia's installer"
answer = qisys.interact.ask_yes_no(mess, default=True)
if not answer:
qtcreator_path = None
else:
ui.warning("QtCreator not found")
if not qtcreator_path:
qtcreator_path = qisys.interact.ask_program(
"Please enter full qtcreator path")
if not qtcreator_path:
ui.warning("Not adding config for QtCreator",
"qibuild open will not work", sep="\n")
return
ide.path = qtcreator_path
qibuild_cfg.add_ide(ide)
def configure_branch(self, name, tracks="origin",
remote_branch=None, default=True,
quiet=False):
""" Configure a branch. If a branch with the same name
already exists, update its tracking remote.
"""
previous_default_branch = self.default_branch
if previous_default_branch and previous_default_branch.name != name:
if not quiet:
ui.warning(self.src, ": default branch changed",
previous_default_branch.name, "->", name)
previous_default_branch.default = False
branch_found = False
for branch in self.branches:
if branch.name == name:
branch_found = True
if branch.tracks != tracks:
if not quiet:
ui.warning(self.src, ":", branch.name, "now tracks", tracks,
"instead of", branch.tracks)
branch.tracks = tracks
branch.default = default
if not branch_found:
branch = qisrc.git_config.Branch()
branch.name = name
branch.tracks = tracks
branch.remote_branch = remote_branch
branch.default = default
self.branches.append(branch)
return branch
def do(args):
"""
Add a package to a toolchain
- Check that there is a current toolchain
- Add the package to the cache
- Add the package from cache to toolchain
"""
toolchain = qitoolchain.parsers.get_toolchain(args)
package_path = args.package_path
legacy = False
try:
if urlparse.urlparse(package_path).scheme:
package_path = qisys.remote.download(package_path, ".")
archive = zipfile.ZipFile(package_path, allowZip64=True)
archive.read("package.xml")
except KeyError:
legacy = True
if legacy and not args.name:
raise Exception("Must specify --name when using legacy format")
if args.name and not legacy:
ui.warning("--name ignored when using modern format")
package = None
if legacy:
package = qitoolchain.qipackage.QiPackage(args.name)
else:
package = qitoolchain.qipackage.from_archive(package_path)
# extract it to the default packages path of the toolchain
tc_name = toolchain.name
tc_packages_path = qitoolchain.toolchain.get_default_packages_path(tc_name)
dest = os.path.join(tc_packages_path, package.name)
qisys.sh.rm(dest)
qitoolchain.qipackage.extract(package_path, dest)
package.path = dest
# add the package to the toolchain
toolchain.add_package(package)
def _configure(self, docs, opts, **kwargs):
try:
templates = kwargs["templates"]
doxylink = kwargs["doxylink"].copy()
except KeyError as err:
raise ConfigureFailedError(self.name, "Keyword argument `{opt}` is missing.".format(opt=err))
rel_doxylink = dict()
for (name, (tag_file, prefix)) in doxylink.iteritems():
full_prefix = os.path.join(self.dest, prefix)
rel_prefix = os.path.relpath(full_prefix, self.dest)
rel_doxylink[name] = (tag_file, rel_prefix)
# Deal with conf.py
conf_py_tmpl = os.path.join(templates, "sphinx", "conf.in.py")
conf_py_in = os.path.join(self.src, "qidoc", "conf.in.py")
if not os.path.exists(conf_py_in):
mess = "Could not configure sphinx sources in: %s\n" % self.src
mess += "qidoc/conf.in.py does not exists"
ui.warning(mess)
return
opts["doxylink"] = str(rel_doxylink)
opts["intersphinx_mapping"] = str(self.get_mapping(docs))
opts["themes_path"] = os.path.join(templates, "sphinx", "_themes")
opts["themes_path"] = qisys.sh.to_posix_path(opts["themes_path"])
opts["ext_path"] = os.path.join(templates, "sphinx", "tools")
opts["ext_path"] = qisys.sh.to_posix_path(opts["ext_path"])
conf_py_out = os.path.join(self.src, "qidoc", "conf.py")
qidoc.templates.configure_file(conf_py_tmpl, conf_py_out, append_file=conf_py_in, opts=opts)
def print_not_on_a_branch(projects):
"""Print list of projects not on a branch."""
not_on_a_branchs = [x for x in projects if x.not_on_a_branch]
if not_on_a_branchs:
ui.info()
ui.warning("Some projects are not on any branch")
for project in not_on_a_branchs:
ui.info(ui.green, " *", ui.reset, ui.blue, project.project.src)
def check(self):
""" Perform a few sanity checks """
# Check that we are not in an other worktree:
parent_worktree = guess_worktree(os.path.join(self.root, ".."))
if parent_worktree and parent_worktree != self.root:
ui.warning("""Nested worktrees detected:
{0} is already in a worktree
(in {1})
""".format(self.root, parent_worktree))
def sigint_handler(signum, frame):
def double_sigint(signum, frame):
ui.warning('Exiting main program without caring (may leave ' + \
'zombies and the like).')
sys.exit(1)
ui.warning('Received keyboard interrupt. Killing all processes ' + \
'. This may take few seconds.')
qisys.command.SIGINT_EVENT.set()
signal.signal(signal.SIGINT, double_sigint)
def do(args):
""" Main entry point. """
if args.list:
ui.warning("`qibuild test --list` is deprecated, use `qitest list` instead")
qisys.script.run_action("qitest.actions.list", forward_args=args)
else:
projects = args.projects
ui.warning("`qibuild test` is deprecated, use `qitest run` instead")
qisys.script.run_action("qitest.actions.run", args=projects,
forward_args=args)
def strip_binary(binary, strip_executable=None, strip_args=None):
if not strip_executable:
strip_executable = qisys.command.find_program("strip", raises=True)
cmd = [strip_executable]
if strip_args:
cmd.extend(strip_args)
cmd.append(binary)
rc = qisys.command.call(cmd, ignore_ret_code=True)
if rc != 0:
ui.warning("Failed to strip symbols for", binary)
def warn_gerrit():
"""Emit a warning telling the user that:
* connection to gerrit has failed
* qisrc push won't work
"""
ui.warning("""Failed to configure gerrit connection
`qisrc push` won't work
When you have resolved this problem, just re-run ``qisrc sync -a``""")
def do(args):
""" Main Entry Point """
try:
test_runners = qitest.parsers.get_test_runners(args)
except qitest.parsers.EmptyTestListException:
if not args.allow_no_test:
raise
test_runners = []
# rule to check for tests which doesn't follow naming convention
expr = re.compile("^test_.*")
warn_name_count = 0
warn_type_count = 0
for test_runner in test_runners:
ui.info("Tests in ", test_runner.project.sdk_directory)
for i, test in enumerate(test_runner.tests):
n = len(test_runner.tests)
name = test["name"]
name_ok = re.match(expr, name)
type_ok = (test.get("pytest") or test.get("gtest"))
if name_ok and type_ok:
ui.info_count(i, n, test["name"])
else:
message = ""
if not name_ok:
warn_name_count += 1
message += "(invalid name) "
if not type_ok:
warn_type_count += 1
message += "(no type)"
ui.info_count(i, n, name, ui.brown, message)
if warn_name_count:
msg = "%i on %i tests do not respect naming convention" % (
warn_name_count, len(test_runner.tests))
ui.warning(msg)
if warn_type_count:
msg = "%i on %i tests do not have any type" % (
warn_type_count, len(test_runner.tests))
ui.warning(msg)
def do(args):
""" Add a package to a toolchain
- Check that there is a current toolchain
- Add the package to the cache
- Add the package from cache to toolchain
"""
toolchain = qitoolchain.parsers.get_toolchain(args)
name = args.name
package_path = args.package_path
legacy = False
try:
archive = zipfile.ZipFile(package_path)
archive.read("package.xml")
except KeyError:
legacy = True
if legacy and not args.name:
raise Exception("Must specify --name when using legacy format")
if args.name and not legacy:
ui.warning("--name ignored when using modern format")
package = None
if legacy:
package = qitoolchain.qipackage.QiPackage(args.name)
else:
package = qitoolchain.qipackage.from_archive(package_path)
# extract it to the default packages path of the toolchain
tc_name = toolchain.name
tc_packages_path = qitoolchain.toolchain.get_default_packages_path(tc_name)
dest = os.path.join(tc_packages_path, package.name)
qisys.sh.rm(dest)
qitoolchain.qipackage.extract(package_path, dest)
package.path = dest
# add the package to the toolchain
toolchain.add_package(package)
def do(args):
"""Main entry point
"""
if "--name" in sys.argv:
ui.warning("--name is deprecated, use --feed-name instead")
feed = args.feed
# Normalize feed path:
if feed and os.path.exists(feed):
feed = qisys.sh.to_native_path(feed)
tc_name = args.name
build_worktree = None
if tc_name in qitoolchain.get_tc_names():
toolchain = qitoolchain.Toolchain(tc_name)
ui.info(tc_name, "already exists,", "updating without removing")
toolchain = qitoolchain.Toolchain(tc_name)
if feed:
toolchain.update(feed, branch=args.branch, name=args.feed_name)
return toolchain
def read_remote_manifest(self,
manifest_xml=None,
warn_if_missing_group=True):
"""
Read the manifest file in .qi/manifests/<name>/manifest.xml using the
settings in .qi/manifest.xml (to know the name and the groups to use).
"""
if not manifest_xml:
manifest_xml = os.path.join(self.manifest_repo, "manifest.xml")
remote_manifest = qisrc.manifest.Manifest(manifest_xml,
review=self.manifest.review)
# parse imported manifest recursively if any
self._read_import_manifest(remote_manifest)
groups = self.manifest.groups
# if self.manifest.groups is empty but there is a default
# group in the manifest, we need to set self.manifest.groups
# so that subsequent calls to qisrc add-group, remove-group
# work
if self.manifest.groups is None:
default_group = remote_manifest.groups.default_group
if default_group:
self.manifest.groups = [default_group.name]
# Maybe some groups were removed from the manifest.
# Only consider those which exist
if groups is None:
groups_to_use = None
else:
groups_to_use = list()
for group in groups:
if group in remote_manifest.groups.group_names:
groups_to_use.append(group)
else:
if warn_if_missing_group:
ui.warning("Group %s not found in the manifest" %
group)
repos = remote_manifest.get_repos(groups=groups_to_use)
return repos
def do(args):
"""Main entry points."""
git_worktree = qisrc.parsers.get_git_worktree(args)
snapshot = None
if args.snapshot:
snapshot = qisrc.snapshot.Snapshot()
snapshot.load(args.snapshot)
if snapshot and snapshot.format_version and snapshot.format_version >= 1:
reset_manifest(git_worktree,
snapshot,
ignore_groups=args.ignore_groups)
git_projects = qisrc.parsers.get_git_projects(git_worktree,
args,
default_all=True,
use_build_deps=True)
errors = list()
for i, git_project in enumerate(git_projects):
ui.info_count(i, len(git_projects), "Reset", git_project.src)
src = git_project.src
git = qisrc.git.Git(git_project.path)
ok, message = git.require_clean_worktree()
if not ok and not args.force:
ui.warning(message)
errors.append(src)
continue
if not git_project.default_branch:
ui.warning(git_project.src, "not in any manifest, skipping")
continue
branch = git_project.default_branch.name
remote = git_project.default_remote.name
git.safe_checkout(branch, remote, force=True)
to_reset = None
if args.snapshot:
to_reset = snapshot.refs.get(src)
if not to_reset:
ui.warning(src, "not found in the snapshot")
continue
elif args.tag:
to_reset = args.tag
else:
to_reset = "%s/%s" % (remote, branch)
try:
qisrc.reset.clever_reset_ref(git_project, to_reset)
except:
errors.append(src)
if not errors:
return
ui.error("Failed to reset some projects")
for error in errors:
ui.info(ui.red, " * ", error)
sys.exit(1)
def split_debug(self, destdir, file_list):
""" Split debug symbols after install. """
if self.using_visual_studio:
raise Exception("split debug not supported on Visual Studio")
ui.info(ui.green, "Splitting debug symbols from binaries ...")
tool_paths = dict()
for name in ["objcopy", "objdump"]:
tool_path = qibuild.cmake.get_binutil(name,
build_dir=self.build_directory,
env=self.build_env)
tool_paths[name] = tool_path
missing = [x for x in tool_paths if not tool_paths[x]]
if missing:
mess = """\
Could not split debug symbols from binaries for project {name}.
The following tools were not found: {missing}\
"""
mess = mess.format(name=self.name, missing=", ".join(missing))
ui.warning(mess)
return
for filename in file_list:
full_path = os.path.join(destdir, filename)
if qibuild.breakpad.is_elf(full_path):
qibuild.gdb.split_debug(full_path, **tool_paths)
def get_known_profiles(self, warns=True):
"""
Parse the remote profiles (coming from qisrc sync),
and the local profiles (written in .qi/qibuild.xml).
Return a dict name -> list of tuple (key, value)
"""
res = dict()
remote_xml = os.path.join(self.root, ".qi", "manifests", "default",
"manifest.xml")
if os.path.exists(remote_xml):
res = qibuild.profile.parse_profiles(remote_xml)
local_xml = self.qibuild_xml
local_profiles = qibuild.profile.parse_profiles(local_xml)
for name in local_profiles:
if name in res:
remote_profile = res[name]
local_profile = local_profiles[name]
if remote_profile != local_profile:
if warns:
ui.warning("Overriding remote profile", name, "\n",
"local: ", local_profile.cmake_flags, "\n",
"remote:", remote_profile.cmake_flags)
res.update(local_profiles)
return res
def get_git_projects(self, groups=None):
""" Get the git projects matching a given group """
if not groups:
return self.git_projects
res = set()
git_project_names = dict()
group_names = groups
for git_project in self.git_projects:
git_project_names[git_project.name] = git_project
groups = qisrc.groups.get_groups(self.worktree)
for group_name in group_names:
warn_for_group = True
project_names = groups.projects(group_name)
for project_name in project_names:
git_project = git_project_names.get(project_name)
if git_project:
res.add(git_project)
else:
if warn_for_group:
ui.warning("Group", group_name, "is not currently in use\n",
"Please use `qisrc add-group %s`" % group_name)
warn_for_group = False
res = list(res)
return sorted(res, key=operator.attrgetter("src"))
def read_remote_config(self, repo, quiet=False):
""" Apply the configuration read from the "repo" setting
of a remote manifest.
Called by WorkTreeSyncer
"""
previous_default = None
if self.default_remote:
previous_default = self.default_remote.name
self.name = repo.project
self.remotes = list()
for remote in repo.remotes:
self.configure_remote(remote)
if repo.default_branch and repo.default_remote:
self.configure_branch(repo.default_branch,
tracks=repo.default_remote.name,
remote_branch=repo.default_branch,
default=True,
quiet=quiet)
new_default = self.default_remote.name
if previous_default is not None and previous_default != new_default:
if not quiet:
ui.warning("Default remote changed", previous_default,
"->", new_default)
if repo.review and not self.review:
# Project is now under code review, try to setup
# gerrit and save success in self.review
# (so that we can retry if gerrit setup did not work)
self.review = bool(qisrc.review.setup_project(self))
if not repo.review and self.review:
# Project was under code review, but no longer is,
# simply set self.review to False so that `qisrc push`
# does not try to push to gerrit
self.review = False
if repo.fixed_ref:
ui.warning("Now using fixed ref:", repo.fixed_ref)
self.fixed_ref = repo.fixed_ref
else:
if self.fixed_ref:
ui.warning("Now instead of fixed ref using branch:",
repo.default_branch)
self.switch_ref_to_branch = True
def do(args):
"""Main entry point"""
standalone = args.standalone
breakpad = args.breakpad
cmake_builder = qibuild.parsers.get_cmake_builder(args)
if breakpad:
cmake_builder.build_config.build_type = "RelWithDebInfo"
projects = cmake_builder.projects
if len(projects) != 1:
ui.fatal("This action can only work on one project")
project = projects[0]
archive_name = project.name
version = args.version
if version:
project.version = version
else:
version = project.version
if not version:
version = "0.1"
if not version:
ui.warning("Could not find project version!",
"Either use --version or fix qiproject.xml",
sep="\n")
build_dir_name = os.path.basename(project.build_directory)
archive_suffix = build_dir_name.replace("build-", "")
if version:
archive_name += "-" + version
archive_name += "-" + archive_suffix
package_dir = os.path.join(cmake_builder.build_worktree.root, "package")
destdir = os.path.join(package_dir, archive_name)
# Clean the destdir just in case the package was already generated
qisys.sh.rm(destdir)
build_type = cmake_builder.build_config.build_type
# Also build in debug on windows when building a package for a toolchain
if sys.platform.startswith("win") and not standalone:
_do_package(cmake_builder,
destdir,
build_type="Debug",
standalone=False)
_do_package(cmake_builder,
destdir,
build_type=build_type,
standalone=standalone)
package_xml_path = os.path.join(destdir, "package.xml")
project.gen_package_xml(package_xml_path)
if breakpad:
symbols_archive_name = archive_name + "-symbols.zip"
symbols_archive = os.path.join(package_dir, symbols_archive_name)
with qisys.sh.TempDir() as tmp:
pool_dir = os.path.join(tmp, "symbols")
qibuild.breakpad.dump_symbols_from_directory(destdir, pool_dir)
qisys.archive.compress(pool_dir, flat=True, output=symbols_archive)
ui.info(ui.blue, "::", ui.reset, ui.bold, "Compressing package ...")
flat = not standalone
archive = qisys.archive.compress(destdir,
algo="zip",
quiet=True,
flat=flat,
output=destdir + ".zip")
# Clean up after ourselves
qisys.sh.rm(destdir)
ui.info(ui.green, "Package generated in", ui.reset, ui.bold, archive)
if breakpad:
ui.info(ui.green, "Symbols package generated in", ui.reset, ui.bold,
symbols_archive)
return archive, symbols_archive
else:
return archive
def _is_runnable(full_path, build_config=None):
"""
Return True if executable:
- has the same architecture (32/64 bits) than current python executable
- on linux, each dynamically linked libraries (found with 'ldd') have the minimum required version
"""
if not full_path:
return False
try:
if platform.architecture(full_path)[0] != platform.architecture(
sys.executable)[0]:
return False
except Exception:
pass
# if a build config is set then we will check for file format
if build_config:
try:
process = subprocess.Popen(['file', '-L', full_path],
stdout=subprocess.PIPE,
env={str("LANG"): str("C")})
output = process.communicate()[0]
if six.PY3:
output = str(output)
ui.debug("Testing %s in %s" %
(platform.processor(), output.split(',')))
if "ASCII text executable" not in output:
try:
bin_proc = output.split(',')[1]
if platform.processor(
) not in bin_proc and platform.processor().replace(
"_", "-") not in bin_proc:
ui.debug("%s not compatible" % (full_path))
return False
except IndexError:
return True
return True
except OSError:
# TODO: Run an equivalent test on mac and on windows
if sys.platform.startswith("linux"):
ui.warning(
"file not available => assuming {} is compatible".format(
full_path))
return True
else:
try:
process = subprocess.Popen(['ldd', full_path],
stdout=subprocess.PIPE,
env={str("LANG"): str("C")})
output = process.communicate()[0]
if six.PY3:
output = str(output)
if process.returncode == 0 and ' not found ' not in output:
pass
elif process.returncode == 1 and 'not a dynamic executable' in output:
pass
else:
return False
except OSError:
# TODO: Run an equivalent test on mac and on windows
if sys.platform.startswith("linux"):
ui.warning(
"ldd not available => assuming {} is runnable".format(
full_path))
return True
