def do(args):
""" Main entry point """
git_worktree = qisrc.parsers.get_git_worktree(args)
git_project = qisrc.parsers.get_one_git_project(git_worktree, args)
git = qisrc.git.Git(git_project.path)
current_branch = git.get_current_branch()
if not current_branch:
ui.error("Not currently on any branch")
sys.exit(2)
if git_project.review:
maintainers = qisrc.maintainers.get(git_project, warn_if_none=True)
reviewers = [x['email'] for x in maintainers]
reviewers.extend(args.reviewers or list())
# Prefer gerrit logins or groups instead of e-mails
reviewers = [x.split("@")[0] for x in reviewers]
qisrc.review.push(git_project, current_branch,
bypass_review=(not args.review),
dry_run=args.dry_run, reviewers=reviewers,
topic=args.topic)
else:
if args.dry_run:
git.push("-n")
else:
if args.review and not args.yes:
mess = """\
The project is not under code review.
Are you sure you want to run `git push` ?
This action cannot be undone\
"""
answer = qisys.interact.ask_yes_no(mess, default=False)
if answer:
git.push()
else:
git.push()
def jar(jar_path, files, paths):
""" Search each files using qibuild find and
add them into a jar using qisys
"""
# Create command line
jar_path = qisys.sh.to_native_path(jar_path)
args = ["cvfM"]
args += [jar_path]
if not files:
raise Exception("Missing arguments : Files to package")
for wanted_file in files:
ui.info("Searching for " + wanted_file + "...")
path = qibuild.find.find(paths, wanted_file, expect_one=False)[0]
if not path:
ui.error("Cannot find " + wanted_file + " in worktree")
return None
ui.debug("Found : " + path)
dirname = os.path.dirname(path)
basename = os.path.basename(path)
args += ["-C", dirname, basename]
ui.debug("Added -C " + dirname + " " + wanted_file +
" to command line")
qisys.command.call(["jar"] + args, ignore_ret_code=False)
return jar_path
def summary(self):
""" Display the tests results.
Called at the end of self.run()
Sets ``self.ok``
"""
if not self.tests:
self.ok = False
return
num_tests = len(self.results)
failures = [x for x in self.results.values() if x.ok is False]
num_failed = len(failures)
message = "Ran %i tests in %is" % (num_tests, self.elapsed_time)
ui.info(message)
self.ok = (not failures) and not self._interrupted
if self.ok:
ui.info(ui.green, "All pass. Congrats!")
return
if num_failed != 0:
ui.error(num_failed, "failures")
if failures:
max_len = max(len(x.test["name"]) for x in failures)
for i, failure in enumerate(failures):
ui.info_count(i, num_failed,
ui.blue, failure.test["name"].ljust(max_len + 2),
ui.reset, *failure.message)
def configure_virtualenv(config, python_worktree, build_worktree=None,
remote_packages=None, site_packages=True):
if not remote_packages:
remote_packages = list()
# create a new virtualenv
python_worktree.config = config
venv_path = python_worktree.venv_path
pip = python_worktree.pip
try:
virtualenv.create_environment(python_worktree.venv_path,
site_packages=site_packages)
except:
ui.error("Failed to create virtualenv")
return
# Install all Python projects using pip install -e .
python_projects = python_worktree.python_projects
for i, project in enumerate(python_projects):
ui.info_count(i, len(python_projects),
ui.green, "Configuring", ui.reset, ui.blue, project.src)
cmd = [pip, "install", "--editable", "."]
qisys.command.call(cmd, cwd=project.path)
# Write a qi.pth file containing path to C/C++ extensions
if build_worktree:
handle_extensions(venv_path, python_worktree, build_worktree)
# Install the extension in the virtualenv
binaries_path = virtualenv.path_locations(venv_path)[-1]
pip_binary = os.path.join(binaries_path, "pip")
if remote_packages:
cmd = [pip_binary, "install"] + remote_packages
subprocess.check_call(cmd)
def main_wrapper(module, args):
"""This wraps the main method of an action so that:
- backtrace is not printed by default
- backtrace is printed is --backtrace was given
- a pdb session is run if --pdb was given
"""
try:
module.do(args)
except Exception as e:
if args.pdb:
traceback = sys.exc_info()[2]
print ""
print "### Exception:", e
print "### Starting a debugger"
try:
import ipdb
ipdb.post_mortem(traceback)
sys.exit(0)
except ImportError:
import pdb
pdb.post_mortem(traceback)
sys.exit(0)
if args.backtrace:
raise
ui.error(e)
sys.exit(2)
def generate_mo_file(self, locale):
""" Generate .mo file for the given locale
"""
ui.info(ui.green, "Generating translation for", ui.reset,
ui.bold, locale)
input_file = self.get_po_file(locale)
if not os.path.exists(input_file):
ui.error("No .po found for locale: ", locale, "\n",
"(looked in %s)" % input_file, "\n",
"Did you run qilinguist update?")
return
output_file = os.path.join(self.mo_path, locale, "LC_MESSAGES",
self.domain + ".mo")
to_make = os.path.dirname(output_file)
qisys.sh.mkdir(to_make, recursive=True)
cmd = ["msgfmt", "--check", "--statistics"]
# required by libqi:
conf_file = os.path.join(self.mo_path, ".confintl")
with open(conf_file, "w") as fp:
fp.write("# THIS FILE IS AUTOGENERATED\n"
"#Do not delete or modify it\n"
"# This file is used to find translation dictionaries\n")
cmd.extend(["--output-file", output_file])
cmd.extend(["--directory", self.po_path])
cmd.append(input_file)
qisys.command.call(cmd)
def do(args):
""" Main entry point """
git_worktree = qisrc.parsers.get_git_worktree(args)
git_project = qisrc.parsers.get_one_git_project(git_worktree, args)
git = qisrc.git.Git(git_project.path)
current_branch = git.get_current_branch()
if not current_branch:
ui.error("Not currently on any branch")
sys.exit(2)
if git_project.review:
maintainers = qisrc.maintainers.get(git_project, warn_if_none=True)
reviewers = [x['email'] for x in maintainers]
reviewers.extend(args.reviewers or list())
# Prefer gerrit logins or groups instead of e-mails
reviewers = [x.split("@")[0] for x in reviewers]
qisrc.review.push(git_project, current_branch,
bypass_review=(not args.review),
dry_run=args.dry_run, reviewers=reviewers,
topic=args.topic)
else:
if args.dry_run:
git.push("-n")
else:
if args.review and not args.yes:
mess = """\
The project is not under code review.
Are you sure you want to run `git push` ?
This action cannot be undone\
"""
answer = qisys.interact.ask_yes_no(mess, default=False)
if answer:
git.push()
else:
git.push()
def checkout(self, branch, force=False):
""" Called by ``qisrc checkout``
For each project, checkout the branch if it is different than
the default branch of the manifest.
"""
ui.info(ui.green, ":: Checkout projects ...")
errors = list()
manifest_xml = os.path.join(self._syncer.manifest_repo, "manifest.xml")
manifest = qisrc.manifest.Manifest(manifest_xml)
max_src = max([len(x.src) for x in self.git_projects])
n = len(self.git_projects)
for i, project in enumerate(self.git_projects):
ui.info_count(i, n, ui.bold, "Checkout",
ui.reset, ui.blue, project.src.ljust(max_src), end="\r")
if project.default_branch is None:
continue
branch_name = project.default_branch.name
remote_name = project.default_remote.name
git = qisrc.git.Git(project.path)
ok, err = git.safe_checkout(branch_name, remote_name, force=force)
if not ok:
errors.append((project.src, err))
if not errors:
return
ui.error("Failed to checkout some projects")
for (project, error) in errors:
ui.info(project, ":", error)
def push_projects(git_projects, dry_run=False):
""" Push Projects """
if not git_projects:
return
ui.info(ui.green, "Pushing ", len(git_projects), "projects")
for i, git_project in enumerate(git_projects):
default_branch = git_project.default_branch.name
remote_branch = git_project.default_branch.remote_branch
ui.info_count(i, len(git_projects), git_project.src)
git = qisrc.git.Git(git_project.path)
if git_project.review:
push_remote = git_project.review_remote
else:
push_remote = git_project.default_remote
remote_ref = "%s/%s" % (push_remote.name, remote_branch)
display_changes(git, default_branch, remote_ref)
answer = qisys.interact.ask_yes_no("OK to push?", default=False)
if not answer:
return
to_push = "%s:%s" % (default_branch, remote_branch)
push_args = [push_remote.name, to_push]
push_args.append("--force")
if dry_run:
push_args.append("--dry-run")
rc, out = git.push(*push_args, raises=False)
if rc == 0:
ui.info(out)
else:
ui.error(out)
def tree_from_feed(feed_location):
""" Returns an ElementTree object from an
feed location
"""
fp = None
tree = None
try:
if os.path.exists(feed_location):
fp = open(feed_location, "r")
else:
if is_url(feed_location):
fp = qisys.remote.open_remote_location(feed_location)
else:
raise Exception(
"Feed location is not an existing path nor an url")
tree = ElementTree.ElementTree()
tree.parse(fp)
except Exception:
ui.error("Could not parse", feed_location)
raise
finally:
if fp:
fp.close()
return tree
def fix_solibs(sdk_dir, paths=None):
"""
Add rpath to relocate the toolchain libraries
"""
# This directory may not exist, so create it:
library_path = "./"
libraries_list = []
if not os.path.exists(sdk_dir):
return
for path in os.listdir(sdk_dir):
if not os.path.exists(path):
continue
lib_dir = os.path.join(sdk_dir, path, "lib")
if os.path.exists(lib_dir):
library_path = library_path + ':' + lib_dir
libs = os.listdir(lib_dir)
for lib in libs:
if ".so" in lib and lib not in libraries_list:
libraries_list.append(os.path.join(lib_dir, lib))
for lib in libraries_list:
patchelf = qisys.command.find_program("patchelf")
if not patchelf:
ui.error("Could not find patchelf for fixing so libraries")
else:
cmd = [patchelf, "--set-rpath", library_path, lib]
qisys.command.call(cmd)
def do(args):
""" Main entry point """
git_worktree = qisrc.parsers.get_git_worktree(args)
git_projects = qisrc.parsers.get_git_projects(git_worktree, args)
for git_project in git_projects:
maintainers = qisrc.maintainers.get(git_project)
if not maintainers:
mess = """\
The project in {src} has no maintainer.
Please edit {qiproject_xml} to silence this warning
"""
ui.warning(mess.format(src=git_project.src,
qiproject_xml=git_project.qiproject_xml),
end="")
reviewers = [x['email'] for x in maintainers]
reviewers.extend(args.reviewers or list())
# Prefer gerrit logins or groups instead of e-mails
reviewers = [x.split("@")[0] for x in reviewers]
git = qisrc.git.Git(git_project.path)
current_branch = git.get_current_branch()
if not current_branch:
ui.error("Not currently on any branch")
sys.exit(2)
if git_project.review:
qisrc.review.push(git_project, current_branch,
bypass_review=(not args.review),
dry_run=args.dry_run, reviewers=reviewers,
topic=args.topic)
else:
if args.dry_run:
git.push("-n")
else:
git.push()
def _run(self, num_jobs=1):
""" Helper function for ._run """
if not self.launcher:
ui.error("test launcher not set, cannot run tests")
return
for i, test in enumerate(self.tests):
self.task_queue.put((test, i))
if num_jobs == 1:
self.test_logger.single_job = True
threads = list()
for i in range(0, num_jobs):
worker = TestWorker(self.task_queue, i)
worker.launcher = self.launcher
worker.launcher.worker_index = i
worker.test_logger = self.test_logger
worker.results = self.results
threads.append(worker)
worker.start()
# Do not use .join() so that this can be interrupted:
while self.task_queue.unfinished_tasks and \
not self._interrupted:
time.sleep(0.1)
for worker_thread in threads:
worker_thread.stop()
worker_thread.join()
def tree_from_feed(feed_location, branch=None, name=None): # pylint: disable=unused-argument
""" Returns an ElementTree object from an
feed location
"""
fp = None
tree = None
try:
if os.path.exists(feed_location):
fp = open(feed_location, "r")
else:
if is_url(feed_location):
fp = qisys.remote.open_remote_location(feed_location)
else:
raise Exception(
"Could not parse %s: Feed location is not an existing path nor an url"
% feed_location)
tree = ElementTree.ElementTree()
tree.parse(fp)
except Exception as e:
ui.error(e.message)
raise
finally:
if fp:
fp.close()
return tree
def jar(jar_path, files, paths):
""" Search each files using qibuild find and
add them into a jar using qisys
"""
# Create command line
jar_path = qisys.sh.to_native_path(jar_path)
args = ["cvfM"]
args += [jar_path]
if not files:
raise Exception("Missing arguments : Files to package")
for wanted_file in files:
ui.info("Searching for " + wanted_file + "...")
path = qibuild.find.find(paths, wanted_file, expect_one=False)[0]
if not path:
ui.error("Cannot find " + wanted_file + " in worktree")
return None
ui.debug("Found : " + path)
dirname = os.path.dirname(path)
basename = os.path.basename(path)
args += ["-C", dirname, basename]
ui.debug("Added -C " + dirname + " " + wanted_file + " to command line")
qisys.command.call(["jar"] + args, ignore_ret_code=False)
return jar_path
def _run(self, num_jobs=1):
""" Helper function for ._run """
if not self.launcher:
ui.error("test launcher not set, cannot run tests")
return
for i, test in enumerate(self.tests):
self.task_queue.put((test, i))
if num_jobs == 1:
self.test_logger.single_job = True
threads = list()
for i in range(0, num_jobs):
worker = TestWorker(self.task_queue, i)
worker.launcher = self.launcher
worker.launcher.worker_index = i
worker.test_logger = self.test_logger
worker.results = self.results
threads.append(worker)
worker.start()
# Do not use .join() so that this can be interrupted:
while self.task_queue.unfinished_tasks and \
not self._interrupted:
time.sleep(0.1)
for worker_thread in threads:
worker_thread.stop()
worker_thread.join()
def main_wrapper(module, args):
"""This wraps the main method of an action so that:
- backtrace is not printed by default
- backtrace is printed is --backtrace was given
- a pdb session is run if --pdb was given
"""
try:
module.do(args)
except Exception as e:
if args.pdb:
traceback = sys.exc_info()[2]
print ""
print "### Exception:", e
print "### Starting a debugger"
try:
#pylint: disable-msg=F0401
import ipdb
ipdb.post_mortem(traceback)
sys.exit(0)
except ImportError:
import pdb
pdb.post_mortem(traceback)
sys.exit(0)
if args.backtrace:
raise
message = str(e)
if message.endswith("\n"):
message = message[:-1]
ui.error(e.__class__.__name__, message)
sys.exit(2)
def clever_reset_ref(git_project, ref, raises=True):
""" Resets only if needed, fetches only if needed """
try:
remote_name = git_project.default_remote.name
except AttributeError:
error_msg = "Project {} has no default remote, defaulting to origin"
ui.error(error_msg.format(git_project.name))
remote_name = "origin"
git = qisrc.git.Git(git_project.path)
# Deals with "refs/" prefixed ref first
if ref.startswith("refs/"):
return _reset_hard_to_refs_prefixed_ref(git, remote_name, ref, raises=raises)
# Else, ref format is a local name (branch or tag)
# Check if this ref exists and if we are already in the expected state
rc, ref_sha1 = git.call("rev-parse", ref, "--", raises=False)
if rc != 0:
# Maybe this is a newly pushed tag, try to fetch:
git.fetch(remote_name)
rc, ref_sha1 = git.call("rev-parse", ref, "--", raises=False)
if rc != 0:
return False, "Could not parse %s as a valid ref" % ref
_, actual_sha1 = git.call("rev-parse", "HEAD", raises=False)
if actual_sha1 == ref_sha1: # Nothing to do
return None if raises else True, ""
# Reset to the ref local name
return _reset_hard_to_local_refs_name(git, remote_name, ref, raises=raises)
def commit_all(self, message):
__rc, out = self.call("status", raises=False) # pylint: disable=unused-variable
for line in out.splitlines():
line = line.strip()
filename = line[8:]
if line.startswith("!"):
self.call("remove", filename)
if line.startswith("?"):
self.call("add", filename)
# Prevent 'Node has unexpectedly changed kind' error
# when a file is replaced by a symlink.
# see http://antoniolorusso.com/blog/2008/09/29/svn-entry-has-unexpectedly-changed-special-status/
if line.startswith("~"):
full_path = os.path.join(self.path, filename)
if os.path.islink(full_path):
target = os.readlink(full_path)
os.remove(full_path)
self.call("remove", filename)
os.symlink(target, full_path)
self.call("add", filename)
else:
ui.error("Could not deal with", filename, "\n",
"Please open a bug report")
self.call("commit", "--message", message)
def clever_reset_ref(git_project, ref, raises=True):
""" Resets only if needed, fetches only if needed """
try:
remote_name = git_project.default_remote.name
except AttributeError:
error_msg = "Project {} has no default remote, defaulting to origin"
ui.error(error_msg.format(git_project.name))
remote_name = "origin"
git = qisrc.git.Git(git_project.path)
# Deals with "refs/" prefixed ref first
if ref.startswith("refs/"):
return _reset_hard_to_refs_prefixed_ref(git,
remote_name,
ref,
raises=raises)
# Else, ref format is a local name (branch or tag)
# Check if this ref exists and if we are already in the expected state
rc, ref_sha1 = git.call("rev-parse", ref, "--", raises=False)
if rc != 0:
# Maybe this is a newly pushed tag, try to fetch:
git.fetch(remote_name, "--prune")
rc, ref_sha1 = git.call("rev-parse", ref, "--", raises=False)
if rc != 0:
return False, "Could not parse %s as a valid ref" % ref
_, actual_sha1 = git.call("rev-parse", "HEAD", raises=False)
if actual_sha1 == ref_sha1: # Nothing to do
return None if raises else True, ""
# Reset to the ref local name
return _reset_hard_to_local_refs_name(git, remote_name, ref, raises=raises)
def copy_jni_resources(build_worktree):
""" Create symlinks (or copy), the jni library and its
dependencies from the build dirs to the resource folder
"""
android = "android" in (build_worktree.build_config.active_config or "")
# FIXME: boost when using a toolchain on linux ?
libs = list()
proj_to_lib = [
("libqi", "qi"),
("qimessaging-jni", "qimessagingjni")]
if android:
proj_to_lib.append(
("qimessaging-jni", "gnustl_shared"))
for project_name, library_name in proj_to_lib:
try:
project = build_worktree.get_build_project(project_name, raises=True)
library_path = qibuild.find.find_lib([project.sdk_directory], library_name)
libs.append(library_path)
except qibuild.find.NotFound as e:
ui.error(e)
ui.error("Make sure qimessaging-jni has been built")
sys.exit(1)
jni_proj = build_worktree.get_build_project("qimessaging-jni")
java_proj = build_worktree.worktree.get_project("sdk/libqi-java/qimessaging")
if android:
dest = os.path.join(java_proj.path, "native-android")
else:
dest = os.path.join(java_proj.path, "native")
qisys.sh.mkdir(dest)
for lib_src in libs:
lib_dest = os.path.join(dest, os.path.basename(lib_src))
copy_or_link(lib_src, lib_dest)
def do(args):
""" Main entry point. """
cmake_builder = qibuild.parsers.get_cmake_builder(args)
if len(cmake_builder.projects) != 1:
raise Exception("This action can only work on one project")
project = cmake_builder.projects[0]
if not os.path.exists(project.build_directory):
ui.error("""It looks like your project has not been configured yet
(The build directory: '%s' does not exist)""" %
project.build_directory)
answer = qisys.interact.ask_yes_no(
"Do you want me to run qibuild configure for you?",
default=True)
if not answer:
sys.exit(2)
else:
cmake_builder.configure()
ide = get_ide(cmake_builder.build_config.qibuild_cfg)
if not ide:
return
if ide.name == "Visual Studio":
open_visual(project)
elif ide.name == "Xcode":
open_xcode(project)
elif ide.name == "QtCreator":
open_qtcreator(project, ide.path)
else:
# Not supported (yet) IDE:
mess = "Invalid ide: %s\n" % ide.name
mess += "Supported IDES are: %s" % ", ".join(SUPPORTED_IDES)
raise Exception(mess)
def do(args):
""" Main entry point """
git_worktree = qisrc.parsers.get_git_worktree(args)
git_projects = qisrc.parsers.get_git_projects(git_worktree, args)
for git_project in git_projects:
maintainers = qisrc.maintainers.get(git_project)
if not maintainers:
mess = """\
The project in {src} has no maintainer.
Please edit {qiproject_xml} to silence this warning
"""
ui.warning(mess.format(src=git_project.src, qiproject_xml=git_project.qiproject_xml), end="")
reviewers = [x["email"] for x in maintainers]
reviewers.extend(args.reviewers or list())
git = qisrc.git.Git(git_project.path)
current_branch = git.get_current_branch()
if not current_branch:
ui.error("Not currently on any branch")
sys.exit(2)
if git_project.review:
qisrc.review.push(
git_project,
current_branch,
bypass_review=(not args.review),
dry_run=args.dry_run,
reviewers=reviewers,
topic=args.topic,
)
else:
if args.dry_run:
git.push("-n")
else:
git.push()
def generate_mo_file(self, locale):
""" Generate .mo file for the given locale """
ui.info(ui.green, "Generating translation for", ui.reset, ui.bold,
locale)
input_file = self.get_po_file(locale)
if not os.path.exists(input_file):
ui.error("No .po found for locale: ", locale, "\n",
"(looked in %s)" % input_file, "\n",
"Did you run qilinguist update?")
return None
output_file = os.path.join(self.mo_path, locale, "LC_MESSAGES",
self.domain + ".mo")
to_make = os.path.dirname(output_file)
qisys.sh.mkdir(to_make, recursive=True)
cmd = ["msgfmt", "--check", "--statistics"]
# required by libqi:
conf_file = os.path.join(self.mo_path, ".confintl")
with open(conf_file, "w") as fp:
fp.write("# THIS FILE IS AUTOGENERATED\n"
"# Do not delete or modify it\n"
"# This file is used to find translation dictionaries\n")
cmd.extend(["--output-file", output_file])
cmd.extend(["--directory", self.po_path])
cmd.append(input_file)
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
__out, err = process.communicate()
ui.info(err.strip())
if "untranslated" in str(err):
return False, "Some untranslated messages were found"
if process.returncode != 0:
return False, "msgfmt failed"
return True, ""
def get_log(self, before_ref, after_ref):
""" Return a list of commits between two refspecs, in
natural order (most recent commits last)
Each commit is a dict containing, 'sha1' and 'message'
FIXME: parse author and date ?
"""
# old git does not support -z and inists on adding \n
# between log entries, so we have no choice but choose
# a custom separator. '\f' looks safe enough
res = list()
rc, out = self.call("log", "--reverse",
"--format=%H%n%B\f",
"%s..%s" % (before_ref, after_ref),
raises=False)
if rc != 0:
ui.error(out)
return res
for log_chunk in out.split('\f'):
if not log_chunk:
continue
log_chunk = log_chunk.strip()
sha1, message = log_chunk.split('\n', 1)
message = message.strip()
commit = {
"sha1" : sha1,
"message" : message,
}
res.append(commit)
return res
def _run(self, num_jobs=1):
""" Helper function for ._run """
if not self.launcher:
ui.error("test launcher not set, cannot run tests")
return
for i, test in enumerate(self.tests):
self.task_queue.put((test, i))
if num_jobs == 1:
self.test_logger.single_job = True
for i in range(0, num_jobs):
worker = TestWorker(self.task_queue, i)
worker.launcher = self.launcher
worker.launcher.worker_index = i
worker.test_logger = self.test_logger
worker.results = self.results
self._workers.append(worker)
worker.start()
while not self.task_queue.empty() and \
not self._interrupted:
time.sleep(0.1)
for worker_thread in self._workers:
worker_thread.join()
def _clone_missing(self, git_project, repo):
""" Clone Missing """
branch = repo.default_branch
fixed_ref = repo.fixed_ref
clone_url = repo.clone_url
qisys.sh.mkdir(git_project.path, recursive=True)
git = qisrc.git.Git(git_project.path)
remote_name = repo.default_remote.name
try:
git.init()
git.remote("add", remote_name, clone_url)
git.fetch(remote_name, "--quiet")
if branch:
git.checkout("-b", branch, "%s/%s" % (remote_name, branch))
if fixed_ref:
git.checkout("-q", fixed_ref)
except Exception:
ui.error("Cloning repo failed")
if git.is_empty():
qisys.sh.rm(git_project.path)
self.worktree.remove_project(repo.src)
return False
self.save_project_config(git_project)
self.load_git_projects()
return True
def main_wrapper(module, args):
"""
This wraps the main method of an action so that:
- backtrace is not printed by default
- backtrace is printed is --backtrace was given
- a pdb session is run if --pdb was given
"""
try:
module.do(args)
except Exception as e:
if args.pdb:
traceback = sys.exc_info()[2]
print("")
print("### Exception:", e)
print("### Starting a debugger")
try:
import ipdb
ipdb.post_mortem(traceback)
sys.exit(0)
except ImportError:
import pdb
pdb.post_mortem(traceback)
sys.exit(0)
if args.backtrace:
raise
if six.PY3:
message = str(e)
else:
message = str(e).decode("utf-8", errors="ignore")
if message.endswith("\n"):
message = message[:-1]
ui.error(e.__class__.__name__, message)
sys.exit(2)
def do(args):
"""Main entry point."""
cmake_builder = qibuild.parsers.get_cmake_builder(args)
if len(cmake_builder.projects) != 1:
raise Exception("This action can only work on one project")
project = cmake_builder.projects[0]
if not os.path.exists(project.build_directory):
ui.error("""It looks like your project has not been configured yet
(The build directory: '%s' does not exist)""" %
project.build_directory)
answer = qisys.interact.ask_yes_no(
"Do you want me to run qibuild configure for you?",
default=True)
if not answer:
sys.exit(2)
else:
cmake_builder.configure()
ide = get_ide(cmake_builder.build_config.qibuild_cfg)
if not ide:
return
if ide.name == "Visual Studio":
open_visual(project)
elif ide.name == "Xcode":
open_xcode(project)
elif ide.name == "QtCreator":
open_qtcreator(project, ide.path)
else:
# Not supported (yet) IDE:
mess = "Invalid ide: %s\n" % ide.name
mess += "Supported IDES are: %s" % ", ".join(SUPPORTED_IDES)
raise Exception(mess)
def generate_mo_file(self, locale):
""" Generate .mo file for the given locale """
ui.info(ui.green, "Generating translation for", ui.reset,
ui.bold, locale)
input_file = self.get_po_file(locale)
if not os.path.exists(input_file):
ui.error("No .po found for locale: ", locale, "\n",
"(looked in %s)" % input_file, "\n",
"Did you run qilinguist update?")
return None
output_file = os.path.join(self.mo_path, locale, "LC_MESSAGES",
self.domain + ".mo")
to_make = os.path.dirname(output_file)
qisys.sh.mkdir(to_make, recursive=True)
cmd = ["msgfmt", "--check", "--statistics"]
# required by libqi:
conf_file = os.path.join(self.mo_path, ".confintl")
with open(conf_file, "w") as fp:
fp.write("# THIS FILE IS AUTOGENERATED\n"
"# Do not delete or modify it\n"
"# This file is used to find translation dictionaries\n")
cmd.extend(["--output-file", output_file])
cmd.extend(["--directory", self.po_path])
cmd.append(input_file)
process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
__out, err = process.communicate()
ui.info(err.strip())
if "untranslated" in err:
return False, "Some untranslated messages were found"
if process.returncode != 0:
return False, "msgfmt failed"
return True, ""
def main_wrapper(module, args):
"""This wraps the main method of an action so that:
- backtrace is not printed by default
- backtrace is printed is --backtrace was given
- a pdb session is run if --pdb was given
"""
try:
module.do(args)
except Exception as e:
if args.pdb:
traceback = sys.exc_info()[2]
print ""
print "### Exception:", e
print "### Starting a debugger"
try:
#pylint: disable-msg=F0401
import ipdb
ipdb.post_mortem(traceback)
sys.exit(0)
except ImportError:
import pdb
pdb.post_mortem(traceback)
sys.exit(0)
if args.backtrace:
raise
ui.error(e.__class__.__name__, e)
sys.exit(2)
def summary(self):
""" Display the tests results.
Called at the end of self.run()
Sets ``self.ok``
"""
if not self.tests:
self.ok = False
return
num_tests = len(self.results)
failures = [x for x in self.results.values() if x.ok is False]
num_failed = len(failures)
message = "Ran %i tests in %is" % (num_tests, self.elapsed_time)
ui.info(message)
self.ok = (not failures) and not self._interrupted
if self.ok:
ui.info(ui.green, "All pass. Congrats!")
else:
if num_failed != 0:
ui.error(num_failed, "failures")
if failures:
max_len = max(len(x.test["name"]) for x in failures)
for i, failure in enumerate(failures):
ui.info_count(i, num_failed, ui.blue,
failure.test["name"].ljust(max_len + 2),
ui.reset, *failure.message)
self.write_failures(failures)
def get_log(self, before_ref, after_ref):
""" Return a list of commits between two refspecs, in
natural order (most recent commits last)
Each commit is a dict containing, 'sha1' and 'message'
FIXME: parse author and date ?
"""
# old git does not support -z and inists on adding \n
# between log entries, so we have no choice but choose
# a custom separator. '\f' looks safe enough
res = list()
rc, out = self.call("log", "--reverse",
"--format=%H%n%B\f",
"%s...%s" % (before_ref, after_ref),
raises=False)
if rc != 0:
ui.error(out)
return res
for log_chunk in out.split('\f'):
if not log_chunk:
continue
log_chunk = log_chunk.strip()
sha1, message = log_chunk.split('\n', 1)
message = message.strip()
commit = {
"sha1" : sha1,
"message" : message,
}
res.append(commit)
return res
def tree_from_feed(feed_location, branch=None, name=None):
""" Returns an ElementTree object from an feed location """
fp = None
tree = None
try:
if feed_location and os.path.exists(feed_location):
fp = open(feed_location, "r")
elif is_url(feed_location):
fp = qisys.remote.open_remote_location(feed_location)
else:
raise Exception(
"Could not parse %s: Feed location is not an existing path nor an url"
% feed_location)
tree = ElementTree.ElementTree()
if six.PY3:
tree.parse(fp, parser=ElementTree.XMLParser(encoding='utf-8'))
else:
tree.parse(fp)
except Exception as e:
if six.PY3:
ui.error(e)
else:
ui.error(e.message)
raise
finally:
if fp:
fp.close()
return tree
def _run(self, num_jobs=1):
""" Helper function for ._run_once """
if not self.launcher:
ui.error("test launcher not set, cannot run tests")
return
for i, test in enumerate(self.tests):
self.task_queue.put((test, i))
if num_jobs == 1:
self.test_logger.single_job = True
for i in range(0, num_jobs):
worker = TestWorker(self.task_queue, i)
worker.launcher = self.launcher
worker.launcher.worker_index = i
worker.test_logger = self.test_logger
worker.results = self.results
self._workers.append(worker)
worker.start()
while not self.task_queue.empty() and \
not self._interrupted:
time.sleep(0.1)
for worker_thread in self._workers:
worker_thread.join()
def build(self, **kwargs):
""" Run sphinx.main() with the correct arguments """
try:
import sphinx
except ImportError, e:
ui.error(e, "skipping build")
return
def _create_virtualenv(config,
python_worktree,
site_packages=True,
python_executable=None,
env=None):
python_worktree.config = config
venv_path = python_worktree.venv_path
# pip = python_worktree.pip
if not python_executable:
python_executable = sys.executable
virtualenv_py = virtualenv.__file__
if virtualenv_py.endswith(".pyc"):
virtualenv_py = virtualenv_py[:-1]
cmd = [python_executable, virtualenv_py, venv_path]
if site_packages:
cmd.append("--system-site-packages")
try:
qisys.command.call(cmd, env=env)
except qisys.command.CommandFailedException:
ui.error("Failed to create virtualenv")
return False
binaries_path = virtualenv.path_locations(venv_path)[-1]
pip_binary = os.path.join(binaries_path, "pip")
return pip_binary, venv_path
def generate_mo_file(self, locale):
""" Generate .mo file for the given locale
"""
ui.info(ui.green, "Generating translation for", ui.reset, ui.bold,
locale)
input_file = self.get_po_file(locale)
if not os.path.exists(input_file):
ui.error("No .po found for locale: ", locale, "\n",
"(looked in %s)" % input_file, "\n",
"Did you run qilinguist update?")
return
output_file = os.path.join(self.mo_path, locale, "LC_MESSAGES",
self.domain + ".mo")
to_make = os.path.dirname(output_file)
qisys.sh.mkdir(to_make, recursive=True)
cmd = ["msgfmt", "--check", "--statistics"]
# required by libqi:
conf_file = os.path.join(self.mo_path, ".confintl")
with open(conf_file, "w") as fp:
fp.write("# THIS FILE IS AUTOGENERATED\n"
"#Do not delete or modify it\n"
"# This file is used to find translation dictionaries\n")
cmd.extend(["--output-file", output_file])
cmd.extend(["--directory", self.po_path])
cmd.append(input_file)
qisys.command.call(cmd)
def dump_symbols_from_binary(binary, pool_dir, dump_syms_executable=None):
""" Dump sympobls from the binary.
Results can be found in
<pool_dir>/<binary name>/<id>/<binary name>.sym
"""
cmd = [dump_syms_executable, binary]
ui.debug(cmd)
process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(out, err) = process.communicate()
if process.returncode != 0:
ui.error("Failed to dump symbols", err)
return
# First line looks like:
# MODULE Linux x86_64 ID binary
lines = out.splitlines()
first_line = lines[0]
uuid = first_line.split()[3]
name = first_line.split()[4]
to_make = os.path.join(pool_dir, name, uuid)
qisys.sh.mkdir(to_make, recursive=True)
with open(os.path.join(to_make, name + ".sym"), "w") as fp:
fp.write(out)
def configure(self, **kwargs):
""" Create a correct conf.py in self.build_dir """
rel_paths = kwargs.get("rel_paths", False)
in_conf_py = os.path.join(self.source_dir, "conf.in.py")
should_use_template = False
if os.path.exists(in_conf_py):
should_use_template = True
else:
in_conf_py = os.path.join(self.source_dir, "conf.py")
if not os.path.exists(in_conf_py):
ui.error("Could not find a conf.py or a conf.in.py in", self.source_dir)
return
with open(in_conf_py) as fp:
conf = fp.read()
if should_use_template:
if self.template_project:
from_template = self.template_project.sphinx_conf
from_template = from_template.format(**kwargs)
conf = from_template + conf
else:
ui.warning("Found a conf.in.py but no template project found "
"in the worktree")
from_conf = dict()
try:
# quick hack if conf.in.py used __file__
from_conf["__file__"] = in_conf_py
exec(conf, from_conf)
conf = conf.replace("__file__", 'r"%s"' % in_conf_py)
except Exception, e:
ui.error("Could not read", in_conf_py, "\n", e)
return
def build(self, **kwargs):
""" Run sphinx.main() with the correct arguments """
try:
import sphinx
except ImportError, e:
ui.error(e, "skipping build")
return
def configure(self, **kwargs):
""" Create a correct conf.py in self.build_dir """
rel_paths = kwargs.get("rel_paths", False)
in_conf_py = os.path.join(self.source_dir, "conf.in.py")
should_use_template = False
if os.path.exists(in_conf_py):
should_use_template = True
else:
in_conf_py = os.path.join(self.source_dir, "conf.py")
if not os.path.exists(in_conf_py):
ui.error("Could not find a conf.py or a conf.in.py in",
self.source_dir)
return
with open(in_conf_py) as fp:
conf = fp.read()
if should_use_template:
if self.template_project:
from_template = self.template_project.sphinx_conf
from_template = from_template.format(**kwargs)
conf = from_template + conf
else:
ui.warning("Found a conf.in.py but no template project found "
"in the worktree")
from_conf = dict()
try:
# quick hack if conf.in.py used __file__
from_conf["__file__"] = in_conf_py
exec(conf, from_conf)
conf = conf.replace("__file__", 'r"%s"' % in_conf_py)
except Exception, e:
ui.error("Could not read", in_conf_py, "\n", e)
return
def commit_all(self, message):
""" Commit All """
__rc, out = self.call("status", raises=False)
for line in out.splitlines():
line = line.strip()
filename = line[8:]
if line.startswith("!"):
self.call("remove", filename)
if line.startswith("?"):
self.call("add", filename)
# Prevent 'Node has unexpectedly changed kind' error
# when a file is replaced by a symlink.
# see http://antoniolorusso.com/blog/2008/09/29/svn-entry-has-unexpectedly-changed-special-status/
if line.startswith("~"):
full_path = os.path.join(self.path, filename)
if os.path.islink(full_path):
target = os.readlink(full_path) # pylint:disable=no-member
os.remove(full_path)
self.call("remove", filename)
os.symlink(target, full_path) # pylint:disable=no-member
self.call("add", filename)
else:
ui.error("Could not deal with", filename, "\n",
"Please open a bug report")
self.call("commit", "--message", message)
def new_func(self, *args, **kwargs):
if not os.path.exists(self.pot_file):
ui.error("No pot file found. Maybe no translatable strings "
"were found?")
return
#pylint: disable-msg=E1102
res = func(self, *args, **kwargs)
return res
def new_func(self, *args, **kwargs):
""" New Func """
if not os.path.exists(self.pot_file):
ui.error(
"No pot file found. Maybe no translatable strings were found?"
)
return None
return func(self, *args, **kwargs)
def build(self, build_type=None, language=None,
spellcheck=False, werror=False, pdb=False):
""" Run sphinx.main() with the correct arguments """
try:
import sphinx
except ImportError, e:
ui.error(e, "skipping build")
return
def new_func(self, *args, **kwargs):
if not os.path.exists(self.pot_file):
ui.error("No pot file found. Maybe no translatable strings "
"were found?")
return
#pylint: disable-msg=E1102
res = func(self, *args, **kwargs)
return res
def build(self, build_type=None, language=None, # pylint: disable=arguments-differ,too-many-branches
spellcheck=False, werror=False, pdb=False):
""" Run sphinx.main() with the correct arguments """
try:
import sphinx
except ImportError, e:
ui.error(e, "skipping build")
return
def do(args):
"""Main entry point """
build_worktree = qibuild.parsers.get_build_worktree(args)
retcode = qibuild.run.run(build_worktree.build_projects, args.binary, args.bin_args)
if retcode < 0:
ui.error("Process crashed: (%s)" % qisys.command.str_from_signal(-retcode))
sys.exit(1)
sys.exit(retcode)
def handle_errors(self, errors):
message = [ui.red]
for error in errors:
message.extend([error.test["name"], ":"])
message.extend(error.message)
message.append("\n")
ui.error(*message)
raise qisys.error.Error()
def configure_virtualenv(config, python_worktree, build_worktree=None,
remote_packages=None, site_packages=True):
""" Main entry point. Called by ``qipy bootstrap``
:param: remote_packages List of third-party packages to add in the virtualenv
:param: site_packages Allow access to global site packages
"""
ui.info(ui.green, "Configuring virtualenv for", ui.reset, ui.bold, python_worktree.root)
if not remote_packages:
remote_packages = list()
# create a new virtualenv
python_worktree.config = config
venv_path = python_worktree.venv_path
pip = python_worktree.pip
try:
virtualenv.create_environment(python_worktree.venv_path,
site_packages=site_packages)
except:
ui.error("Failed to create virtualenv")
return False
ui.info("Adding python projects")
# Write a qi.pth file containing path to C/C++ extensions and
# path to pure python modules or packages
pure_python_ok = handle_pure_python(venv_path, python_worktree)
if build_worktree:
handle_extensions(venv_path, python_worktree, build_worktree)
ui.info("Adding other requirements: " + ", ".join(remote_packages))
binaries_path = virtualenv.path_locations(venv_path)[-1]
pip_binary = os.path.join(binaries_path, "pip")
remote_ok = True
if remote_packages:
cmd = [pip_binary, "install"] + remote_packages
rc = qisys.command.call(cmd, ignore_ret_code=True)
remote_ok = (rc == 0)
if pure_python_ok and remote_ok:
ui.info(ui.green, "Done")
if not pure_python_ok:
ui.info(ui.red, "Failed to add some python projects")
if not remote_ok:
ui.info(ui.red, "Failed to add some third party requirements")
requirements_ok = True
for project in python_worktree.python_projects:
path = os.path.join(project.path, "requirements.txt")
if os.path.isfile( path ):
ui.info(ui.green, " * Installing dependencies from " + path)
cmd = [pip_binary, "install", "--requirement", path]
rc = qisys.command.call(cmd, ignore_ret_code=True)
requirements_ok = (rc == 0)
else:
ui.debug(ui.yellow, " * missing " + path)
return (pure_python_ok and remote_ok and requirements_ok)
def build(self, **kwargs):
""" Run sphinx.main() with the correct arguments """
try:
import sphinx
except ImportError as e:
ui.error(e, "skipping build")
return
build_type = kwargs.get("build_type", None)
language = kwargs.get("language", None)
spellcheck = kwargs.get("spellcheck", False)
werror = kwargs.get("werror", False)
pdb = kwargs.get("pdb", False)
if self.prebuild_script:
ui.info(ui.green, "Running pre-build script:",
ui.white, self.prebuild_script)
cmd = [sys.executable, self.prebuild_script]
qisys.command.call(cmd, cwd=self.path)
ui.info()
self.generate_examples_zips()
if self.translated and language and language != "en" \
and language not in self.linguas:
raise UnknownLingua(self, language)
if self.translated:
self.intl_build(language)
qisys.sh.mkdir(self.html_dir, recursive=True)
spell_dir = os.path.join(self.build_dir, "spellcheck")
qisys.sh.mkdir(spell_dir, recursive=True)
cmd = [sys.executable, "-c", self.build_dir]
if spellcheck:
cmd.extend(("-b", "spelling"))
else:
cmd.extend(("-b", "html"))
if werror:
cmd.append("-W")
if language:
cmd.append("-Dlanguage=%s" % language)
if pdb:
cmd.append("-P")
cmd.append(self.source_dir)
if spellcheck:
cmd.append(spell_dir)
else:
cmd.append(self.html_dir)
if build_type:
os.environ["build_type"] = build_type
ui.debug("launching:", cmd)
rc = 0
try:
sphinx.main(argv=cmd)
except SystemExit as e:
rc = e.code
if spellcheck:
num_errors = get_num_spellcheck_errors(self.build_dir)
if num_errors != 0:
raise SphinxBuildError(self)
if rc != 0:
raise SphinxBuildError(self)
def build(self, **kwargs):
""" Run sphinx.main() with the correct arguments """
try:
import sphinx
except ImportError as e:
ui.error(e, "skipping build")
return
build_type = kwargs.get("build_type", None)
language = kwargs.get("language", None)
spellcheck = kwargs.get("spellcheck", False)
werror = kwargs.get("werror", False)
pdb = kwargs.get("pdb", False)
if self.prebuild_script:
ui.info(ui.green, "Running pre-build script:", ui.white,
self.prebuild_script)
cmd = [sys.executable, self.prebuild_script]
qisys.command.call(cmd, cwd=self.path)
ui.info()
self.generate_examples_zips()
if self.translated and language and language != "en" \
and language not in self.linguas:
raise UnknownLingua(self, language)
if self.translated:
self.intl_build(language)
qisys.sh.mkdir(self.html_dir, recursive=True)
spell_dir = os.path.join(self.build_dir, "spellcheck")
qisys.sh.mkdir(spell_dir, recursive=True)
cmd = [sys.executable, "-c", self.build_dir]
if spellcheck:
cmd.extend(("-b", "spelling"))
else:
cmd.extend(("-b", "html"))
if werror:
cmd.append("-W")
if language:
cmd.append("-Dlanguage=%s" % language)
if pdb:
cmd.append("-P")
cmd.append(self.source_dir)
if spellcheck:
cmd.append(spell_dir)
else:
cmd.append(self.html_dir)
if build_type:
os.environ["build_type"] = build_type
ui.debug("launching:", cmd)
rc = 0
try:
sphinx.main(argv=cmd)
except SystemExit as e:
rc = e.code
if spellcheck:
num_errors = get_num_spellcheck_errors(self.build_dir)
if num_errors != 0:
raise SphinxBuildError(self)
if rc != 0:
raise SphinxBuildError(self)
def do(args):
"""Main entry points."""
git_worktree = qisrc.parsers.get_git_worktree(args)
snapshot = None
if args.snapshot:
snapshot = qisrc.snapshot.Snapshot()
snapshot.load(args.snapshot)
if snapshot and snapshot.format_version and snapshot.format_version >= 1:
reset_manifest(git_worktree,
snapshot,
ignore_groups=args.ignore_groups)
git_projects = qisrc.parsers.get_git_projects(git_worktree,
args,
default_all=True,
use_build_deps=True)
errors = list()
for i, git_project in enumerate(git_projects):
ui.info_count(i, len(git_projects), "Reset", git_project.src)
src = git_project.src
git = qisrc.git.Git(git_project.path)
ok, message = git.require_clean_worktree()
if not ok and not args.force:
ui.warning(message)
errors.append(src)
continue
if not git_project.default_branch:
ui.warning(git_project.src, "not in any manifest, skipping")
continue
branch = git_project.default_branch.name
remote = git_project.default_remote.name
git.safe_checkout(branch, remote, force=True)
to_reset = None
if args.snapshot:
to_reset = snapshot.refs.get(src)
if not to_reset:
ui.warning(src, "not found in the snapshot")
continue
elif args.tag:
to_reset = args.tag
else:
to_reset = "%s/%s" % (remote, branch)
try:
qisrc.reset.clever_reset_ref(git_project, to_reset)
except:
errors.append(src)
if not errors:
return
ui.error("Failed to reset some projects")
for error in errors:
ui.info(ui.red, " * ", error)
sys.exit(1)
def tests(self):
res = [x for x in self._tests if match_patterns(self.patterns, x["name"])]
# Perf tests are run alone
res = [x for x in res if x.get("perf", False) == self.perf]
# But nightly tests are run along with the normal tests
if not self.nightly:
res = [x for x in res if x.get("nightly", False) is False]
if not res and self.patterns:
ui.error("No test found matching pattern")
return res
def _load_deprecated(self, source):
for line in source.splitlines():
try:
(src, sha1) = line.split(":")
except ValueError:
ui.error("could not parse", line)
continue
src = src.strip()
sha1 = sha1.strip()
self.refs[src] = sha1
def to_test_project(self):
if not os.path.exists(self.qitest_json):
raise Exception("qitest.json not found. Did you run qibuild configure?")
res = qitest.project.TestProject(self.qitest_json)
if not res.tests:
ui.error("No tests were found in qitest.json.\n" +
"Add some in the CMakeLists using " +
"qi_add_test() or similar")
res.name = self.name
return res
def to_test_project(self):
if not os.path.exists(self.qitest_json):
raise NoQiTestJson()
res = qitest.project.TestProject(self.qitest_json)
if not res.tests:
ui.error("No tests were found in qitest.json.\n" +
"Add some in the CMakeLists using " +
"qi_add_test() or similar")
res.name = self.name
return res
