def do(args): """ Main entry point """ git_worktree = qisrc.parsers.get_git_worktree(args) git_project = qisrc.parsers.get_one_git_project(git_worktree, args) git = qisrc.git.Git(git_project.path) current_branch = git.get_current_branch() if not current_branch: ui.error("Not currently on any branch") sys.exit(2) if git_project.review: maintainers = qisrc.maintainers.get(git_project, warn_if_none=True) reviewers = [x['email'] for x in maintainers] reviewers.extend(args.reviewers or list()) # Prefer gerrit logins or groups instead of e-mails reviewers = [x.split("@")[0] for x in reviewers] qisrc.review.push(git_project, current_branch, bypass_review=(not args.review), dry_run=args.dry_run, reviewers=reviewers, topic=args.topic) else: if args.dry_run: git.push("-n") else: if args.review and not args.yes: mess = """\ The project is not under code review. Are you sure you want to run `git push` ? This action cannot be undone\ """ answer = qisys.interact.ask_yes_no(mess, default=False) if answer: git.push() else: git.push()
def jar(jar_path, files, paths): """ Search each files using qibuild find and add them into a jar using qisys """ # Create command line jar_path = qisys.sh.to_native_path(jar_path) args = ["cvfM"] args += [jar_path] if not files: raise Exception("Missing arguments : Files to package") for wanted_file in files: ui.info("Searching for " + wanted_file + "...") path = qibuild.find.find(paths, wanted_file, expect_one=False)[0] if not path: ui.error("Cannot find " + wanted_file + " in worktree") return None ui.debug("Found : " + path) dirname = os.path.dirname(path) basename = os.path.basename(path) args += ["-C", dirname, basename] ui.debug("Added -C " + dirname + " " + wanted_file + " to command line") qisys.command.call(["jar"] + args, ignore_ret_code=False) return jar_path
def summary(self): """ Display the tests results. Called at the end of self.run() Sets ``self.ok`` """ if not self.tests: self.ok = False return num_tests = len(self.results) failures = [x for x in self.results.values() if x.ok is False] num_failed = len(failures) message = "Ran %i tests in %is" % (num_tests, self.elapsed_time) ui.info(message) self.ok = (not failures) and not self._interrupted if self.ok: ui.info(ui.green, "All pass. Congrats!") return if num_failed != 0: ui.error(num_failed, "failures") if failures: max_len = max(len(x.test["name"]) for x in failures) for i, failure in enumerate(failures): ui.info_count(i, num_failed, ui.blue, failure.test["name"].ljust(max_len + 2), ui.reset, *failure.message)
def configure_virtualenv(config, python_worktree, build_worktree=None, remote_packages=None, site_packages=True): if not remote_packages: remote_packages = list() # create a new virtualenv python_worktree.config = config venv_path = python_worktree.venv_path pip = python_worktree.pip try: virtualenv.create_environment(python_worktree.venv_path, site_packages=site_packages) except: ui.error("Failed to create virtualenv") return # Install all Python projects using pip install -e . python_projects = python_worktree.python_projects for i, project in enumerate(python_projects): ui.info_count(i, len(python_projects), ui.green, "Configuring", ui.reset, ui.blue, project.src) cmd = [pip, "install", "--editable", "."] qisys.command.call(cmd, cwd=project.path) # Write a qi.pth file containing path to C/C++ extensions if build_worktree: handle_extensions(venv_path, python_worktree, build_worktree) # Install the extension in the virtualenv binaries_path = virtualenv.path_locations(venv_path)[-1] pip_binary = os.path.join(binaries_path, "pip") if remote_packages: cmd = [pip_binary, "install"] + remote_packages subprocess.check_call(cmd)
def main_wrapper(module, args): """This wraps the main method of an action so that: - backtrace is not printed by default - backtrace is printed is --backtrace was given - a pdb session is run if --pdb was given """ try: module.do(args) except Exception as e: if args.pdb: traceback = sys.exc_info()[2] print "" print "### Exception:", e print "### Starting a debugger" try: import ipdb ipdb.post_mortem(traceback) sys.exit(0) except ImportError: import pdb pdb.post_mortem(traceback) sys.exit(0) if args.backtrace: raise ui.error(e) sys.exit(2)
def generate_mo_file(self, locale): """ Generate .mo file for the given locale """ ui.info(ui.green, "Generating translation for", ui.reset, ui.bold, locale) input_file = self.get_po_file(locale) if not os.path.exists(input_file): ui.error("No .po found for locale: ", locale, "\n", "(looked in %s)" % input_file, "\n", "Did you run qilinguist update?") return output_file = os.path.join(self.mo_path, locale, "LC_MESSAGES", self.domain + ".mo") to_make = os.path.dirname(output_file) qisys.sh.mkdir(to_make, recursive=True) cmd = ["msgfmt", "--check", "--statistics"] # required by libqi: conf_file = os.path.join(self.mo_path, ".confintl") with open(conf_file, "w") as fp: fp.write("# THIS FILE IS AUTOGENERATED\n" "#Do not delete or modify it\n" "# This file is used to find translation dictionaries\n") cmd.extend(["--output-file", output_file]) cmd.extend(["--directory", self.po_path]) cmd.append(input_file) qisys.command.call(cmd)
def checkout(self, branch, force=False): """ Called by ``qisrc checkout`` For each project, checkout the branch if it is different than the default branch of the manifest. """ ui.info(ui.green, ":: Checkout projects ...") errors = list() manifest_xml = os.path.join(self._syncer.manifest_repo, "manifest.xml") manifest = qisrc.manifest.Manifest(manifest_xml) max_src = max([len(x.src) for x in self.git_projects]) n = len(self.git_projects) for i, project in enumerate(self.git_projects): ui.info_count(i, n, ui.bold, "Checkout", ui.reset, ui.blue, project.src.ljust(max_src), end="\r") if project.default_branch is None: continue branch_name = project.default_branch.name remote_name = project.default_remote.name git = qisrc.git.Git(project.path) ok, err = git.safe_checkout(branch_name, remote_name, force=force) if not ok: errors.append((project.src, err)) if not errors: return ui.error("Failed to checkout some projects") for (project, error) in errors: ui.info(project, ":", error)
def push_projects(git_projects, dry_run=False): """ Push Projects """ if not git_projects: return ui.info(ui.green, "Pushing ", len(git_projects), "projects") for i, git_project in enumerate(git_projects): default_branch = git_project.default_branch.name remote_branch = git_project.default_branch.remote_branch ui.info_count(i, len(git_projects), git_project.src) git = qisrc.git.Git(git_project.path) if git_project.review: push_remote = git_project.review_remote else: push_remote = git_project.default_remote remote_ref = "%s/%s" % (push_remote.name, remote_branch) display_changes(git, default_branch, remote_ref) answer = qisys.interact.ask_yes_no("OK to push?", default=False) if not answer: return to_push = "%s:%s" % (default_branch, remote_branch) push_args = [push_remote.name, to_push] push_args.append("--force") if dry_run: push_args.append("--dry-run") rc, out = git.push(*push_args, raises=False) if rc == 0: ui.info(out) else: ui.error(out)
def tree_from_feed(feed_location): """ Returns an ElementTree object from an feed location """ fp = None tree = None try: if os.path.exists(feed_location): fp = open(feed_location, "r") else: if is_url(feed_location): fp = qisys.remote.open_remote_location(feed_location) else: raise Exception( "Feed location is not an existing path nor an url") tree = ElementTree.ElementTree() tree.parse(fp) except Exception: ui.error("Could not parse", feed_location) raise finally: if fp: fp.close() return tree
def fix_solibs(sdk_dir, paths=None): """ Add rpath to relocate the toolchain libraries """ # This directory may not exist, so create it: library_path = "./" libraries_list = [] if not os.path.exists(sdk_dir): return for path in os.listdir(sdk_dir): if not os.path.exists(path): continue lib_dir = os.path.join(sdk_dir, path, "lib") if os.path.exists(lib_dir): library_path = library_path + ':' + lib_dir libs = os.listdir(lib_dir) for lib in libs: if ".so" in lib and lib not in libraries_list: libraries_list.append(os.path.join(lib_dir, lib)) for lib in libraries_list: patchelf = qisys.command.find_program("patchelf") if not patchelf: ui.error("Could not find patchelf for fixing so libraries") else: cmd = [patchelf, "--set-rpath", library_path, lib] qisys.command.call(cmd)
def do(args): """ Main entry point """ git_worktree = qisrc.parsers.get_git_worktree(args) git_projects = qisrc.parsers.get_git_projects(git_worktree, args) for git_project in git_projects: maintainers = qisrc.maintainers.get(git_project) if not maintainers: mess = """\ The project in {src} has no maintainer. Please edit {qiproject_xml} to silence this warning """ ui.warning(mess.format(src=git_project.src, qiproject_xml=git_project.qiproject_xml), end="") reviewers = [x['email'] for x in maintainers] reviewers.extend(args.reviewers or list()) # Prefer gerrit logins or groups instead of e-mails reviewers = [x.split("@")[0] for x in reviewers] git = qisrc.git.Git(git_project.path) current_branch = git.get_current_branch() if not current_branch: ui.error("Not currently on any branch") sys.exit(2) if git_project.review: qisrc.review.push(git_project, current_branch, bypass_review=(not args.review), dry_run=args.dry_run, reviewers=reviewers, topic=args.topic) else: if args.dry_run: git.push("-n") else: git.push()
def _run(self, num_jobs=1): """ Helper function for ._run """ if not self.launcher: ui.error("test launcher not set, cannot run tests") return for i, test in enumerate(self.tests): self.task_queue.put((test, i)) if num_jobs == 1: self.test_logger.single_job = True threads = list() for i in range(0, num_jobs): worker = TestWorker(self.task_queue, i) worker.launcher = self.launcher worker.launcher.worker_index = i worker.test_logger = self.test_logger worker.results = self.results threads.append(worker) worker.start() # Do not use .join() so that this can be interrupted: while self.task_queue.unfinished_tasks and \ not self._interrupted: time.sleep(0.1) for worker_thread in threads: worker_thread.stop() worker_thread.join()
def tree_from_feed(feed_location, branch=None, name=None): # pylint: disable=unused-argument """ Returns an ElementTree object from an feed location """ fp = None tree = None try: if os.path.exists(feed_location): fp = open(feed_location, "r") else: if is_url(feed_location): fp = qisys.remote.open_remote_location(feed_location) else: raise Exception( "Could not parse %s: Feed location is not an existing path nor an url" % feed_location) tree = ElementTree.ElementTree() tree.parse(fp) except Exception as e: ui.error(e.message) raise finally: if fp: fp.close() return tree
def main_wrapper(module, args): """This wraps the main method of an action so that: - backtrace is not printed by default - backtrace is printed is --backtrace was given - a pdb session is run if --pdb was given """ try: module.do(args) except Exception as e: if args.pdb: traceback = sys.exc_info()[2] print "" print "### Exception:", e print "### Starting a debugger" try: #pylint: disable-msg=F0401 import ipdb ipdb.post_mortem(traceback) sys.exit(0) except ImportError: import pdb pdb.post_mortem(traceback) sys.exit(0) if args.backtrace: raise message = str(e) if message.endswith("\n"): message = message[:-1] ui.error(e.__class__.__name__, message) sys.exit(2)
def clever_reset_ref(git_project, ref, raises=True): """ Resets only if needed, fetches only if needed """ try: remote_name = git_project.default_remote.name except AttributeError: error_msg = "Project {} has no default remote, defaulting to origin" ui.error(error_msg.format(git_project.name)) remote_name = "origin" git = qisrc.git.Git(git_project.path) # Deals with "refs/" prefixed ref first if ref.startswith("refs/"): return _reset_hard_to_refs_prefixed_ref(git, remote_name, ref, raises=raises) # Else, ref format is a local name (branch or tag) # Check if this ref exists and if we are already in the expected state rc, ref_sha1 = git.call("rev-parse", ref, "--", raises=False) if rc != 0: # Maybe this is a newly pushed tag, try to fetch: git.fetch(remote_name) rc, ref_sha1 = git.call("rev-parse", ref, "--", raises=False) if rc != 0: return False, "Could not parse %s as a valid ref" % ref _, actual_sha1 = git.call("rev-parse", "HEAD", raises=False) if actual_sha1 == ref_sha1: # Nothing to do return None if raises else True, "" # Reset to the ref local name return _reset_hard_to_local_refs_name(git, remote_name, ref, raises=raises)
def commit_all(self, message): __rc, out = self.call("status", raises=False) # pylint: disable=unused-variable for line in out.splitlines(): line = line.strip() filename = line[8:] if line.startswith("!"): self.call("remove", filename) if line.startswith("?"): self.call("add", filename) # Prevent 'Node has unexpectedly changed kind' error # when a file is replaced by a symlink. # see http://antoniolorusso.com/blog/2008/09/29/svn-entry-has-unexpectedly-changed-special-status/ if line.startswith("~"): full_path = os.path.join(self.path, filename) if os.path.islink(full_path): target = os.readlink(full_path) os.remove(full_path) self.call("remove", filename) os.symlink(target, full_path) self.call("add", filename) else: ui.error("Could not deal with", filename, "\n", "Please open a bug report") self.call("commit", "--message", message)
def clever_reset_ref(git_project, ref, raises=True): """ Resets only if needed, fetches only if needed """ try: remote_name = git_project.default_remote.name except AttributeError: error_msg = "Project {} has no default remote, defaulting to origin" ui.error(error_msg.format(git_project.name)) remote_name = "origin" git = qisrc.git.Git(git_project.path) # Deals with "refs/" prefixed ref first if ref.startswith("refs/"): return _reset_hard_to_refs_prefixed_ref(git, remote_name, ref, raises=raises) # Else, ref format is a local name (branch or tag) # Check if this ref exists and if we are already in the expected state rc, ref_sha1 = git.call("rev-parse", ref, "--", raises=False) if rc != 0: # Maybe this is a newly pushed tag, try to fetch: git.fetch(remote_name, "--prune") rc, ref_sha1 = git.call("rev-parse", ref, "--", raises=False) if rc != 0: return False, "Could not parse %s as a valid ref" % ref _, actual_sha1 = git.call("rev-parse", "HEAD", raises=False) if actual_sha1 == ref_sha1: # Nothing to do return None if raises else True, "" # Reset to the ref local name return _reset_hard_to_local_refs_name(git, remote_name, ref, raises=raises)
def copy_jni_resources(build_worktree): """ Create symlinks (or copy), the jni library and its dependencies from the build dirs to the resource folder """ android = "android" in (build_worktree.build_config.active_config or "") # FIXME: boost when using a toolchain on linux ? libs = list() proj_to_lib = [ ("libqi", "qi"), ("qimessaging-jni", "qimessagingjni")] if android: proj_to_lib.append( ("qimessaging-jni", "gnustl_shared")) for project_name, library_name in proj_to_lib: try: project = build_worktree.get_build_project(project_name, raises=True) library_path = qibuild.find.find_lib([project.sdk_directory], library_name) libs.append(library_path) except qibuild.find.NotFound as e: ui.error(e) ui.error("Make sure qimessaging-jni has been built") sys.exit(1) jni_proj = build_worktree.get_build_project("qimessaging-jni") java_proj = build_worktree.worktree.get_project("sdk/libqi-java/qimessaging") if android: dest = os.path.join(java_proj.path, "native-android") else: dest = os.path.join(java_proj.path, "native") qisys.sh.mkdir(dest) for lib_src in libs: lib_dest = os.path.join(dest, os.path.basename(lib_src)) copy_or_link(lib_src, lib_dest)
def do(args): """ Main entry point. """ cmake_builder = qibuild.parsers.get_cmake_builder(args) if len(cmake_builder.projects) != 1: raise Exception("This action can only work on one project") project = cmake_builder.projects[0] if not os.path.exists(project.build_directory): ui.error("""It looks like your project has not been configured yet (The build directory: '%s' does not exist)""" % project.build_directory) answer = qisys.interact.ask_yes_no( "Do you want me to run qibuild configure for you?", default=True) if not answer: sys.exit(2) else: cmake_builder.configure() ide = get_ide(cmake_builder.build_config.qibuild_cfg) if not ide: return if ide.name == "Visual Studio": open_visual(project) elif ide.name == "Xcode": open_xcode(project) elif ide.name == "QtCreator": open_qtcreator(project, ide.path) else: # Not supported (yet) IDE: mess = "Invalid ide: %s\n" % ide.name mess += "Supported IDES are: %s" % ", ".join(SUPPORTED_IDES) raise Exception(mess)
def do(args): """ Main entry point """ git_worktree = qisrc.parsers.get_git_worktree(args) git_projects = qisrc.parsers.get_git_projects(git_worktree, args) for git_project in git_projects: maintainers = qisrc.maintainers.get(git_project) if not maintainers: mess = """\ The project in {src} has no maintainer. Please edit {qiproject_xml} to silence this warning """ ui.warning(mess.format(src=git_project.src, qiproject_xml=git_project.qiproject_xml), end="") reviewers = [x["email"] for x in maintainers] reviewers.extend(args.reviewers or list()) git = qisrc.git.Git(git_project.path) current_branch = git.get_current_branch() if not current_branch: ui.error("Not currently on any branch") sys.exit(2) if git_project.review: qisrc.review.push( git_project, current_branch, bypass_review=(not args.review), dry_run=args.dry_run, reviewers=reviewers, topic=args.topic, ) else: if args.dry_run: git.push("-n") else: git.push()
def generate_mo_file(self, locale): """ Generate .mo file for the given locale """ ui.info(ui.green, "Generating translation for", ui.reset, ui.bold, locale) input_file = self.get_po_file(locale) if not os.path.exists(input_file): ui.error("No .po found for locale: ", locale, "\n", "(looked in %s)" % input_file, "\n", "Did you run qilinguist update?") return None output_file = os.path.join(self.mo_path, locale, "LC_MESSAGES", self.domain + ".mo") to_make = os.path.dirname(output_file) qisys.sh.mkdir(to_make, recursive=True) cmd = ["msgfmt", "--check", "--statistics"] # required by libqi: conf_file = os.path.join(self.mo_path, ".confintl") with open(conf_file, "w") as fp: fp.write("# THIS FILE IS AUTOGENERATED\n" "# Do not delete or modify it\n" "# This file is used to find translation dictionaries\n") cmd.extend(["--output-file", output_file]) cmd.extend(["--directory", self.po_path]) cmd.append(input_file) process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) __out, err = process.communicate() ui.info(err.strip()) if "untranslated" in str(err): return False, "Some untranslated messages were found" if process.returncode != 0: return False, "msgfmt failed" return True, ""
def get_log(self, before_ref, after_ref): """ Return a list of commits between two refspecs, in natural order (most recent commits last) Each commit is a dict containing, 'sha1' and 'message' FIXME: parse author and date ? """ # old git does not support -z and inists on adding \n # between log entries, so we have no choice but choose # a custom separator. '\f' looks safe enough res = list() rc, out = self.call("log", "--reverse", "--format=%H%n%B\f", "%s..%s" % (before_ref, after_ref), raises=False) if rc != 0: ui.error(out) return res for log_chunk in out.split('\f'): if not log_chunk: continue log_chunk = log_chunk.strip() sha1, message = log_chunk.split('\n', 1) message = message.strip() commit = { "sha1" : sha1, "message" : message, } res.append(commit) return res
def _run(self, num_jobs=1): """ Helper function for ._run """ if not self.launcher: ui.error("test launcher not set, cannot run tests") return for i, test in enumerate(self.tests): self.task_queue.put((test, i)) if num_jobs == 1: self.test_logger.single_job = True for i in range(0, num_jobs): worker = TestWorker(self.task_queue, i) worker.launcher = self.launcher worker.launcher.worker_index = i worker.test_logger = self.test_logger worker.results = self.results self._workers.append(worker) worker.start() while not self.task_queue.empty() and \ not self._interrupted: time.sleep(0.1) for worker_thread in self._workers: worker_thread.join()
def _clone_missing(self, git_project, repo): """ Clone Missing """ branch = repo.default_branch fixed_ref = repo.fixed_ref clone_url = repo.clone_url qisys.sh.mkdir(git_project.path, recursive=True) git = qisrc.git.Git(git_project.path) remote_name = repo.default_remote.name try: git.init() git.remote("add", remote_name, clone_url) git.fetch(remote_name, "--quiet") if branch: git.checkout("-b", branch, "%s/%s" % (remote_name, branch)) if fixed_ref: git.checkout("-q", fixed_ref) except Exception: ui.error("Cloning repo failed") if git.is_empty(): qisys.sh.rm(git_project.path) self.worktree.remove_project(repo.src) return False self.save_project_config(git_project) self.load_git_projects() return True
def main_wrapper(module, args): """ This wraps the main method of an action so that: - backtrace is not printed by default - backtrace is printed is --backtrace was given - a pdb session is run if --pdb was given """ try: module.do(args) except Exception as e: if args.pdb: traceback = sys.exc_info()[2] print("") print("### Exception:", e) print("### Starting a debugger") try: import ipdb ipdb.post_mortem(traceback) sys.exit(0) except ImportError: import pdb pdb.post_mortem(traceback) sys.exit(0) if args.backtrace: raise if six.PY3: message = str(e) else: message = str(e).decode("utf-8", errors="ignore") if message.endswith("\n"): message = message[:-1] ui.error(e.__class__.__name__, message) sys.exit(2)
def do(args): """Main entry point.""" cmake_builder = qibuild.parsers.get_cmake_builder(args) if len(cmake_builder.projects) != 1: raise Exception("This action can only work on one project") project = cmake_builder.projects[0] if not os.path.exists(project.build_directory): ui.error("""It looks like your project has not been configured yet (The build directory: '%s' does not exist)""" % project.build_directory) answer = qisys.interact.ask_yes_no( "Do you want me to run qibuild configure for you?", default=True) if not answer: sys.exit(2) else: cmake_builder.configure() ide = get_ide(cmake_builder.build_config.qibuild_cfg) if not ide: return if ide.name == "Visual Studio": open_visual(project) elif ide.name == "Xcode": open_xcode(project) elif ide.name == "QtCreator": open_qtcreator(project, ide.path) else: # Not supported (yet) IDE: mess = "Invalid ide: %s\n" % ide.name mess += "Supported IDES are: %s" % ", ".join(SUPPORTED_IDES) raise Exception(mess)
def generate_mo_file(self, locale): """ Generate .mo file for the given locale """ ui.info(ui.green, "Generating translation for", ui.reset, ui.bold, locale) input_file = self.get_po_file(locale) if not os.path.exists(input_file): ui.error("No .po found for locale: ", locale, "\n", "(looked in %s)" % input_file, "\n", "Did you run qilinguist update?") return None output_file = os.path.join(self.mo_path, locale, "LC_MESSAGES", self.domain + ".mo") to_make = os.path.dirname(output_file) qisys.sh.mkdir(to_make, recursive=True) cmd = ["msgfmt", "--check", "--statistics"] # required by libqi: conf_file = os.path.join(self.mo_path, ".confintl") with open(conf_file, "w") as fp: fp.write("# THIS FILE IS AUTOGENERATED\n" "# Do not delete or modify it\n" "# This file is used to find translation dictionaries\n") cmd.extend(["--output-file", output_file]) cmd.extend(["--directory", self.po_path]) cmd.append(input_file) process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) __out, err = process.communicate() ui.info(err.strip()) if "untranslated" in err: return False, "Some untranslated messages were found" if process.returncode != 0: return False, "msgfmt failed" return True, ""
def main_wrapper(module, args): """This wraps the main method of an action so that: - backtrace is not printed by default - backtrace is printed is --backtrace was given - a pdb session is run if --pdb was given """ try: module.do(args) except Exception as e: if args.pdb: traceback = sys.exc_info()[2] print "" print "### Exception:", e print "### Starting a debugger" try: #pylint: disable-msg=F0401 import ipdb ipdb.post_mortem(traceback) sys.exit(0) except ImportError: import pdb pdb.post_mortem(traceback) sys.exit(0) if args.backtrace: raise ui.error(e.__class__.__name__, e) sys.exit(2)
def summary(self): """ Display the tests results. Called at the end of self.run() Sets ``self.ok`` """ if not self.tests: self.ok = False return num_tests = len(self.results) failures = [x for x in self.results.values() if x.ok is False] num_failed = len(failures) message = "Ran %i tests in %is" % (num_tests, self.elapsed_time) ui.info(message) self.ok = (not failures) and not self._interrupted if self.ok: ui.info(ui.green, "All pass. Congrats!") else: if num_failed != 0: ui.error(num_failed, "failures") if failures: max_len = max(len(x.test["name"]) for x in failures) for i, failure in enumerate(failures): ui.info_count(i, num_failed, ui.blue, failure.test["name"].ljust(max_len + 2), ui.reset, *failure.message) self.write_failures(failures)
def get_log(self, before_ref, after_ref): """ Return a list of commits between two refspecs, in natural order (most recent commits last) Each commit is a dict containing, 'sha1' and 'message' FIXME: parse author and date ? """ # old git does not support -z and inists on adding \n # between log entries, so we have no choice but choose # a custom separator. '\f' looks safe enough res = list() rc, out = self.call("log", "--reverse", "--format=%H%n%B\f", "%s...%s" % (before_ref, after_ref), raises=False) if rc != 0: ui.error(out) return res for log_chunk in out.split('\f'): if not log_chunk: continue log_chunk = log_chunk.strip() sha1, message = log_chunk.split('\n', 1) message = message.strip() commit = { "sha1" : sha1, "message" : message, } res.append(commit) return res
def tree_from_feed(feed_location, branch=None, name=None): """ Returns an ElementTree object from an feed location """ fp = None tree = None try: if feed_location and os.path.exists(feed_location): fp = open(feed_location, "r") elif is_url(feed_location): fp = qisys.remote.open_remote_location(feed_location) else: raise Exception( "Could not parse %s: Feed location is not an existing path nor an url" % feed_location) tree = ElementTree.ElementTree() if six.PY3: tree.parse(fp, parser=ElementTree.XMLParser(encoding='utf-8')) else: tree.parse(fp) except Exception as e: if six.PY3: ui.error(e) else: ui.error(e.message) raise finally: if fp: fp.close() return tree
def _run(self, num_jobs=1): """ Helper function for ._run_once """ if not self.launcher: ui.error("test launcher not set, cannot run tests") return for i, test in enumerate(self.tests): self.task_queue.put((test, i)) if num_jobs == 1: self.test_logger.single_job = True for i in range(0, num_jobs): worker = TestWorker(self.task_queue, i) worker.launcher = self.launcher worker.launcher.worker_index = i worker.test_logger = self.test_logger worker.results = self.results self._workers.append(worker) worker.start() while not self.task_queue.empty() and \ not self._interrupted: time.sleep(0.1) for worker_thread in self._workers: worker_thread.join()
def build(self, **kwargs): """ Run sphinx.main() with the correct arguments """ try: import sphinx except ImportError, e: ui.error(e, "skipping build") return
def _create_virtualenv(config, python_worktree, site_packages=True, python_executable=None, env=None): python_worktree.config = config venv_path = python_worktree.venv_path # pip = python_worktree.pip if not python_executable: python_executable = sys.executable virtualenv_py = virtualenv.__file__ if virtualenv_py.endswith(".pyc"): virtualenv_py = virtualenv_py[:-1] cmd = [python_executable, virtualenv_py, venv_path] if site_packages: cmd.append("--system-site-packages") try: qisys.command.call(cmd, env=env) except qisys.command.CommandFailedException: ui.error("Failed to create virtualenv") return False binaries_path = virtualenv.path_locations(venv_path)[-1] pip_binary = os.path.join(binaries_path, "pip") return pip_binary, venv_path
def dump_symbols_from_binary(binary, pool_dir, dump_syms_executable=None): """ Dump sympobls from the binary. Results can be found in <pool_dir>/<binary name>/<id>/<binary name>.sym """ cmd = [dump_syms_executable, binary] ui.debug(cmd) process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) (out, err) = process.communicate() if process.returncode != 0: ui.error("Failed to dump symbols", err) return # First line looks like: # MODULE Linux x86_64 ID binary lines = out.splitlines() first_line = lines[0] uuid = first_line.split()[3] name = first_line.split()[4] to_make = os.path.join(pool_dir, name, uuid) qisys.sh.mkdir(to_make, recursive=True) with open(os.path.join(to_make, name + ".sym"), "w") as fp: fp.write(out)
def configure(self, **kwargs): """ Create a correct conf.py in self.build_dir """ rel_paths = kwargs.get("rel_paths", False) in_conf_py = os.path.join(self.source_dir, "conf.in.py") should_use_template = False if os.path.exists(in_conf_py): should_use_template = True else: in_conf_py = os.path.join(self.source_dir, "conf.py") if not os.path.exists(in_conf_py): ui.error("Could not find a conf.py or a conf.in.py in", self.source_dir) return with open(in_conf_py) as fp: conf = fp.read() if should_use_template: if self.template_project: from_template = self.template_project.sphinx_conf from_template = from_template.format(**kwargs) conf = from_template + conf else: ui.warning("Found a conf.in.py but no template project found " "in the worktree") from_conf = dict() try: # quick hack if conf.in.py used __file__ from_conf["__file__"] = in_conf_py exec(conf, from_conf) conf = conf.replace("__file__", 'r"%s"' % in_conf_py) except Exception, e: ui.error("Could not read", in_conf_py, "\n", e) return
def commit_all(self, message): """ Commit All """ __rc, out = self.call("status", raises=False) for line in out.splitlines(): line = line.strip() filename = line[8:] if line.startswith("!"): self.call("remove", filename) if line.startswith("?"): self.call("add", filename) # Prevent 'Node has unexpectedly changed kind' error # when a file is replaced by a symlink. # see http://antoniolorusso.com/blog/2008/09/29/svn-entry-has-unexpectedly-changed-special-status/ if line.startswith("~"): full_path = os.path.join(self.path, filename) if os.path.islink(full_path): target = os.readlink(full_path) # pylint:disable=no-member os.remove(full_path) self.call("remove", filename) os.symlink(target, full_path) # pylint:disable=no-member self.call("add", filename) else: ui.error("Could not deal with", filename, "\n", "Please open a bug report") self.call("commit", "--message", message)
def new_func(self, *args, **kwargs): if not os.path.exists(self.pot_file): ui.error("No pot file found. Maybe no translatable strings " "were found?") return #pylint: disable-msg=E1102 res = func(self, *args, **kwargs) return res
def new_func(self, *args, **kwargs): """ New Func """ if not os.path.exists(self.pot_file): ui.error( "No pot file found. Maybe no translatable strings were found?" ) return None return func(self, *args, **kwargs)
def build(self, build_type=None, language=None, spellcheck=False, werror=False, pdb=False): """ Run sphinx.main() with the correct arguments """ try: import sphinx except ImportError, e: ui.error(e, "skipping build") return
def build(self, build_type=None, language=None, # pylint: disable=arguments-differ,too-many-branches spellcheck=False, werror=False, pdb=False): """ Run sphinx.main() with the correct arguments """ try: import sphinx except ImportError, e: ui.error(e, "skipping build") return
def do(args): """Main entry point """ build_worktree = qibuild.parsers.get_build_worktree(args) retcode = qibuild.run.run(build_worktree.build_projects, args.binary, args.bin_args) if retcode < 0: ui.error("Process crashed: (%s)" % qisys.command.str_from_signal(-retcode)) sys.exit(1) sys.exit(retcode)
def handle_errors(self, errors): message = [ui.red] for error in errors: message.extend([error.test["name"], ":"]) message.extend(error.message) message.append("\n") ui.error(*message) raise qisys.error.Error()
def configure_virtualenv(config, python_worktree, build_worktree=None, remote_packages=None, site_packages=True): """ Main entry point. Called by ``qipy bootstrap`` :param: remote_packages List of third-party packages to add in the virtualenv :param: site_packages Allow access to global site packages """ ui.info(ui.green, "Configuring virtualenv for", ui.reset, ui.bold, python_worktree.root) if not remote_packages: remote_packages = list() # create a new virtualenv python_worktree.config = config venv_path = python_worktree.venv_path pip = python_worktree.pip try: virtualenv.create_environment(python_worktree.venv_path, site_packages=site_packages) except: ui.error("Failed to create virtualenv") return False ui.info("Adding python projects") # Write a qi.pth file containing path to C/C++ extensions and # path to pure python modules or packages pure_python_ok = handle_pure_python(venv_path, python_worktree) if build_worktree: handle_extensions(venv_path, python_worktree, build_worktree) ui.info("Adding other requirements: " + ", ".join(remote_packages)) binaries_path = virtualenv.path_locations(venv_path)[-1] pip_binary = os.path.join(binaries_path, "pip") remote_ok = True if remote_packages: cmd = [pip_binary, "install"] + remote_packages rc = qisys.command.call(cmd, ignore_ret_code=True) remote_ok = (rc == 0) if pure_python_ok and remote_ok: ui.info(ui.green, "Done") if not pure_python_ok: ui.info(ui.red, "Failed to add some python projects") if not remote_ok: ui.info(ui.red, "Failed to add some third party requirements") requirements_ok = True for project in python_worktree.python_projects: path = os.path.join(project.path, "requirements.txt") if os.path.isfile( path ): ui.info(ui.green, " * Installing dependencies from " + path) cmd = [pip_binary, "install", "--requirement", path] rc = qisys.command.call(cmd, ignore_ret_code=True) requirements_ok = (rc == 0) else: ui.debug(ui.yellow, " * missing " + path) return (pure_python_ok and remote_ok and requirements_ok)
def build(self, **kwargs): """ Run sphinx.main() with the correct arguments """ try: import sphinx except ImportError as e: ui.error(e, "skipping build") return build_type = kwargs.get("build_type", None) language = kwargs.get("language", None) spellcheck = kwargs.get("spellcheck", False) werror = kwargs.get("werror", False) pdb = kwargs.get("pdb", False) if self.prebuild_script: ui.info(ui.green, "Running pre-build script:", ui.white, self.prebuild_script) cmd = [sys.executable, self.prebuild_script] qisys.command.call(cmd, cwd=self.path) ui.info() self.generate_examples_zips() if self.translated and language and language != "en" \ and language not in self.linguas: raise UnknownLingua(self, language) if self.translated: self.intl_build(language) qisys.sh.mkdir(self.html_dir, recursive=True) spell_dir = os.path.join(self.build_dir, "spellcheck") qisys.sh.mkdir(spell_dir, recursive=True) cmd = [sys.executable, "-c", self.build_dir] if spellcheck: cmd.extend(("-b", "spelling")) else: cmd.extend(("-b", "html")) if werror: cmd.append("-W") if language: cmd.append("-Dlanguage=%s" % language) if pdb: cmd.append("-P") cmd.append(self.source_dir) if spellcheck: cmd.append(spell_dir) else: cmd.append(self.html_dir) if build_type: os.environ["build_type"] = build_type ui.debug("launching:", cmd) rc = 0 try: sphinx.main(argv=cmd) except SystemExit as e: rc = e.code if spellcheck: num_errors = get_num_spellcheck_errors(self.build_dir) if num_errors != 0: raise SphinxBuildError(self) if rc != 0: raise SphinxBuildError(self)
def do(args): """Main entry points.""" git_worktree = qisrc.parsers.get_git_worktree(args) snapshot = None if args.snapshot: snapshot = qisrc.snapshot.Snapshot() snapshot.load(args.snapshot) if snapshot and snapshot.format_version and snapshot.format_version >= 1: reset_manifest(git_worktree, snapshot, ignore_groups=args.ignore_groups) git_projects = qisrc.parsers.get_git_projects(git_worktree, args, default_all=True, use_build_deps=True) errors = list() for i, git_project in enumerate(git_projects): ui.info_count(i, len(git_projects), "Reset", git_project.src) src = git_project.src git = qisrc.git.Git(git_project.path) ok, message = git.require_clean_worktree() if not ok and not args.force: ui.warning(message) errors.append(src) continue if not git_project.default_branch: ui.warning(git_project.src, "not in any manifest, skipping") continue branch = git_project.default_branch.name remote = git_project.default_remote.name git.safe_checkout(branch, remote, force=True) to_reset = None if args.snapshot: to_reset = snapshot.refs.get(src) if not to_reset: ui.warning(src, "not found in the snapshot") continue elif args.tag: to_reset = args.tag else: to_reset = "%s/%s" % (remote, branch) try: qisrc.reset.clever_reset_ref(git_project, to_reset) except: errors.append(src) if not errors: return ui.error("Failed to reset some projects") for error in errors: ui.info(ui.red, " * ", error) sys.exit(1)
def tests(self): res = [x for x in self._tests if match_patterns(self.patterns, x["name"])] # Perf tests are run alone res = [x for x in res if x.get("perf", False) == self.perf] # But nightly tests are run along with the normal tests if not self.nightly: res = [x for x in res if x.get("nightly", False) is False] if not res and self.patterns: ui.error("No test found matching pattern") return res
def _load_deprecated(self, source): for line in source.splitlines(): try: (src, sha1) = line.split(":") except ValueError: ui.error("could not parse", line) continue src = src.strip() sha1 = sha1.strip() self.refs[src] = sha1
def to_test_project(self): if not os.path.exists(self.qitest_json): raise Exception("qitest.json not found. Did you run qibuild configure?") res = qitest.project.TestProject(self.qitest_json) if not res.tests: ui.error("No tests were found in qitest.json.\n" + "Add some in the CMakeLists using " + "qi_add_test() or similar") res.name = self.name return res
def to_test_project(self): if not os.path.exists(self.qitest_json): raise NoQiTestJson() res = qitest.project.TestProject(self.qitest_json) if not res.tests: ui.error("No tests were found in qitest.json.\n" + "Add some in the CMakeLists using " + "qi_add_test() or similar") res.name = self.name return res