Example #1
0
 def run(self, timeout=None):
     def target():
         ui.debug("Starting thread.")
         ui.debug("Calling:", " ".join(self.cmd))
         try:
             opts = dict()
             if os.name == 'posix':
                 opts = {
                     'preexec_fn': os.setsid,
                     'close_fds': True
                 }
             elif os.name == 'nt':
                 opts = {
                     'creationflags': subprocess.CREATE_NEW_PROCESS_GROUP,
                 }
             self._process = subprocess.Popen(self.cmd,
                 stdout=subprocess.PIPE,
                 stderr=subprocess.STDOUT,
                 cwd=self.cwd,
                 env=self.env,
                 **opts)
         except Exception, e:
             self.exception = e
             self.return_type = Process.NOT_RUN
             return
         self.out = self._process.communicate()[0]
         self.returncode = self._process.returncode
         if self.returncode == 0:
             self.return_type = Process.OK
         ui.debug("Thread terminated.")
Example #2
0
File: jar.py Project: Grimy/qibuild
def jar(jar_path, files, paths):
    """ Search each files using qibuild find and
        add them into a jar using qisys
    """

    # Create command line
    jar_path = qisys.sh.to_native_path(jar_path)
    args = ["cvfM"]
    args += [jar_path]

    if not files:
        raise Exception("Missing arguments : Files to package")
    for wanted_file in files:
        ui.info("Searching for " + wanted_file + "...")
        path = qibuild.find.find(paths, wanted_file, expect_one=False)[0]
        if not path:
            ui.error("Cannot find " + wanted_file + " in worktree")
            return None
        ui.debug("Found : " + path)
        dirname = os.path.dirname(path)
        basename = os.path.basename(path)
        args += ["-C", dirname, basename]
        ui.debug("Added -C " + dirname + " " + wanted_file +
                 " to command line")

    qisys.command.call(["jar"] + args, ignore_ret_code=False)
    return jar_path
Example #3
0
 def call(self, *args, **kwargs):
     if "cwd" not in kwargs.keys():
         kwargs["cwd"] = self.path
     ui.debug("svn", " ".join(args), "in", kwargs["cwd"])
     if "quiet" not in kwargs.keys():
         kwargs["quiet"] = False
     svn = qisys.command.find_program("svn", raises=True)
     cmd = [svn]
     cmd.extend(args)
     raises = kwargs.get("raises")
     if raises is False:
         del kwargs["raises"]
         del kwargs["quiet"]
         process = subprocess.Popen(cmd,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.STDOUT,
                                    **kwargs)
         out = process.communicate()[0]
         # Don't want useless blank lines
         out = out.rstrip("\n")
         ui.debug("out:", out)
         return process.returncode, out
     else:
         if "raises" in kwargs:
             del kwargs["raises"]
         qisys.command.call(cmd, **kwargs)
Example #4
0
def run(projects, binary, bin_args, env=None, exec_=True):
    """ Find binary in worktree and
        exec it with given arguments.
    """
    paths = list()
    for proj in projects:
        paths += [proj.sdk_directory]
    full_path = qisys.sh.to_native_path(binary)
    if os.path.isfile(full_path) and os.access(full_path, os.X_OK):
        bin_path = full_path
    else:
        bin_path = None
        candidates = qibuild.find.find_bin(paths, binary, expect_one=False)
        if len(candidates) == 1:
            bin_path = candidates[0]
        if len(candidates) > 1:
            bin_path = qisys.interact.ask_choice(
                candidates, "Please select a binary to run")
    if not bin_path:
        bin_path = qisys.command.find_program(binary)
    if not bin_path:
        raise Exception("Cannot find " + binary + " binary")
    cmd = [bin_path] + bin_args
    if exec_:
        ui.debug("exec", cmd)
        os.execve(bin_path, cmd, env)
    else:
        qisys.command.call(cmd, env=env)
Example #5
0
 def _call(self, *args, **kwargs):
     """ Helper for self.call """
     ui.debug("git", " ".join(args), "in", self.repo)
     if not "cwd" in kwargs.keys():
         kwargs["cwd"] = self.repo
     if not "quiet" in kwargs.keys():
         kwargs["quiet"] = False
     git = qisys.command.find_program("git", raises=True)
     cmd = [git]
     cmd.extend(args)
     raises = kwargs.get("raises")
     env = os.environ.copy()
     for key in env.keys():
         if key.startswith("GIT_"):
             del env[key]
     if raises is False:
         del kwargs["raises"]
         del kwargs["quiet"]
         process = subprocess.Popen(cmd,
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
             env=env,
             **kwargs)
         out = process.communicate()[0]
         # Don't want useless blank lines
         out = out.rstrip("\n")
         ui.debug("out:", out)
         return (process.returncode, out)
     else:
         if "raises" in kwargs:
             del kwargs["raises"]
         qisys.command.call(cmd, env=env, **kwargs)
Example #6
0
def find_program(executable, env=None, raises=False, build_config=None):
    """
    Get the full path of an executable by
    looking at PATH environment variable
    (and PATHEXT on windows)
    Toolchain binaries from build_config are also prepend to path.
    :return: None if program was not found,
      the full path to executable otherwise
    """
    if executable in _FIND_PROGRAM_CACHE:
        return _FIND_PROGRAM_CACHE[executable]
    res = None
    if not env:
        env = dict(qibuild.config.get_build_env())
        if not env:
            env = dict(os.environ)
    toolchain_paths = get_toolchain_binary_paths(build_config)
    if toolchain_paths:
        env["PATH"] = os.pathsep.join((toolchain_paths, env.get("PATH", "")))
    for path in env["PATH"].split(os.pathsep):
        res = _find_program_in_path(executable, path)
        if res and _is_runnable(res):
            ui.debug("Use %s from: %s" % (executable, res))
            _FIND_PROGRAM_CACHE[executable] = res
            return res
    if raises:
        raise NotInPath(executable, env=env)
    return None
Example #7
0
def _compress_tar(directory,
                  output=None,
                  algo=None,
                  quiet=True,
                  verbose=False):
    """Compress directory in a .tar.* archive

    :param directory:        directory to add to the archive
    :param archive_basepath: output archive basepath (without extension)
    :param algo:             compression method
    :param quiet:            quiet mode (print nothing)
    :param verbose:          verbose mode (print all the archive content)

    :return: path to the generated archive (archive_basepath.tar.*)

    """
    if quiet and verbose:
        mess = """Unconsistent arguments: both 'quiet' and 'verbose' options are set.
Please set only one of these two options to 'True'
"""
        raise ValueError(mess)
    ui.debug("Compressing", directory, "to", output)
    cmd = _get_tar_command("compress", algo, output, directory, quiet)
    try:
        if verbose:
            printed = qisys.command.check_output(cmd, stderr=subprocess.STDOUT)
        else:
            unused_output, printed = qisys.command.check_output_error(cmd)
    except qisys.command.CommandFailedException as err:
        mess = "Could not compress directory %s\n" % directory
        mess += "(algo: %s)\n" % algo
        mess += "Calling tar failed\n"
        mess += str(err)
        raise Exception(mess)
    return output
Example #8
0
def _compress_tar(directory, output=None, algo=None,
                  quiet=True, verbose=False, flat=False):
    """Compress directory in a .tar.* archive

    :param directory:        directory to add to the archive
    :param archive_basepath: output archive basepath (without extension)
    :param algo:             compression method
    :param quiet:            quiet mode (print nothing)
    :param verbose:          verbose mode (print all the archive content)
    :param flat:             if False, put all files in a common top dir
                             (default: False)

    :return: path to the generated archive (archive_basepath.tar.*)

    """
    if quiet and verbose:
        mess = """Unconsistent arguments: both 'quiet' and 'verbose' options are set.
Please set only one of these two options to 'True'
"""
        raise ValueError(mess)
    ui.debug("Compressing", directory, "to", output)
    cmd = _get_tar_command("compress", algo, output, directory, quiet, flat=flat)
    try:
        if verbose:
            printed = qisys.command.check_output(cmd, stderr=subprocess.STDOUT)
        else:
            unused_output, printed = qisys.command.check_output_error(cmd)
    except qisys.command.CommandFailedException as err:
        mess  = "Could not compress directory %s\n" % directory
        mess += "(algo: %s)\n" % algo
        mess += "Calling tar failed\n"
        mess += str(err)
        raise Exception(mess)
    return output
 def __init__(self, name):
     """ Toolchain Init """
     self.name = name
     self.feed_url = None
     # Used when feed_url is a git URL
     self.feed_name = None
     self.feed_branch = None
     self.config_path = qisys.sh.get_config_path("qi", "toolchains", "%s.xml" % self.name)
     self.register()
     self.load()
     self.toolchain_path = os.path.join(qisys.sh.get_share_path("qi", "toolchains"), self.name)
     self.toolchain_file = qisys.sh.get_share_path(
         "qi",
         "toolchains",
         self.name,
         "toolchain-%s.cmake" % self.name
     )
     db_path = qisys.sh.get_share_path("qi", "toolchains", "%s.xml" % self.name)
     if not os.path.exists(db_path):
         with open(db_path, "w") as fp:
             fp.write("<toolchain />")
     self.db = qitoolchain.database.DataBase(name, db_path)
     self.build_target = self.db.target
     ui.debug("Get target from database target", self.build_target)
     self.generate_toolchain_file()
Example #10
0
def find_installed_cmake_qibuild_dir(python_dir):
    ui.debug("looking for cmake code from", python_dir)
    for candidate in [
        # python in qibuild/python, cmake in qibuild/cmake
        ("..", "..", "cmake"),
        # python in lib/python-2.7/{dist,site}-packages,
        # cmake in share/cmake/
        # (default pip)
        ("..", "..", "..", "..", "share", "cmake"),
        # python in local/lib/python-2.7/{dist,site}-packages,
        # cmake in share/cmake
        # (debian's pip)
        ("..", "..", "..", "..", "..", "share", "cmake"),
        # python in Python27\Lib\{dist,site}-packages
        # cmake in Python27\share\cmake
        # (windows' pip)
        ("..", "..", "..", "share", "cmake"),
        # python in qibuild.egg/qibuild,
        # cmake in qibuild.egg/share/cmake
        # (pip with setuptools)
        ("..", "share", "cmake"),
        # pip on mac
        (sys.prefix, "share", "cmake")
        ]:

        rel_path = os.path.join(*candidate)
        res = os.path.join(python_dir, rel_path)
        res = qisys.sh.to_native_path(res)
        qibuild_config = os.path.join(res, "qibuild", "qibuild-config.cmake")
        ui.debug("trying", qibuild_config)
        if os.path.exists(qibuild_config):
            return res
Example #11
0
def find_libs(directory, info=None):
    """ Find Libs """
    lib_directory = os.path.join(directory, "lib")
    res = list()
    if not os.path.exists(lib_directory):
        return list()
    candidates = os.listdir(lib_directory)
    ui.debug("info:", info)
    for candidate in candidates:
        if info:
            clues = info.get("libs")
            clues.append(info.get("name"))
            for c in clues:
                candidate_path = "lib/{}".format(candidate)
                if candidate.endswith((".so", ".a", ".lib", ".dylib")) and \
                   candidate.lower().startswith(c.lower()) or \
                   "lib{}".format(c.lower()) in candidate.lower() or \
                   "{}lib".format(c.lower()) in candidate.lower() or \
                   "{}.".format(c.lower()) in candidate.lower():
                    ui.debug("search:", c, "found:", candidate)
                    res.append(candidate_path)
                    break
        else:
            if candidate.endswith((".so", ".a", ".lib", ".dylib")):
                res.append("lib/" + candidate)
    return sorted(res)
Example #12
0
def jar(jar_path, files, paths):
    """ Search each files using qibuild find and
        add them into a jar using qisys
    """

    # Create command line
    jar_path = qisys.sh.to_native_path(jar_path)
    args = ["cvfM"]
    args += [jar_path]

    if not files:
        raise Exception("Missing arguments : Files to package")
    for wanted_file in files:
        ui.info("Searching for " + wanted_file + "...")
        path = qibuild.find.find(paths, wanted_file, expect_one=False)[0]
        if not path:
            ui.error("Cannot find " + wanted_file + " in worktree")
            return None
        ui.debug("Found : " + path)
        dirname = os.path.dirname(path)
        basename = os.path.basename(path)
        args += ["-C", dirname, basename]
        ui.debug("Added -C " + dirname + " " + wanted_file + " to command line")

    qisys.command.call(["jar"] + args, ignore_ret_code=False)
    return jar_path
Example #13
0
def find_installed_cmake_qibuild_dir(python_dir):
    """ Find CMake QiBuild Dir """
    ui.debug("looking for cmake code from", python_dir)
    candidates = [
        # python in qibuild/python, cmake in qibuild/cmake
        ("..", "..", "cmake"),
        # python in lib/python-2.7/{dist,site}-packages,
        # cmake in share/cmake/
        # (default pip)
        ("..", "..", "..", "..", "share", "cmake"),
        # python in local/lib/python-2.7/{dist,site}-packages,
        # cmake in share/cmake
        # (debian's pip)
        ("..", "..", "..", "..", "..", "share", "cmake"),
        # python in Python27\Lib\{dist,site}-packages
        # cmake in Python27\share\cmake
        # (windows' pip)
        ("..", "..", "..", "share", "cmake"),
        # python in qibuild.egg/qibuild,
        # cmake in qibuild.egg/share/cmake
        # (pip with setuptools)
        ("..", "share", "cmake"),
        # pip on mac
        (sys.prefix, "share", "cmake")
    ]
    for candidate in candidates:
        rel_path = os.path.join(*candidate)
        res = os.path.join(python_dir, rel_path)
        res = qisys.sh.to_native_path(res)
        qibuild_config = os.path.join(res, "qibuild", "qibuild-config.cmake")
        ui.debug("trying", qibuild_config)
        if os.path.exists(qibuild_config):
            return res
    return None
Example #14
0
def _compress_zip(directory, quiet=True, verbose=False, flat=False, output=None):
    """Compress directory in a .zip file

    :param directory:        directory to add to the archive
    :param archive_basepath: output archive basepath (without extension)
    :param quiet:            quiet mode (print nothing)

    :return: path to the generated archive (archive_basepath.zip)

    """
    if quiet and verbose:
        mess = """Unconsistent arguments: both 'quiet' and 'verbose' options are set.
Please set only one of these two options to 'True'
"""
        raise ValueError(mess)
    ui.debug("Compressing", directory, "to", output)
    archive = zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED)
    for root, _, filenames in os.walk(directory):
        for filename in filenames:
            full_path = os.path.join(root, filename)
            # Do not zip ourselves
            if full_path == output:
                continue
            rel_path  = os.path.relpath(full_path, directory)
            if flat:
                arcname = rel_path
            else:
                arcname = os.path.join(os.path.basename(directory), rel_path)
            if not quiet:
                sys.stdout.write("adding {0}\n".format(rel_path))
                sys.stdout.flush()
            if not qisys.sh.broken_symlink(full_path):
                archive.write(full_path, arcname)
    archive.close()
    return output
Example #15
0
 def _call(self, *args, **kwargs):
     """ Helper for self.call """
     ui.debug("git", " ".join(args))
     if not "cwd" in kwargs.keys():
         kwargs["cwd"] = self.repo
     if not "quiet" in kwargs.keys():
         kwargs["quiet"] = False
     git = qisys.command.find_program("git", raises=True)
     cmd = [git]
     cmd.extend(args)
     raises = kwargs.get("raises")
     if raises is False:
         del kwargs["raises"]
         del kwargs["quiet"]
         process = subprocess.Popen(cmd,
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
             **kwargs)
         out = process.communicate()[0]
         # Don't want useless blank lines
         out = out.rstrip("\n")
         ui.debug("out:", out)
         return (process.returncode, out)
     else:
         if "raises" in kwargs:
             del kwargs["raises"]
         qisys.command.call(cmd, **kwargs)
Example #16
0
 def get_server_access(self, server_name):
     """ Return the access settings of a server. """
     server = self.servers.get(server_name)
     ui.debug("access for", server_name, ":", server)
     if not server:
         return None
     return server.access
Example #17
0
 def run(self, timeout=None):
     def target():
         ui.debug("Starting thread.")
         ui.debug("Calling:", subprocess.list2cmdline(self.cmd))
         try:
             opts = dict()
             if os.name == 'posix':
                 opts = {'preexec_fn': os.setsid, 'close_fds': True}
             elif os.name == 'nt':
                 opts = {
                     'creationflags': subprocess.CREATE_NEW_PROCESS_GROUP,
                 }
             self._process = subprocess.Popen(self.cmd,
                                              stdout=subprocess.PIPE,
                                              stderr=subprocess.STDOUT,
                                              cwd=self.cwd,
                                              env=self.env,
                                              **opts)
         except Exception, e:
             self.exception = e
             self.return_type = Process.NOT_RUN
             return
         self.out = self._process.communicate()[0]
         self.returncode = self._process.returncode
         if self.returncode == 0:
             ui.debug("Setting return code to Process.OK")
             self.return_type = Process.OK
         ui.debug("Thread terminated.")
Example #18
0
def dump_symbols_from_binary(binary, pool_dir, dump_syms_executable=None):
    """ Dump sympobls from the binary.
    Results can be found in
    <pool_dir>/<binary name>/<id>/<binary name>.sym

    """
    cmd = [dump_syms_executable, binary]
    ui.debug(cmd)
    process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                    stderr=subprocess.PIPE)
    (out, err) = process.communicate()
    if process.returncode != 0:
        ui.error("Failed to dump symbols", err)
        return

    # First line looks like:
    # MODULE Linux x86_64  ID  binary
    lines = out.splitlines()
    first_line = lines[0]
    uuid = first_line.split()[3]
    name = first_line.split()[4]

    to_make = os.path.join(pool_dir, name, uuid)
    qisys.sh.mkdir(to_make, recursive=True)
    with open(os.path.join(to_make, name + ".sym"), "w") as fp:
        fp.write(out)
Example #19
0
 def call(self, *args, **kwargs):
     """ Call """
     if "cwd" not in kwargs.keys():
         kwargs["cwd"] = self.path
     ui.debug("svn", " ".join(args), "in", kwargs["cwd"])
     if "quiet" not in kwargs.keys():
         kwargs["quiet"] = False
     svn = qisys.command.find_program("svn", raises=True)
     cmd = [svn]
     cmd.extend(args)
     raises = kwargs.get("raises")
     if raises is False:
         del kwargs["raises"]
         del kwargs["quiet"]
         process = subprocess.Popen(cmd,
                                    stdout=subprocess.PIPE,
                                    stderr=subprocess.STDOUT,
                                    **kwargs)
         out = process.communicate()[0]
         # Don't want useless blank lines
         out = out.rstrip("\n")
         ui.debug("out:", out)
         return process.returncode, out
     else:
         if "raises" in kwargs:
             del kwargs["raises"]
         qisys.command.call(cmd, **kwargs)
     return None
Example #20
0
def _find_program_in_toolchain_path(executable, build_config=None):
    try:
        path = build_config.toolchain.toolchain_path
    except Exception:
        return None
    executable = os.path.basename(executable)
    ui.debug("_find_program_in_toolchain_path(%s): %s" % (executable, path))
    try:
        env = {str("LANG"): str("C")}
        process = subprocess.Popen(['find', path, '-name', executable],
                                   stdout=subprocess.PIPE,
                                   env=env)
        output = process.communicate()[0]
        if six.PY3:
            output = str(output)
        splitted = output.split()
        splitted.sort(key=len)
        for p in splitted:
            ui.debug("_find_program_in_toolchain_path(%s): Testing %s" %
                     (executable, os.path.dirname(p)))
            res = _check_access(executable, os.path.dirname(p))
            if res and _is_runnable(res, build_config):
                return res
    except OSError:
        # TODO: Run an equivalent test on mac and on windows
        if sys.platform.startswith("linux"):
            ui.warning(
                "find not available => assuming {} is not in the toolchain".
                format(executable))
        return None
    return None
Example #21
0
def run(projects, binary, bin_args, env=None, exec_=True):
    """ Find binary in worktree and
        exec it with given arguments.
    """
    paths = list()
    for proj in projects:
        paths += [proj.sdk_directory]
    if os.path.exists(binary):
        bin_path = qisys.sh.to_native_path(binary)
    else:
        bin_path = None
        candidates = qibuild.find.find_bin(paths, binary, expect_one=False)
        if len(candidates) == 1:
            bin_path = candidates[0]
        if len(candidates) > 1:
            bin_path = qisys.interact.ask_choice(candidates,
                                                 "Please select a binary to run")
    if not bin_path:
        bin_path = qisys.command.find_program(binary)
    if not bin_path:
        raise Exception("Cannot find " + binary + " binary")
    cmd = [bin_path] + bin_args
    if exec_:
      ui.debug("exec", cmd)
      os.execve(bin_path,  cmd, env)
    else:
      qisys.command.call(cmd, env=env)
Example #22
0
 def on_dependent_job_finished(self, job):
     with self.lock:
         try:
             self.deps.remove(job)
         except ValueError:
             ui.debug(ui.red, "Job not in the deps list!", self.deps, job)
             pass
Example #23
0
 def on_dependent_job_finished(self, job):
     with self.lock:
         try:
             self.deps.remove(job)
         except ValueError:
             ui.debug(ui.red, "Job not in the deps list!", self.deps, job)
             pass
Example #24
0
def handle_pure_python(venv_path, python_worktree, env=None):
    """ Add the paths of all python projects to the virtualenv """
    lib_path = virtualenv.path_locations(venv_path)[1]
    qi_pth_dest = os.path.join(venv_path, lib_path, "site-packages/qi.pth")
    res = True
    with open(qi_pth_dest, "w") as fp:
        fp.write("")
        for i, project in enumerate(python_worktree.python_projects):
            ui.info_count(i, len(python_worktree.python_projects), ui.blue,
                          project.name)
            if project.setup_with_distutils:
                cmd = [python_worktree.pip, "install"]
                if not ui.CONFIG["verbose"]:
                    cmd.append("--quiet")
                cmd.extend(["--editable", "."])
                rc = qisys.command.call(cmd,
                                        cwd=project.path,
                                        ignore_ret_code=True,
                                        env=env)
                if rc != 0:
                    ui.warning("Failed to run pip install on", project.src)
                    res = False
            else:
                ui.debug("Adding python path for project", project.name, ":\n",
                         project.python_path)
                for path in project.python_path:
                    fp.write(path + "\n")
    return res
Example #25
0
 def get_server_access(self, server_name):
     """ Return the access settings of a server. """
     server = self.servers.get(server_name)
     ui.debug("access for", server_name, ":", server)
     if not server:
         return None
     return server.access
Example #26
0
 def target():
     ui.debug("Starting thread.")
     ui.debug("Calling:", subprocess.list2cmdline(self.cmd))
     try:
         opts = dict()
         if os.name == 'posix':
             opts = {
                 'preexec_fn': os.setsid,
                 'close_fds': True
             }
         elif os.name == 'nt':
             opts = {
                 'creationflags': subprocess.CREATE_NEW_PROCESS_GROUP,
             }
         kwargs = {
                 "cwd": self.cwd,
                 "env" : self.env
         }
         kwargs.update(opts)
         if self.capture:
             kwargs["stdout"] = subprocess.PIPE
             kwargs["stderr"] = subprocess.STDOUT
         self._process = subprocess.Popen(self.cmd, **kwargs)
     except Exception, e:
         self.exception = e
         self.return_type = Process.NOT_RUN
         return
Example #27
0
def handle_pure_python(venv_path, python_worktree, env=None):
    """ Add the paths of all python projects to the virtualenv """
    lib_path = virtualenv.path_locations(venv_path)[1]
    qi_pth_dest = os.path.join(venv_path, lib_path, "site-packages/qi.pth")
    res = True
    with open(qi_pth_dest, "w") as fp:
        fp.write("")
        for i, project in enumerate(python_worktree.python_projects):
            ui.info_count(i, len(python_worktree.python_projects),
                          ui.blue, project.name)
            if project.setup_with_distutils:
                cmd = [python_worktree.pip, "install"]
                if not ui.CONFIG["verbose"]:
                    cmd.append("--quiet")
                cmd.extend(["--editable", "."])
                rc = qisys.command.call(cmd, cwd=project.path, ignore_ret_code=True,
                                        env=env)
                if rc != 0:
                    ui.warning("Failed to run pip install on", project.src)
                    res = False
            else:
                ui.debug("Adding python path for project", project.name, ":\n",
                         project.python_path)
                for path in project.python_path:
                    fp.write(path + "\n")
    return res
Example #28
0
def _compress_zip(directory, quiet=True, verbose=False, display_progress=False,
                  flat=False, output=None):
    """
    Compress directory in a .zip file
    :param directory:        directory to add to the archive
    :param archive_basepath: output archive basepath (without extension)
    :param quiet:            quiet mode (print nothing)
    :return: path to the generated archive (archive_basepath.zip)
    """
    if quiet and verbose:
        mess = """Unconsistent arguments: both 'quiet' and 'verbose' options are set.
Please set only one of these two options to 'True'
"""
        raise ValueError(mess)
    ui.debug("Compressing", directory, "to", output)
    archive = zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED, allowZip64=True)
    # a list of tuple src, arcname to be added in the archive
    to_add = list()
    for root, directories, filenames in os.walk(directory):
        entries = directories
        entries.extend(filenames)
        for entry in entries:
            full_path = os.path.join(root, entry)
            # Do not zip ourselves
            if full_path == output:
                continue
            rel_path = os.path.relpath(full_path, directory)
            if flat:
                arcname = rel_path
            else:
                arcname = os.path.join(os.path.basename(directory), rel_path)
            to_add.append((full_path, arcname))
    for i, (full_path, arcname) in enumerate(to_add):
        if os.path.islink(full_path):
            content = os.readlink(full_path)  # pylint:disable=no-member
            attr = zipfile.ZipInfo(arcname)
            attr.create_system = 3
            # long type of hex val of '0xA1ED0000L',
            # say, symlink attr magic..
            attr.external_attr = 2716663808
            zip_call = archive.writestr
        elif os.path.isdir(full_path):
            continue
        else:
            attr = full_path
            content = arcname
            zip_call = archive.write
        if not quiet and not display_progress:
            rel_path = os.path.relpath(full_path, directory)
            sys.stdout.write("adding {0}\n".format(rel_path.encode('ascii', "ignore")))
            sys.stdout.flush()
        if display_progress:
            ui.info_progress(i, len(to_add), "Done")
        if six.PY3:
            zip_call(attr, content)
        else:
            zip_call(attr, content.encode('ascii', "ignore"))
    archive.close()
    return output
Example #29
0
    def install(self, destdir, prefix="/", components=None, split_debug=False):
        """ Install the project

        :param project: project name.
        :param destdir: destination. Note that when using `qibuild install`,
          we will first call `cmake` to make sure `CMAKE_INSTALL_PREFIX` is
          ``/``. But this function simply calls ``cmake --target install``
          in the simple case.
        :param runtime: Whether to install the project as a runtime
           package or not.
           (see :ref:`cmake-install` section for the details)
        :package split_debug: split the debug symbols out of the binaries
            useful for `qibuild deploy`

        """
        installed = list()
        if components is None:
            components = list()
        # DESTDIR=/tmp/foo and CMAKE_PREFIX="/usr/local" means
        # dest = /tmp/foo/usr/local
        destdir = qisys.sh.to_native_path(destdir)
        build_env = self.build_env.copy()
        build_env["DESTDIR"] = destdir
        # Must make sure prefix is not seen as an absolute path here:
        dest = os.path.join(destdir, prefix[1:])
        dest = qisys.sh.to_native_path(dest)

        cprefix = qibuild.cmake.get_cached_var(self.build_directory,
                                               "CMAKE_INSTALL_PREFIX")
        if cprefix != prefix:
            qibuild.cmake.cmake(self.path,
                                self.build_directory,
                                ['-DCMAKE_INSTALL_PREFIX=%s' % prefix],
                                clean_first=False,
                                env=build_env)
        else:
            mess = "Skipping configuration of project %s\n" % self.name
            mess += "CMAKE_INSTALL_PREFIX is already correct"
            ui.debug(mess)

        # Hack for http://www.cmake.org/Bug/print_bug_page.php?bug_id=13934
        if "Unix Makefiles" in self.cmake_generator:
            self.build(target="preinstall", env=build_env)
        if components:
            for component in components:
                files = self._install_component(destdir, component)
                installed.extend(files)
        else:
            self.build(target="install", env=build_env)
            manifest_path = os.path.join(self.build_directory,
                                         "install_manifest.txt")
            installed.extend(read_install_manifest(manifest_path))
        if "test" in components:
            self._install_qitest_json(destdir)

        if split_debug:
            self.split_debug(destdir, file_list=installed)

        return installed
Example #30
0
 def _resolve_job_build_dependencies(self, job):
     for p in job.project.build_depends:
         dep_job = self._find_job_by_name(p)
         if dep_job:
             job.add_dependency(dep_job)
         else:
             ui.debug("Job {job}: Couldn't find the job for the project dependency {dep}". \
                 format(job=job.project.name, dep=p))
Example #31
0
 def _kill_subprocess(self):
     if self._thread and self._process:
         ui.debug('Terminating process.')
         self._process.terminate()
         self.return_type = Process.TIME_OUT
         self._thread.join(5)
         if self._thread.is_alive():
             self._destroy_zombie()
Example #32
0
 def _resolve_job_build_dependencies(self, job):
     for p in job.project.build_depends:
         dep_job = self._find_job_by_name(p)
         if dep_job:
             job.add_dependency(dep_job)
         else:
             ui.debug("Job {job}: Couldn't find the job for the project dependency {dep}".
                      format(job=job.project.name, dep=p))
Example #33
0
def configure_virtualenv(config, python_worktree,  build_worktree=None,
                         remote_packages=None, site_packages=True):
    """ Main entry point. Called by ``qipy bootstrap``

    :param: remote_packages List of third-party packages to add in the virtualenv
    :param: site_packages Allow access to global site packages

    """
    ui.info(ui.green, "Configuring virtualenv for", ui.reset, ui.bold, python_worktree.root)
    if not remote_packages:
        remote_packages = list()

    # create a new virtualenv
    python_worktree.config = config
    venv_path = python_worktree.venv_path
    pip = python_worktree.pip

    try:
        virtualenv.create_environment(python_worktree.venv_path,
                                      site_packages=site_packages)
    except:
        ui.error("Failed to create virtualenv")
        return False


    ui.info("Adding python projects")
    # Write a qi.pth file containing path to C/C++ extensions and
    # path to pure python modules or packages
    pure_python_ok = handle_pure_python(venv_path, python_worktree)
    if build_worktree:
        handle_extensions(venv_path, python_worktree, build_worktree)

    ui.info("Adding other requirements: " + ", ".join(remote_packages))
    binaries_path = virtualenv.path_locations(venv_path)[-1]
    pip_binary = os.path.join(binaries_path, "pip")
    remote_ok = True
    if remote_packages:
        cmd = [pip_binary, "install"] + remote_packages
        rc = qisys.command.call(cmd, ignore_ret_code=True)
        remote_ok = (rc == 0)
    if pure_python_ok and remote_ok:
        ui.info(ui.green, "Done")
    if not pure_python_ok:
        ui.info(ui.red, "Failed to add some python projects")
    if not remote_ok:
        ui.info(ui.red, "Failed to add some third party requirements")

    requirements_ok = True
    for project in python_worktree.python_projects:
        path = os.path.join(project.path, "requirements.txt")
        if os.path.isfile( path ):
            ui.info(ui.green, " * Installing dependencies from " + path)
            cmd = [pip_binary, "install", "--requirement", path]
            rc = qisys.command.call(cmd, ignore_ret_code=True)
            requirements_ok = (rc == 0)
        else:
            ui.debug(ui.yellow, " * missing " + path)
    return (pure_python_ok and remote_ok and requirements_ok)
Example #34
0
    def install(self, destdir, prefix="/", components=None, num_jobs=1,
                split_debug=False):
        """ Install the project

        :param project: project name.
        :param destdir: destination. Note that when using `qibuild install`,
          we will first call `cmake` to make sure `CMAKE_INSTALL_PREFIX` is
          ``/``. But this function simply calls ``cmake --target install``
          in the simple case.
        :param runtime: Whether to install the project as a runtime
           package or not.
           (see :ref:`cmake-install` section for the details)
        :package split_debug: split the debug symbols out of the binaries
            useful for `qibuild deploy`

        """
        installed = list()
        if components is None:
            components = list()
        # DESTDIR=/tmp/foo and CMAKE_PREFIX="/usr/local" means
        # dest = /tmp/foo/usr/local
        destdir = qisys.sh.to_native_path(destdir)
        build_env = self.build_env.copy()
        build_env["DESTDIR"] = destdir
        # Must make sure prefix is not seen as an absolute path here:
        dest = os.path.join(destdir, prefix[1:])
        dest = qisys.sh.to_native_path(dest)

        cprefix = qibuild.cmake.get_cached_var(self.build_directory,
                                               "CMAKE_INSTALL_PREFIX")
        if cprefix != prefix:
            qibuild.cmake.cmake(self.path, self.build_directory,
                ['-DCMAKE_INSTALL_PREFIX=%s' % prefix],
                clean_first=False,
                env=build_env)
        else:
            mess = "Skipping configuration of project %s\n" % self.name
            mess += "CMAKE_INSTALL_PREFIX is already correct"
            ui.debug(mess)

        # Hack for http://www.cmake.org/Bug/print_bug_page.php?bug_id=13934
        if "Unix Makefiles" in self.cmake_generator:
            self.build(target="preinstall", num_jobs=num_jobs, env=build_env)
        if components:
            for component in components:
                files = self._install_component(destdir, component)
                installed.extend(files)
        else:
            self.build(target="install", env=build_env)
            manifest_path = os.path.join(self.build_directory, "install_manifest.txt")
            installed.extend(read_install_manifest(manifest_path, destdir))
        if "test" in components:
            self._install_qitest_json(destdir)

        if split_debug:
            self.split_debug(destdir, file_list=installed)

        return installed
Example #35
0
 def build(self, **kwargs):
     """ Run sphinx.main() with the correct arguments """
     try:
         import sphinx
     except ImportError as e:
         ui.error(e, "skipping build")
         return
     build_type = kwargs.get("build_type", None)
     language = kwargs.get("language", None)
     spellcheck = kwargs.get("spellcheck", False)
     werror = kwargs.get("werror", False)
     pdb = kwargs.get("pdb", False)
     if self.prebuild_script:
         ui.info(ui.green, "Running pre-build script:",
                 ui.white, self.prebuild_script)
         cmd = [sys.executable, self.prebuild_script]
         qisys.command.call(cmd, cwd=self.path)
         ui.info()
     self.generate_examples_zips()
     if self.translated and language and language != "en" \
             and language not in self.linguas:
         raise UnknownLingua(self, language)
     if self.translated:
         self.intl_build(language)
     qisys.sh.mkdir(self.html_dir, recursive=True)
     spell_dir = os.path.join(self.build_dir, "spellcheck")
     qisys.sh.mkdir(spell_dir, recursive=True)
     cmd = [sys.executable, "-c", self.build_dir]
     if spellcheck:
         cmd.extend(("-b", "spelling"))
     else:
         cmd.extend(("-b", "html"))
     if werror:
         cmd.append("-W")
     if language:
         cmd.append("-Dlanguage=%s" % language)
     if pdb:
         cmd.append("-P")
     cmd.append(self.source_dir)
     if spellcheck:
         cmd.append(spell_dir)
     else:
         cmd.append(self.html_dir)
     if build_type:
         os.environ["build_type"] = build_type
     ui.debug("launching:", cmd)
     rc = 0
     try:
         sphinx.main(argv=cmd)
     except SystemExit as e:
         rc = e.code
     if spellcheck:
         num_errors = get_num_spellcheck_errors(self.build_dir)
         if num_errors != 0:
             raise SphinxBuildError(self)
     if rc != 0:
         raise SphinxBuildError(self)
Example #36
0
def _compress_zip(directory,
                  quiet=True,
                  verbose=False,
                  flat=False,
                  output=None):
    """Compress directory in a .zip file

    :param directory:        directory to add to the archive
    :param archive_basepath: output archive basepath (without extension)
    :param quiet:            quiet mode (print nothing)

    :return: path to the generated archive (archive_basepath.zip)

    """

    if quiet and verbose:
        mess = """Unconsistent arguments: both 'quiet' and 'verbose' options are set.
Please set only one of these two options to 'True'
"""
        raise ValueError(mess)
    ui.debug("Compressing", directory, "to", output)
    archive = zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED)
    for root, directories, filenames in os.walk(directory):
        entries = directories
        entries.extend(filenames)
        for entry in entries:
            full_path = os.path.join(root, entry)
            # Do not zip ourselves
            if full_path == output:
                continue

            rel_path = os.path.relpath(full_path, directory)
            if flat:
                arcname = rel_path
            else:
                arcname = os.path.join(os.path.basename(directory), rel_path)
            if os.path.islink(full_path):
                content = os.readlink(full_path)
                attr = zipfile.ZipInfo(arcname)
                attr.create_system = 3
                # long type of hex val of '0xA1ED0000L',
                # say, symlink attr magic..
                attr.external_attr = 2716663808L
                zip_call = archive.writestr
            elif os.path.isdir(full_path):
                continue
            else:
                attr = full_path
                content = arcname
                zip_call = archive.write
            if not quiet:
                sys.stdout.write("adding {0}\n".format(rel_path))
                sys.stdout.flush()
            zip_call(attr, content)

    archive.close()
    return output
Example #37
0
 def build(self, **kwargs):
     """ Run sphinx.main() with the correct arguments """
     try:
         import sphinx
     except ImportError as e:
         ui.error(e, "skipping build")
         return
     build_type = kwargs.get("build_type", None)
     language = kwargs.get("language", None)
     spellcheck = kwargs.get("spellcheck", False)
     werror = kwargs.get("werror", False)
     pdb = kwargs.get("pdb", False)
     if self.prebuild_script:
         ui.info(ui.green, "Running pre-build script:", ui.white,
                 self.prebuild_script)
         cmd = [sys.executable, self.prebuild_script]
         qisys.command.call(cmd, cwd=self.path)
         ui.info()
     self.generate_examples_zips()
     if self.translated and language and language != "en" \
             and language not in self.linguas:
         raise UnknownLingua(self, language)
     if self.translated:
         self.intl_build(language)
     qisys.sh.mkdir(self.html_dir, recursive=True)
     spell_dir = os.path.join(self.build_dir, "spellcheck")
     qisys.sh.mkdir(spell_dir, recursive=True)
     cmd = [sys.executable, "-c", self.build_dir]
     if spellcheck:
         cmd.extend(("-b", "spelling"))
     else:
         cmd.extend(("-b", "html"))
     if werror:
         cmd.append("-W")
     if language:
         cmd.append("-Dlanguage=%s" % language)
     if pdb:
         cmd.append("-P")
     cmd.append(self.source_dir)
     if spellcheck:
         cmd.append(spell_dir)
     else:
         cmd.append(self.html_dir)
     if build_type:
         os.environ["build_type"] = build_type
     ui.debug("launching:", cmd)
     rc = 0
     try:
         sphinx.main(argv=cmd)
     except SystemExit as e:
         rc = e.code
     if spellcheck:
         num_errors = get_num_spellcheck_errors(self.build_dir)
         if num_errors != 0:
             raise SphinxBuildError(self)
     if rc != 0:
         raise SphinxBuildError(self)
Example #38
0
def _compress_zip(directory, quiet=True, verbose=False, display_progress=False,
                  flat=False, output=None):
    """
    Compress directory in a .zip file
    :param directory:        directory to add to the archive
    :param archive_basepath: output archive basepath (without extension)
    :param quiet:            quiet mode (print nothing)
    :return: path to the generated archive (archive_basepath.zip)
    """
    if quiet and verbose:
        mess = """Unconsistent arguments: both 'quiet' and 'verbose' options are set.
Please set only one of these two options to 'True'
"""
        raise ValueError(mess)
    ui.debug("Compressing", directory, "to", output)
    archive = zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED, allowZip64=True)
    # a list of tuple src, arcname to be added in the archive
    to_add = list()
    for root, directories, filenames in os.walk(directory):
        entries = directories
        entries.extend(filenames)
        for entry in entries:
            full_path = os.path.join(root, entry)
            # Do not zip ourselves
            if full_path == output:
                continue
            rel_path = os.path.relpath(full_path, directory)
            if flat:
                arcname = rel_path
            else:
                arcname = os.path.join(os.path.basename(directory), rel_path)
            to_add.append((full_path, arcname))
    for i, (full_path, arcname) in enumerate(to_add):
        if os.path.islink(full_path):
            content = os.readlink(full_path)  # pylint:disable=no-member
            attr = zipfile.ZipInfo(arcname)
            attr.create_system = 3
            # long type of hex val of '0xA1ED0000L',
            # say, symlink attr magic..
            attr.external_attr = 2716663808
            zip_call = archive.writestr
        elif os.path.isdir(full_path):
            continue
        else:
            attr = full_path
            content = arcname
            zip_call = archive.write
        if not quiet and not display_progress:
            rel_path = os.path.relpath(full_path, directory)
            sys.stdout.write("adding {0}\n".format(rel_path.encode('ascii', "ignore")))
            sys.stdout.flush()
        if display_progress:
            ui.info_progress(i, len(to_add), "Done")
        zip_call(attr, content.encode('ascii', "ignore"))
    archive.close()
    return output
Example #39
0
def install(src, dest, filter_fun=None, quiet=False):
    """
    Install a directory or a file to a destination.
    If filter_fun is not None, then the file will only be
    installed if filter_fun(relative/path/to/file) returns True.
    If ``dest`` does not exist, it will be created first.
    When installing files, if the destination already exists,
    it will be removed first, then overwritten by the new file.
    This function will preserve relative symlinks between directories,
    used for instance in Mac frameworks::
        |__ Versions
            |__ Current  -> 4.0
            |__ 4        -> 4.0
            |__ 4.0
    Return the list of files installed (with relative paths)
    """
    installed = list()
    # FIXME: add a `safe mode` ala install?
    if not os.path.exists(src):
        mess = "Could not install '%s' to '%s'\n" % (src, dest)
        mess += '%s does not exist' % src
        raise Exception(mess)
    src = to_native_path(src, normcase=False)
    dest = to_native_path(dest, normcase=False)
    ui.debug("Installing", src, "->", dest)
    if filter_fun is None:

        def no_filter_fun(_unused):
            """ Filter Function Always True """
            return True

        filter_fun = no_filter_fun
    if os.path.isdir(src):
        if src == dest:
            raise Exception("source and destination are the same directory")
        for (root, dirs, files) in os.walk(src):
            dirs = _handle_dirs(src, dest, root, dirs, filter_fun, quiet)
            files = _handle_files(src, dest, root, files, filter_fun, quiet)
            installed.extend(files)
    else:
        # Emulate posix `install' behavior:
        # if dest is a dir, install in the directory, else
        # simply copy the file.
        if os.path.isdir(dest):
            dest = os.path.join(dest, os.path.basename(src))
        if src == dest:
            raise Exception("source and destination are the same file")
        mkdir(os.path.dirname(dest), recursive=True)
        if sys.stdout.isatty() and not quiet:
            print("-- Installing %s" % dest)
        # We do not want to fail if dest exists but is read only
        # (following what `install` does, but not what `cp` does)
        rm(dest)
        shutil.copy(src, dest)
        installed.append(os.path.basename(src))
    return installed
Example #40
0
def dump_symbols_from_binary(binary,
                             pool_dir,
                             build_config=None,
                             dump_syms=None):
    """
    Dump symbols from the binary.
    Results can be found in
    <pool_dir>/<binary name>/<id>/<binary name>.sym
    """
    if not dump_syms:
        dump_syms = qisys.command.find_program("dump_syms",
                                               raises=True,
                                               build_config=build_config)
    if sys.platform == "darwin":
        dsym = gen_dsym(binary)
        cmd = [dump_syms, dsym]
    else:
        cmd = [dump_syms, binary]
    ui.debug("dump_symbols_from_binary() cmd: %s" % cmd)
    try:
        process = subprocess.Popen(cmd,
                                   stdout=subprocess.PIPE,
                                   stderr=subprocess.PIPE)
    except OSError as e:
        ui.error("Could not dump symbols", cmd, e)
        return False
    dump_ok = True
    (out, err) = process.communicate()
    if process.returncode != 0:
        ui.error("Failed to dump symbols", err)
        dump_ok = False
    if sys.platform == "darwin":
        qisys.sh.rm(dsym)
    if not dump_ok:
        return False
    out = out.decode("utf-8")
    # First line looks like:
    # MODULE Linux x86_64  ID  foo on linux
    # MODULE windows x86 ID foo.pdb on windows
    # path should be
    # pool/foo.pdb/ID/foo.sym on windows,
    # pool/foo/ID/foo.sym on linux
    lines = out.splitlines()
    first_line = lines[0]
    uuid = first_line.split()[3]
    name = first_line.split()[4]
    if os.name == "nt":
        basename = name.replace(".pdb", "")
    else:
        basename = name
    to_make = os.path.join(pool_dir, name, uuid)
    qisys.sh.mkdir(to_make, recursive=True)
    with open(os.path.join(to_make, basename + ".sym"), "w") as fp:
        fp.write(out)
    fp.close()
    return True
Example #41
0
def mv(src, dest):
    """ Move a file into a directory, but do not crash if dest/src exists """
    if src == dest:
        return
    if os.path.isdir(dest):
        dest = os.path.join(dest, os.path.basename(src))
    if os.path.exists(dest):
        rm(dest)
    ui.debug(src, "->", dest)
    shutil.move(src, dest)
Example #42
0
def _fix_rpaths(package_path):
    """ Search all dylib in lib directory and fix rpaths. """
    ui.info("Fix RPATH for", package_path, "librairies")
    # pylint: disable=W0612
    for root, dirs, files in os.walk(package_path):
        for basename in files:
            if basename.endswith("dylib"):
                ui.debug("Fixing RPATH for", basename)
                filename = os.path.join(root, basename)
                _fix_rpath(filename, package_path)
Example #43
0
 def package(self, *args, **kwargs):
     """
     Generate a package containing every project.
     :param: with_breakpad generate debug symbols for usage with breakpad
     :param: force make package even if it does not satisfy default package requirements
     :param install_tc_packages also install toolchain packages
     """
     output = kwargs.get('output', None)
     force = kwargs.get('force', False)
     with_breakpad = kwargs.get('with_breakpad', False)
     python_minify = kwargs.get('python_minify', False)
     strip = kwargs.get('strip', True)
     install_tc_packages = kwargs.get('install_tc_packages', False)
     strip_args = kwargs.get('strip_args', None)
     strip_exe = kwargs.get('strip_exe', None)
     # If the package is not valid, do not go further
     if not self.validator.is_valid and not force:
         raise Exception("Given package does not satisfy "
                         "default package requirements.\n"
                         "Use option '--force' to bypass this validation")
     # Make sure self.stage_path exists and is empty
     qisys.sh.rm(self.stage_path)
     qisys.sh.mkdir(self.stage_path, recursive=True)
     if not output:
         output = os.path.join(os.getcwd(), self.pkg_name + ".pkg")
     # Add everything from the staged path
     self.install(self.stage_path,
                  install_tc_packages=install_tc_packages,
                  python_minify=python_minify)
     symbols_archive = None
     if with_breakpad and self.build_project:
         ui.info(ui.bold, "-> Generating breakpad symbols ...")
         dirname = os.path.dirname(output)
         build_config = kwargs.get('build_config',
                                   self.cmake_builder.build_config)
         ui.debug("Use breakpad build_config:", build_config)
         symbols_archive = os.path.join(dirname,
                                        self.pkg_name + "-symbols.zip")
         qibuild.breakpad.gen_symbol_archive(base_dir=self.stage_path,
                                             output=symbols_archive,
                                             strip=strip,
                                             strip_exe=strip_exe,
                                             strip_args=strip_args,
                                             build_config=build_config)
         ui.info(ui.bold, "-> Symbols generated in", symbols_archive)
     ui.info(ui.bold, "-> Package generated in", output, "\n")
     ui.info(ui.bold, "-> Compressing package ...")
     qisys.archive.compress(self.stage_path,
                            output=output,
                            flat=True,
                            display_progress=True)
     qisys.sh.rm(self.stage_path)
     if symbols_archive:
         return [output, symbols_archive]
     return output
Example #44
0
 def read(self, cfg_path=None, create_if_missing=False):
     """ Read from a config location. """
     if not cfg_path:
         cfg_path = get_global_cfg_path()
     if create_if_missing:
         if not os.path.exists(cfg_path):
             dirname = os.path.dirname(cfg_path)
             qisys.sh.mkdir(dirname, recursive=True)
             with open(cfg_path, "w") as fp:
                 fp.write('<qibuild />\n')
     ui.debug("Reading config from", cfg_path)
     try:
         self.tree.parse(cfg_path)
     except Exception as e:
         mess = "Could not parse config from %s\n" % cfg_path
         mess += "Error was: %s" % str(e)
         raise Exception(mess)
     # Parse defaults:
     defaults_tree = self.tree.find("defaults")
     if defaults_tree is not None:
         self.defaults.parse(defaults_tree)
     # Parse configs:
     config_trees = self.tree.findall("config")
     for config_tree in config_trees:
         config = BuildConfig()
         config.parse(config_tree)
         self.configs[config.name] = config
     # Parse IDES:
     ide_trees = self.tree.findall("ide")
     for ide_tree in ide_trees:
         ide = IDE()
         ide.parse(ide_tree)
         self.ides[ide.name] = ide
     # Parse servers:
     server_trees = self.tree.findall("server")
     for server_tree in server_trees:
         server = Server()
         server.parse(server_tree)
         self.servers[server.name] = server
     # Parse worktrees
     worktree_trees = self.tree.findall("worktree")
     for worktree_tree in worktree_trees:
         worktree = WorkTree()
         worktree.parse(worktree_tree)
         self.worktrees[worktree.path] = worktree
     self.cmake.generator = self.defaults.cmake.generator
     self.env.bat_file = self.defaults.env.bat_file
     self.env.editor = self.defaults.env.editor
     self.env.path = self.defaults.env.path
     self.ide = None
     current_ide = self.defaults.ide
     if current_ide:
         matching_ide = self.ides.get(current_ide)
         if matching_ide:
             self.ide = matching_ide
Example #45
0
 def read(self, cfg_path=None, create_if_missing=False):
     """ Read from a config location. """
     if not cfg_path:
         cfg_path = get_global_cfg_path()
     if create_if_missing:
         if not os.path.exists(cfg_path):
             dirname = os.path.dirname(cfg_path)
             qisys.sh.mkdir(dirname, recursive=True)
             with open(cfg_path, "w") as fp:
                 fp.write('<qibuild />\n')
     ui.debug("Reading config from", cfg_path)
     try:
         self.tree.parse(cfg_path)
     except Exception as e:
         mess = "Could not parse config from %s\n" % cfg_path
         mess += "Error was: %s" % str(e)
         raise Exception(mess)
     # Parse defaults:
     defaults_tree = self.tree.find("defaults")
     if defaults_tree is not None:
         self.defaults.parse(defaults_tree)
     # Parse configs:
     config_trees = self.tree.findall("config")
     for config_tree in config_trees:
         config = BuildConfig()
         config.parse(config_tree)
         self.configs[config.name] = config
     # Parse IDES:
     ide_trees = self.tree.findall("ide")
     for ide_tree in ide_trees:
         ide = IDE()
         ide.parse(ide_tree)
         self.ides[ide.name] = ide
     # Parse servers:
     server_trees = self.tree.findall("server")
     for server_tree in server_trees:
         server = Server()
         server.parse(server_tree)
         self.servers[server.name] = server
     # Parse worktrees
     worktree_trees = self.tree.findall("worktree")
     for worktree_tree in worktree_trees:
         worktree = WorkTree()
         worktree.parse(worktree_tree)
         self.worktrees[worktree.path] = worktree
     self.cmake.generator = self.defaults.cmake.generator
     self.env.bat_file = self.defaults.env.bat_file
     self.env.editor = self.defaults.env.editor
     self.env.path = self.defaults.env.path
     self.ide = None
     current_ide = self.defaults.ide
     if current_ide:
         matching_ide = self.ides.get(current_ide)
         if matching_ide:
             self.ide = matching_ide
Example #46
0
def mv(src, dest):
    """Move a file into a directory, but do not crash
    if dest/src exists

    """
    if os.path.isdir(dest):
        dest = os.path.join(dest, os.path.basename(src))
    if os.path.exists(dest):
        rm(dest)
    ui.debug(src, "->", dest)
    shutil.move(src, dest)
Example #47
0
 def _destroy_zombie(self):
     ui.debug('Process was a zombie...')
     if not self._process:
         pass
     elif os.name == 'posix':
         os.killpg(self._process.pid, signal.SIGKILL)
     elif os.name == 'nt':
         # pylint: disable-msg=E1101
         os.kill(self._process.pid, signal.CTRL_BREAK_EVENT)
     self.return_type = Process.ZOMBIE
     self._thread.join()
Example #48
0
def _dump_arguments(name, args):
    """ Dump an argparser namespace to log """
    output = ""
    keys = sorted(args.__dict__.keys())
    max_len = max(len(k) for k in keys)
    for k in keys:
        value = args.__dict__[k]
        output += "  " + k.ljust(max_len) + " = %s\n" % (value, )
    if output[-1] == "\n":
        output = output[:-1]
    ui.debug("[%s] arguments:\n%s" % (name, output))
Example #49
0
def run_action(module_name, args=None, forward_args=None):
    """
    Run an action using its module path and a list of arguments.
    If forward_args is given, it must be an argparse.Namespace object.
    This namespace will be merged with args before being
    passed to the do() method of module_name.
    """
    if not args:
        args = list()
    ui.debug("running", module_name, " ".join(args))
    action_name = module_name.split(".")[-1]
    package_name = ".".join(module_name.split(".")[:-1])
    if not isinstance(action_name, str):
        action_name = str(action_name)
    try:
        _tmp = __import__(package_name, globals(), locals(), [action_name])
    except ImportError as err:
        raise InvalidAction(module_name, str(err))
    try:
        module = getattr(_tmp, action_name)
    except AttributeError as err:
        raise InvalidAction(
            module_name, "Could not find module %s in package %s" %
            (module_name, package_name))
    check_module(module)
    parser = argparse.ArgumentParser()
    module.configure_parser(parser)

    # Quick hack to prevent argparse.parse_args to
    #  - print usage to the console
    #  - call SystemExit
    # Instead, raise a nice Exception

    def custom_exit():
        """ Custom Exit """
        return

    parser.exit = custom_exit

    def error(message):
        """ Raise an Error Message """
        mess = "Invalid arguments when calling run_action(%s)\n" % module_name
        mess += message + "\n"
        mess += "args: %s\n" % " ".join(args)
        mess += "forward_args: %s\n" % forward_args
        raise Exception(mess)

    parser.error = error
    if forward_args:
        parsed_args = parser.parse_args(args=args,
                                        namespace=copy.deepcopy(forward_args))
    else:
        parsed_args = parser.parse_args(args=args)
    return module.do(parsed_args)
Example #50
0
def _dump_arguments(name, args):
    """ Dump an argparser namespace to log """
    output = ""
    keys = sorted(args.__dict__.keys())
    max_len = max(len(k) for k in keys)
    for k in keys:
        value = args.__dict__[k]
        output += "  " + k.ljust(max_len) + " = %s\n" % (value,)
    if output[-1] == "\n":
        output = output[:-1]
    ui.debug("[%s] arguments:\n%s" % (name, output))
Example #51
0
 def save(self):
     """ Save the packages in the xml file """
     root = etree.Element("toolchain")
     if self.build_target:
         root.set("target", self.build_target)
         ui.debug("Save target in database xml", self.build_target, "in",
                  self.db_path)
     tree = etree.ElementTree(root)
     for package in self.packages.values():
         element = package.to_xml()
         root.append(element)
     qisys.qixml.write(tree, self.db_path)
Example #52
0
 def load(self):
     """ Load the packages from the xml file """
     tree = qisys.qixml.read(self.db_path)
     self.build_target = tree.getroot().get("target")
     ui.debug("Load target from database xml", self.build_target, "in",
              self.db_path)
     for element in tree.findall("package"):
         to_add = qitoolchain.qipackage.from_xml(element)
         self.packages[to_add.name] = to_add
     for svn_elem in tree.findall("svn_package"):
         to_add = qitoolchain.qipackage.from_xml(svn_elem)
         self.packages[to_add.name] = to_add
Example #53
0
def main():
    ui.info(ui.red, "This is a an error message\n",
        ui.reset, "And here are the details")
    ui.error("could not build")
    ui.warning("-j ignored for this generator")
    ui.info("building foo")
    ui.debug("debug message")
    ui.info(ui.brown, "this is brown")
    ui.info(ui.bold, ui.brown, "this is bold brown")
    ui.info(ui.red, "red is dead")
    ui.info(ui.darkred, "darkred is really dead")
    ui.info(ui.yellow, "this is yellow")
Example #54
0
def write_file_if_different(data, out_path, mode="w"):
    """ Write the data to out_path if the content is different
    """
    try:
        with open(out_path, "r") as outr:
            out_prev = outr.read()
        if out_prev == data:
            ui.debug("skipping write to %s: same content" % (out_path))
            return
    except:
        pass
    with open(out_path, mode) as out_file:
        out_file.write(data)
Example #55
0
def _dump_arguments(name, args):
    """ Dump an argparser namespace to log """
    output = ""
    max_len = 0
    for k in args.__dict__.keys():
        if len(k) > max_len:
            max_len = len(k)
    for k, v in args.__dict__.iteritems():
        pad = " " * (max_len - len(k))
        output += "  %s%s = %s\n" % (str(k), pad, str(v))
    if output[-1] == "\n":
        output = output[:-1]
    ui.debug("[%s] arguments:\n%s" % (name, output))
Example #56
0
def dump_symbols_from_binary(binary, pool_dir):
    """ Dump sympobls from the binary.
    Results can be found in
    <pool_dir>/<binary name>/<id>/<binary name>.sym

    """
    dump_syms = qisys.command.find_program("dump_syms", raises=True)
    if sys.platform == "darwin":
        dsym = gen_dsym(binary)
        cmd = [dump_syms, dsym]
    else:
        cmd = [dump_syms, binary]
    ui.debug(cmd)
    with qisys.sh.PreserveFileMetadata(binary):
        mode_rw = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO
        os.chmod(binary, mode_rw)
        process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                        stderr=subprocess.PIPE)

    dump_ok = True
    (out, err) = process.communicate()
    if process.returncode != 0:
        ui.error("Failed to dump symbols", err)
        dump_ok = False

    if sys.platform == "darwin":
        qisys.sh.rm(dsym)

    if not dump_ok:
        return

    # First line looks like:
    # MODULE Linux x86_64  ID  foo on linux
    # MODULE windows x86 ID foo.pdb on windows
    # path should be
    # pool/foo.pdb/ID/foo.sym on windows,
    # pool/foo/ID/foo.sym on linux
    lines = out.splitlines()
    first_line = lines[0]
    uuid = first_line.split()[3]
    name = first_line.split()[4]
    if os.name == "nt":
        basename = name.replace(".pdb", "")
    else:
        basename = name

    to_make = os.path.join(pool_dir, name, uuid)
    qisys.sh.mkdir(to_make, recursive=True)

    with open(os.path.join(to_make, basename + ".sym"), "w") as fp:
        fp.write(out)
Example #57
0
def run_action(module_name, args=None, forward_args=None):
    """
    Run an action using its module path and a list of arguments.
    If forward_args is given, it must be an argparse.Namespace object.
    This namespace will be merged with args before being
    passed to the do() method of module_name.
    """
    if not args:
        args = list()
    ui.debug("running", module_name, " ".join(args))
    action_name = module_name.split(".")[-1]
    package_name = ".".join(module_name.split(".")[:-1])
    if not isinstance(action_name, str):
        action_name = str(action_name)
    try:
        _tmp = __import__(package_name, globals(), locals(), [action_name])
    except ImportError as err:
        raise InvalidAction(module_name, str(err))
    try:
        module = getattr(_tmp, action_name)
    except AttributeError as err:
        raise InvalidAction(module_name, "Could not find module %s in package %s" %
                            (module_name, package_name))
    check_module(module)
    parser = argparse.ArgumentParser()
    module.configure_parser(parser)
    # Quick hack to prevent argparse.parse_args to
    #  - print usage to the console
    #  - call SystemExit
    # Instead, raise a nice Exception

    def custom_exit():
        """ Custom Exit """
        return

    parser.exit = custom_exit

    def error(message):
        """ Raise an Error Message """
        mess = "Invalid arguments when calling run_action(%s)\n" % module_name
        mess += message + "\n"
        mess += "args: %s\n" % " ".join(args)
        mess += "forward_args: %s\n" % forward_args
        raise Exception(mess)

    parser.error = error
    if forward_args:
        parsed_args = parser.parse_args(args=args, namespace=copy.deepcopy(forward_args))
    else:
        parsed_args = parser.parse_args(args=args)
    return module.do(parsed_args)