def run(self, timeout=None):
def target():
ui.debug("Starting thread.")
ui.debug("Calling:", " ".join(self.cmd))
try:
opts = dict()
if os.name == 'posix':
opts = {
'preexec_fn': os.setsid,
'close_fds': True
}
elif os.name == 'nt':
opts = {
'creationflags': subprocess.CREATE_NEW_PROCESS_GROUP,
}
self._process = subprocess.Popen(self.cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=self.cwd,
env=self.env,
**opts)
except Exception, e:
self.exception = e
self.return_type = Process.NOT_RUN
return
self.out = self._process.communicate()[0]
self.returncode = self._process.returncode
if self.returncode == 0:
self.return_type = Process.OK
ui.debug("Thread terminated.")
def jar(jar_path, files, paths):
""" Search each files using qibuild find and
add them into a jar using qisys
"""
# Create command line
jar_path = qisys.sh.to_native_path(jar_path)
args = ["cvfM"]
args += [jar_path]
if not files:
raise Exception("Missing arguments : Files to package")
for wanted_file in files:
ui.info("Searching for " + wanted_file + "...")
path = qibuild.find.find(paths, wanted_file, expect_one=False)[0]
if not path:
ui.error("Cannot find " + wanted_file + " in worktree")
return None
ui.debug("Found : " + path)
dirname = os.path.dirname(path)
basename = os.path.basename(path)
args += ["-C", dirname, basename]
ui.debug("Added -C " + dirname + " " + wanted_file +
" to command line")
qisys.command.call(["jar"] + args, ignore_ret_code=False)
return jar_path
def call(self, *args, **kwargs):
if "cwd" not in kwargs.keys():
kwargs["cwd"] = self.path
ui.debug("svn", " ".join(args), "in", kwargs["cwd"])
if "quiet" not in kwargs.keys():
kwargs["quiet"] = False
svn = qisys.command.find_program("svn", raises=True)
cmd = [svn]
cmd.extend(args)
raises = kwargs.get("raises")
if raises is False:
del kwargs["raises"]
del kwargs["quiet"]
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
**kwargs)
out = process.communicate()[0]
# Don't want useless blank lines
out = out.rstrip("\n")
ui.debug("out:", out)
return process.returncode, out
else:
if "raises" in kwargs:
del kwargs["raises"]
qisys.command.call(cmd, **kwargs)
def run(projects, binary, bin_args, env=None, exec_=True):
""" Find binary in worktree and
exec it with given arguments.
"""
paths = list()
for proj in projects:
paths += [proj.sdk_directory]
full_path = qisys.sh.to_native_path(binary)
if os.path.isfile(full_path) and os.access(full_path, os.X_OK):
bin_path = full_path
else:
bin_path = None
candidates = qibuild.find.find_bin(paths, binary, expect_one=False)
if len(candidates) == 1:
bin_path = candidates[0]
if len(candidates) > 1:
bin_path = qisys.interact.ask_choice(
candidates, "Please select a binary to run")
if not bin_path:
bin_path = qisys.command.find_program(binary)
if not bin_path:
raise Exception("Cannot find " + binary + " binary")
cmd = [bin_path] + bin_args
if exec_:
ui.debug("exec", cmd)
os.execve(bin_path, cmd, env)
else:
qisys.command.call(cmd, env=env)
def _call(self, *args, **kwargs):
""" Helper for self.call """
ui.debug("git", " ".join(args), "in", self.repo)
if not "cwd" in kwargs.keys():
kwargs["cwd"] = self.repo
if not "quiet" in kwargs.keys():
kwargs["quiet"] = False
git = qisys.command.find_program("git", raises=True)
cmd = [git]
cmd.extend(args)
raises = kwargs.get("raises")
env = os.environ.copy()
for key in env.keys():
if key.startswith("GIT_"):
del env[key]
if raises is False:
del kwargs["raises"]
del kwargs["quiet"]
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env,
**kwargs)
out = process.communicate()[0]
# Don't want useless blank lines
out = out.rstrip("\n")
ui.debug("out:", out)
return (process.returncode, out)
else:
if "raises" in kwargs:
del kwargs["raises"]
qisys.command.call(cmd, env=env, **kwargs)
def find_program(executable, env=None, raises=False, build_config=None):
"""
Get the full path of an executable by
looking at PATH environment variable
(and PATHEXT on windows)
Toolchain binaries from build_config are also prepend to path.
:return: None if program was not found,
the full path to executable otherwise
"""
if executable in _FIND_PROGRAM_CACHE:
return _FIND_PROGRAM_CACHE[executable]
res = None
if not env:
env = dict(qibuild.config.get_build_env())
if not env:
env = dict(os.environ)
toolchain_paths = get_toolchain_binary_paths(build_config)
if toolchain_paths:
env["PATH"] = os.pathsep.join((toolchain_paths, env.get("PATH", "")))
for path in env["PATH"].split(os.pathsep):
res = _find_program_in_path(executable, path)
if res and _is_runnable(res):
ui.debug("Use %s from: %s" % (executable, res))
_FIND_PROGRAM_CACHE[executable] = res
return res
if raises:
raise NotInPath(executable, env=env)
return None
def _compress_tar(directory,
output=None,
algo=None,
quiet=True,
verbose=False):
"""Compress directory in a .tar.* archive
:param directory: directory to add to the archive
:param archive_basepath: output archive basepath (without extension)
:param algo: compression method
:param quiet: quiet mode (print nothing)
:param verbose: verbose mode (print all the archive content)
:return: path to the generated archive (archive_basepath.tar.*)
"""
if quiet and verbose:
mess = """Unconsistent arguments: both 'quiet' and 'verbose' options are set.
Please set only one of these two options to 'True'
"""
raise ValueError(mess)
ui.debug("Compressing", directory, "to", output)
cmd = _get_tar_command("compress", algo, output, directory, quiet)
try:
if verbose:
printed = qisys.command.check_output(cmd, stderr=subprocess.STDOUT)
else:
unused_output, printed = qisys.command.check_output_error(cmd)
except qisys.command.CommandFailedException as err:
mess = "Could not compress directory %s\n" % directory
mess += "(algo: %s)\n" % algo
mess += "Calling tar failed\n"
mess += str(err)
raise Exception(mess)
return output
def _compress_tar(directory, output=None, algo=None,
quiet=True, verbose=False, flat=False):
"""Compress directory in a .tar.* archive
:param directory: directory to add to the archive
:param archive_basepath: output archive basepath (without extension)
:param algo: compression method
:param quiet: quiet mode (print nothing)
:param verbose: verbose mode (print all the archive content)
:param flat: if False, put all files in a common top dir
(default: False)
:return: path to the generated archive (archive_basepath.tar.*)
"""
if quiet and verbose:
mess = """Unconsistent arguments: both 'quiet' and 'verbose' options are set.
Please set only one of these two options to 'True'
"""
raise ValueError(mess)
ui.debug("Compressing", directory, "to", output)
cmd = _get_tar_command("compress", algo, output, directory, quiet, flat=flat)
try:
if verbose:
printed = qisys.command.check_output(cmd, stderr=subprocess.STDOUT)
else:
unused_output, printed = qisys.command.check_output_error(cmd)
except qisys.command.CommandFailedException as err:
mess = "Could not compress directory %s\n" % directory
mess += "(algo: %s)\n" % algo
mess += "Calling tar failed\n"
mess += str(err)
raise Exception(mess)
return output
def __init__(self, name):
""" Toolchain Init """
self.name = name
self.feed_url = None
# Used when feed_url is a git URL
self.feed_name = None
self.feed_branch = None
self.config_path = qisys.sh.get_config_path("qi", "toolchains", "%s.xml" % self.name)
self.register()
self.load()
self.toolchain_path = os.path.join(qisys.sh.get_share_path("qi", "toolchains"), self.name)
self.toolchain_file = qisys.sh.get_share_path(
"qi",
"toolchains",
self.name,
"toolchain-%s.cmake" % self.name
)
db_path = qisys.sh.get_share_path("qi", "toolchains", "%s.xml" % self.name)
if not os.path.exists(db_path):
with open(db_path, "w") as fp:
fp.write("<toolchain />")
self.db = qitoolchain.database.DataBase(name, db_path)
self.build_target = self.db.target
ui.debug("Get target from database target", self.build_target)
self.generate_toolchain_file()
def find_installed_cmake_qibuild_dir(python_dir):
ui.debug("looking for cmake code from", python_dir)
for candidate in [
# python in qibuild/python, cmake in qibuild/cmake
("..", "..", "cmake"),
# python in lib/python-2.7/{dist,site}-packages,
# cmake in share/cmake/
# (default pip)
("..", "..", "..", "..", "share", "cmake"),
# python in local/lib/python-2.7/{dist,site}-packages,
# cmake in share/cmake
# (debian's pip)
("..", "..", "..", "..", "..", "share", "cmake"),
# python in Python27\Lib\{dist,site}-packages
# cmake in Python27\share\cmake
# (windows' pip)
("..", "..", "..", "share", "cmake"),
# python in qibuild.egg/qibuild,
# cmake in qibuild.egg/share/cmake
# (pip with setuptools)
("..", "share", "cmake"),
# pip on mac
(sys.prefix, "share", "cmake")
]:
rel_path = os.path.join(*candidate)
res = os.path.join(python_dir, rel_path)
res = qisys.sh.to_native_path(res)
qibuild_config = os.path.join(res, "qibuild", "qibuild-config.cmake")
ui.debug("trying", qibuild_config)
if os.path.exists(qibuild_config):
return res
def find_libs(directory, info=None):
""" Find Libs """
lib_directory = os.path.join(directory, "lib")
res = list()
if not os.path.exists(lib_directory):
return list()
candidates = os.listdir(lib_directory)
ui.debug("info:", info)
for candidate in candidates:
if info:
clues = info.get("libs")
clues.append(info.get("name"))
for c in clues:
candidate_path = "lib/{}".format(candidate)
if candidate.endswith((".so", ".a", ".lib", ".dylib")) and \
candidate.lower().startswith(c.lower()) or \
"lib{}".format(c.lower()) in candidate.lower() or \
"{}lib".format(c.lower()) in candidate.lower() or \
"{}.".format(c.lower()) in candidate.lower():
ui.debug("search:", c, "found:", candidate)
res.append(candidate_path)
break
else:
if candidate.endswith((".so", ".a", ".lib", ".dylib")):
res.append("lib/" + candidate)
return sorted(res)
def jar(jar_path, files, paths):
""" Search each files using qibuild find and
add them into a jar using qisys
"""
# Create command line
jar_path = qisys.sh.to_native_path(jar_path)
args = ["cvfM"]
args += [jar_path]
if not files:
raise Exception("Missing arguments : Files to package")
for wanted_file in files:
ui.info("Searching for " + wanted_file + "...")
path = qibuild.find.find(paths, wanted_file, expect_one=False)[0]
if not path:
ui.error("Cannot find " + wanted_file + " in worktree")
return None
ui.debug("Found : " + path)
dirname = os.path.dirname(path)
basename = os.path.basename(path)
args += ["-C", dirname, basename]
ui.debug("Added -C " + dirname + " " + wanted_file + " to command line")
qisys.command.call(["jar"] + args, ignore_ret_code=False)
return jar_path
def find_installed_cmake_qibuild_dir(python_dir):
""" Find CMake QiBuild Dir """
ui.debug("looking for cmake code from", python_dir)
candidates = [
# python in qibuild/python, cmake in qibuild/cmake
("..", "..", "cmake"),
# python in lib/python-2.7/{dist,site}-packages,
# cmake in share/cmake/
# (default pip)
("..", "..", "..", "..", "share", "cmake"),
# python in local/lib/python-2.7/{dist,site}-packages,
# cmake in share/cmake
# (debian's pip)
("..", "..", "..", "..", "..", "share", "cmake"),
# python in Python27\Lib\{dist,site}-packages
# cmake in Python27\share\cmake
# (windows' pip)
("..", "..", "..", "share", "cmake"),
# python in qibuild.egg/qibuild,
# cmake in qibuild.egg/share/cmake
# (pip with setuptools)
("..", "share", "cmake"),
# pip on mac
(sys.prefix, "share", "cmake")
]
for candidate in candidates:
rel_path = os.path.join(*candidate)
res = os.path.join(python_dir, rel_path)
res = qisys.sh.to_native_path(res)
qibuild_config = os.path.join(res, "qibuild", "qibuild-config.cmake")
ui.debug("trying", qibuild_config)
if os.path.exists(qibuild_config):
return res
return None
def _compress_zip(directory, quiet=True, verbose=False, flat=False, output=None):
"""Compress directory in a .zip file
:param directory: directory to add to the archive
:param archive_basepath: output archive basepath (without extension)
:param quiet: quiet mode (print nothing)
:return: path to the generated archive (archive_basepath.zip)
"""
if quiet and verbose:
mess = """Unconsistent arguments: both 'quiet' and 'verbose' options are set.
Please set only one of these two options to 'True'
"""
raise ValueError(mess)
ui.debug("Compressing", directory, "to", output)
archive = zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED)
for root, _, filenames in os.walk(directory):
for filename in filenames:
full_path = os.path.join(root, filename)
# Do not zip ourselves
if full_path == output:
continue
rel_path = os.path.relpath(full_path, directory)
if flat:
arcname = rel_path
else:
arcname = os.path.join(os.path.basename(directory), rel_path)
if not quiet:
sys.stdout.write("adding {0}\n".format(rel_path))
sys.stdout.flush()
if not qisys.sh.broken_symlink(full_path):
archive.write(full_path, arcname)
archive.close()
return output
def _call(self, *args, **kwargs):
""" Helper for self.call """
ui.debug("git", " ".join(args))
if not "cwd" in kwargs.keys():
kwargs["cwd"] = self.repo
if not "quiet" in kwargs.keys():
kwargs["quiet"] = False
git = qisys.command.find_program("git", raises=True)
cmd = [git]
cmd.extend(args)
raises = kwargs.get("raises")
if raises is False:
del kwargs["raises"]
del kwargs["quiet"]
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
**kwargs)
out = process.communicate()[0]
# Don't want useless blank lines
out = out.rstrip("\n")
ui.debug("out:", out)
return (process.returncode, out)
else:
if "raises" in kwargs:
del kwargs["raises"]
qisys.command.call(cmd, **kwargs)
def get_server_access(self, server_name):
""" Return the access settings of a server. """
server = self.servers.get(server_name)
ui.debug("access for", server_name, ":", server)
if not server:
return None
return server.access
def run(self, timeout=None):
def target():
ui.debug("Starting thread.")
ui.debug("Calling:", subprocess.list2cmdline(self.cmd))
try:
opts = dict()
if os.name == 'posix':
opts = {'preexec_fn': os.setsid, 'close_fds': True}
elif os.name == 'nt':
opts = {
'creationflags': subprocess.CREATE_NEW_PROCESS_GROUP,
}
self._process = subprocess.Popen(self.cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=self.cwd,
env=self.env,
**opts)
except Exception, e:
self.exception = e
self.return_type = Process.NOT_RUN
return
self.out = self._process.communicate()[0]
self.returncode = self._process.returncode
if self.returncode == 0:
ui.debug("Setting return code to Process.OK")
self.return_type = Process.OK
ui.debug("Thread terminated.")
def dump_symbols_from_binary(binary, pool_dir, dump_syms_executable=None):
""" Dump sympobls from the binary.
Results can be found in
<pool_dir>/<binary name>/<id>/<binary name>.sym
"""
cmd = [dump_syms_executable, binary]
ui.debug(cmd)
process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
(out, err) = process.communicate()
if process.returncode != 0:
ui.error("Failed to dump symbols", err)
return
# First line looks like:
# MODULE Linux x86_64 ID binary
lines = out.splitlines()
first_line = lines[0]
uuid = first_line.split()[3]
name = first_line.split()[4]
to_make = os.path.join(pool_dir, name, uuid)
qisys.sh.mkdir(to_make, recursive=True)
with open(os.path.join(to_make, name + ".sym"), "w") as fp:
fp.write(out)
def call(self, *args, **kwargs):
""" Call """
if "cwd" not in kwargs.keys():
kwargs["cwd"] = self.path
ui.debug("svn", " ".join(args), "in", kwargs["cwd"])
if "quiet" not in kwargs.keys():
kwargs["quiet"] = False
svn = qisys.command.find_program("svn", raises=True)
cmd = [svn]
cmd.extend(args)
raises = kwargs.get("raises")
if raises is False:
del kwargs["raises"]
del kwargs["quiet"]
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
**kwargs)
out = process.communicate()[0]
# Don't want useless blank lines
out = out.rstrip("\n")
ui.debug("out:", out)
return process.returncode, out
else:
if "raises" in kwargs:
del kwargs["raises"]
qisys.command.call(cmd, **kwargs)
return None
def _find_program_in_toolchain_path(executable, build_config=None):
try:
path = build_config.toolchain.toolchain_path
except Exception:
return None
executable = os.path.basename(executable)
ui.debug("_find_program_in_toolchain_path(%s): %s" % (executable, path))
try:
env = {str("LANG"): str("C")}
process = subprocess.Popen(['find', path, '-name', executable],
stdout=subprocess.PIPE,
env=env)
output = process.communicate()[0]
if six.PY3:
output = str(output)
splitted = output.split()
splitted.sort(key=len)
for p in splitted:
ui.debug("_find_program_in_toolchain_path(%s): Testing %s" %
(executable, os.path.dirname(p)))
res = _check_access(executable, os.path.dirname(p))
if res and _is_runnable(res, build_config):
return res
except OSError:
# TODO: Run an equivalent test on mac and on windows
if sys.platform.startswith("linux"):
ui.warning(
"find not available => assuming {} is not in the toolchain".
format(executable))
return None
return None
def run(projects, binary, bin_args, env=None, exec_=True):
""" Find binary in worktree and
exec it with given arguments.
"""
paths = list()
for proj in projects:
paths += [proj.sdk_directory]
if os.path.exists(binary):
bin_path = qisys.sh.to_native_path(binary)
else:
bin_path = None
candidates = qibuild.find.find_bin(paths, binary, expect_one=False)
if len(candidates) == 1:
bin_path = candidates[0]
if len(candidates) > 1:
bin_path = qisys.interact.ask_choice(candidates,
"Please select a binary to run")
if not bin_path:
bin_path = qisys.command.find_program(binary)
if not bin_path:
raise Exception("Cannot find " + binary + " binary")
cmd = [bin_path] + bin_args
if exec_:
ui.debug("exec", cmd)
os.execve(bin_path, cmd, env)
else:
qisys.command.call(cmd, env=env)
def on_dependent_job_finished(self, job):
with self.lock:
try:
self.deps.remove(job)
except ValueError:
ui.debug(ui.red, "Job not in the deps list!", self.deps, job)
pass
def on_dependent_job_finished(self, job):
with self.lock:
try:
self.deps.remove(job)
except ValueError:
ui.debug(ui.red, "Job not in the deps list!", self.deps, job)
pass
def handle_pure_python(venv_path, python_worktree, env=None):
""" Add the paths of all python projects to the virtualenv """
lib_path = virtualenv.path_locations(venv_path)[1]
qi_pth_dest = os.path.join(venv_path, lib_path, "site-packages/qi.pth")
res = True
with open(qi_pth_dest, "w") as fp:
fp.write("")
for i, project in enumerate(python_worktree.python_projects):
ui.info_count(i, len(python_worktree.python_projects), ui.blue,
project.name)
if project.setup_with_distutils:
cmd = [python_worktree.pip, "install"]
if not ui.CONFIG["verbose"]:
cmd.append("--quiet")
cmd.extend(["--editable", "."])
rc = qisys.command.call(cmd,
cwd=project.path,
ignore_ret_code=True,
env=env)
if rc != 0:
ui.warning("Failed to run pip install on", project.src)
res = False
else:
ui.debug("Adding python path for project", project.name, ":\n",
project.python_path)
for path in project.python_path:
fp.write(path + "\n")
return res
def get_server_access(self, server_name):
""" Return the access settings of a server. """
server = self.servers.get(server_name)
ui.debug("access for", server_name, ":", server)
if not server:
return None
return server.access
def target():
ui.debug("Starting thread.")
ui.debug("Calling:", subprocess.list2cmdline(self.cmd))
try:
opts = dict()
if os.name == 'posix':
opts = {
'preexec_fn': os.setsid,
'close_fds': True
}
elif os.name == 'nt':
opts = {
'creationflags': subprocess.CREATE_NEW_PROCESS_GROUP,
}
kwargs = {
"cwd": self.cwd,
"env" : self.env
}
kwargs.update(opts)
if self.capture:
kwargs["stdout"] = subprocess.PIPE
kwargs["stderr"] = subprocess.STDOUT
self._process = subprocess.Popen(self.cmd, **kwargs)
except Exception, e:
self.exception = e
self.return_type = Process.NOT_RUN
return
def handle_pure_python(venv_path, python_worktree, env=None):
""" Add the paths of all python projects to the virtualenv """
lib_path = virtualenv.path_locations(venv_path)[1]
qi_pth_dest = os.path.join(venv_path, lib_path, "site-packages/qi.pth")
res = True
with open(qi_pth_dest, "w") as fp:
fp.write("")
for i, project in enumerate(python_worktree.python_projects):
ui.info_count(i, len(python_worktree.python_projects),
ui.blue, project.name)
if project.setup_with_distutils:
cmd = [python_worktree.pip, "install"]
if not ui.CONFIG["verbose"]:
cmd.append("--quiet")
cmd.extend(["--editable", "."])
rc = qisys.command.call(cmd, cwd=project.path, ignore_ret_code=True,
env=env)
if rc != 0:
ui.warning("Failed to run pip install on", project.src)
res = False
else:
ui.debug("Adding python path for project", project.name, ":\n",
project.python_path)
for path in project.python_path:
fp.write(path + "\n")
return res
def _compress_zip(directory, quiet=True, verbose=False, display_progress=False,
flat=False, output=None):
"""
Compress directory in a .zip file
:param directory: directory to add to the archive
:param archive_basepath: output archive basepath (without extension)
:param quiet: quiet mode (print nothing)
:return: path to the generated archive (archive_basepath.zip)
"""
if quiet and verbose:
mess = """Unconsistent arguments: both 'quiet' and 'verbose' options are set.
Please set only one of these two options to 'True'
"""
raise ValueError(mess)
ui.debug("Compressing", directory, "to", output)
archive = zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED, allowZip64=True)
# a list of tuple src, arcname to be added in the archive
to_add = list()
for root, directories, filenames in os.walk(directory):
entries = directories
entries.extend(filenames)
for entry in entries:
full_path = os.path.join(root, entry)
# Do not zip ourselves
if full_path == output:
continue
rel_path = os.path.relpath(full_path, directory)
if flat:
arcname = rel_path
else:
arcname = os.path.join(os.path.basename(directory), rel_path)
to_add.append((full_path, arcname))
for i, (full_path, arcname) in enumerate(to_add):
if os.path.islink(full_path):
content = os.readlink(full_path) # pylint:disable=no-member
attr = zipfile.ZipInfo(arcname)
attr.create_system = 3
# long type of hex val of '0xA1ED0000L',
# say, symlink attr magic..
attr.external_attr = 2716663808
zip_call = archive.writestr
elif os.path.isdir(full_path):
continue
else:
attr = full_path
content = arcname
zip_call = archive.write
if not quiet and not display_progress:
rel_path = os.path.relpath(full_path, directory)
sys.stdout.write("adding {0}\n".format(rel_path.encode('ascii', "ignore")))
sys.stdout.flush()
if display_progress:
ui.info_progress(i, len(to_add), "Done")
if six.PY3:
zip_call(attr, content)
else:
zip_call(attr, content.encode('ascii', "ignore"))
archive.close()
return output
def install(self, destdir, prefix="/", components=None, split_debug=False):
""" Install the project
:param project: project name.
:param destdir: destination. Note that when using `qibuild install`,
we will first call `cmake` to make sure `CMAKE_INSTALL_PREFIX` is
``/``. But this function simply calls ``cmake --target install``
in the simple case.
:param runtime: Whether to install the project as a runtime
package or not.
(see :ref:`cmake-install` section for the details)
:package split_debug: split the debug symbols out of the binaries
useful for `qibuild deploy`
"""
installed = list()
if components is None:
components = list()
# DESTDIR=/tmp/foo and CMAKE_PREFIX="/usr/local" means
# dest = /tmp/foo/usr/local
destdir = qisys.sh.to_native_path(destdir)
build_env = self.build_env.copy()
build_env["DESTDIR"] = destdir
# Must make sure prefix is not seen as an absolute path here:
dest = os.path.join(destdir, prefix[1:])
dest = qisys.sh.to_native_path(dest)
cprefix = qibuild.cmake.get_cached_var(self.build_directory,
"CMAKE_INSTALL_PREFIX")
if cprefix != prefix:
qibuild.cmake.cmake(self.path,
self.build_directory,
['-DCMAKE_INSTALL_PREFIX=%s' % prefix],
clean_first=False,
env=build_env)
else:
mess = "Skipping configuration of project %s\n" % self.name
mess += "CMAKE_INSTALL_PREFIX is already correct"
ui.debug(mess)
# Hack for http://www.cmake.org/Bug/print_bug_page.php?bug_id=13934
if "Unix Makefiles" in self.cmake_generator:
self.build(target="preinstall", env=build_env)
if components:
for component in components:
files = self._install_component(destdir, component)
installed.extend(files)
else:
self.build(target="install", env=build_env)
manifest_path = os.path.join(self.build_directory,
"install_manifest.txt")
installed.extend(read_install_manifest(manifest_path))
if "test" in components:
self._install_qitest_json(destdir)
if split_debug:
self.split_debug(destdir, file_list=installed)
return installed
def _resolve_job_build_dependencies(self, job):
for p in job.project.build_depends:
dep_job = self._find_job_by_name(p)
if dep_job:
job.add_dependency(dep_job)
else:
ui.debug("Job {job}: Couldn't find the job for the project dependency {dep}". \
format(job=job.project.name, dep=p))
def _kill_subprocess(self):
if self._thread and self._process:
ui.debug('Terminating process.')
self._process.terminate()
self.return_type = Process.TIME_OUT
self._thread.join(5)
if self._thread.is_alive():
self._destroy_zombie()
def _resolve_job_build_dependencies(self, job):
for p in job.project.build_depends:
dep_job = self._find_job_by_name(p)
if dep_job:
job.add_dependency(dep_job)
else:
ui.debug("Job {job}: Couldn't find the job for the project dependency {dep}".
format(job=job.project.name, dep=p))
def configure_virtualenv(config, python_worktree, build_worktree=None,
remote_packages=None, site_packages=True):
""" Main entry point. Called by ``qipy bootstrap``
:param: remote_packages List of third-party packages to add in the virtualenv
:param: site_packages Allow access to global site packages
"""
ui.info(ui.green, "Configuring virtualenv for", ui.reset, ui.bold, python_worktree.root)
if not remote_packages:
remote_packages = list()
# create a new virtualenv
python_worktree.config = config
venv_path = python_worktree.venv_path
pip = python_worktree.pip
try:
virtualenv.create_environment(python_worktree.venv_path,
site_packages=site_packages)
except:
ui.error("Failed to create virtualenv")
return False
ui.info("Adding python projects")
# Write a qi.pth file containing path to C/C++ extensions and
# path to pure python modules or packages
pure_python_ok = handle_pure_python(venv_path, python_worktree)
if build_worktree:
handle_extensions(venv_path, python_worktree, build_worktree)
ui.info("Adding other requirements: " + ", ".join(remote_packages))
binaries_path = virtualenv.path_locations(venv_path)[-1]
pip_binary = os.path.join(binaries_path, "pip")
remote_ok = True
if remote_packages:
cmd = [pip_binary, "install"] + remote_packages
rc = qisys.command.call(cmd, ignore_ret_code=True)
remote_ok = (rc == 0)
if pure_python_ok and remote_ok:
ui.info(ui.green, "Done")
if not pure_python_ok:
ui.info(ui.red, "Failed to add some python projects")
if not remote_ok:
ui.info(ui.red, "Failed to add some third party requirements")
requirements_ok = True
for project in python_worktree.python_projects:
path = os.path.join(project.path, "requirements.txt")
if os.path.isfile( path ):
ui.info(ui.green, " * Installing dependencies from " + path)
cmd = [pip_binary, "install", "--requirement", path]
rc = qisys.command.call(cmd, ignore_ret_code=True)
requirements_ok = (rc == 0)
else:
ui.debug(ui.yellow, " * missing " + path)
return (pure_python_ok and remote_ok and requirements_ok)
def install(self, destdir, prefix="/", components=None, num_jobs=1,
split_debug=False):
""" Install the project
:param project: project name.
:param destdir: destination. Note that when using `qibuild install`,
we will first call `cmake` to make sure `CMAKE_INSTALL_PREFIX` is
``/``. But this function simply calls ``cmake --target install``
in the simple case.
:param runtime: Whether to install the project as a runtime
package or not.
(see :ref:`cmake-install` section for the details)
:package split_debug: split the debug symbols out of the binaries
useful for `qibuild deploy`
"""
installed = list()
if components is None:
components = list()
# DESTDIR=/tmp/foo and CMAKE_PREFIX="/usr/local" means
# dest = /tmp/foo/usr/local
destdir = qisys.sh.to_native_path(destdir)
build_env = self.build_env.copy()
build_env["DESTDIR"] = destdir
# Must make sure prefix is not seen as an absolute path here:
dest = os.path.join(destdir, prefix[1:])
dest = qisys.sh.to_native_path(dest)
cprefix = qibuild.cmake.get_cached_var(self.build_directory,
"CMAKE_INSTALL_PREFIX")
if cprefix != prefix:
qibuild.cmake.cmake(self.path, self.build_directory,
['-DCMAKE_INSTALL_PREFIX=%s' % prefix],
clean_first=False,
env=build_env)
else:
mess = "Skipping configuration of project %s\n" % self.name
mess += "CMAKE_INSTALL_PREFIX is already correct"
ui.debug(mess)
# Hack for http://www.cmake.org/Bug/print_bug_page.php?bug_id=13934
if "Unix Makefiles" in self.cmake_generator:
self.build(target="preinstall", num_jobs=num_jobs, env=build_env)
if components:
for component in components:
files = self._install_component(destdir, component)
installed.extend(files)
else:
self.build(target="install", env=build_env)
manifest_path = os.path.join(self.build_directory, "install_manifest.txt")
installed.extend(read_install_manifest(manifest_path, destdir))
if "test" in components:
self._install_qitest_json(destdir)
if split_debug:
self.split_debug(destdir, file_list=installed)
return installed
def build(self, **kwargs):
""" Run sphinx.main() with the correct arguments """
try:
import sphinx
except ImportError as e:
ui.error(e, "skipping build")
return
build_type = kwargs.get("build_type", None)
language = kwargs.get("language", None)
spellcheck = kwargs.get("spellcheck", False)
werror = kwargs.get("werror", False)
pdb = kwargs.get("pdb", False)
if self.prebuild_script:
ui.info(ui.green, "Running pre-build script:",
ui.white, self.prebuild_script)
cmd = [sys.executable, self.prebuild_script]
qisys.command.call(cmd, cwd=self.path)
ui.info()
self.generate_examples_zips()
if self.translated and language and language != "en" \
and language not in self.linguas:
raise UnknownLingua(self, language)
if self.translated:
self.intl_build(language)
qisys.sh.mkdir(self.html_dir, recursive=True)
spell_dir = os.path.join(self.build_dir, "spellcheck")
qisys.sh.mkdir(spell_dir, recursive=True)
cmd = [sys.executable, "-c", self.build_dir]
if spellcheck:
cmd.extend(("-b", "spelling"))
else:
cmd.extend(("-b", "html"))
if werror:
cmd.append("-W")
if language:
cmd.append("-Dlanguage=%s" % language)
if pdb:
cmd.append("-P")
cmd.append(self.source_dir)
if spellcheck:
cmd.append(spell_dir)
else:
cmd.append(self.html_dir)
if build_type:
os.environ["build_type"] = build_type
ui.debug("launching:", cmd)
rc = 0
try:
sphinx.main(argv=cmd)
except SystemExit as e:
rc = e.code
if spellcheck:
num_errors = get_num_spellcheck_errors(self.build_dir)
if num_errors != 0:
raise SphinxBuildError(self)
if rc != 0:
raise SphinxBuildError(self)
def _compress_zip(directory,
quiet=True,
verbose=False,
flat=False,
output=None):
"""Compress directory in a .zip file
:param directory: directory to add to the archive
:param archive_basepath: output archive basepath (without extension)
:param quiet: quiet mode (print nothing)
:return: path to the generated archive (archive_basepath.zip)
"""
if quiet and verbose:
mess = """Unconsistent arguments: both 'quiet' and 'verbose' options are set.
Please set only one of these two options to 'True'
"""
raise ValueError(mess)
ui.debug("Compressing", directory, "to", output)
archive = zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED)
for root, directories, filenames in os.walk(directory):
entries = directories
entries.extend(filenames)
for entry in entries:
full_path = os.path.join(root, entry)
# Do not zip ourselves
if full_path == output:
continue
rel_path = os.path.relpath(full_path, directory)
if flat:
arcname = rel_path
else:
arcname = os.path.join(os.path.basename(directory), rel_path)
if os.path.islink(full_path):
content = os.readlink(full_path)
attr = zipfile.ZipInfo(arcname)
attr.create_system = 3
# long type of hex val of '0xA1ED0000L',
# say, symlink attr magic..
attr.external_attr = 2716663808L
zip_call = archive.writestr
elif os.path.isdir(full_path):
continue
else:
attr = full_path
content = arcname
zip_call = archive.write
if not quiet:
sys.stdout.write("adding {0}\n".format(rel_path))
sys.stdout.flush()
zip_call(attr, content)
archive.close()
return output
def build(self, **kwargs):
""" Run sphinx.main() with the correct arguments """
try:
import sphinx
except ImportError as e:
ui.error(e, "skipping build")
return
build_type = kwargs.get("build_type", None)
language = kwargs.get("language", None)
spellcheck = kwargs.get("spellcheck", False)
werror = kwargs.get("werror", False)
pdb = kwargs.get("pdb", False)
if self.prebuild_script:
ui.info(ui.green, "Running pre-build script:", ui.white,
self.prebuild_script)
cmd = [sys.executable, self.prebuild_script]
qisys.command.call(cmd, cwd=self.path)
ui.info()
self.generate_examples_zips()
if self.translated and language and language != "en" \
and language not in self.linguas:
raise UnknownLingua(self, language)
if self.translated:
self.intl_build(language)
qisys.sh.mkdir(self.html_dir, recursive=True)
spell_dir = os.path.join(self.build_dir, "spellcheck")
qisys.sh.mkdir(spell_dir, recursive=True)
cmd = [sys.executable, "-c", self.build_dir]
if spellcheck:
cmd.extend(("-b", "spelling"))
else:
cmd.extend(("-b", "html"))
if werror:
cmd.append("-W")
if language:
cmd.append("-Dlanguage=%s" % language)
if pdb:
cmd.append("-P")
cmd.append(self.source_dir)
if spellcheck:
cmd.append(spell_dir)
else:
cmd.append(self.html_dir)
if build_type:
os.environ["build_type"] = build_type
ui.debug("launching:", cmd)
rc = 0
try:
sphinx.main(argv=cmd)
except SystemExit as e:
rc = e.code
if spellcheck:
num_errors = get_num_spellcheck_errors(self.build_dir)
if num_errors != 0:
raise SphinxBuildError(self)
if rc != 0:
raise SphinxBuildError(self)
def _compress_zip(directory, quiet=True, verbose=False, display_progress=False,
flat=False, output=None):
"""
Compress directory in a .zip file
:param directory: directory to add to the archive
:param archive_basepath: output archive basepath (without extension)
:param quiet: quiet mode (print nothing)
:return: path to the generated archive (archive_basepath.zip)
"""
if quiet and verbose:
mess = """Unconsistent arguments: both 'quiet' and 'verbose' options are set.
Please set only one of these two options to 'True'
"""
raise ValueError(mess)
ui.debug("Compressing", directory, "to", output)
archive = zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED, allowZip64=True)
# a list of tuple src, arcname to be added in the archive
to_add = list()
for root, directories, filenames in os.walk(directory):
entries = directories
entries.extend(filenames)
for entry in entries:
full_path = os.path.join(root, entry)
# Do not zip ourselves
if full_path == output:
continue
rel_path = os.path.relpath(full_path, directory)
if flat:
arcname = rel_path
else:
arcname = os.path.join(os.path.basename(directory), rel_path)
to_add.append((full_path, arcname))
for i, (full_path, arcname) in enumerate(to_add):
if os.path.islink(full_path):
content = os.readlink(full_path) # pylint:disable=no-member
attr = zipfile.ZipInfo(arcname)
attr.create_system = 3
# long type of hex val of '0xA1ED0000L',
# say, symlink attr magic..
attr.external_attr = 2716663808
zip_call = archive.writestr
elif os.path.isdir(full_path):
continue
else:
attr = full_path
content = arcname
zip_call = archive.write
if not quiet and not display_progress:
rel_path = os.path.relpath(full_path, directory)
sys.stdout.write("adding {0}\n".format(rel_path.encode('ascii', "ignore")))
sys.stdout.flush()
if display_progress:
ui.info_progress(i, len(to_add), "Done")
zip_call(attr, content.encode('ascii', "ignore"))
archive.close()
return output
def install(src, dest, filter_fun=None, quiet=False):
"""
Install a directory or a file to a destination.
If filter_fun is not None, then the file will only be
installed if filter_fun(relative/path/to/file) returns True.
If ``dest`` does not exist, it will be created first.
When installing files, if the destination already exists,
it will be removed first, then overwritten by the new file.
This function will preserve relative symlinks between directories,
used for instance in Mac frameworks::
|__ Versions
|__ Current -> 4.0
|__ 4 -> 4.0
|__ 4.0
Return the list of files installed (with relative paths)
"""
installed = list()
# FIXME: add a `safe mode` ala install?
if not os.path.exists(src):
mess = "Could not install '%s' to '%s'\n" % (src, dest)
mess += '%s does not exist' % src
raise Exception(mess)
src = to_native_path(src, normcase=False)
dest = to_native_path(dest, normcase=False)
ui.debug("Installing", src, "->", dest)
if filter_fun is None:
def no_filter_fun(_unused):
""" Filter Function Always True """
return True
filter_fun = no_filter_fun
if os.path.isdir(src):
if src == dest:
raise Exception("source and destination are the same directory")
for (root, dirs, files) in os.walk(src):
dirs = _handle_dirs(src, dest, root, dirs, filter_fun, quiet)
files = _handle_files(src, dest, root, files, filter_fun, quiet)
installed.extend(files)
else:
# Emulate posix `install' behavior:
# if dest is a dir, install in the directory, else
# simply copy the file.
if os.path.isdir(dest):
dest = os.path.join(dest, os.path.basename(src))
if src == dest:
raise Exception("source and destination are the same file")
mkdir(os.path.dirname(dest), recursive=True)
if sys.stdout.isatty() and not quiet:
print("-- Installing %s" % dest)
# We do not want to fail if dest exists but is read only
# (following what `install` does, but not what `cp` does)
rm(dest)
shutil.copy(src, dest)
installed.append(os.path.basename(src))
return installed
def dump_symbols_from_binary(binary,
pool_dir,
build_config=None,
dump_syms=None):
"""
Dump symbols from the binary.
Results can be found in
<pool_dir>/<binary name>/<id>/<binary name>.sym
"""
if not dump_syms:
dump_syms = qisys.command.find_program("dump_syms",
raises=True,
build_config=build_config)
if sys.platform == "darwin":
dsym = gen_dsym(binary)
cmd = [dump_syms, dsym]
else:
cmd = [dump_syms, binary]
ui.debug("dump_symbols_from_binary() cmd: %s" % cmd)
try:
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
except OSError as e:
ui.error("Could not dump symbols", cmd, e)
return False
dump_ok = True
(out, err) = process.communicate()
if process.returncode != 0:
ui.error("Failed to dump symbols", err)
dump_ok = False
if sys.platform == "darwin":
qisys.sh.rm(dsym)
if not dump_ok:
return False
out = out.decode("utf-8")
# First line looks like:
# MODULE Linux x86_64 ID foo on linux
# MODULE windows x86 ID foo.pdb on windows
# path should be
# pool/foo.pdb/ID/foo.sym on windows,
# pool/foo/ID/foo.sym on linux
lines = out.splitlines()
first_line = lines[0]
uuid = first_line.split()[3]
name = first_line.split()[4]
if os.name == "nt":
basename = name.replace(".pdb", "")
else:
basename = name
to_make = os.path.join(pool_dir, name, uuid)
qisys.sh.mkdir(to_make, recursive=True)
with open(os.path.join(to_make, basename + ".sym"), "w") as fp:
fp.write(out)
fp.close()
return True
def mv(src, dest):
""" Move a file into a directory, but do not crash if dest/src exists """
if src == dest:
return
if os.path.isdir(dest):
dest = os.path.join(dest, os.path.basename(src))
if os.path.exists(dest):
rm(dest)
ui.debug(src, "->", dest)
shutil.move(src, dest)
def _fix_rpaths(package_path):
""" Search all dylib in lib directory and fix rpaths. """
ui.info("Fix RPATH for", package_path, "librairies")
# pylint: disable=W0612
for root, dirs, files in os.walk(package_path):
for basename in files:
if basename.endswith("dylib"):
ui.debug("Fixing RPATH for", basename)
filename = os.path.join(root, basename)
_fix_rpath(filename, package_path)
def package(self, *args, **kwargs):
"""
Generate a package containing every project.
:param: with_breakpad generate debug symbols for usage with breakpad
:param: force make package even if it does not satisfy default package requirements
:param install_tc_packages also install toolchain packages
"""
output = kwargs.get('output', None)
force = kwargs.get('force', False)
with_breakpad = kwargs.get('with_breakpad', False)
python_minify = kwargs.get('python_minify', False)
strip = kwargs.get('strip', True)
install_tc_packages = kwargs.get('install_tc_packages', False)
strip_args = kwargs.get('strip_args', None)
strip_exe = kwargs.get('strip_exe', None)
# If the package is not valid, do not go further
if not self.validator.is_valid and not force:
raise Exception("Given package does not satisfy "
"default package requirements.\n"
"Use option '--force' to bypass this validation")
# Make sure self.stage_path exists and is empty
qisys.sh.rm(self.stage_path)
qisys.sh.mkdir(self.stage_path, recursive=True)
if not output:
output = os.path.join(os.getcwd(), self.pkg_name + ".pkg")
# Add everything from the staged path
self.install(self.stage_path,
install_tc_packages=install_tc_packages,
python_minify=python_minify)
symbols_archive = None
if with_breakpad and self.build_project:
ui.info(ui.bold, "-> Generating breakpad symbols ...")
dirname = os.path.dirname(output)
build_config = kwargs.get('build_config',
self.cmake_builder.build_config)
ui.debug("Use breakpad build_config:", build_config)
symbols_archive = os.path.join(dirname,
self.pkg_name + "-symbols.zip")
qibuild.breakpad.gen_symbol_archive(base_dir=self.stage_path,
output=symbols_archive,
strip=strip,
strip_exe=strip_exe,
strip_args=strip_args,
build_config=build_config)
ui.info(ui.bold, "-> Symbols generated in", symbols_archive)
ui.info(ui.bold, "-> Package generated in", output, "\n")
ui.info(ui.bold, "-> Compressing package ...")
qisys.archive.compress(self.stage_path,
output=output,
flat=True,
display_progress=True)
qisys.sh.rm(self.stage_path)
if symbols_archive:
return [output, symbols_archive]
return output
def read(self, cfg_path=None, create_if_missing=False):
""" Read from a config location. """
if not cfg_path:
cfg_path = get_global_cfg_path()
if create_if_missing:
if not os.path.exists(cfg_path):
dirname = os.path.dirname(cfg_path)
qisys.sh.mkdir(dirname, recursive=True)
with open(cfg_path, "w") as fp:
fp.write('<qibuild />\n')
ui.debug("Reading config from", cfg_path)
try:
self.tree.parse(cfg_path)
except Exception as e:
mess = "Could not parse config from %s\n" % cfg_path
mess += "Error was: %s" % str(e)
raise Exception(mess)
# Parse defaults:
defaults_tree = self.tree.find("defaults")
if defaults_tree is not None:
self.defaults.parse(defaults_tree)
# Parse configs:
config_trees = self.tree.findall("config")
for config_tree in config_trees:
config = BuildConfig()
config.parse(config_tree)
self.configs[config.name] = config
# Parse IDES:
ide_trees = self.tree.findall("ide")
for ide_tree in ide_trees:
ide = IDE()
ide.parse(ide_tree)
self.ides[ide.name] = ide
# Parse servers:
server_trees = self.tree.findall("server")
for server_tree in server_trees:
server = Server()
server.parse(server_tree)
self.servers[server.name] = server
# Parse worktrees
worktree_trees = self.tree.findall("worktree")
for worktree_tree in worktree_trees:
worktree = WorkTree()
worktree.parse(worktree_tree)
self.worktrees[worktree.path] = worktree
self.cmake.generator = self.defaults.cmake.generator
self.env.bat_file = self.defaults.env.bat_file
self.env.editor = self.defaults.env.editor
self.env.path = self.defaults.env.path
self.ide = None
current_ide = self.defaults.ide
if current_ide:
matching_ide = self.ides.get(current_ide)
if matching_ide:
self.ide = matching_ide
def read(self, cfg_path=None, create_if_missing=False):
""" Read from a config location. """
if not cfg_path:
cfg_path = get_global_cfg_path()
if create_if_missing:
if not os.path.exists(cfg_path):
dirname = os.path.dirname(cfg_path)
qisys.sh.mkdir(dirname, recursive=True)
with open(cfg_path, "w") as fp:
fp.write('<qibuild />\n')
ui.debug("Reading config from", cfg_path)
try:
self.tree.parse(cfg_path)
except Exception as e:
mess = "Could not parse config from %s\n" % cfg_path
mess += "Error was: %s" % str(e)
raise Exception(mess)
# Parse defaults:
defaults_tree = self.tree.find("defaults")
if defaults_tree is not None:
self.defaults.parse(defaults_tree)
# Parse configs:
config_trees = self.tree.findall("config")
for config_tree in config_trees:
config = BuildConfig()
config.parse(config_tree)
self.configs[config.name] = config
# Parse IDES:
ide_trees = self.tree.findall("ide")
for ide_tree in ide_trees:
ide = IDE()
ide.parse(ide_tree)
self.ides[ide.name] = ide
# Parse servers:
server_trees = self.tree.findall("server")
for server_tree in server_trees:
server = Server()
server.parse(server_tree)
self.servers[server.name] = server
# Parse worktrees
worktree_trees = self.tree.findall("worktree")
for worktree_tree in worktree_trees:
worktree = WorkTree()
worktree.parse(worktree_tree)
self.worktrees[worktree.path] = worktree
self.cmake.generator = self.defaults.cmake.generator
self.env.bat_file = self.defaults.env.bat_file
self.env.editor = self.defaults.env.editor
self.env.path = self.defaults.env.path
self.ide = None
current_ide = self.defaults.ide
if current_ide:
matching_ide = self.ides.get(current_ide)
if matching_ide:
self.ide = matching_ide
def mv(src, dest):
"""Move a file into a directory, but do not crash
if dest/src exists
"""
if os.path.isdir(dest):
dest = os.path.join(dest, os.path.basename(src))
if os.path.exists(dest):
rm(dest)
ui.debug(src, "->", dest)
shutil.move(src, dest)
def _destroy_zombie(self):
ui.debug('Process was a zombie...')
if not self._process:
pass
elif os.name == 'posix':
os.killpg(self._process.pid, signal.SIGKILL)
elif os.name == 'nt':
# pylint: disable-msg=E1101
os.kill(self._process.pid, signal.CTRL_BREAK_EVENT)
self.return_type = Process.ZOMBIE
self._thread.join()
def _dump_arguments(name, args):
""" Dump an argparser namespace to log """
output = ""
keys = sorted(args.__dict__.keys())
max_len = max(len(k) for k in keys)
for k in keys:
value = args.__dict__[k]
output += " " + k.ljust(max_len) + " = %s\n" % (value, )
if output[-1] == "\n":
output = output[:-1]
ui.debug("[%s] arguments:\n%s" % (name, output))
def run_action(module_name, args=None, forward_args=None):
"""
Run an action using its module path and a list of arguments.
If forward_args is given, it must be an argparse.Namespace object.
This namespace will be merged with args before being
passed to the do() method of module_name.
"""
if not args:
args = list()
ui.debug("running", module_name, " ".join(args))
action_name = module_name.split(".")[-1]
package_name = ".".join(module_name.split(".")[:-1])
if not isinstance(action_name, str):
action_name = str(action_name)
try:
_tmp = __import__(package_name, globals(), locals(), [action_name])
except ImportError as err:
raise InvalidAction(module_name, str(err))
try:
module = getattr(_tmp, action_name)
except AttributeError as err:
raise InvalidAction(
module_name, "Could not find module %s in package %s" %
(module_name, package_name))
check_module(module)
parser = argparse.ArgumentParser()
module.configure_parser(parser)
# Quick hack to prevent argparse.parse_args to
# - print usage to the console
# - call SystemExit
# Instead, raise a nice Exception
def custom_exit():
""" Custom Exit """
return
parser.exit = custom_exit
def error(message):
""" Raise an Error Message """
mess = "Invalid arguments when calling run_action(%s)\n" % module_name
mess += message + "\n"
mess += "args: %s\n" % " ".join(args)
mess += "forward_args: %s\n" % forward_args
raise Exception(mess)
parser.error = error
if forward_args:
parsed_args = parser.parse_args(args=args,
namespace=copy.deepcopy(forward_args))
else:
parsed_args = parser.parse_args(args=args)
return module.do(parsed_args)
def _dump_arguments(name, args):
""" Dump an argparser namespace to log """
output = ""
keys = sorted(args.__dict__.keys())
max_len = max(len(k) for k in keys)
for k in keys:
value = args.__dict__[k]
output += " " + k.ljust(max_len) + " = %s\n" % (value,)
if output[-1] == "\n":
output = output[:-1]
ui.debug("[%s] arguments:\n%s" % (name, output))
def save(self):
""" Save the packages in the xml file """
root = etree.Element("toolchain")
if self.build_target:
root.set("target", self.build_target)
ui.debug("Save target in database xml", self.build_target, "in",
self.db_path)
tree = etree.ElementTree(root)
for package in self.packages.values():
element = package.to_xml()
root.append(element)
qisys.qixml.write(tree, self.db_path)
def load(self):
""" Load the packages from the xml file """
tree = qisys.qixml.read(self.db_path)
self.build_target = tree.getroot().get("target")
ui.debug("Load target from database xml", self.build_target, "in",
self.db_path)
for element in tree.findall("package"):
to_add = qitoolchain.qipackage.from_xml(element)
self.packages[to_add.name] = to_add
for svn_elem in tree.findall("svn_package"):
to_add = qitoolchain.qipackage.from_xml(svn_elem)
self.packages[to_add.name] = to_add
def main():
ui.info(ui.red, "This is a an error message\n",
ui.reset, "And here are the details")
ui.error("could not build")
ui.warning("-j ignored for this generator")
ui.info("building foo")
ui.debug("debug message")
ui.info(ui.brown, "this is brown")
ui.info(ui.bold, ui.brown, "this is bold brown")
ui.info(ui.red, "red is dead")
ui.info(ui.darkred, "darkred is really dead")
ui.info(ui.yellow, "this is yellow")
def write_file_if_different(data, out_path, mode="w"):
""" Write the data to out_path if the content is different
"""
try:
with open(out_path, "r") as outr:
out_prev = outr.read()
if out_prev == data:
ui.debug("skipping write to %s: same content" % (out_path))
return
except:
pass
with open(out_path, mode) as out_file:
out_file.write(data)
def _dump_arguments(name, args):
""" Dump an argparser namespace to log """
output = ""
max_len = 0
for k in args.__dict__.keys():
if len(k) > max_len:
max_len = len(k)
for k, v in args.__dict__.iteritems():
pad = " " * (max_len - len(k))
output += " %s%s = %s\n" % (str(k), pad, str(v))
if output[-1] == "\n":
output = output[:-1]
ui.debug("[%s] arguments:\n%s" % (name, output))
def dump_symbols_from_binary(binary, pool_dir):
""" Dump sympobls from the binary.
Results can be found in
<pool_dir>/<binary name>/<id>/<binary name>.sym
"""
dump_syms = qisys.command.find_program("dump_syms", raises=True)
if sys.platform == "darwin":
dsym = gen_dsym(binary)
cmd = [dump_syms, dsym]
else:
cmd = [dump_syms, binary]
ui.debug(cmd)
with qisys.sh.PreserveFileMetadata(binary):
mode_rw = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO
os.chmod(binary, mode_rw)
process = subprocess.Popen(cmd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
dump_ok = True
(out, err) = process.communicate()
if process.returncode != 0:
ui.error("Failed to dump symbols", err)
dump_ok = False
if sys.platform == "darwin":
qisys.sh.rm(dsym)
if not dump_ok:
return
# First line looks like:
# MODULE Linux x86_64 ID foo on linux
# MODULE windows x86 ID foo.pdb on windows
# path should be
# pool/foo.pdb/ID/foo.sym on windows,
# pool/foo/ID/foo.sym on linux
lines = out.splitlines()
first_line = lines[0]
uuid = first_line.split()[3]
name = first_line.split()[4]
if os.name == "nt":
basename = name.replace(".pdb", "")
else:
basename = name
to_make = os.path.join(pool_dir, name, uuid)
qisys.sh.mkdir(to_make, recursive=True)
with open(os.path.join(to_make, basename + ".sym"), "w") as fp:
fp.write(out)
def run_action(module_name, args=None, forward_args=None):
"""
Run an action using its module path and a list of arguments.
If forward_args is given, it must be an argparse.Namespace object.
This namespace will be merged with args before being
passed to the do() method of module_name.
"""
if not args:
args = list()
ui.debug("running", module_name, " ".join(args))
action_name = module_name.split(".")[-1]
package_name = ".".join(module_name.split(".")[:-1])
if not isinstance(action_name, str):
action_name = str(action_name)
try:
_tmp = __import__(package_name, globals(), locals(), [action_name])
except ImportError as err:
raise InvalidAction(module_name, str(err))
try:
module = getattr(_tmp, action_name)
except AttributeError as err:
raise InvalidAction(module_name, "Could not find module %s in package %s" %
(module_name, package_name))
check_module(module)
parser = argparse.ArgumentParser()
module.configure_parser(parser)
# Quick hack to prevent argparse.parse_args to
# - print usage to the console
# - call SystemExit
# Instead, raise a nice Exception
def custom_exit():
""" Custom Exit """
return
parser.exit = custom_exit
def error(message):
""" Raise an Error Message """
mess = "Invalid arguments when calling run_action(%s)\n" % module_name
mess += message + "\n"
mess += "args: %s\n" % " ".join(args)
mess += "forward_args: %s\n" % forward_args
raise Exception(mess)
parser.error = error
if forward_args:
parsed_args = parser.parse_args(args=args, namespace=copy.deepcopy(forward_args))
else:
parsed_args = parser.parse_args(args=args)
return module.do(parsed_args)