def extract_packages(packages, num_jobs=8):
t0 = time.time()
cmds = []
for package in packages:
cmd = " ".join(
[
"bash",
"do_extract.sh",
package["orig_name"],
package["tar_name"],
package["work_dir"],
]
)
cmds.append(cmd)
# using command line parallel is much faster than pytho multiprocessing.
fname = os.path.join(gettmpdir(), "gnu_compile_script_cmds.txt")
with open(fname, "w") as f:
f.write("\n".join(cmds) + "\n")
if len(cmds) < num_jobs:
num_jobs = len(cmds)
os.system("parallel -j {0} :::: {1}".format(num_jobs, fname))
logger.info("done. %0.3fs", time.time() - t0)
def compile_package(
packages,
to_compile,
opti_list,
arch_list,
compiler_list,
suffix,
echo=False,
num_jobs=8,
):
t0 = time.time()
cmds = []
logger.info("[+] start compiling %d packages ...", len(to_compile))
for package in packages:
package_name = package["name"]
if package_name not in to_compile or len(
to_compile[package_name]) == 0:
# no need to compile this package since it is already checked!
continue
opts = to_compile[package_name]
logger.debug("%s: %d options left", package_name, len(opts))
for opt in opts:
compiler, arch, opti = opt.split(":")
if (compiler not in compiler_list or arch not in arch_list
or opti not in opti_list):
continue
cmd = " ".join([
"timeout",
str(60 * 30),
"bash",
"do_compile_utils.sh",
package["orig_name"],
package["tar_name"],
package["version"],
package["work_dir"],
package["out_dir"],
package["log_dir"],
opti,
arch,
compiler,
suffix,
str(echo),
])
cmds.append(cmd)
# using command line parallel is much faster than pytho multiprocessing.
fname = os.path.join(gettmpdir(), "gnu_compile_script_cmds.txt")
with open(fname, "w") as f:
f.write("\n".join(cmds) + "\n")
if len(cmds) < num_jobs:
num_jobs = len(cmds)
os.system("parallel -j {0} :::: {1}".format(num_jobs, fname))
logger.info("done. %0.3fs", time.time() - t0)
def remove_files(remove_list):
if not remove_list:
return
logger.warning("removing unnecessary files ...")
fname = os.path.join(gettmpdir(), "gnu_compile_remove_list.txt")
with open(fname, "w") as f:
for r in remove_list:
f.write(r + "\n")
logger.warning("%d files and directories have been removed.",
len(remove_list))
# command line program is much faster.
os.system("cat {0} | xargs rm -rf".format(fname))