def get_installed_packages(dependencies):
installed_packages = list()
cmds = {
"apt-get": "dpkg -s {}",
"pacman": "pacman -Qs {} | grep \"/{} \"",
"dnf": "rpm -qa | grep {}",
"yum": [""]
# "emerge": [],
# "zypper": []
}
for dep in dependencies:
try:
status = subprocess.call(
[cmds[tset.PKG_MANAGER][0].format(dep, dep)],
shell=True,
universal_newlines=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
#status = subprocess.call([cmds.get("apt-get").format(dep, dep)], shell=True, universal_newlines=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)
if (status == 0):
installed_packages.append(dep)
except subprocess.CalledProcessError:
tcom.pprint(1, "Unable to build the list of installed packages.")
return None
return installed_packages
def write_bdd_to_csv():
csv_writer = csv.writer(csvfile,
delimiter=';',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
try:
socket = MySQLdb.connect(tset.HOST, tset.DB_USER, tset.DB_PASSWD,
"depML_DB")
query = "SELECT DISTINCT depML_environnement.config_file,depML_environnement.environnement,packages.missing_files,packages.missing_packages,packages.candidate_missing_packages,packages.resolution_successful FROM packages , depML_environnement WHERE packages.cid=depML_environnement.cid"
socket.query(query)
res = socket.store_result()
tuples = res.fetch_row(maxrows=0)
first_row = True
for t in tuples:
optionName = list() #Column names for compilation options.
optionValue = list()
envDict = ast.literal_eval(tuples[0][1])
#Column names for environment details
csv_env_col_names = list(
chain(*[list(envDict.keys()) for d in list(envDict.values())]))
for option in t[0].split('\n'):
if '#' in option or len(option.split('=')) < 2:
# We ignore the comment lines and the empty lines.
continue
else:
option_name = option.split('=')[0]
option_value = option.split('=')[1]
#print(option_name+" " + option_value)
if first_row:
optionName.append(option_name)
optionValue.append(option_value)
if first_row:
#We write the header only once in the first iteration.
csv_col_names = optionName + csv_env_col_names
csv_col_names.append("missing_files")
csv_col_names.append("missing_packages")
csv_col_names.append("candidate_missing_packages")
csv_col_names.append("resolution_successful")
#We write the header here, i.e the column names.
csv_writer.writerow(csv_col_names)
first_row = False
#For each column, we write the values.
csv_col_values = optionValue + list(
chain(
*[list(envDict.values()) for d in list(envDict.values())]))
csv_col_values.append(t[2])
csv_col_values.append(t[3])
csv_col_values.append(t[4])
csv_col_values.append(t[5])
csv_writer.writerow(csv_col_values)
except MySQLdb.Error as err:
tcom.pprint(1, "Can't retrieve info from db : {}".format(err))
return -1
def get_kernel_size():
possible_filenames = [
"vmlinux", "vmlinux.bin", "vmlinuz", "zImage", "bzImage"
]
for filename in possible_filenames:
full_filename = tset.PATH + "/" + filename
if os.path.isfile(full_filename):
tcom.pprint(2, "kernel found: " + filename)
return os.path.getsize(full_filename)
return 0
def get_environment_details():
tcom.pprint(2, "Getting environment details")
env = {
"system": get_os_details(),
"hardware": get_hardware_details(),
"compilation": get_compilation_details()
}
if tset.VERBOSE > 1:
environment_pprinter(env)
export_as_csv(env["system"], env["hardware"], env["compilation"])
return env
def boot_try():
tcom.pprint(2, "Launching boot test on kernel")
try:
sbStatus = subprocess.call(["mkinitramfs","-o", tset.PATH + "/arch/x86_64/boot/initrd.img-4.13.3"], stdout=tset.OUTPUT, stderr=tset.OUTPUT)
except Exception:
sbStatus = -1;
cmd = "qemu-system-x86_64"
if sbStatus == 0:
# procId = subprocess.Popen([cmd, "-kernel", tset.PATH + "/arch/x86_64/boot/bzImage", "-initrd", tset.PATH + "/arch/x86_64/boot/initrd.img-4.13.3", "-m", "1G", "-append", "console=ttyS0,38400", "-serial", "file:serial.out"], stdout=tset.OUTPUT, stderr=tset.OUTPUT)
procId = subprocess.run([cmd, "-kernel", tset.PATH + "/arch/x86_64/boot/bzImage", "-initrd", tset.PATH + "/arch/x86_64/boot/initrd.img-4.13.3", "-m", "1G", "-append", "console=ttyS0,38400", "-serial", "file:serial.out"], stdout=subprocess.DEVNULL, stderr=tset.OUTPUT)
rndCounter = 1
status = 1
time.sleep(5)
try:
outFile = open("serial.out",mode='r')
except OSError:
tcom.pprint(1, "Unable to open output file, assuming subprocess call failed !")
return -3
while status == 1:
time.sleep(10)
# tcom.pprint(3, "Reading output file for boot end, attempt".format(rndCounter))
fileData = outFile.read()
if re.search("(initramfs)", fileData):
tcom.pprint(0, "Shell detected, kernel successfully booted")
status = 0
if re.search("end Kernel panic", fileData):
tcom.pprint(1, "Kernel panic detected, kernel probably failed to boot")
status = -1
if rndCounter == 11: # default 51
tcom.pprint(1, "More than 60 attempts at reading, possible infinite loop in boot process, interrupting")
status = -2
rndCounter = rndCounter + 1
# procId.terminate() # <-- terminate the process
outFile.close()
return status
else:
return -3
def sendToDB():
try:
socket = MySQLdb.connect(tset.HOST, tset.DB_USER, tset.DB_PASSWD,
"depML_DB")
cursor = socket.cursor()
# Values for request
args_env = {
"config_file": open(tset.PATH + "/.config", "r").read(),
"environnement": tset.TUXML_ENV,
}
keys = ",".join(args_env.keys())
values = ','.join(['%s'] * len(args_env.values()))
query = "INSERT INTO depML_environnement({}) VALUES({})".format(
keys, values)
cursor.execute(query, list(args_env.values()))
socket.commit()
for missing_file in tdep.tdepLogger.log.keys():
args_pkg = {
"cid":
cursor.lastrowid,
"missing_files":
str(missing_file),
"missing_packages":
tdep.tdepLogger.log.get(missing_file),
"candidate_missing_packages":
tdep.tdepLogger.candidates.get(missing_file).split(':')[0],
"resolution_successful":
tdep.tdepLogger.status.get(missing_file)
}
keys = ",".join(args_pkg.keys())
values = ','.join(['%s'] * len(args_pkg.values()))
query = "INSERT INTO packages({}) VALUES({})".format(keys, values)
print("QUERY IS: " + query)
cursor.execute(query, list(args_pkg.values()))
socket.commit()
socket.close()
# file_upload(logfiles, date)
tcom.pprint(0, "Successfully sent info to db")
return 0
except MySQLdb.Error as err:
tcom.pprint(1, "Can't send info to db : {}".format(err))
return -1
def build_dependencies(missing_files, missing_packages):
cmds = {
"apt-get": ["apt-file search {}", "dpkg-query -l | grep {}"],
"pacman": ["pkgfile -d {}", "pacman -Fs {}"],
"dnf": ["dnf whatprovides *{}", "rpm -qa | grep {}"],
"yum": ["yum whatprovides *{}", "rpm -qa | grep {}"]
# "emerge": [],
# "zypper": []
}
if tset.VERBOSE > 0 and len(missing_files) > 0:
tcom.pprint(3, "Those files are missing :")
for mf in missing_files:
if tset.VERBOSE > 0:
print(" " * 3 + mf)
if (tset.PKG_MANAGER is "pacman"):
mf = mf.replace("/", " ")
try:
output = subprocess.check_output(
[cmds[tset.PKG_MANAGER][0].format(mf)],
shell=True,
universal_newlines=True)
except subprocess.CalledProcessError:
tcom.pprint(1, "Unable to find the missing package(s)")
return -1
# Sometimes the output gives several packages. The program takes the
# first one and check if the package is already installed. If not, tuxml
# installs it. Else it installs the next one
lines = output.splitlines()
i = 0
status = 0
while i < len(lines) and status == 0:
package = lines[i].split(":")[0]
# 0: package already installed
# 1: package not installed
status = subprocess.call(
[cmds[tset.PKG_MANAGER][1].format(package)],
stdout=tset.OUTPUT,
stderr=tset.OUTPUT,
shell=True)
if status == 1:
missing_packages.append(package)
i += 1
# if tuxml reaches the end of the packages list without installing any package
# it means that there is a problem with mf, so it returns an error
if i > len(lines) and status == 0:
tcom.pprint(1, "Unable to find the missing package(s)")
return -1
tcom.pprint(0, "Dependencies built")
return 0
def install_default_dependencies():
# Install packages common to all distro
tcom.pprint(2, "Installing default dependencies")
common_pkgs = [
"gcc", "make", "binutils", "util-linux", "kmod", "e2fsprogs",
"jfsutils", "xfsprogs", "btrfs-progs", "pcmciautils", "ppp", "grub",
"iptables", "openssl", "bc"
]
# Now installation of packages with name that vary amongs distributions
# TODO ajouter les paquets python3-pip, mysql-client?, libmariadbclient-dev, mysql-server?
debian_specific = [
"reiserfsprogs", "squashfs-tools", "quotatool", "nfs-kernel-server",
"procps", "mcelog", "libcrypto++6", "apt-utils"
]
arch_specific = [
"reiserfsprogs", "squashfs-tools", "quota-tools", "isdn4k-utils",
"nfs-utils", "procps-ng", "oprofile"
]
redHat_specific = [
"reiserfs-utils", "squashfs-tools", "quotatool", "isdn4k-utils",
"nfs-utils", "procps-ng", "oprofile", "mcelog"
]
gentoo_specific = [
"reiserfsprogs", "squashfs-tools", "quotatool", "nfs-utils", "procps",
"mcelog", "oprofile"
]
suse_specific = ["reiserfs", "quota", "nfs-client", "procps"]
specific_pkgs = {
"apt-get": debian_specific,
"pacman": arch_specific,
"dnf": redHat_specific,
"yum": redHat_specific,
"emerge": gentoo_specific,
"zypper": suse_specific
}
if tcom.install_packages(common_pkgs +
specific_pkgs[tset.PKG_MANAGER]) != 0:
return -1
else:
return 0
def send_outputlog(cid, outputfilename, databasename):
try:
socket = MySQLdb.connect(tset.HOST, tset.DB_USER, tset.DB_PASSWD, databasename)
cursor = socket.cursor()
bzoutput = bz2.compress(open(outputfilename, "rb").read())
query = "UPDATE Compilations SET output_file = %s WHERE cid = %s"
for c in cid:
data = (bzoutput, c)
cursor.execute(query, data)
socket.commit()
socket.close()
return cid
except MySQLdb.Error as err:
tcom.pprint(1, "Can't send info to db : {}".format(err))
return -1
def get_compilation_details():
brim = ["N/A", "N/A"]
try:
with open(tset.CONF_FILE, "r") as conf_file:
i = 0
for line in conf_file:
brim[i] = line.split("=")[1][1:-1] #format : OPTION = value
i += 1
except EnvironmentError:
tcom.pprint(1, "Unable to find {}".format(tset.CONF_FILE))
comp = {
"tuxml_version": __get_tuxml_version(),
"libc_version": __get_libc_version(),
"gcc_version": __get_gcc_version(),
"core_used": str(tset.NB_CORES),
"incremental_mod": str(tset.INCREMENTAL_MOD),
"git_branch": brim[0],
"docker_image": brim[1]
}
return comp
def launch_compilations():
status = -1
while status == -1:
missing_packages = []
missing_files = []
if tml.compilation() == -1:
if tml.log_analysis(missing_files, missing_packages) == 0:
if tml.install_missing_packages(missing_files,
missing_packages) == 0:
tcom.pprint(0, "Restarting compilation")
status = -1
else:
status = -3
else:
status = -2
else:
status = 0
if status == 0:
tcom.pprint(0, "Successfully compiled")
else:
tcom.pprint(
1, "Unable to compile using this KCONFIG_FILE, status={}".format(
status))
def send_data(path, err_log_file, compile_time):
tcom.pprint(2, "Sending config file and status to database")
# date
today = datetime.datetime.today()
dateFormatted = '{0:%Y-%m-%d}'.format(today)
# Config file
config_path = path + "/.config"
if not os.path.isfile(config_path):
tcom.pprint(1, ".config not found")
return 0
config_file = open(config_path, "r+b")
# Error log
has_compiled = compile_time > 0
err_log = open(path +
err_log_file, "r+b").read() if not has_compiled else ""
try:
conn = MySQLdb.connect(**irmaDBCredentials.info)
cursor = conn.cursor()
# Request
entry_sql = (
"INSERT INTO TuxML"
"(compilation_time, config_file, core_size, error)"
"VALUES (%(compilation_time)s, %(config_file)s, %(core_size)s, %(error)s)"
)
# Values for request
entry_values = {
"compilation_time": compile_time,
"config_file": config_file.read(),
"core_size": get_kernel_size(path),
"date": dateFormatted,
"error": err_log
}
cursor.execute(entry_sql, entry_values)
conn.commit()
tcom.pprint(0, "Successfully sent info to db")
return 1
except MySQLdb.Error as err:
tcom.pprint(1, "Can't send info to db : {}".format(err.args[1]))
finally:
conn.close()
return 0
def log_analysis(missing_files, missing_packages):
tcom.pprint(2, "Analyzing error log file")
with open(tset.PATH + tset.ERR_LOG_FILE, "r") as err_logs:
for line in err_logs:
if re.search("fatal error", line):
# case "file.c:48:19: fatal error: <file.h>: No such file or directory"
missing_files.append(line.split(":")[4])
elif re.search("Command not found", line):
# case "make[4]: <command> : command not found"
missing_packages.append(line.split(":")[1])
elif re.search("not found", line):
if len(line.split(":")) == 4:
# case "/bin/sh: 1: <command>: not found"
missing_files.append(line.split(":")[2])
else:
# ./scripts/gcc-plugin.sh: 11: ./scripts/gcc-plugin.sh: <package>: not found
missing_packages.append(line.split(":")[3])
else:
pass
if len(missing_files) > 0 or len(missing_packages) > 0:
tcom.pprint(0, "Missing file(s)/package(s) found")
return 0
else:
tcom.pprint(1, "Unable to find the missing package(s)")
return -1
def install_minimal_dependencies():
tcom.pprint(2, "Installing minimal dependencies")
minimal_pkgs = ["gcc", "make", "binutils", "util-linux", "e2fsprogs", "bc"]
if tcom.install_packages(minimal_pkgs) != 0:
tcom.pprint(2, "Unable to install minimal dependencies.")
return -1
else:
tcom.pprint(2, "Minimal dependencies successfully installed.")
return 0
def main():
args_handler()
# get environment details
tset.TUXML_ENV = tenv.get_environment_details()
# get the package manager
tset.PKG_MANAGER = tcom.get_package_manager()
if tset.PKG_MANAGER == None:
sys.exit(-1)
# updating package database
if tcom.update_system() != 0:
sys.exit(-1)
# install default packages
if tdep.install_default_dependencies() != 0:
sys.exit(-1)
# launching compilation
start_time = time.time()
status = -1
while status == -1:
missing_packages = []
missing_files = []
if compilation() == -1:
if log_analysis(missing_files, missing_packages) == 0:
if install_missing_packages(missing_files,
missing_packages) == 0:
tcom.pprint(0, "Restarting compilation")
status = -1
else:
status = -3
else:
status = -2
else:
status = 0
end_time = time.time()
# testing kernel
if status == 0:
status = end_time - start_time
compile_time = time.strftime("%H:%M:%S", time.gmtime(status))
tcom.pprint(0, "Successfully compiled in {}".format(compile_time))
# TODO tests
else:
tcom.pprint(
1, "Unable to compile using this KCONFIG_FILE, status={}".format(
status))
# sending data to IrmaDB
tsen.send_data(tset.PATH, tset.ERR_LOG_FILE, status)
def install_default_dependencies():
if (install_minimal_dependencies() != 0):
return -1
# Install packages common to all distro
tcom.pprint(2, "Installing default dependencies")
common_pkgs = [
"kmod", "jfsutils", "xfsprogs", "btrfs-progs", "pcmciautils", "ppp",
"grub", "iptables", "openssl"
]
# Now installation of packages with name that vary amongs distributions
debian_specific = [
"reiserfsprogs", "squashfs-tools", "quotatool", "nfs-kernel-server",
"procps", "mcelog", "libcrypto++6", "apt-utils", "gcc-6-plugin-dev",
"libssl-dev"
]
arch_specific = [
"reiserfsprogs", "squashfs-tools", "quota-tools", "isdn4k-utils",
"nfs-utils", "procps-ng", "oprofile"
]
redHat_specific = [
"reiserfs-utils", "squashfs-tools", "quotatool", "isdn4k-utils",
"nfs-utils", "procps-ng", "oprofile", "mcelog"
]
gentoo_specific = [
"reiserfsprogs", "squashfs-tools", "quotatool", "nfs-utils", "procps",
"mcelog", "oprofile"
]
suse_specific = ["reiserfs", "quota", "nfs-client", "procps"]
specific_pkgs = {
"apt-get": debian_specific,
"pacman": arch_specific,
"dnf": redHat_specific,
"yum": redHat_specific,
"emerge": gentoo_specific,
"zypper": suse_specific
}
if tcom.install_packages(common_pkgs +
specific_pkgs[tset.PKG_MANAGER]) != 0:
tcom.pprint(2, "Unable to install default dependencies.")
return -1
else:
tcom.pprint(2, "Default dependencies successfully installed.")
return 0
def compilation():
tcom.pprint(2, "Compilation in progress")
if not os.path.exists(tset.PATH + tset.LOG_DIR):
os.makedirs(tset.PATH + tset.LOG_DIR)
with open(tset.PATH + tset.STD_LOG_FILE,
"w") as std_logs, open(tset.PATH + tset.ERR_LOG_FILE,
"w") as err_logs:
status = subprocess.call("make -C " + tset.PATH + " -j" +
str(tset.NB_CORES) + " | ts -s",
shell=True,
stdout=std_logs,
stderr=err_logs)
if status == 0:
tcom.pprint(0, "Compilation done")
return 0
else:
tcom.pprint(2, "Compilation failed, exit status : {}".format(status))
return -1
def main():
targs.args_handler()
# init launch
if init_launcher() != 0:
sys.exit(-1)
# launching compilation
gen_config(tset.KCONFIG1)
tset.BASE_CONFIG_ID = launcher()
if tset.BASE_CONFIG_ID < 0:
sys.exit(-1)
if tset.KCONFIG2:
tset.INCREMENTAL_MOD = 1
gen_config(tset.KCONFIG2)
if launcher() < 0:
sys.exit(-1)
sys.exit(0)
cid_array = []
for i in range(0, tset.INCITERS):
tset.INCREMENTAL_MOD = 1
tset.TUXML_ENV["compilation"]["incremental_mod"] = "1"
tcom.pprint(0, "Launching incremental compilation #" + str(i + 1))
gen_config()
tmp_cid = launcher()
if tmp_cid < 0:
sys.exit(-1)
cid_array.append(tmp_cid)
tcom.pprint(0, "DATABASE CONFIGURATION ID={}".format(tset.BASE_CONFIG_ID))
for c in range(len(cid_array)):
tcom.pprint(
0, "INCREMENTAL CONFIGURATION ID #{}={}".format(c, cid_array[c]))
sys.exit(0)
def compilation():
tcom.pprint(2, "Compilation in progress")
# TODO barre de chargement [ ##########-------------------- ] 33%
if not os.path.exists(tset.PATH + tset.LOG_DIR):
os.makedirs(tset.PATH + tset.LOG_DIR)
with open(tset.PATH + tset.STD_LOG_FILE,
"w") as std_logs, open(tset.PATH + tset.ERR_LOG_FILE,
"w") as err_logs:
status = subprocess.call(
["make", "-C", tset.PATH, "-j" + str(tset.NB_CORES)],
stdout=std_logs,
stderr=err_logs)
if status == 0:
tcom.pprint(0, "Compilation done")
return 0
else:
tcom.pprint(2, "Compilation failed, exit status : {}".format(status))
return -1
def get_compressed_sizes():
tcom.pprint(2, "Computing compressed kernels")
compression = ["GZIP", "BZIP2", "LZMA", "XZ", "LZO", "LZ4"]
extension = [".gz", ".bz2", ".lzma", ".xz", ".lzo", ".lz4"]
res = ""
for c in compression:
if compress.enable(c, tset.PATH) == -1:
if res == "":
res = res + c + " : -1"
else:
res = res + " , " + c + " : -1"
else:
subprocess.run("make -C " + tset.PATH + " -j" + str(tset.NB_CORES),
shell=True,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL)
size = ""
vm = ""
bzImage = ""
try:
size = subprocess.check_output(
"wc -c " + tset.PATH + "/arch/x86/boot/compressed/*" +
extension[compression.index(c)],
shell=True,
stderr=subprocess.DEVNULL).decode().replace("\n",
"").split()[0]
except:
size = ""
try:
vm = subprocess.check_output(
"wc -c " + tset.PATH + "/arch/x86/boot/compressed/vmlinux",
shell=True,
stderr=subprocess.DEVNULL).decode().replace("\n",
"").split()[0]
except:
vm = ""
try:
bzImage = subprocess.check_output(
"wc -c " + tset.PATH + "/arch/x86/boot/bzImage",
shell=True,
stderr=subprocess.DEVNULL).decode().replace("\n",
"").split()[0]
except:
bzImage = ""
if size == "":
size = "-1"
if vm == "":
vm = "-1"
if bzImage == "":
bzImage = "-1"
if res == "":
res = res + c + "-bzImage : " + bzImage + " , " + c + "-vmlinux : " + vm + " , " + c + " : " + size
else:
res = res + " , " + c + "-bzImage : " + bzImage + " , " + c + "-vmlinux : " + vm + " , " + c + " : " + size
return res
def launcher():
start_compil_time = time.time()
install_time = 0
status = -1
while status == -1:
missing_packages = []
missing_files = []
if compilation() == -1:
start_install_time = time.time()
if log_analysis(missing_files, missing_packages) == 0:
if install_missing_packages(missing_files,
missing_packages) == 0:
tcom.pprint(0, "Restarting compilation")
status = -1
else:
status = -3
else:
status = -2
stop_install_time = time.time()
install_time += stop_install_time - start_install_time
if (tset.VERBOSE > 1):
tcom.pprint(
2, "TuxML has spent {} to install missing packages".format(
time.strftime(
"%H:%M:%S",
time.gmtime(stop_install_time -
start_install_time))))
else:
status = 0
end_compil_time = time.time()
if status == 0:
compile_time = end_compil_time - start_compil_time - install_time
duration = time.strftime("%H:%M:%S", time.gmtime(compile_time))
tcom.pprint(0, "Successfully compiled in {}".format(duration))
# launching tests
start_time = time.time()
status = tbch.boot_try()
end_time = time.time()
boot_time = -1
if status == 0:
boot_time = end_time - start_time
else:
boot_time = status
else:
compile_time = status
boot_time = -1
tcom.pprint(
1, "Unable to compile using this KCONFIG_FILE, status={}".format(
status))
# sending data to IrmaDB
cid = tsen.send_data(compile_time, boot_time)
if cid <= 0:
return -1
return cid
def gen_config(Kconfig=None):
if Kconfig:
try:
# debug config with given KCONFIG_SEED
int(Kconfig, 16)
tcom.pprint(2,
"Generating config file with KCONFIG_SEED=" + Kconfig)
status = subprocess.call([
"KCONFIG_SEED=" + Kconfig + " make -C " + tset.PATH +
" randconfig"
],
stdout=tset.OUTPUT,
stderr=tset.OUTPUT,
shell=True)
except ValueError:
# debug config with given KCONFIG_FILE
if os.path.exists(Kconfig):
tcom.pprint(2, "Using KCONFIG_FILE " + Kconfig)
shutil.copyfile(Kconfig, tset.PATH + "/.config")
return 0
else:
tcom.pprint(1, "KCONFIG_FILE doesn't exist")
return -1
else:
# generating new KConfig file
tcom.pprint(2, "Randomising new KCONFIG_FILE")
if not tset.TUXCONFIG == "tuxml.config":
tcom.pprint(2, "\"" + tset.TUXCONFIG + "\" used")
tcom.pprint(
2,
"KCONFIG_ALLCONFIG=" + os.path.dirname(os.path.abspath(__file__)) +
"/" + tset.TUXCONFIG + " make -C " + tset.PATH + " randconfig")
status = subprocess.call([
"KCONFIG_ALLCONFIG=" + os.path.dirname(os.path.abspath(__file__)) +
"/" + tset.TUXCONFIG + " make -C " + tset.PATH + " randconfig"
],
stdout=subprocess.DEVNULL,
stderr=tset.OUTPUT,
shell=True)
# testing status after subprocess call to make randconfig
if status != 0:
tcom.pprint(1, "Unable to generate KCONFIG_FILE")
return -1
return 0
def args_handler():
msg = "Welcome, this is the TuxML core program.\n\n"
msg += "The goal of TuxML is to automatically compile Linux kernel sources in order to\n"
msg += "build a database for a machine learning algorithm.\n"
msg += "If the compilation crashes, TuxML analyzes the error log file to determine the\n"
msg += "causes. There are two possible ways:\n"
msg += " * it is a missing package : TuxML will install it and resume the compilation\n"
msg += " * the error can't be fixed : the compilation stops\n"
msg += "Then TuxML sends the results of the compilation to the database.\n\n"
msg += "Keep in mind that the program is currently in developpement stage. Please visit\n"
msg += "our Github at <https://github.com/TuxML> in order to report any issue.\n"
msg += "Thanks !\n\n"
p_help = "path to the Linux source directory"
v_help = "increase or decrease output verbosity\n"
v_help += " " * 2 + "1 : very quiet\n"
v_help += " " * 2 + "2 : quiet\n"
v_help += " " * 2 + "3 : chatty (default)\n"
v_help += " " * 2 + "4 : very chatty\n"
V_help = "display TuxML version and exit"
d_help = "generate a new kconfig file with the KCONFIG_SEED or use\n"
d_help = "the KCONFIG_FILE given.\n"
c_help = "define the number of CPU cores to use during the\n"
c_help += "compilation. By default TuxML use all the availables\n"
c_help += "cores on the system."
i_help = "incremental mod does not erase files from previous\n"
i_help += "compilations. The I parameter corresponds to the number\n"
i_help += "of incremental compilation to launch."
j_help = "incremental mod with two specifics KCONFIG"
s_help = "choose on which database send the compilation results"
t_help = "choose to use the tiny_tuxml.config pre-set file"
parser = argparse.ArgumentParser(
description=msg, formatter_class=argparse.RawTextHelpFormatter)
gexcl1 = parser.add_mutually_exclusive_group()
parser.add_argument("source_path", help=p_help)
parser.add_argument("-v",
"--verbose",
help=v_help,
type=int,
choices=range(1, 5))
parser.add_argument("-V",
"--version",
help=V_help,
action='version',
version='%(prog)s pre-alpha v0.2')
parser.add_argument("-c",
"--cores",
help=c_help,
type=int,
metavar="NB_CORES")
parser.add_argument("-d",
"--debug",
help=d_help,
type=str,
metavar="KCONFIG")
gexcl1.add_argument("--incremental", help=i_help, type=int, metavar="NINC")
gexcl1.add_argument("--incrementalVS",
help=j_help,
type=str,
metavar="KCONFIG",
nargs=2)
parser.add_argument("--database",
help=s_help,
type=str,
default='prod',
choices=['prod', 'dev'])
parser.add_argument("--tiny", help=t_help, action="store_true")
args = parser.parse_args()
# ask root credentials
if os.getuid() != 0:
sudo_args = ["sudo", sys.executable] + sys.argv + [os.environ]
os.execlpe('sudo', *sudo_args)
sys.exit(-1)
# setting up the database
tset.DB_NAME += args.database
if args.tiny:
tset.TUXCONFIG = "tiny_tuxml.config"
# manage level of verbosity
if args.verbose:
tset.VERBOSE = args.verbose
if tset.VERBOSE > 3:
tset.OUTPUT = sys.__stdout__
else:
tset.VERBOSE = 3
# store the linux source path in a global var
if not os.path.exists(args.source_path):
tcom.pprint(1, "This path doesn't exist")
sys.exit(-1)
else:
tset.PATH = args.source_path
# enable or disable incremental mod (clean or not clean, that's the question)
if args.incremental:
tset.INCITERS = args.incremental
else:
# cleaning previous compilation
tset.INCREMENTAL_MOD = 0
tcom.pprint(2, "Cleaning previous compilation")
status = subprocess.call(["make", "-C", tset.PATH, "clean"],
stdout=tset.OUTPUT,
stderr=tset.OUTPUT)
if status != 0:
tcom.pprint(1, "Unable to clean previous compilation")
sys.exit(-1)
# handle debug mode
if args.debug:
tset.KCONFIG1 = args.debug
# handle incremental versus mod
if args.incrementalVS:
tset.KCONFIG1 = args.incrementalVS[0]
tset.KCONFIG2 = args.incrementalVS[1]
print(tset.KCONFIG1, ' ', tset.KCONFIG2)
# set the number of cores
if args.cores:
tset.NB_CORES = args.cores
else:
try:
tset.NB_CORES = int(tenv.get_hardware_details()["cpu_cores"])
except ValueError:
tcom.pprint(1, "Wrong number of CPU cores value")
sys.exit(-1)
def args_handler():
msg = "Welcome, this is the TuxML core program.\n\n"
msg += "The goal of TuxML is to automatically compile Linux kernel sources in order to\n"
msg += "build a database for a machine learning algorithm.\n"
msg += "If the compilation crashes, TuxML analyzes the error log file to determine the\n"
msg += "causes. There are two possible ways:\n"
msg += " * it is a missing package : TuxML will install it and resume the compilation\n"
msg += " * the error can't be fixed : the compilation stops\n"
msg += "Then TuxML sends the results of the compilation to the database.\n\n"
msg += "Keep in mind that the program is currently in developpement stage. Please visit\n"
msg += "our Github at <https://github.com/TuxML> in order to report any issue.\n"
msg += "Thanks !\n\n"
p_help = "path to the Linux source directory"
v_help = "increase or decrease output verbosity\n"
v_help += " " * 2 + "0 : quiet\n"
v_help += " " * 2 + "1 : normal (default)\n"
v_help += " " * 2 + "2 : chatty\n"
V_help = "display TuxML version and exit"
d_help = "debug a given KCONFIG_SEED or KCONFIG_FILE. If no seed\n"
d_help += "or file are given, the script will use the existing\n"
d_help += "KCONFIG_FILE in the linux source directory"
c_help = "define the number of CPU cores to use during the\n"
c_help += "compilation. By default TuxML use all the availables\n"
c_help += "cores on the system."
nc_help = "do not erase files from previous compilations"
parser = argparse.ArgumentParser(
description=msg, formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument("source_path", help=p_help)
parser.add_argument("-v",
"--verbose",
help=v_help,
type=int,
choices=[0, 1, 2])
parser.add_argument("-V",
"--version",
help=V_help,
action='version',
version='%(prog)s pre-alpha v0.2')
parser.add_argument("-c",
"--cores",
help=c_help,
type=int,
metavar="NB_CORES")
parser.add_argument("-d",
"--debug",
help=d_help,
type=str,
metavar="KCONFIG",
nargs='?',
const=-1)
parser.add_argument("--no-clean", help=nc_help, action="store_true")
args = parser.parse_args()
# ask root credentials
if os.getuid() != 0:
sudo_args = ["sudo", "-k", sys.executable] + sys.argv + [os.environ]
os.execlpe('sudo', *sudo_args)
sys.exit(-1)
# manage level of verbosity
if args.verbose:
tset.VERBOSE = args.verbose
if tset.VERBOSE > 1:
tset.OUTPUT = sys.__stdout__
else:
tset.VERBOSE = 1
# store the linux source path in a global var
if not os.path.exists(args.source_path):
tcom.pprint(1, "This path doesn't exist")
sys.exit(-1)
else:
tset.PATH = args.source_path
# enable or disable incremental mod (clean or not clean, that's the question)
tset.INCREMENTAL_MOD = args.no_clean
if not args.no_clean:
# cleaning previous compilation
tcom.pprint(2, "Cleaning previous compilation")
subprocess.call(["make", "-C", tset.PATH, "mrproper"],
stdout=tset.OUTPUT,
stderr=tset.OUTPUT)
# handle debug mode
if args.debug:
if args.debug == -1:
# use previous config file
if not os.path.exists(tset.PATH + "/.config"):
tcom.pprint(1, "KCONFIG_FILE not found")
sys.exit(-1)
else:
tcom.pprint(2, "Using previous KCONFIG_FILE")
else:
# debug config with given KCONFIG_SEED
try:
kconfig_seed = int(args.debug, 16)
except ValueError:
kconfig_seed = -1
if kconfig_seed != -1:
tcom.pprint(
2,
"Generating config file with KCONFIG_SEED=" + args.debug)
output = subprocess.call([
"KCONFIG_SEED=" + args.debug + " make -C " + tset.PATH +
" randconfig"
],
stdout=tset.OUTPUT,
stderr=tset.OUTPUT,
shell=True)
else:
# debug config with given KCONFIG_FILE
if os.path.exists(args.debug):
tcom.pprint(2, "Using KCONFIG_FILE " + args.debug)
shutil.copyfile(args.debug, tset.PATH +
"/.config") # TODO maybe a better way ?
else:
tcom.pprint(
1,
"Invalid KCONFIG_SEED or KCONFIG_FILE doesn't exist")
sys.exit(-1)
else:
# generating new KConfig file
tcom.pprint(2, "Randomising new config file")
output = subprocess.call([
"KCONFIG_ALLCONFIG=" + os.path.dirname(os.path.abspath(__file__)) +
"/tuxml.config make -C " + tset.PATH + " randconfig"
],
stdout=tset.OUTPUT,
stderr=tset.OUTPUT,
shell=True)
# set the number of cores
if args.cores:
tset.NB_CORES = args.cores
else:
try:
tset.NB_CORES = int(tenv.get_hardware_details()["cpu_cores"])
except ValueError:
tcom.pprint(1, "Wrong number of CPU cores value")
sys.exit(-1)
def send_data(compile_time, boot_time):
tcom.pprint(2, "Sending compilation and tests results to database")
# Log files
logfiles = [
tset.PATH + "/.config", tset.PATH + tset.STD_LOG_FILE,
tset.PATH + tset.ERR_LOG_FILE
]
for logfile in logfiles:
if not os.path.isfile(logfile):
tcom.pprint(1, "{} not found".format(logfile))
return -1
try:
socket = MySQLdb.connect(tset.HOST, tset.DB_USER, tset.DB_PASSWD,
tset.DB_NAME)
cursor = socket.cursor()
# Values for request
# date = time.gmtime(time.time())
sizes_compressed = get_compressed_sizes()
date = time.localtime(time.time())
args = {
"compilation_date": time.strftime("%Y-%m-%d %H:%M:%S", date),
"compilation_time": str(compile_time),
"config_file": bz2.compress(open(logfiles[0], "rb").read()),
"stdlog_file": bz2.compress(open(logfiles[1], "rb").read()),
"errlog_file": bz2.compress(open(logfiles[2], "rb").read()),
"core_size": str(get_size_kernel()),
"compressed_sizes": sizes_compressed,
"dependencies": open("/TuxML/dependences.txt", "r").read()
}
for dico in tset.TUXML_ENV:
args.update(tset.TUXML_ENV[dico])
keys = ",".join(args.keys())
values = ','.join(['%s'] * len(args.values()))
query = "INSERT INTO Compilations({}) VALUES({})".format(keys, values)
cursor.execute(query, list(args.values()))
query = "SELECT cid FROM Compilations ORDER BY cid DESC LIMIT 1"
cursor.execute(query)
cid = cursor.fetchone()[0]
if tset.INCREMENTAL_MOD and tset.BASE_CONFIG_ID != 0:
query = "INSERT INTO Incremental_compilations(cid_incmod, cid_origin) VALUES (%s, %s)"
cursor.execute(query, [cid, tset.BASE_CONFIG_ID])
query = "INSERT INTO Tests(cid, test_date, boot_time) VALUES (%s, %s, %s)"
cursor.execute(
query,
[cid, time.strftime("%Y-%m-%d %H:%M:%S", date), boot_time])
socket.commit()
socket.close()
# file_upload(logfiles, date)
tcom.pprint(0, "Successfully sent info to db")
return cid
except MySQLdb.Error as err:
tcom.pprint(1, "Can't send info to db : {}".format(err))
return -1
