def get_installed_packages(dependencies): installed_packages = list() cmds = { "apt-get": "dpkg -s {}", "pacman": "pacman -Qs {} | grep \"/{} \"", "dnf": "rpm -qa | grep {}", "yum": [""] # "emerge": [], # "zypper": [] } for dep in dependencies: try: status = subprocess.call( [cmds[tset.PKG_MANAGER][0].format(dep, dep)], shell=True, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) #status = subprocess.call([cmds.get("apt-get").format(dep, dep)], shell=True, universal_newlines=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE) if (status == 0): installed_packages.append(dep) except subprocess.CalledProcessError: tcom.pprint(1, "Unable to build the list of installed packages.") return None return installed_packages
def write_bdd_to_csv(): csv_writer = csv.writer(csvfile, delimiter=';', quotechar='|', quoting=csv.QUOTE_MINIMAL) try: socket = MySQLdb.connect(tset.HOST, tset.DB_USER, tset.DB_PASSWD, "depML_DB") query = "SELECT DISTINCT depML_environnement.config_file,depML_environnement.environnement,packages.missing_files,packages.missing_packages,packages.candidate_missing_packages,packages.resolution_successful FROM packages , depML_environnement WHERE packages.cid=depML_environnement.cid" socket.query(query) res = socket.store_result() tuples = res.fetch_row(maxrows=0) first_row = True for t in tuples: optionName = list() #Column names for compilation options. optionValue = list() envDict = ast.literal_eval(tuples[0][1]) #Column names for environment details csv_env_col_names = list( chain(*[list(envDict.keys()) for d in list(envDict.values())])) for option in t[0].split('\n'): if '#' in option or len(option.split('=')) < 2: # We ignore the comment lines and the empty lines. continue else: option_name = option.split('=')[0] option_value = option.split('=')[1] #print(option_name+" " + option_value) if first_row: optionName.append(option_name) optionValue.append(option_value) if first_row: #We write the header only once in the first iteration. csv_col_names = optionName + csv_env_col_names csv_col_names.append("missing_files") csv_col_names.append("missing_packages") csv_col_names.append("candidate_missing_packages") csv_col_names.append("resolution_successful") #We write the header here, i.e the column names. csv_writer.writerow(csv_col_names) first_row = False #For each column, we write the values. csv_col_values = optionValue + list( chain( *[list(envDict.values()) for d in list(envDict.values())])) csv_col_values.append(t[2]) csv_col_values.append(t[3]) csv_col_values.append(t[4]) csv_col_values.append(t[5]) csv_writer.writerow(csv_col_values) except MySQLdb.Error as err: tcom.pprint(1, "Can't retrieve info from db : {}".format(err)) return -1
def get_kernel_size(): possible_filenames = [ "vmlinux", "vmlinux.bin", "vmlinuz", "zImage", "bzImage" ] for filename in possible_filenames: full_filename = tset.PATH + "/" + filename if os.path.isfile(full_filename): tcom.pprint(2, "kernel found: " + filename) return os.path.getsize(full_filename) return 0
def get_environment_details(): tcom.pprint(2, "Getting environment details") env = { "system": get_os_details(), "hardware": get_hardware_details(), "compilation": get_compilation_details() } if tset.VERBOSE > 1: environment_pprinter(env) export_as_csv(env["system"], env["hardware"], env["compilation"]) return env
def boot_try(): tcom.pprint(2, "Launching boot test on kernel") try: sbStatus = subprocess.call(["mkinitramfs","-o", tset.PATH + "/arch/x86_64/boot/initrd.img-4.13.3"], stdout=tset.OUTPUT, stderr=tset.OUTPUT) except Exception: sbStatus = -1; cmd = "qemu-system-x86_64" if sbStatus == 0: # procId = subprocess.Popen([cmd, "-kernel", tset.PATH + "/arch/x86_64/boot/bzImage", "-initrd", tset.PATH + "/arch/x86_64/boot/initrd.img-4.13.3", "-m", "1G", "-append", "console=ttyS0,38400", "-serial", "file:serial.out"], stdout=tset.OUTPUT, stderr=tset.OUTPUT) procId = subprocess.run([cmd, "-kernel", tset.PATH + "/arch/x86_64/boot/bzImage", "-initrd", tset.PATH + "/arch/x86_64/boot/initrd.img-4.13.3", "-m", "1G", "-append", "console=ttyS0,38400", "-serial", "file:serial.out"], stdout=subprocess.DEVNULL, stderr=tset.OUTPUT) rndCounter = 1 status = 1 time.sleep(5) try: outFile = open("serial.out",mode='r') except OSError: tcom.pprint(1, "Unable to open output file, assuming subprocess call failed !") return -3 while status == 1: time.sleep(10) # tcom.pprint(3, "Reading output file for boot end, attempt".format(rndCounter)) fileData = outFile.read() if re.search("(initramfs)", fileData): tcom.pprint(0, "Shell detected, kernel successfully booted") status = 0 if re.search("end Kernel panic", fileData): tcom.pprint(1, "Kernel panic detected, kernel probably failed to boot") status = -1 if rndCounter == 11: # default 51 tcom.pprint(1, "More than 60 attempts at reading, possible infinite loop in boot process, interrupting") status = -2 rndCounter = rndCounter + 1 # procId.terminate() # <-- terminate the process outFile.close() return status else: return -3
def sendToDB(): try: socket = MySQLdb.connect(tset.HOST, tset.DB_USER, tset.DB_PASSWD, "depML_DB") cursor = socket.cursor() # Values for request args_env = { "config_file": open(tset.PATH + "/.config", "r").read(), "environnement": tset.TUXML_ENV, } keys = ",".join(args_env.keys()) values = ','.join(['%s'] * len(args_env.values())) query = "INSERT INTO depML_environnement({}) VALUES({})".format( keys, values) cursor.execute(query, list(args_env.values())) socket.commit() for missing_file in tdep.tdepLogger.log.keys(): args_pkg = { "cid": cursor.lastrowid, "missing_files": str(missing_file), "missing_packages": tdep.tdepLogger.log.get(missing_file), "candidate_missing_packages": tdep.tdepLogger.candidates.get(missing_file).split(':')[0], "resolution_successful": tdep.tdepLogger.status.get(missing_file) } keys = ",".join(args_pkg.keys()) values = ','.join(['%s'] * len(args_pkg.values())) query = "INSERT INTO packages({}) VALUES({})".format(keys, values) print("QUERY IS: " + query) cursor.execute(query, list(args_pkg.values())) socket.commit() socket.close() # file_upload(logfiles, date) tcom.pprint(0, "Successfully sent info to db") return 0 except MySQLdb.Error as err: tcom.pprint(1, "Can't send info to db : {}".format(err)) return -1
def build_dependencies(missing_files, missing_packages): cmds = { "apt-get": ["apt-file search {}", "dpkg-query -l | grep {}"], "pacman": ["pkgfile -d {}", "pacman -Fs {}"], "dnf": ["dnf whatprovides *{}", "rpm -qa | grep {}"], "yum": ["yum whatprovides *{}", "rpm -qa | grep {}"] # "emerge": [], # "zypper": [] } if tset.VERBOSE > 0 and len(missing_files) > 0: tcom.pprint(3, "Those files are missing :") for mf in missing_files: if tset.VERBOSE > 0: print(" " * 3 + mf) if (tset.PKG_MANAGER is "pacman"): mf = mf.replace("/", " ") try: output = subprocess.check_output( [cmds[tset.PKG_MANAGER][0].format(mf)], shell=True, universal_newlines=True) except subprocess.CalledProcessError: tcom.pprint(1, "Unable to find the missing package(s)") return -1 # Sometimes the output gives several packages. The program takes the # first one and check if the package is already installed. If not, tuxml # installs it. Else it installs the next one lines = output.splitlines() i = 0 status = 0 while i < len(lines) and status == 0: package = lines[i].split(":")[0] # 0: package already installed # 1: package not installed status = subprocess.call( [cmds[tset.PKG_MANAGER][1].format(package)], stdout=tset.OUTPUT, stderr=tset.OUTPUT, shell=True) if status == 1: missing_packages.append(package) i += 1 # if tuxml reaches the end of the packages list without installing any package # it means that there is a problem with mf, so it returns an error if i > len(lines) and status == 0: tcom.pprint(1, "Unable to find the missing package(s)") return -1 tcom.pprint(0, "Dependencies built") return 0
def install_default_dependencies(): # Install packages common to all distro tcom.pprint(2, "Installing default dependencies") common_pkgs = [ "gcc", "make", "binutils", "util-linux", "kmod", "e2fsprogs", "jfsutils", "xfsprogs", "btrfs-progs", "pcmciautils", "ppp", "grub", "iptables", "openssl", "bc" ] # Now installation of packages with name that vary amongs distributions # TODO ajouter les paquets python3-pip, mysql-client?, libmariadbclient-dev, mysql-server? debian_specific = [ "reiserfsprogs", "squashfs-tools", "quotatool", "nfs-kernel-server", "procps", "mcelog", "libcrypto++6", "apt-utils" ] arch_specific = [ "reiserfsprogs", "squashfs-tools", "quota-tools", "isdn4k-utils", "nfs-utils", "procps-ng", "oprofile" ] redHat_specific = [ "reiserfs-utils", "squashfs-tools", "quotatool", "isdn4k-utils", "nfs-utils", "procps-ng", "oprofile", "mcelog" ] gentoo_specific = [ "reiserfsprogs", "squashfs-tools", "quotatool", "nfs-utils", "procps", "mcelog", "oprofile" ] suse_specific = ["reiserfs", "quota", "nfs-client", "procps"] specific_pkgs = { "apt-get": debian_specific, "pacman": arch_specific, "dnf": redHat_specific, "yum": redHat_specific, "emerge": gentoo_specific, "zypper": suse_specific } if tcom.install_packages(common_pkgs + specific_pkgs[tset.PKG_MANAGER]) != 0: return -1 else: return 0
def send_outputlog(cid, outputfilename, databasename): try: socket = MySQLdb.connect(tset.HOST, tset.DB_USER, tset.DB_PASSWD, databasename) cursor = socket.cursor() bzoutput = bz2.compress(open(outputfilename, "rb").read()) query = "UPDATE Compilations SET output_file = %s WHERE cid = %s" for c in cid: data = (bzoutput, c) cursor.execute(query, data) socket.commit() socket.close() return cid except MySQLdb.Error as err: tcom.pprint(1, "Can't send info to db : {}".format(err)) return -1
def get_compilation_details(): brim = ["N/A", "N/A"] try: with open(tset.CONF_FILE, "r") as conf_file: i = 0 for line in conf_file: brim[i] = line.split("=")[1][1:-1] #format : OPTION = value i += 1 except EnvironmentError: tcom.pprint(1, "Unable to find {}".format(tset.CONF_FILE)) comp = { "tuxml_version": __get_tuxml_version(), "libc_version": __get_libc_version(), "gcc_version": __get_gcc_version(), "core_used": str(tset.NB_CORES), "incremental_mod": str(tset.INCREMENTAL_MOD), "git_branch": brim[0], "docker_image": brim[1] } return comp
def launch_compilations(): status = -1 while status == -1: missing_packages = [] missing_files = [] if tml.compilation() == -1: if tml.log_analysis(missing_files, missing_packages) == 0: if tml.install_missing_packages(missing_files, missing_packages) == 0: tcom.pprint(0, "Restarting compilation") status = -1 else: status = -3 else: status = -2 else: status = 0 if status == 0: tcom.pprint(0, "Successfully compiled") else: tcom.pprint( 1, "Unable to compile using this KCONFIG_FILE, status={}".format( status))
def send_data(path, err_log_file, compile_time): tcom.pprint(2, "Sending config file and status to database") # date today = datetime.datetime.today() dateFormatted = '{0:%Y-%m-%d}'.format(today) # Config file config_path = path + "/.config" if not os.path.isfile(config_path): tcom.pprint(1, ".config not found") return 0 config_file = open(config_path, "r+b") # Error log has_compiled = compile_time > 0 err_log = open(path + err_log_file, "r+b").read() if not has_compiled else "" try: conn = MySQLdb.connect(**irmaDBCredentials.info) cursor = conn.cursor() # Request entry_sql = ( "INSERT INTO TuxML" "(compilation_time, config_file, core_size, error)" "VALUES (%(compilation_time)s, %(config_file)s, %(core_size)s, %(error)s)" ) # Values for request entry_values = { "compilation_time": compile_time, "config_file": config_file.read(), "core_size": get_kernel_size(path), "date": dateFormatted, "error": err_log } cursor.execute(entry_sql, entry_values) conn.commit() tcom.pprint(0, "Successfully sent info to db") return 1 except MySQLdb.Error as err: tcom.pprint(1, "Can't send info to db : {}".format(err.args[1])) finally: conn.close() return 0
def log_analysis(missing_files, missing_packages): tcom.pprint(2, "Analyzing error log file") with open(tset.PATH + tset.ERR_LOG_FILE, "r") as err_logs: for line in err_logs: if re.search("fatal error", line): # case "file.c:48:19: fatal error: <file.h>: No such file or directory" missing_files.append(line.split(":")[4]) elif re.search("Command not found", line): # case "make[4]: <command> : command not found" missing_packages.append(line.split(":")[1]) elif re.search("not found", line): if len(line.split(":")) == 4: # case "/bin/sh: 1: <command>: not found" missing_files.append(line.split(":")[2]) else: # ./scripts/gcc-plugin.sh: 11: ./scripts/gcc-plugin.sh: <package>: not found missing_packages.append(line.split(":")[3]) else: pass if len(missing_files) > 0 or len(missing_packages) > 0: tcom.pprint(0, "Missing file(s)/package(s) found") return 0 else: tcom.pprint(1, "Unable to find the missing package(s)") return -1
def install_minimal_dependencies(): tcom.pprint(2, "Installing minimal dependencies") minimal_pkgs = ["gcc", "make", "binutils", "util-linux", "e2fsprogs", "bc"] if tcom.install_packages(minimal_pkgs) != 0: tcom.pprint(2, "Unable to install minimal dependencies.") return -1 else: tcom.pprint(2, "Minimal dependencies successfully installed.") return 0
def main(): args_handler() # get environment details tset.TUXML_ENV = tenv.get_environment_details() # get the package manager tset.PKG_MANAGER = tcom.get_package_manager() if tset.PKG_MANAGER == None: sys.exit(-1) # updating package database if tcom.update_system() != 0: sys.exit(-1) # install default packages if tdep.install_default_dependencies() != 0: sys.exit(-1) # launching compilation start_time = time.time() status = -1 while status == -1: missing_packages = [] missing_files = [] if compilation() == -1: if log_analysis(missing_files, missing_packages) == 0: if install_missing_packages(missing_files, missing_packages) == 0: tcom.pprint(0, "Restarting compilation") status = -1 else: status = -3 else: status = -2 else: status = 0 end_time = time.time() # testing kernel if status == 0: status = end_time - start_time compile_time = time.strftime("%H:%M:%S", time.gmtime(status)) tcom.pprint(0, "Successfully compiled in {}".format(compile_time)) # TODO tests else: tcom.pprint( 1, "Unable to compile using this KCONFIG_FILE, status={}".format( status)) # sending data to IrmaDB tsen.send_data(tset.PATH, tset.ERR_LOG_FILE, status)
def install_default_dependencies(): if (install_minimal_dependencies() != 0): return -1 # Install packages common to all distro tcom.pprint(2, "Installing default dependencies") common_pkgs = [ "kmod", "jfsutils", "xfsprogs", "btrfs-progs", "pcmciautils", "ppp", "grub", "iptables", "openssl" ] # Now installation of packages with name that vary amongs distributions debian_specific = [ "reiserfsprogs", "squashfs-tools", "quotatool", "nfs-kernel-server", "procps", "mcelog", "libcrypto++6", "apt-utils", "gcc-6-plugin-dev", "libssl-dev" ] arch_specific = [ "reiserfsprogs", "squashfs-tools", "quota-tools", "isdn4k-utils", "nfs-utils", "procps-ng", "oprofile" ] redHat_specific = [ "reiserfs-utils", "squashfs-tools", "quotatool", "isdn4k-utils", "nfs-utils", "procps-ng", "oprofile", "mcelog" ] gentoo_specific = [ "reiserfsprogs", "squashfs-tools", "quotatool", "nfs-utils", "procps", "mcelog", "oprofile" ] suse_specific = ["reiserfs", "quota", "nfs-client", "procps"] specific_pkgs = { "apt-get": debian_specific, "pacman": arch_specific, "dnf": redHat_specific, "yum": redHat_specific, "emerge": gentoo_specific, "zypper": suse_specific } if tcom.install_packages(common_pkgs + specific_pkgs[tset.PKG_MANAGER]) != 0: tcom.pprint(2, "Unable to install default dependencies.") return -1 else: tcom.pprint(2, "Default dependencies successfully installed.") return 0
def compilation(): tcom.pprint(2, "Compilation in progress") if not os.path.exists(tset.PATH + tset.LOG_DIR): os.makedirs(tset.PATH + tset.LOG_DIR) with open(tset.PATH + tset.STD_LOG_FILE, "w") as std_logs, open(tset.PATH + tset.ERR_LOG_FILE, "w") as err_logs: status = subprocess.call("make -C " + tset.PATH + " -j" + str(tset.NB_CORES) + " | ts -s", shell=True, stdout=std_logs, stderr=err_logs) if status == 0: tcom.pprint(0, "Compilation done") return 0 else: tcom.pprint(2, "Compilation failed, exit status : {}".format(status)) return -1
def main(): targs.args_handler() # init launch if init_launcher() != 0: sys.exit(-1) # launching compilation gen_config(tset.KCONFIG1) tset.BASE_CONFIG_ID = launcher() if tset.BASE_CONFIG_ID < 0: sys.exit(-1) if tset.KCONFIG2: tset.INCREMENTAL_MOD = 1 gen_config(tset.KCONFIG2) if launcher() < 0: sys.exit(-1) sys.exit(0) cid_array = [] for i in range(0, tset.INCITERS): tset.INCREMENTAL_MOD = 1 tset.TUXML_ENV["compilation"]["incremental_mod"] = "1" tcom.pprint(0, "Launching incremental compilation #" + str(i + 1)) gen_config() tmp_cid = launcher() if tmp_cid < 0: sys.exit(-1) cid_array.append(tmp_cid) tcom.pprint(0, "DATABASE CONFIGURATION ID={}".format(tset.BASE_CONFIG_ID)) for c in range(len(cid_array)): tcom.pprint( 0, "INCREMENTAL CONFIGURATION ID #{}={}".format(c, cid_array[c])) sys.exit(0)
def compilation(): tcom.pprint(2, "Compilation in progress") # TODO barre de chargement [ ##########-------------------- ] 33% if not os.path.exists(tset.PATH + tset.LOG_DIR): os.makedirs(tset.PATH + tset.LOG_DIR) with open(tset.PATH + tset.STD_LOG_FILE, "w") as std_logs, open(tset.PATH + tset.ERR_LOG_FILE, "w") as err_logs: status = subprocess.call( ["make", "-C", tset.PATH, "-j" + str(tset.NB_CORES)], stdout=std_logs, stderr=err_logs) if status == 0: tcom.pprint(0, "Compilation done") return 0 else: tcom.pprint(2, "Compilation failed, exit status : {}".format(status)) return -1
def get_compressed_sizes(): tcom.pprint(2, "Computing compressed kernels") compression = ["GZIP", "BZIP2", "LZMA", "XZ", "LZO", "LZ4"] extension = [".gz", ".bz2", ".lzma", ".xz", ".lzo", ".lz4"] res = "" for c in compression: if compress.enable(c, tset.PATH) == -1: if res == "": res = res + c + " : -1" else: res = res + " , " + c + " : -1" else: subprocess.run("make -C " + tset.PATH + " -j" + str(tset.NB_CORES), shell=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) size = "" vm = "" bzImage = "" try: size = subprocess.check_output( "wc -c " + tset.PATH + "/arch/x86/boot/compressed/*" + extension[compression.index(c)], shell=True, stderr=subprocess.DEVNULL).decode().replace("\n", "").split()[0] except: size = "" try: vm = subprocess.check_output( "wc -c " + tset.PATH + "/arch/x86/boot/compressed/vmlinux", shell=True, stderr=subprocess.DEVNULL).decode().replace("\n", "").split()[0] except: vm = "" try: bzImage = subprocess.check_output( "wc -c " + tset.PATH + "/arch/x86/boot/bzImage", shell=True, stderr=subprocess.DEVNULL).decode().replace("\n", "").split()[0] except: bzImage = "" if size == "": size = "-1" if vm == "": vm = "-1" if bzImage == "": bzImage = "-1" if res == "": res = res + c + "-bzImage : " + bzImage + " , " + c + "-vmlinux : " + vm + " , " + c + " : " + size else: res = res + " , " + c + "-bzImage : " + bzImage + " , " + c + "-vmlinux : " + vm + " , " + c + " : " + size return res
def launcher(): start_compil_time = time.time() install_time = 0 status = -1 while status == -1: missing_packages = [] missing_files = [] if compilation() == -1: start_install_time = time.time() if log_analysis(missing_files, missing_packages) == 0: if install_missing_packages(missing_files, missing_packages) == 0: tcom.pprint(0, "Restarting compilation") status = -1 else: status = -3 else: status = -2 stop_install_time = time.time() install_time += stop_install_time - start_install_time if (tset.VERBOSE > 1): tcom.pprint( 2, "TuxML has spent {} to install missing packages".format( time.strftime( "%H:%M:%S", time.gmtime(stop_install_time - start_install_time)))) else: status = 0 end_compil_time = time.time() if status == 0: compile_time = end_compil_time - start_compil_time - install_time duration = time.strftime("%H:%M:%S", time.gmtime(compile_time)) tcom.pprint(0, "Successfully compiled in {}".format(duration)) # launching tests start_time = time.time() status = tbch.boot_try() end_time = time.time() boot_time = -1 if status == 0: boot_time = end_time - start_time else: boot_time = status else: compile_time = status boot_time = -1 tcom.pprint( 1, "Unable to compile using this KCONFIG_FILE, status={}".format( status)) # sending data to IrmaDB cid = tsen.send_data(compile_time, boot_time) if cid <= 0: return -1 return cid
def gen_config(Kconfig=None): if Kconfig: try: # debug config with given KCONFIG_SEED int(Kconfig, 16) tcom.pprint(2, "Generating config file with KCONFIG_SEED=" + Kconfig) status = subprocess.call([ "KCONFIG_SEED=" + Kconfig + " make -C " + tset.PATH + " randconfig" ], stdout=tset.OUTPUT, stderr=tset.OUTPUT, shell=True) except ValueError: # debug config with given KCONFIG_FILE if os.path.exists(Kconfig): tcom.pprint(2, "Using KCONFIG_FILE " + Kconfig) shutil.copyfile(Kconfig, tset.PATH + "/.config") return 0 else: tcom.pprint(1, "KCONFIG_FILE doesn't exist") return -1 else: # generating new KConfig file tcom.pprint(2, "Randomising new KCONFIG_FILE") if not tset.TUXCONFIG == "tuxml.config": tcom.pprint(2, "\"" + tset.TUXCONFIG + "\" used") tcom.pprint( 2, "KCONFIG_ALLCONFIG=" + os.path.dirname(os.path.abspath(__file__)) + "/" + tset.TUXCONFIG + " make -C " + tset.PATH + " randconfig") status = subprocess.call([ "KCONFIG_ALLCONFIG=" + os.path.dirname(os.path.abspath(__file__)) + "/" + tset.TUXCONFIG + " make -C " + tset.PATH + " randconfig" ], stdout=subprocess.DEVNULL, stderr=tset.OUTPUT, shell=True) # testing status after subprocess call to make randconfig if status != 0: tcom.pprint(1, "Unable to generate KCONFIG_FILE") return -1 return 0
def args_handler(): msg = "Welcome, this is the TuxML core program.\n\n" msg += "The goal of TuxML is to automatically compile Linux kernel sources in order to\n" msg += "build a database for a machine learning algorithm.\n" msg += "If the compilation crashes, TuxML analyzes the error log file to determine the\n" msg += "causes. There are two possible ways:\n" msg += " * it is a missing package : TuxML will install it and resume the compilation\n" msg += " * the error can't be fixed : the compilation stops\n" msg += "Then TuxML sends the results of the compilation to the database.\n\n" msg += "Keep in mind that the program is currently in developpement stage. Please visit\n" msg += "our Github at <https://github.com/TuxML> in order to report any issue.\n" msg += "Thanks !\n\n" p_help = "path to the Linux source directory" v_help = "increase or decrease output verbosity\n" v_help += " " * 2 + "1 : very quiet\n" v_help += " " * 2 + "2 : quiet\n" v_help += " " * 2 + "3 : chatty (default)\n" v_help += " " * 2 + "4 : very chatty\n" V_help = "display TuxML version and exit" d_help = "generate a new kconfig file with the KCONFIG_SEED or use\n" d_help = "the KCONFIG_FILE given.\n" c_help = "define the number of CPU cores to use during the\n" c_help += "compilation. By default TuxML use all the availables\n" c_help += "cores on the system." i_help = "incremental mod does not erase files from previous\n" i_help += "compilations. The I parameter corresponds to the number\n" i_help += "of incremental compilation to launch." j_help = "incremental mod with two specifics KCONFIG" s_help = "choose on which database send the compilation results" t_help = "choose to use the tiny_tuxml.config pre-set file" parser = argparse.ArgumentParser( description=msg, formatter_class=argparse.RawTextHelpFormatter) gexcl1 = parser.add_mutually_exclusive_group() parser.add_argument("source_path", help=p_help) parser.add_argument("-v", "--verbose", help=v_help, type=int, choices=range(1, 5)) parser.add_argument("-V", "--version", help=V_help, action='version', version='%(prog)s pre-alpha v0.2') parser.add_argument("-c", "--cores", help=c_help, type=int, metavar="NB_CORES") parser.add_argument("-d", "--debug", help=d_help, type=str, metavar="KCONFIG") gexcl1.add_argument("--incremental", help=i_help, type=int, metavar="NINC") gexcl1.add_argument("--incrementalVS", help=j_help, type=str, metavar="KCONFIG", nargs=2) parser.add_argument("--database", help=s_help, type=str, default='prod', choices=['prod', 'dev']) parser.add_argument("--tiny", help=t_help, action="store_true") args = parser.parse_args() # ask root credentials if os.getuid() != 0: sudo_args = ["sudo", sys.executable] + sys.argv + [os.environ] os.execlpe('sudo', *sudo_args) sys.exit(-1) # setting up the database tset.DB_NAME += args.database if args.tiny: tset.TUXCONFIG = "tiny_tuxml.config" # manage level of verbosity if args.verbose: tset.VERBOSE = args.verbose if tset.VERBOSE > 3: tset.OUTPUT = sys.__stdout__ else: tset.VERBOSE = 3 # store the linux source path in a global var if not os.path.exists(args.source_path): tcom.pprint(1, "This path doesn't exist") sys.exit(-1) else: tset.PATH = args.source_path # enable or disable incremental mod (clean or not clean, that's the question) if args.incremental: tset.INCITERS = args.incremental else: # cleaning previous compilation tset.INCREMENTAL_MOD = 0 tcom.pprint(2, "Cleaning previous compilation") status = subprocess.call(["make", "-C", tset.PATH, "clean"], stdout=tset.OUTPUT, stderr=tset.OUTPUT) if status != 0: tcom.pprint(1, "Unable to clean previous compilation") sys.exit(-1) # handle debug mode if args.debug: tset.KCONFIG1 = args.debug # handle incremental versus mod if args.incrementalVS: tset.KCONFIG1 = args.incrementalVS[0] tset.KCONFIG2 = args.incrementalVS[1] print(tset.KCONFIG1, ' ', tset.KCONFIG2) # set the number of cores if args.cores: tset.NB_CORES = args.cores else: try: tset.NB_CORES = int(tenv.get_hardware_details()["cpu_cores"]) except ValueError: tcom.pprint(1, "Wrong number of CPU cores value") sys.exit(-1)
def args_handler(): msg = "Welcome, this is the TuxML core program.\n\n" msg += "The goal of TuxML is to automatically compile Linux kernel sources in order to\n" msg += "build a database for a machine learning algorithm.\n" msg += "If the compilation crashes, TuxML analyzes the error log file to determine the\n" msg += "causes. There are two possible ways:\n" msg += " * it is a missing package : TuxML will install it and resume the compilation\n" msg += " * the error can't be fixed : the compilation stops\n" msg += "Then TuxML sends the results of the compilation to the database.\n\n" msg += "Keep in mind that the program is currently in developpement stage. Please visit\n" msg += "our Github at <https://github.com/TuxML> in order to report any issue.\n" msg += "Thanks !\n\n" p_help = "path to the Linux source directory" v_help = "increase or decrease output verbosity\n" v_help += " " * 2 + "0 : quiet\n" v_help += " " * 2 + "1 : normal (default)\n" v_help += " " * 2 + "2 : chatty\n" V_help = "display TuxML version and exit" d_help = "debug a given KCONFIG_SEED or KCONFIG_FILE. If no seed\n" d_help += "or file are given, the script will use the existing\n" d_help += "KCONFIG_FILE in the linux source directory" c_help = "define the number of CPU cores to use during the\n" c_help += "compilation. By default TuxML use all the availables\n" c_help += "cores on the system." nc_help = "do not erase files from previous compilations" parser = argparse.ArgumentParser( description=msg, formatter_class=argparse.RawTextHelpFormatter) parser.add_argument("source_path", help=p_help) parser.add_argument("-v", "--verbose", help=v_help, type=int, choices=[0, 1, 2]) parser.add_argument("-V", "--version", help=V_help, action='version', version='%(prog)s pre-alpha v0.2') parser.add_argument("-c", "--cores", help=c_help, type=int, metavar="NB_CORES") parser.add_argument("-d", "--debug", help=d_help, type=str, metavar="KCONFIG", nargs='?', const=-1) parser.add_argument("--no-clean", help=nc_help, action="store_true") args = parser.parse_args() # ask root credentials if os.getuid() != 0: sudo_args = ["sudo", "-k", sys.executable] + sys.argv + [os.environ] os.execlpe('sudo', *sudo_args) sys.exit(-1) # manage level of verbosity if args.verbose: tset.VERBOSE = args.verbose if tset.VERBOSE > 1: tset.OUTPUT = sys.__stdout__ else: tset.VERBOSE = 1 # store the linux source path in a global var if not os.path.exists(args.source_path): tcom.pprint(1, "This path doesn't exist") sys.exit(-1) else: tset.PATH = args.source_path # enable or disable incremental mod (clean or not clean, that's the question) tset.INCREMENTAL_MOD = args.no_clean if not args.no_clean: # cleaning previous compilation tcom.pprint(2, "Cleaning previous compilation") subprocess.call(["make", "-C", tset.PATH, "mrproper"], stdout=tset.OUTPUT, stderr=tset.OUTPUT) # handle debug mode if args.debug: if args.debug == -1: # use previous config file if not os.path.exists(tset.PATH + "/.config"): tcom.pprint(1, "KCONFIG_FILE not found") sys.exit(-1) else: tcom.pprint(2, "Using previous KCONFIG_FILE") else: # debug config with given KCONFIG_SEED try: kconfig_seed = int(args.debug, 16) except ValueError: kconfig_seed = -1 if kconfig_seed != -1: tcom.pprint( 2, "Generating config file with KCONFIG_SEED=" + args.debug) output = subprocess.call([ "KCONFIG_SEED=" + args.debug + " make -C " + tset.PATH + " randconfig" ], stdout=tset.OUTPUT, stderr=tset.OUTPUT, shell=True) else: # debug config with given KCONFIG_FILE if os.path.exists(args.debug): tcom.pprint(2, "Using KCONFIG_FILE " + args.debug) shutil.copyfile(args.debug, tset.PATH + "/.config") # TODO maybe a better way ? else: tcom.pprint( 1, "Invalid KCONFIG_SEED or KCONFIG_FILE doesn't exist") sys.exit(-1) else: # generating new KConfig file tcom.pprint(2, "Randomising new config file") output = subprocess.call([ "KCONFIG_ALLCONFIG=" + os.path.dirname(os.path.abspath(__file__)) + "/tuxml.config make -C " + tset.PATH + " randconfig" ], stdout=tset.OUTPUT, stderr=tset.OUTPUT, shell=True) # set the number of cores if args.cores: tset.NB_CORES = args.cores else: try: tset.NB_CORES = int(tenv.get_hardware_details()["cpu_cores"]) except ValueError: tcom.pprint(1, "Wrong number of CPU cores value") sys.exit(-1)
def send_data(compile_time, boot_time): tcom.pprint(2, "Sending compilation and tests results to database") # Log files logfiles = [ tset.PATH + "/.config", tset.PATH + tset.STD_LOG_FILE, tset.PATH + tset.ERR_LOG_FILE ] for logfile in logfiles: if not os.path.isfile(logfile): tcom.pprint(1, "{} not found".format(logfile)) return -1 try: socket = MySQLdb.connect(tset.HOST, tset.DB_USER, tset.DB_PASSWD, tset.DB_NAME) cursor = socket.cursor() # Values for request # date = time.gmtime(time.time()) sizes_compressed = get_compressed_sizes() date = time.localtime(time.time()) args = { "compilation_date": time.strftime("%Y-%m-%d %H:%M:%S", date), "compilation_time": str(compile_time), "config_file": bz2.compress(open(logfiles[0], "rb").read()), "stdlog_file": bz2.compress(open(logfiles[1], "rb").read()), "errlog_file": bz2.compress(open(logfiles[2], "rb").read()), "core_size": str(get_size_kernel()), "compressed_sizes": sizes_compressed, "dependencies": open("/TuxML/dependences.txt", "r").read() } for dico in tset.TUXML_ENV: args.update(tset.TUXML_ENV[dico]) keys = ",".join(args.keys()) values = ','.join(['%s'] * len(args.values())) query = "INSERT INTO Compilations({}) VALUES({})".format(keys, values) cursor.execute(query, list(args.values())) query = "SELECT cid FROM Compilations ORDER BY cid DESC LIMIT 1" cursor.execute(query) cid = cursor.fetchone()[0] if tset.INCREMENTAL_MOD and tset.BASE_CONFIG_ID != 0: query = "INSERT INTO Incremental_compilations(cid_incmod, cid_origin) VALUES (%s, %s)" cursor.execute(query, [cid, tset.BASE_CONFIG_ID]) query = "INSERT INTO Tests(cid, test_date, boot_time) VALUES (%s, %s, %s)" cursor.execute( query, [cid, time.strftime("%Y-%m-%d %H:%M:%S", date), boot_time]) socket.commit() socket.close() # file_upload(logfiles, date) tcom.pprint(0, "Successfully sent info to db") return cid except MySQLdb.Error as err: tcom.pprint(1, "Can't send info to db : {}".format(err)) return -1