command_template = "java -ea $JAVA_ARGS xtc.lang.cpp.FunctionChecker %s %s %s %s %s" if args.server and args.port: command_template += " %s %s" % (args.server, args.port) if arch == "x86": arch = linuxinstance.get_x86_archname(version) job_index = args.job_index if args.job_index != None else "0" functions_datafile = kmaxdata.functions_datafile(version, arch) functions_pc_datafile = kmaxdata.functions_pc_datafile(version, arch) unit_pc_datafile = kmaxdata.unit_pc_datafile(version, arch) clauses_datafile = kmaxdata.clauses_datafile(version, arch) model_assumptions_datafile = kmaxdata.model_assumptions_datafile(version, arch) linker_errors_datafile = kmaxdata.linker_errors_datafile(version, arch) command = command_template % (functions_datafile, functions_pc_datafile, unit_pc_datafile, clauses_datafile, model_assumptions_datafile) sys.stderr.write("%s\n" % (command)) if job_index == "0": kmaxdata.backup_existing_file(linker_errors_datafile) with tempfile.NamedTemporaryFile(dir=kmaxdata.kmax_scratch) as tmp: popen = subprocess.call(command, shell=True, stdout=tmp, stderr=tmp) tmp.seek(0, os.SEEK_SET) with lockfile.LockFile(linker_errors_datafile): with open(linker_errors_datafile, 'ab') as dataf: for line in tmp: dataf.write(line)
f.write(compilation_unit + '\n') datafile = kmaxdata.buildsystem_datafile(version, arch) with lockfile.LockFile(datafile): if os.path.exists(datafile): with tempfile.NamedTemporaryFile(dir=os.path.dirname(datafile), prefix=os.path.basename(datafile) + ".back", delete=False) as tmp: os.rename(datafile, tmp.name) with open(datafile, "wb") as f: pickle.dump(buildsystemdata, f) # get presence conditions unit_pc_datafile = kmaxdata.unit_pc_datafile(version, arch) kmaxdata.backup_existing_file(unit_pc_datafile) with tempfile.NamedTemporaryFile(dir=kmaxdata.kmax_scratch) as tmp: pc_command = compunit_command + ' --get-presence-conditions' + remaining_arguments print pc_command popen = subprocess.call(pc_command, shell=True, stdout=tmp) tmp.seek(0, os.SEEK_SET) with lockfile.LockFile(unit_pc_datafile): with open(unit_pc_datafile, 'ab') as dataf: for line in tmp: dataf.write(line) # get every c files with open(kmaxdata.everycfile_datafile(version), "wb") as f: command = 'find -type f | grep "\.c$" | sort | uniq | cut -c3-' popen = subprocess.Popen(command, shell=True, stdout=f)
golem_command = os.path.join(undertaker_path, "python/golem -v -i") golem_models = "/work/pcg234/golem" golem_datafile = kmaxdata.golem_datafile(version, arch) instance = linuxinstance.LinuxInstance() instance.enter(version) instance.init_arch(arch) if os.path.exists("./models"): shutil.rmtree("./models") shutil.copytree(os.path.join(golem_models, version, "models"), "./models") with tempfile.NamedTemporaryFile(dir=kmaxdata.kmax_scratch) as tmp: print tmp.name command = 'ARCH=%s python %s' % (arch, golem_command) print command popen = subprocess.call(command, shell=True, stderr=tmp, stdout=tmp) tmp.seek(0, os.SEEK_SET) units = set() for line in tmp: if line.startswith("FILE_"): # name is first field, trim "FILE_" from it units.add(line.split()[0][5:]) kmaxdata.backup_existing_file(golem_datafile) with lockfile.LockFile(golem_datafile): with open(golem_datafile, 'ab') as dataf: for unit in units: dataf.write("%s\n" % (unit)) instance.exit()
if arch == "x86": arch = linuxinstance.get_x86_archname(version) job_index = args.job_index if args.job_index != None else "0" if args.worklist != None: compilation_units_datafile = args.worklist else: compilation_units_datafile = kmaxdata.compilation_units_datafile(version, arch) compilation_units_worklist = os.path.join(kmaxdata.kmax_data, "checkers_worklist_" + version + arch) checkers_datafile = kmaxdata.checkers_datafile(version, arch) if job_index == "0": kmaxdata.backup_existing_file(compilation_units_worklist) with lockfile.LockFile(compilation_units_worklist): with open(compilation_units_worklist, 'wb') as outf: with open(compilation_units_datafile, 'rb') as inf: for line in inf: outf.write(line) kmaxdata.backup_existing_file(checkers_datafile) # -D CONFIG_X86 -D CONFIG_X86_64 -D CONFIG_64BIT clauses_datafile = kmaxdata.clauses_datafile(version, arch) model_assumptions_datafile = kmaxdata.model_assumptions_datafile(version, arch) superc_args = "-restrictConfigToPrefix CONFIG_ -U __ASSEMBLY__ -D __KERNEL__ -D CONFIG_64BIT -D CONFIG_X86 -D CONFIG_64BIT -D CONFIG_KASAN_SHADOW_OFFSETUL=0xdffffc0000000000UL -checkers -featureModel %s -modelAssumptions %s" % (clauses_datafile, model_assumptions_datafile) superc_args += linuxinstance.get_superc_args(version)
arch = linuxinstance.get_x86_archname(version) job_index = args.job_index if args.job_index != None else "0" if args.worklist != None: compilation_units_datafile = args.worklist else: compilation_units_datafile = kmaxdata.compilation_units_datafile( version, arch) compilation_units_worklist = os.path.join( kmaxdata.kmax_data, "dynamicanalysis_worklist_" + version + arch) dynamicanalysis_datafile = kmaxdata.dynamicanalysis_datafile(version, arch) if job_index == "0": kmaxdata.backup_existing_file(compilation_units_worklist) with lockfile.LockFile(compilation_units_worklist): with open(compilation_units_worklist, 'wb') as outf: with open(compilation_units_datafile, 'rb') as inf: for line in inf: outf.write(line) kmaxdata.backup_existing_file(dynamicanalysis_datafile) stat_parms = { 'preprocessor': '-preprocessor -preprocessorStatistics -size -configurationVariables -headerGuards', 'parser': '-parserStatistics -languageStatistics -conditionGranularity' } superc_args = linuxinstance.get_superc_args(version)
golem_command = os.path.join(undertaker_path, "python/golem -v -i") golem_models = "/work/pcg234/golem" golem_datafile = kmaxdata.golem_datafile(version, arch) instance = linuxinstance.LinuxInstance() instance.enter(version) instance.init_arch(arch) if os.path.exists("./models"): shutil.rmtree("./models") shutil.copytree(os.path.join(golem_models, version, "models"), "./models") with tempfile.NamedTemporaryFile(dir=kmaxdata.kmax_scratch) as tmp: print tmp.name command = 'ARCH=%s python %s' % (arch, golem_command) print command popen = subprocess.call(command, shell=True, stderr=tmp, stdout=tmp) tmp.seek(0, os.SEEK_SET) units = set() for line in tmp: if line.startswith("FILE_"): # name is first field, trim "FILE_" from it units.add(line.split()[0][5:]) kmaxdata.backup_existing_file(golem_datafile) with lockfile.LockFile(golem_datafile): with open(golem_datafile, 'ab') as dataf: for unit in units: dataf.write("%s\n" % (unit)) instance.exit()
arch = args.arch job_index = args.job_index if args.job_index != None else "0" if args.worklist != None: compilation_units_datafile = args.worklist else: compilation_units_datafile = kmaxdata.compilation_units_datafile( version, arch) compilation_units_worklist = os.path.join( kmaxdata.kmax_data, "cppstats_worklist_" + version + arch) cppstats_datafile = kmaxdata.cppstats_datafile(version, arch) if job_index == "0": kmaxdata.backup_existing_file(compilation_units_worklist) with lockfile.LockFile(compilation_units_worklist): with open(compilation_units_worklist, 'wb') as outf: with open(compilation_units_datafile, 'rb') as inf: for line in inf: outf.write(line) kmaxdata.backup_existing_file(cppstats_datafile) superc_args = '-preprocessor -size -configurationVariables -headerGuards' superc_args += linuxinstance.get_superc_args(version) print 'version = %s, arch = %s' % (version, arch) print 'compilation_units_datafile = %s' % (compilation_units_datafile) print 'compilation_units_worklist = %s' % (compilation_units_worklist) instance = linuxinstance.LinuxInstance()
n_files_per_lock = 100 # Records zlib-compressed presence condition expression strings presenceconds_datafile = kmaxdata.presenceconds_datafile(version, arch) # Records the hashcode, file start location, and length of each # compressed presence condition in the presenceconds_datafile presencecond_hashes_datafile = kmaxdata.presencecond_hashes_datafile( version, arch) # Records the presence condition used at each conditional presencecond_usages_datafile = kmaxdata.presencecond_usages_datafile( version, arch) # # Use a separate usages file for each node to avoid the io bottleneck # presencecond_usages_datafile += "." + job_index if job_index == "0": kmaxdata.backup_existing_file(compilation_units_worklist) with lockfile.LockFile(compilation_units_worklist): with open(compilation_units_worklist, 'wb') as outf: with open(compilation_units_datafile, 'rb') as inf: for line in inf: outf.write(line) kmaxdata.backup_existing_file(presenceconds_datafile) kmaxdata.backup_existing_file(presencecond_hashes_datafile) kmaxdata.backup_existing_file(presencecond_usages_datafile) superc_args = "-presenceConditions -preprocessor" superc_args += linuxinstance.get_superc_args(version) print 'version = %s, arch = %s' % (version, arch) print 'compilation_units_datafile = %s' % (compilation_units_datafile) print 'compilation_units_worklist = %s' % (compilation_units_worklist)
if arch == "x86": arch = linuxinstance.get_x86_archname(version) job_index = args.job_index if args.job_index != None else "0" if args.worklist != None: compilation_units_datafile = args.worklist else: compilation_units_datafile = kmaxdata.compilation_units_datafile(version, arch) compilation_units_worklist = os.path.join(kmaxdata.kmax_data, "dynamicanalysis_worklist_" + version + arch) dynamicanalysis_datafile = kmaxdata.dynamicanalysis_datafile(version, arch) if job_index == "0": kmaxdata.backup_existing_file(compilation_units_worklist) with lockfile.LockFile(compilation_units_worklist): with open(compilation_units_worklist, 'wb') as outf: with open(compilation_units_datafile, 'rb') as inf: for line in inf: outf.write(line) kmaxdata.backup_existing_file(dynamicanalysis_datafile) stat_parms = { 'preprocessor': '-preprocessor -preprocessorStatistics -size -configurationVariables -headerGuards', 'parser' : '-parserStatistics -languageStatistics -conditionGranularity' } superc_args = linuxinstance.get_superc_args(version) print 'version = %s, arch = %s' % (version, arch) print 'compilation_units_datafile = %s' % (compilation_units_datafile) print 'compilation_units_worklist = %s' % (compilation_units_worklist)
if arch == "x86": arch = linuxinstance.get_x86_archname(version) job_index = args.job_index if args.job_index != None else "0" if args.worklist != None: compilation_units_datafile = args.worklist else: compilation_units_datafile = kmaxdata.compilation_units_datafile(version, arch) compilation_units_worklist = os.path.join(kmaxdata.kmax_data, "functions_worklist_" + version + arch) functions_datafile = kmaxdata.functions_datafile(version, arch) functions_pc_datafile = kmaxdata.functions_pc_datafile(version, arch) if job_index == "0": kmaxdata.backup_existing_file(compilation_units_worklist) with lockfile.LockFile(compilation_units_worklist): with open(compilation_units_worklist, 'wb') as outf: with open(compilation_units_datafile, 'rb') as inf: for line in inf: outf.write(line) kmaxdata.backup_existing_file(functions_datafile) kmaxdata.backup_existing_file(functions_pc_datafile) # superc_args = '-restrictFreeToPrefix CONFIG_ -U __ASSEMBLY__ -D __KERNEL__ -D CONFIG_X86 -D CONFIG_X86_64 -D CONFIG_64BIT -U CONFIG_X86_32 -D CONFIG_KASAN_SHADOW_OFFSETUL=0xdffffc0000000000UL -functionAnalysis -printErrorConditions' # superc_args = '-U __ASSEMBLY__ -D __KERNEL__ -D CONFIG_X86 -D CONFIG_X86_64 -D CONFIG_64BIT -U CONFIG_X86_32 -D CONFIG_KASAN_SHADOW_OFFSETUL=0xdffffc0000000000UL -functionAnalysis -printErrorConditions' # superc_args = '-D __KERNEL__ -D CONFIG_KASAN_SHADOW_OFFSETUL=0xdffffc0000000000UL -functionAnalysis -printErrorConditions' # superc_args = '-restrictConfigToPrefix CONFIG_ -D __KERNEL__ -U __ASSEMBLY__ -D __KERNEL__ -D CONFIG_X86 -D CONFIG_X86_64 -D CONFIG_64BIT -U CONFIG_X86_32 -D CONFIG_KASAN_SHADOW_OFFSETUL=0xdffffc0000000000UL -functionAnalysis -printErrorConditions' # superc_args = '-restrictConfigToPrefix CONFIG_ -D __KERNEL__ -D CONFIG_KASAN_SHADOW_OFFSETUL=0xdffffc0000000000UL -functionAnalysis -printErrorConditions' superc_args = '-U __ASSEMBLY__ -D __KERNEL__ -D CONFIG_KASAN_SHADOW_OFFSETUL=0xdffffc0000000000UL -functionAnalysis -printErrorConditions'