command_template = "java -ea $JAVA_ARGS xtc.lang.cpp.FunctionChecker %s %s %s %s %s"
if args.server and args.port:
command_template += " %s %s" % (args.server, args.port)
if arch == "x86":
arch = linuxinstance.get_x86_archname(version)
job_index = args.job_index if args.job_index != None else "0"
functions_datafile = kmaxdata.functions_datafile(version, arch)
functions_pc_datafile = kmaxdata.functions_pc_datafile(version, arch)
unit_pc_datafile = kmaxdata.unit_pc_datafile(version, arch)
clauses_datafile = kmaxdata.clauses_datafile(version, arch)
model_assumptions_datafile = kmaxdata.model_assumptions_datafile(version, arch)
linker_errors_datafile = kmaxdata.linker_errors_datafile(version, arch)
command = command_template % (functions_datafile, functions_pc_datafile, unit_pc_datafile, clauses_datafile, model_assumptions_datafile)
sys.stderr.write("%s\n" % (command))
if job_index == "0":
kmaxdata.backup_existing_file(linker_errors_datafile)
with tempfile.NamedTemporaryFile(dir=kmaxdata.kmax_scratch) as tmp:
popen = subprocess.call(command, shell=True, stdout=tmp, stderr=tmp)
tmp.seek(0, os.SEEK_SET)
with lockfile.LockFile(linker_errors_datafile):
with open(linker_errors_datafile, 'ab') as dataf:
for line in tmp:
dataf.write(line)
f.write(compilation_unit + '\n')
datafile = kmaxdata.buildsystem_datafile(version, arch)
with lockfile.LockFile(datafile):
if os.path.exists(datafile):
with tempfile.NamedTemporaryFile(dir=os.path.dirname(datafile),
prefix=os.path.basename(datafile) + ".back",
delete=False) as tmp:
os.rename(datafile, tmp.name)
with open(datafile, "wb") as f:
pickle.dump(buildsystemdata, f)
# get presence conditions
unit_pc_datafile = kmaxdata.unit_pc_datafile(version, arch)
kmaxdata.backup_existing_file(unit_pc_datafile)
with tempfile.NamedTemporaryFile(dir=kmaxdata.kmax_scratch) as tmp:
pc_command = compunit_command + ' --get-presence-conditions' + remaining_arguments
print pc_command
popen = subprocess.call(pc_command, shell=True, stdout=tmp)
tmp.seek(0, os.SEEK_SET)
with lockfile.LockFile(unit_pc_datafile):
with open(unit_pc_datafile, 'ab') as dataf:
for line in tmp:
dataf.write(line)
# get every c files
with open(kmaxdata.everycfile_datafile(version), "wb") as f:
command = 'find -type f | grep "\.c$" | sort | uniq | cut -c3-'
popen = subprocess.Popen(command, shell=True, stdout=f)
golem_command = os.path.join(undertaker_path, "python/golem -v -i")
golem_models = "/work/pcg234/golem"
golem_datafile = kmaxdata.golem_datafile(version, arch)
instance = linuxinstance.LinuxInstance()
instance.enter(version)
instance.init_arch(arch)
if os.path.exists("./models"):
shutil.rmtree("./models")
shutil.copytree(os.path.join(golem_models, version, "models"), "./models")
with tempfile.NamedTemporaryFile(dir=kmaxdata.kmax_scratch) as tmp:
print tmp.name
command = 'ARCH=%s python %s' % (arch, golem_command)
print command
popen = subprocess.call(command, shell=True, stderr=tmp, stdout=tmp)
tmp.seek(0, os.SEEK_SET)
units = set()
for line in tmp:
if line.startswith("FILE_"):
# name is first field, trim "FILE_" from it
units.add(line.split()[0][5:])
kmaxdata.backup_existing_file(golem_datafile)
with lockfile.LockFile(golem_datafile):
with open(golem_datafile, 'ab') as dataf:
for unit in units:
dataf.write("%s\n" % (unit))
instance.exit()
if arch == "x86":
arch = linuxinstance.get_x86_archname(version)
job_index = args.job_index if args.job_index != None else "0"
if args.worklist != None:
compilation_units_datafile = args.worklist
else:
compilation_units_datafile = kmaxdata.compilation_units_datafile(version, arch)
compilation_units_worklist = os.path.join(kmaxdata.kmax_data, "checkers_worklist_" + version + arch)
checkers_datafile = kmaxdata.checkers_datafile(version, arch)
if job_index == "0":
kmaxdata.backup_existing_file(compilation_units_worklist)
with lockfile.LockFile(compilation_units_worklist):
with open(compilation_units_worklist, 'wb') as outf:
with open(compilation_units_datafile, 'rb') as inf:
for line in inf:
outf.write(line)
kmaxdata.backup_existing_file(checkers_datafile)
# -D CONFIG_X86 -D CONFIG_X86_64 -D CONFIG_64BIT
clauses_datafile = kmaxdata.clauses_datafile(version, arch)
model_assumptions_datafile = kmaxdata.model_assumptions_datafile(version, arch)
superc_args = "-restrictConfigToPrefix CONFIG_ -U __ASSEMBLY__ -D __KERNEL__ -D CONFIG_64BIT -D CONFIG_X86 -D CONFIG_64BIT -D CONFIG_KASAN_SHADOW_OFFSETUL=0xdffffc0000000000UL -checkers -featureModel %s -modelAssumptions %s" % (clauses_datafile, model_assumptions_datafile)
superc_args += linuxinstance.get_superc_args(version)
arch = linuxinstance.get_x86_archname(version)
job_index = args.job_index if args.job_index != None else "0"
if args.worklist != None:
compilation_units_datafile = args.worklist
else:
compilation_units_datafile = kmaxdata.compilation_units_datafile(
version, arch)
compilation_units_worklist = os.path.join(
kmaxdata.kmax_data, "dynamicanalysis_worklist_" + version + arch)
dynamicanalysis_datafile = kmaxdata.dynamicanalysis_datafile(version, arch)
if job_index == "0":
kmaxdata.backup_existing_file(compilation_units_worklist)
with lockfile.LockFile(compilation_units_worklist):
with open(compilation_units_worklist, 'wb') as outf:
with open(compilation_units_datafile, 'rb') as inf:
for line in inf:
outf.write(line)
kmaxdata.backup_existing_file(dynamicanalysis_datafile)
stat_parms = {
'preprocessor':
'-preprocessor -preprocessorStatistics -size -configurationVariables -headerGuards',
'parser': '-parserStatistics -languageStatistics -conditionGranularity'
}
superc_args = linuxinstance.get_superc_args(version)
golem_command = os.path.join(undertaker_path, "python/golem -v -i")
golem_models = "/work/pcg234/golem"
golem_datafile = kmaxdata.golem_datafile(version, arch)
instance = linuxinstance.LinuxInstance()
instance.enter(version)
instance.init_arch(arch)
if os.path.exists("./models"):
shutil.rmtree("./models")
shutil.copytree(os.path.join(golem_models, version, "models"), "./models")
with tempfile.NamedTemporaryFile(dir=kmaxdata.kmax_scratch) as tmp:
print tmp.name
command = 'ARCH=%s python %s' % (arch, golem_command)
print command
popen = subprocess.call(command, shell=True, stderr=tmp, stdout=tmp)
tmp.seek(0, os.SEEK_SET)
units = set()
for line in tmp:
if line.startswith("FILE_"):
# name is first field, trim "FILE_" from it
units.add(line.split()[0][5:])
kmaxdata.backup_existing_file(golem_datafile)
with lockfile.LockFile(golem_datafile):
with open(golem_datafile, 'ab') as dataf:
for unit in units:
dataf.write("%s\n" % (unit))
instance.exit()
arch = args.arch
job_index = args.job_index if args.job_index != None else "0"
if args.worklist != None:
compilation_units_datafile = args.worklist
else:
compilation_units_datafile = kmaxdata.compilation_units_datafile(
version, arch)
compilation_units_worklist = os.path.join(
kmaxdata.kmax_data, "cppstats_worklist_" + version + arch)
cppstats_datafile = kmaxdata.cppstats_datafile(version, arch)
if job_index == "0":
kmaxdata.backup_existing_file(compilation_units_worklist)
with lockfile.LockFile(compilation_units_worklist):
with open(compilation_units_worklist, 'wb') as outf:
with open(compilation_units_datafile, 'rb') as inf:
for line in inf:
outf.write(line)
kmaxdata.backup_existing_file(cppstats_datafile)
superc_args = '-preprocessor -size -configurationVariables -headerGuards'
superc_args += linuxinstance.get_superc_args(version)
print 'version = %s, arch = %s' % (version, arch)
print 'compilation_units_datafile = %s' % (compilation_units_datafile)
print 'compilation_units_worklist = %s' % (compilation_units_worklist)
instance = linuxinstance.LinuxInstance()
n_files_per_lock = 100
# Records zlib-compressed presence condition expression strings
presenceconds_datafile = kmaxdata.presenceconds_datafile(version, arch)
# Records the hashcode, file start location, and length of each
# compressed presence condition in the presenceconds_datafile
presencecond_hashes_datafile = kmaxdata.presencecond_hashes_datafile(
version, arch)
# Records the presence condition used at each conditional
presencecond_usages_datafile = kmaxdata.presencecond_usages_datafile(
version, arch)
# # Use a separate usages file for each node to avoid the io bottleneck
# presencecond_usages_datafile += "." + job_index
if job_index == "0":
kmaxdata.backup_existing_file(compilation_units_worklist)
with lockfile.LockFile(compilation_units_worklist):
with open(compilation_units_worklist, 'wb') as outf:
with open(compilation_units_datafile, 'rb') as inf:
for line in inf:
outf.write(line)
kmaxdata.backup_existing_file(presenceconds_datafile)
kmaxdata.backup_existing_file(presencecond_hashes_datafile)
kmaxdata.backup_existing_file(presencecond_usages_datafile)
superc_args = "-presenceConditions -preprocessor"
superc_args += linuxinstance.get_superc_args(version)
print 'version = %s, arch = %s' % (version, arch)
print 'compilation_units_datafile = %s' % (compilation_units_datafile)
print 'compilation_units_worklist = %s' % (compilation_units_worklist)
if arch == "x86":
arch = linuxinstance.get_x86_archname(version)
job_index = args.job_index if args.job_index != None else "0"
if args.worklist != None:
compilation_units_datafile = args.worklist
else:
compilation_units_datafile = kmaxdata.compilation_units_datafile(version, arch)
compilation_units_worklist = os.path.join(kmaxdata.kmax_data, "dynamicanalysis_worklist_" + version + arch)
dynamicanalysis_datafile = kmaxdata.dynamicanalysis_datafile(version, arch)
if job_index == "0":
kmaxdata.backup_existing_file(compilation_units_worklist)
with lockfile.LockFile(compilation_units_worklist):
with open(compilation_units_worklist, 'wb') as outf:
with open(compilation_units_datafile, 'rb') as inf:
for line in inf:
outf.write(line)
kmaxdata.backup_existing_file(dynamicanalysis_datafile)
stat_parms = { 'preprocessor': '-preprocessor -preprocessorStatistics -size -configurationVariables -headerGuards',
'parser' : '-parserStatistics -languageStatistics -conditionGranularity' }
superc_args = linuxinstance.get_superc_args(version)
print 'version = %s, arch = %s' % (version, arch)
print 'compilation_units_datafile = %s' % (compilation_units_datafile)
print 'compilation_units_worklist = %s' % (compilation_units_worklist)
if arch == "x86":
arch = linuxinstance.get_x86_archname(version)
job_index = args.job_index if args.job_index != None else "0"
if args.worklist != None:
compilation_units_datafile = args.worklist
else:
compilation_units_datafile = kmaxdata.compilation_units_datafile(version, arch)
compilation_units_worklist = os.path.join(kmaxdata.kmax_data, "functions_worklist_" + version + arch)
functions_datafile = kmaxdata.functions_datafile(version, arch)
functions_pc_datafile = kmaxdata.functions_pc_datafile(version, arch)
if job_index == "0":
kmaxdata.backup_existing_file(compilation_units_worklist)
with lockfile.LockFile(compilation_units_worklist):
with open(compilation_units_worklist, 'wb') as outf:
with open(compilation_units_datafile, 'rb') as inf:
for line in inf:
outf.write(line)
kmaxdata.backup_existing_file(functions_datafile)
kmaxdata.backup_existing_file(functions_pc_datafile)
# superc_args = '-restrictFreeToPrefix CONFIG_ -U __ASSEMBLY__ -D __KERNEL__ -D CONFIG_X86 -D CONFIG_X86_64 -D CONFIG_64BIT -U CONFIG_X86_32 -D CONFIG_KASAN_SHADOW_OFFSETUL=0xdffffc0000000000UL -functionAnalysis -printErrorConditions'
# superc_args = '-U __ASSEMBLY__ -D __KERNEL__ -D CONFIG_X86 -D CONFIG_X86_64 -D CONFIG_64BIT -U CONFIG_X86_32 -D CONFIG_KASAN_SHADOW_OFFSETUL=0xdffffc0000000000UL -functionAnalysis -printErrorConditions'
# superc_args = '-D __KERNEL__ -D CONFIG_KASAN_SHADOW_OFFSETUL=0xdffffc0000000000UL -functionAnalysis -printErrorConditions'
# superc_args = '-restrictConfigToPrefix CONFIG_ -D __KERNEL__ -U __ASSEMBLY__ -D __KERNEL__ -D CONFIG_X86 -D CONFIG_X86_64 -D CONFIG_64BIT -U CONFIG_X86_32 -D CONFIG_KASAN_SHADOW_OFFSETUL=0xdffffc0000000000UL -functionAnalysis -printErrorConditions'
# superc_args = '-restrictConfigToPrefix CONFIG_ -D __KERNEL__ -D CONFIG_KASAN_SHADOW_OFFSETUL=0xdffffc0000000000UL -functionAnalysis -printErrorConditions'
superc_args = '-U __ASSEMBLY__ -D __KERNEL__ -D CONFIG_KASAN_SHADOW_OFFSETUL=0xdffffc0000000000UL -functionAnalysis -printErrorConditions'