def mkconfig( test_command ):
with mktemp() as config:
check_call( [ "cp", configfile, config ] )
infomsg( "using test-command:", test_command )
with open( config, 'a' ) as fh:
infomsg( "--test-command", test_command, file = fh )
yield config
def build_variant( self, genome ):
if len( genome ) == 0:
genome = [ "original" ]
seed = self.config.get( "--config", "0" )
with mktemp() as genome_file:
with open( genome_file, 'w' ) as fh:
print( "--oracle-genome", " ".join( genome ), file = fh )
cmd = [
self.genprog, self.configfile, genome_file,
"--seed", seed,
"--keep-source",
"--no-test-cache",
"--search", "oracle",
"--test-command", "true",
]
keepfiles = [ "000000", "repair.debug." + seed ]
tmpdir = tempfile.mkdtemp( dir = "." )
try:
for fname in keepfiles:
if os.path.exists( fname ):
os.rename( fname, os.path.join( tmpdir, fname ) )
with mktemp() as log:
try:
with open( log, 'w' ) as fh:
check_call( cmd, stdout = fh, stderr = fh )
except CalledProcessError:
with open( log ) as fh:
infomsg( fh.read(), file = sys.stderr )
raise
if os.path.exists( genome_file ):
os.remove( genome_file )
if os.path.exists( "000000/000000" ):
yield "000000/000000"
else:
yield None
finally:
for fname in keepfiles:
if os.path.exists( fname ):
check_call( [ "rm", "-rf", fname ] )
if os.path.exists( os.path.join( tmpdir, fname ) ):
os.rename( os.path.join( tmpdir, fname ), fname )
check_call( [ "rm", "-rf", tmpdir ] )
def run_test( self, exe ):
cmd = self.config[ "--test-command" ]
cmd = cmd.replace( "__EXE_NAME__", exe )
with mktemp() as fitnessfile:
tmp = cmd.replace( "__FITNESS_FILE__", fitnessfile )
call( [ "sh", "-c", tmp ] )
with open( fitnessfile ) as fh:
return float( fh.next().strip() )
def run_test( self, exe ):
cmd = self.config[ "--test-command" ]
cmd = cmd.replace( "__EXE_NAME__", exe )
with mktemp() as fitnessfile:
tmp = cmd.replace( "__FITNESS_FILE__", fitnessfile )
call( [ "sh", "-c", tmp ] )
with open( fitnessfile ) as fh:
yielded = False
for line in fh:
yield map( float, line.split() )
yielded = True
if not yielded:
yield [ 0.0 ]
def save_temp( fname ):
if not os.path.exists( fname ):
raise OSError( 2, "No such file or directory", fname )
with mktemp( dir = os.path.dirname( fname ) ) as tmp:
os.rename( fname, tmp )
try:
yield tmp
finally:
if os.path.exists( fname ):
call( [ "rm", "-rf", fname ] )
if os.path.exists( tmp ):
os.rename( tmp, fname )
def get_minimized_fitness( genome ):
infomsg( "getting minimized genome for", genome )
if not os.path.exists( genome + ".min" ):
record.time( "minimize.py", check_call, [
os.path.join( root, "bin", "minimize.py" ), genprog, config,
"--genome-file", genome,
"--cache", genome + ".cache",
"--save-binary", genome + ".bin",
"--save-source", genome + ".src",
"--save-genome", genome + ".min"
] )
if options.regenerate is not None:
cfg = Config()
cfg.load( config )
cmd = cfg[ "--test-command" ]
cmd = cmd.replace( "__EXE_NAME__", genome + ".bin" )
if options.regenerate == "wall":
cmd += " -j 1 --wall --repeat 100 --no-limit"
with record.context( "fitness eval" ):
fitnesses = list()
for i in range( options.rows ):
with mktemp() as fitnessfile:
# use call instead of check_call because test scripts always
# return non-zero status
tmp = cmd.replace( "__FITNESS_FILE__", fitnessfile )
call( [ "sh", "-c", tmp ] )
with open( fitnessfile ) as fh:
for line in fh:
value = float( line.split()[ 0 ] )
infomsg( " ", value )
fitnesses.append( value )
break
else:
infomsg(
"ERROR: no fitness for", genome + ".bin",
file = sys.stderr
)
exit( 2 )
return fitnesses
else:
with open( genome + ".min" ) as fh:
key = " ".join( fh.readlines() )
key = " ".join( key.split() )
d = shelve.open( genome + ".cache" )
return d[ key ]
def profile( cmd, **kw ):
f = 0
while f < 100 or 1900 < f:
f = int( random.gauss( 1000, 100 ) )
with mktemp() as tmp:
record = [
"perf", "record",
"--freq", str( f ),
"--quiet",
"--output", tmp,
"--"
] + cmd
check_call( record, **kw )
return get_metrics( tmp )
def getParallelFitness( self, *args ):
with mktemp( prefix = "vips" ) as tmpexe:
oldexe = self.exe
shutil.copyfile( self.exe, tmpexe )
shutil.copymode( self.exe, tmpexe )
try:
self.exe = tmpexe
results = ParallelTest.getParallelFitness( self, *args )
if self.options.error:
if results == [ [ 0 ] ]:
return [ [ 0 ], [ 0 ] ]
results.append( self.error )
return results
else:
return results
finally:
self.exe = oldexe
def saving( filename ):
if not os.path.exists( filename ):
yield
elif os.path.isdir( filename ):
tmp = tempfile.mkdtemp( dir = os.path.dirname( filename ) )
try:
tmpname = os.path.join( tmp, os.path.basename( filename ) )
try:
os.rename( filename, tmpname )
yield
finally:
os.rename( tmpname, filename )
finally:
os.rmdir( tmp )
else:
with mktemp( dir = os.path.dirname( filename ) ) as tmp:
os.rename( filename, tmp )
try:
yield
finally:
os.rename( tmp, filename )
def __enter__(self):
self.fnames = list()
for i in range(self.num):
self.fnames.append(mktemp(self.suffix).__enter__())
return self
print( "ERROR: you must use --pin-root or set PIN_ROOT", file = sys.stderr )
exit( 1 )
pin = os.path.join( os.environ[ "PIN_ROOT" ], "pin" )
check_call( [ "make", "-C", os.path.join( root, "pintool" ) ] )
pintool = os.path.join( root, "pintool", "obj-intel64", "print_lines.so" )
sources = list()
for d, dnames, fnames in os.walk( options.source ):
for fname in fnames:
fname = os.path.join( d, fname )
if os.path.splitext( fname )[ 1 ] == ".s":
sources.append( fname )
with mktemp() as exe:
infomsg( "INFO: relabeling source files" )
with renumber_files( sources ):
cmd = config[ "--compiler-command" ]
cmd = cmd.replace( "__SOURCE_NAME__", sources[ 0 ] )
cmd = cmd.replace( "__EXE_NAME__", exe )
infomsg( "INFO: compiling binary" )
check_call( [ "sh", "-c", cmd ] )
cmd = config[ "--test-command" ]
cmd = cmd.replace( "__EXE_NAME__", exe )
cmd = cmd.replace( "__FITNESS_FILE__", "/dev/null" )
with mktemp() as lines_file:
prefix = [
pin, "-injection", "child",
def process_genome( best ):
########
# Write the genome to disk so that minimize and maximize can use it
########
if options.stop_after is None:
genome = "genome.%d" % best[ 0 ]
else:
genome = "genome.%d-%d" % ( options.stop_after, best[ 0 ] )
this_genome = os.path.join( results, genome ) + ".orig"
if not os.path.exists( this_genome ) or options.force:
infomsg( "INFO: saving genome to", this_genome )
with open( this_genome, 'w' ) as fh:
infomsg( " ".join( lower_genome( best[ 1 ].split() ) ), file = fh )
########
# Maximize the genome for all inputs
########
if input_size not in options.inputs:
options.inputs = [ input_size ] + options.inputs
infomsg( "INFO: maximizing genome for inputs:", *options.inputs )
with saving( "multi.cache" ):
for test_input in options.inputs:
next_genome = this_genome + "." + test_input
if not os.path.exists( next_genome ) or options.force:
with mkconfig( test_cmd( test_input ) ) as config:
check_call( [
maximize, genprog, config,
"--genome-file", this_genome,
"--save-genome", next_genome
] )
with open( '/dev/null', 'w' ) as null:
status = call(
[ "diff", this_genome, next_genome ],
stdout = null, stderr = null
)
if status != 0:
this_genome = next_genome
infomsg( "INFO: final genome stored at", this_genome )
########
# Minimize the genome for the original input
########
if not options.skip_minimize:
min_genome = this_genome + ".min"
min_binary = min_genome + ".bin"
min_source = min_genome + ".src"
fnames = [ min_genome, min_binary, min_source ]
if options.force or any(
[ not os.path.exists( f ) for f in fnames ]
):
with mkconfig( test_cmd ) as config:
cmd = [
minimize, genprog, config,
"--genome-file", this_genome,
"--save-binary", min_binary,
"--save-genome", min_genome,
"--save-sources", min_source
]
if options.low_error is not None:
cmd += [ "--low-error", str( options.low_error ) ]
check_call( cmd )
this_genome = min_genome
########
# Measure the improvement over the original
########
if options.stop_after is None:
cache = lambda i: os.path.join(
results, "genome.%s-%s.cache" % ( options.filter, i )
)
else:
cache = lambda i: os.path.join(
results, "genome.%d-%s-%s.cache" % (
options.stop_after, options.filter, i
)
)
genome_key = os.path.basename( this_genome )
with ImprovementTable( os.path.join( results, "improvement" ) ) as imprv:
for test_input in options.inputs:
if not options.force and ( genome_key, test_input ) in imprv:
continue
with mkconfig( test_cmd( test_input ) ) as config:
cmd = [
minimize, genprog, config,
"--search", "none",
"--genome-file", this_genome,
]
if not options.force:
cmd += [ "--cache", cache( test_input ) ]
if options.low_error is not None:
cmd += [ "--low-error", str( options.low_error ) ]
with mktemp() as log:
try:
pipeline( [ cmd, [ "tee", log ] ] )
except CalledProcessError:
if ( genome_key, test_input ) in imprv:
del imprv[ genome_key, test_input ]
continue
imprv[ genome_key, test_input ] = " ".join(
map( str, get_improvement( log ) )
)
# Get the best genomes from the search...
########
if options.search_csv is None or not os.path.exists( options.search_csv ):
if options.logfile is None:
logfiles = glob( os.path.join( results, "repair.debug.*" ) )
if len( logfiles ) != 1:
infomsg(
"ERROR: could not identify logfile. Use --logfile to specify"
)
for fname in logfiles:
infomsg( " ", fname )
exit( 1 )
options.logfile = logfiles[ 0 ]
if options.search_csv is None:
with mktemp() as csvfile:
parse_log( options.logfile, csvfile )
genomes = get_genomes( csvfile )
else:
parse_log( options.logfile, options.search_csv )
genomes = get_genomes( options.search_csv )
else:
genomes = get_genomes( options.search_csv )
if len( genomes ) == 0:
infomsg(
"ERROR: no variants found! Is %s a valid file?" %
options.logfile if options.search_csv is None else options.search_csv
)
exit( 1 )
def process_genome( best ):
for d, dnames, fnames in os.walk(buildir):
for fname in fnames:
if fname != "flags.make":
continue
fname = os.path.join(d, fname)
with open(fname) as fh:
lines = fh.readlines()
with open(fname, "w") as fh:
for line in lines:
line = line.rstrip()
if line.startswith("C_FLAGS") or line.startswith("CXX_FLAGS"):
line = line + " -save-temps=obj"
print(line, file=fh)
infomsg("INFO: running make")
with mktemp() as log:
pipeline([["make", "-C", buildir, "VERBOSE=1", "V=1", "install"], ["tee", log]])
infomsg("INFO: collecting assembly files")
asmfiles = dict()
with open("multi.txt", "w") as fh:
for d, dnames, fnames in os.walk(buildir):
for fname in fnames:
if os.path.splitext(fname)[1] == ".s":
fname = os.path.join(d, fname)
relpath = fname[len(buildir) + 1 :]
print(relpath, file=fh)
objname = os.path.splitext(relpath)[0] + ".o"
asmfiles[objname] = relpath
depgraph = dict()
check_path( srcdir )
infomsg( "checking that est-energy.py works..." )
check_path( est_energy )
check_cmd( [ est_energy, "-o", "/dev/null", "--", "true" ] )
infomsg( "pass" )
infomsg( "checking that limit command works..." )
check_path( limit )
check_cmd( [ limit, "1", "true" ] )
infomsg( "pass" )
config = Config()
config.load( configfile )
with mktemp( dir = "." ) as exe:
infomsg( "checking that we can compile the program..." )
compile_cmd = config[ "--compiler-command" ]
compile_cmd = compile_cmd.replace( "__SOURCE_NAME__", "src/000000.s" )
compile_cmd = compile_cmd.replace( "__EXE_NAME__", exe )
status = call( [ "sh", "-c", compile_cmd ] )
if status != 0:
infomsg( "ERROR: compile command failed!", file = sys.stderr )
infomsg( "ERROR:", compile_cmd, file = sys.stderr )
exit( status )
infomsg( "pass" )
with mktemp() as fitnessfile:
infomsg( "checking that the test command works..." )