Beispiel #1
0
def mkconfig( test_command ):
    with mktemp() as config:
        check_call( [ "cp", configfile, config ] )
        infomsg( "using test-command:", test_command )
        with open( config, 'a' ) as fh:
            infomsg( "--test-command", test_command, file = fh )
        yield config
Beispiel #2
0
    def build_variant( self, genome ):
        if len( genome ) == 0:
            genome = [ "original" ]

        seed = self.config.get( "--config", "0" )

        with mktemp() as genome_file:
            with open( genome_file, 'w' ) as fh:
                print( "--oracle-genome", " ".join( genome ), file = fh )
            cmd = [
                self.genprog, self.configfile, genome_file,
                    "--seed", seed,
                    "--keep-source",
                    "--no-test-cache",
                    "--search", "oracle",
                    "--test-command", "true",
            ]

            keepfiles = [ "000000", "repair.debug." + seed ]

            tmpdir = tempfile.mkdtemp( dir = "." )
            try:
                for fname in keepfiles:
                    if os.path.exists( fname ):
                        os.rename( fname, os.path.join( tmpdir, fname ) )
                with mktemp() as log:
                    try:
                        with open( log, 'w' ) as fh:
                            check_call( cmd, stdout = fh, stderr = fh )
                    except CalledProcessError:
                        with open( log ) as fh:
                            infomsg( fh.read(), file = sys.stderr )
                        raise
                if os.path.exists( genome_file ):
                    os.remove( genome_file )

                if os.path.exists( "000000/000000" ):
                    yield "000000/000000"
                else:
                    yield None
            finally:
                for fname in keepfiles:
                    if os.path.exists( fname ):
                        check_call( [ "rm", "-rf", fname ] )
                    if os.path.exists( os.path.join( tmpdir, fname ) ):
                        os.rename( os.path.join( tmpdir, fname ), fname )
                check_call( [ "rm", "-rf", tmpdir ] )
Beispiel #3
0
 def run_test( self, exe ):
     cmd = self.config[ "--test-command" ]
     cmd = cmd.replace( "__EXE_NAME__", exe )
     with mktemp() as fitnessfile:
         tmp = cmd.replace( "__FITNESS_FILE__", fitnessfile )
         call( [ "sh", "-c", tmp ] )
         with open( fitnessfile ) as fh:
             return float( fh.next().strip() )
Beispiel #4
0
 def run_test( self, exe ):
     cmd = self.config[ "--test-command" ]
     cmd = cmd.replace( "__EXE_NAME__", exe )
     with mktemp() as fitnessfile:
         tmp = cmd.replace( "__FITNESS_FILE__", fitnessfile )
         call( [ "sh", "-c", tmp ] )
         with open( fitnessfile ) as fh:
             yielded = False
             for line in fh:
                 yield map( float, line.split() )
                 yielded = True
             if not yielded:
                 yield [ 0.0 ]
def save_temp( fname ):
    if not os.path.exists( fname ):
        raise OSError( 2, "No such file or directory", fname )

    with mktemp( dir = os.path.dirname( fname ) ) as tmp:
        os.rename( fname, tmp )

        try:
            yield tmp
        finally:
            if os.path.exists( fname ):
                call( [ "rm", "-rf", fname ] )
            if os.path.exists( tmp ):
                os.rename( tmp, fname )
Beispiel #6
0
def get_minimized_fitness( genome ):
    infomsg( "getting minimized genome for", genome )

    if not os.path.exists( genome + ".min" ):
        record.time( "minimize.py", check_call, [
            os.path.join( root, "bin", "minimize.py" ), genprog, config,
                "--genome-file", genome,
                "--cache", genome + ".cache",
                "--save-binary", genome + ".bin",
                "--save-source", genome + ".src",
                "--save-genome", genome + ".min"
        ] )

    if options.regenerate is not None:
        cfg = Config()
        cfg.load( config )
        cmd = cfg[ "--test-command" ]
        cmd = cmd.replace( "__EXE_NAME__", genome + ".bin" )
        if options.regenerate == "wall":
            cmd += " -j 1 --wall --repeat 100 --no-limit"

        with record.context( "fitness eval" ):
            fitnesses = list()
            for i in range( options.rows ):
                with mktemp() as fitnessfile:
                    # use call instead of check_call because test scripts always
                    # return non-zero status

                    tmp = cmd.replace( "__FITNESS_FILE__", fitnessfile )
                    call( [ "sh", "-c", tmp ] )
                    with open( fitnessfile ) as fh:
                        for line in fh:
                            value = float( line.split()[ 0 ] )
                            infomsg( "   ", value )
                            fitnesses.append( value )
                            break
                        else:
                            infomsg(
                                "ERROR: no fitness for", genome + ".bin",
                                file = sys.stderr
                            )
                            exit( 2 )
            return fitnesses
    else:
        with open( genome + ".min" ) as fh:
            key = " ".join( fh.readlines() )
        key = " ".join( key.split() )

        d = shelve.open( genome + ".cache" )
        return d[ key ]
Beispiel #7
0
def profile( cmd, **kw ):
    f = 0
    while f < 100 or 1900 < f:
        f = int( random.gauss( 1000, 100 ) )
    with mktemp() as tmp:
        record = [
            "perf", "record",
                "--freq", str( f ),
                "--quiet",
                "--output", tmp,
                "--"
        ] + cmd
        check_call( record, **kw )

        return get_metrics( tmp )
Beispiel #8
0
 def getParallelFitness( self, *args ):
     with mktemp( prefix = "vips" ) as tmpexe:
         oldexe = self.exe
         shutil.copyfile( self.exe, tmpexe )
         shutil.copymode( self.exe, tmpexe )
         try:
             self.exe = tmpexe
             results = ParallelTest.getParallelFitness( self, *args )
             if self.options.error:
                 if results == [ [ 0 ] ]:
                     return [ [ 0 ], [ 0 ] ]
                 results.append( self.error )
                 return results
             else:
                 return results
         finally:
             self.exe = oldexe
Beispiel #9
0
def saving( filename ):
    if not os.path.exists( filename ):
        yield
    elif os.path.isdir( filename ):
        tmp = tempfile.mkdtemp( dir = os.path.dirname( filename ) )
        try:
            tmpname = os.path.join( tmp, os.path.basename( filename ) )
            try:
                os.rename( filename, tmpname )
                yield
            finally:
                os.rename( tmpname, filename )
        finally:
            os.rmdir( tmp )
    else:
        with mktemp( dir = os.path.dirname( filename ) ) as tmp:
            os.rename( filename, tmp )
            try:
                yield
            finally:
                os.rename( tmp, filename )
Beispiel #10
0
 def __enter__(self):
     self.fnames = list()
     for i in range(self.num):
         self.fnames.append(mktemp(self.suffix).__enter__())
     return self
    print( "ERROR: you must use --pin-root or set PIN_ROOT", file = sys.stderr )
    exit( 1 )
pin = os.path.join( os.environ[ "PIN_ROOT" ], "pin" )

check_call( [ "make", "-C", os.path.join( root, "pintool" ) ] )

pintool = os.path.join( root, "pintool", "obj-intel64", "print_lines.so" )

sources = list()
for d, dnames, fnames in os.walk( options.source ):
    for fname in fnames:
        fname = os.path.join( d, fname )
        if os.path.splitext( fname )[ 1 ] == ".s":
            sources.append( fname )

with mktemp() as exe:
    infomsg( "INFO: relabeling source files" )
    with renumber_files( sources ):
        cmd = config[ "--compiler-command" ]
        cmd = cmd.replace( "__SOURCE_NAME__", sources[ 0 ] )
        cmd = cmd.replace( "__EXE_NAME__", exe )

        infomsg( "INFO: compiling binary" )
        check_call( [ "sh", "-c", cmd ] )

    cmd = config[ "--test-command" ]
    cmd = cmd.replace( "__EXE_NAME__", exe )
    cmd = cmd.replace( "__FITNESS_FILE__", "/dev/null" )
    with mktemp() as lines_file:
        prefix = [
            pin, "-injection", "child",
Beispiel #12
0
def process_genome( best ):

########
# Write the genome to disk so that minimize and maximize can use it
########

    if options.stop_after is None:
        genome = "genome.%d" % best[ 0 ]
    else:
        genome = "genome.%d-%d" % ( options.stop_after, best[ 0 ] )
    this_genome = os.path.join( results, genome ) + ".orig"
    if not os.path.exists( this_genome ) or options.force:
        infomsg( "INFO: saving genome to", this_genome )
        with open( this_genome, 'w' ) as fh:
            infomsg( " ".join( lower_genome( best[ 1 ].split() ) ), file = fh )

########
# Maximize the genome for all inputs
########

    if input_size not in options.inputs:
        options.inputs = [ input_size ] + options.inputs
    infomsg( "INFO: maximizing genome for inputs:", *options.inputs )
    with saving( "multi.cache" ):
        for test_input in options.inputs:
            next_genome = this_genome + "." + test_input
            if not os.path.exists( next_genome ) or options.force:
                with mkconfig( test_cmd( test_input ) ) as config:
                    check_call( [
                        maximize, genprog, config,
                            "--genome-file", this_genome,
                            "--save-genome", next_genome
                    ] )
            with open( '/dev/null', 'w' ) as null:
                status = call(
                    [ "diff", this_genome, next_genome ],
                    stdout = null, stderr = null
                )
                if status != 0:
                    this_genome = next_genome
        infomsg( "INFO: final genome stored at", this_genome )

########
# Minimize the genome for the original input
########

        if not options.skip_minimize:
            min_genome = this_genome + ".min"
            min_binary = min_genome + ".bin"
            min_source = min_genome + ".src"
            fnames = [ min_genome, min_binary, min_source ]
            if options.force or any(
                    [ not os.path.exists( f ) for f in fnames ]
                ):
                with mkconfig( test_cmd ) as config:
                    cmd = [
                        minimize, genprog, config,
                            "--genome-file", this_genome,
                            "--save-binary", min_binary,
                            "--save-genome", min_genome,
                            "--save-sources", min_source
                    ]
                    if options.low_error is not None:
                        cmd += [ "--low-error", str( options.low_error ) ]
                    check_call( cmd )
            this_genome = min_genome

########
# Measure the improvement over the original
########

        if options.stop_after is None:
            cache = lambda i: os.path.join(
                results, "genome.%s-%s.cache" % ( options.filter, i )
            )
        else:
            cache = lambda i: os.path.join(
                results, "genome.%d-%s-%s.cache" % (
                    options.stop_after, options.filter, i
                )
            )
        genome_key = os.path.basename( this_genome )
        with ImprovementTable( os.path.join( results, "improvement" ) ) as imprv:
            for test_input in options.inputs:
                if not options.force and ( genome_key, test_input ) in imprv:
                    continue
                with mkconfig( test_cmd( test_input ) ) as config:
                    cmd = [
                        minimize, genprog, config,
                            "--search", "none",
                            "--genome-file", this_genome,
                    ]
                    if not options.force:
                        cmd += [ "--cache", cache( test_input ) ]
                    if options.low_error is not None:
                        cmd += [ "--low-error", str( options.low_error ) ]
                    with mktemp() as log:
                        try:
                            pipeline( [ cmd, [ "tee", log ] ] )
                        except CalledProcessError:
                            if ( genome_key, test_input ) in imprv:
                                del imprv[ genome_key, test_input ]
                            continue
                        imprv[ genome_key, test_input ] = " ".join(
                            map( str, get_improvement( log ) )
                        )
Beispiel #13
0
# Get the best genomes from the search...
########

if options.search_csv is None or not os.path.exists( options.search_csv ):
    if options.logfile is None:
        logfiles = glob( os.path.join( results, "repair.debug.*" ) )
        if len( logfiles ) != 1:
            infomsg(
                "ERROR: could not identify logfile. Use --logfile to specify"
            )
            for fname in logfiles:
                infomsg( "   ", fname )
            exit( 1 )
        options.logfile = logfiles[ 0 ]
    if options.search_csv is None:
        with mktemp() as csvfile:
            parse_log( options.logfile, csvfile )
            genomes = get_genomes( csvfile )
    else:
        parse_log( options.logfile, options.search_csv )
        genomes = get_genomes( options.search_csv )
else:
    genomes = get_genomes( options.search_csv )
if len( genomes ) == 0:
    infomsg(
        "ERROR: no variants found! Is %s a valid file?" %
            options.logfile if options.search_csv is None else options.search_csv
    )
    exit( 1 )

def process_genome( best ):
Beispiel #14
0
for d, dnames, fnames in os.walk(buildir):
    for fname in fnames:
        if fname != "flags.make":
            continue
        fname = os.path.join(d, fname)
        with open(fname) as fh:
            lines = fh.readlines()
        with open(fname, "w") as fh:
            for line in lines:
                line = line.rstrip()
                if line.startswith("C_FLAGS") or line.startswith("CXX_FLAGS"):
                    line = line + " -save-temps=obj"
                print(line, file=fh)

infomsg("INFO: running make")
with mktemp() as log:
    pipeline([["make", "-C", buildir, "VERBOSE=1", "V=1", "install"], ["tee", log]])

    infomsg("INFO: collecting assembly files")
    asmfiles = dict()
    with open("multi.txt", "w") as fh:
        for d, dnames, fnames in os.walk(buildir):
            for fname in fnames:
                if os.path.splitext(fname)[1] == ".s":
                    fname = os.path.join(d, fname)
                    relpath = fname[len(buildir) + 1 :]
                    print(relpath, file=fh)
                    objname = os.path.splitext(relpath)[0] + ".o"
                    asmfiles[objname] = relpath

    depgraph = dict()
Beispiel #15
0
check_path( srcdir )

infomsg( "checking that est-energy.py works..." )
check_path( est_energy )
check_cmd( [ est_energy, "-o", "/dev/null", "--", "true" ] )
infomsg( "pass" )

infomsg( "checking that limit command works..." )
check_path( limit )
check_cmd( [ limit, "1", "true" ] )
infomsg( "pass" )

config = Config()
config.load( configfile )

with mktemp( dir = "." ) as exe:
    infomsg( "checking that we can compile the program..." )

    compile_cmd = config[ "--compiler-command" ]
    compile_cmd = compile_cmd.replace( "__SOURCE_NAME__", "src/000000.s" )
    compile_cmd = compile_cmd.replace( "__EXE_NAME__", exe )

    status = call( [ "sh", "-c", compile_cmd ] )
    if status != 0:
        infomsg( "ERROR: compile command failed!", file = sys.stderr )
        infomsg( "ERROR:", compile_cmd, file = sys.stderr )
        exit( status )
    infomsg( "pass" )

    with mktemp() as fitnessfile:
        infomsg( "checking that the test command works..." )