Exemple #1
0
def rankUsingPrograms():
    results = pickle.load(open(arguments.name, 'rb'))
    print " [+] Loaded %d synthesis results from %s." % (len(results),
                                                         arguments.name)

    def getProgramForParse(sequence):
        for r in results:
            if sequence == r.parse and r.usedPrior():
                return r
        return None

    def featuresOfParticle(p):
        r = getProgramForParse(p.sequence())
        if r != None and r.cost != None and r.source != None:
            programFeatures = mergeDictionaries({'failure': 0.0},
                                                parseSketchOutput(
                                                    r.source).features())
        else:
            programFeatures = {'failure': 1.0}
        parseFeatures = {
            'distance': p.distance[0] + p.distance[1],
            'logPrior': p.sequence().logPrior(),
            'logLikelihood': p.logLikelihood
        }
        return mergeDictionaries(parseFeatures, programFeatures)

    k = arguments.learnToRank
    topParticles = [
        loadTopParticles('drawings/expert-%d-parses' % j, k)
        for j in range(100)
    ]
    learningProblems = []
    for j, ps in enumerate(topParticles):
        gt = getGroundTruthParse('drawings/expert-%d.png' % j)
        positives = []
        negatives = []
        for p in ps:
            if p.sequence() == gt: positives.append(p)
            else: negatives.append(p)
        if positives != [] and negatives != []:
            learningProblems.append(
                (map(featuresOfParticle,
                     positives), map(featuresOfParticle, negatives)))

    featureIndices = list(
        set([
            f for pn in learningProblems for exs in pn for ex in exs
            for f in ex.keys()
        ]))

    def dictionaryToVector(featureMap):
        return [featureMap.get(f, 0.0) for f in featureIndices]

    learningProblems = [(map(dictionaryToVector,
                             positives), map(dictionaryToVector, negatives))
                        for positives, negatives in learningProblems]
    parameters = learnToRank(learningProblems)
    for f, p in zip(featureIndices, parameters):
        print f, p

    # showcases where it succeeds
    programAccuracy = 0
    oldAccuracy = 0
    for j, tp in enumerate(topParticles):
        if tp == []: continue

        gt = getGroundTruthParse('drawings/expert-%d.png' % j)
        # the_top_particles_according_to_the_learned_weights
        featureVectors = np.array(
            [dictionaryToVector(featuresOfParticle(p)) for p in tp])
        particleScores = featureVectors.dot(parameters)
        bestParticleUsingPrograms = max(zip(particleScores.tolist(), tp))[1]
        programPredictionCorrect = False
        if bestParticleUsingPrograms.sequence() == gt:
            print "Prediction using the program is correct."
            programPredictionCorrect = True
            programAccuracy += 1
        else:
            print "Prediction using the program is incorrect."
        oldPredictionCorrect = tp[0].sequence() == gt
        print "Was the old prediction correct?", oldPredictionCorrect
        oldAccuracy += int(oldPredictionCorrect)

        visualization = np.zeros((256, 256 * 3))
        visualization[:, :256] = 1 - frameImageNicely(
            loadImage('drawings/expert-%d.png' % j))
        visualization[:, 256:(256 * 2)] = frameImageNicely(
            fastRender(tp[0].sequence()))
        visualization[:, (256 * 2):(256 * 3)] = frameImageNicely(
            fastRender(bestParticleUsingPrograms.sequence()))
        visualization[:, 256] = 0.5
        visualization[:, 256 * 2] = 0.5
        visualization = 255 * visualization

        if not oldPredictionCorrect and programPredictionCorrect:
            fp = "../TikZpaper/figures/programSuccess%d.png" % j
            print "Great success! see %s" % fp
            saveMatrixAsImage(visualization, fp)

        if oldPredictionCorrect and not programPredictionCorrect:
            print "Minor setback!"
            print particleScores

    print programAccuracy, "vs", oldAccuracy
Exemple #2
0
def viewSynthesisResults(arguments):
    results = pickle.load(open(arguments.name,'rb'))
    print " [+] Loaded %d synthesis results."%(len(results))

    interestingExtrapolations = [7,
                                 #14,
                                 17,
                                 29,
                                 #35,
                                 52,
                                 57,
                                 63,
                                 70,
                                 72,
                                 88,
                                 #99]
                                 ]
    interestingExtrapolations = [(14,0),
                                 (23,0),
                                 (31,12),
                                 (35,0),
                                 (38,0),
                                 (39,10),
                                 (58,11),
                                 (75,0),
                                 (93,0),
                                  (99,0)]
    interestingExtrapolations = list(range(100))
                                 

    latex = []
    extrapolationMatrix = []

    programFeatures = {}

    for expertIndex in range(100):

        if arguments.extrapolate:
            if not any([ e == expertIndex or isinstance(e,tuple) and e[0] == expertIndex
                         for e in interestingExtrapolations ]):
                continue
            
        
        f = 'drawings/expert-%d.png'%expertIndex
        parse = getGroundTruthParse(f)
        if parse == None:
            print "No ground truth for %d"%expertIndex
            assert False
            
        parts = set(map(str,parse.lines))
        result = None
        for r in results:
            if isinstance(r,SynthesisResult) and r.originalDrawing == f:
                if set(map(str,r.parse.lines)) == parts and r.usedPrior() == (not arguments.noPrior):
                    result = r
                    break
        if expertIndex == 38: result = icingResult
        if result == None:
            print "No synthesis result for %s"%f
            if arguments.extrapolate: continue

        if result.source == None:
            print "Synthesis failure for %s"%f
            if arguments.extrapolate: continue

        print " [+] %s"%f
        print "\t(synthesis time: %f)"%(result.time)
        print

        if arguments.debug:
            print result.source

        if result != None and result.source != None:
            syntaxTree = parseSketchOutput(result.source)
            syntaxTree = syntaxTree.fixReflections(result.parse.canonicalTranslation())
            print syntaxTree
            print syntaxTree.features()
            print syntaxTree.convertToPython()
            print syntaxTree.convertToSequence()
            #showImage(fastRender(syntaxTree.convertToSequence()) + loadImage(f)*0.5 + fastRender(result.parse))
            programFeatures[f] = syntaxTree.features()

        extrapolations = []
        if arguments.extrapolate:
            syntaxTree = syntaxTree.explode()
            trace = syntaxTree.convertToSequence()
            print trace
            originalHasCollisions = result.parse.hasCollisions()
            print "COLLISIONS",originalHasCollisions

            framedExtrapolations = []
            for e in syntaxTree.extrapolations():
                t = e.convertToSequence()
                if not originalHasCollisions and t.removeDuplicates().hasCollisions(): continue
                if t == trace: continue
                if any([t == o for o in extrapolations ]): continue
                extrapolations.append(t)

                framedExtrapolations.append(1 - frameImageNicely(1 - t.framedRendering(result.parse)))

                if len(framedExtrapolations) > 20:
                    break
                
            if framedExtrapolations != []:
                for crap in interestingExtrapolations:
                    if isinstance(crap,tuple) and crap[0] == expertIndex:
                        framedExtrapolations = [framedExtrapolations[crap[1]]]
                        break
                
                        
                if arguments.debug:
                    framedExtrapolations = [loadImage(f), fastRender(syntaxTree.convertToSequence())] + framedExtrapolations
                else:
                    framedExtrapolations = [frameImageNicely(loadImage(f))] + framedExtrapolations
                a = np.zeros((256*len(framedExtrapolations),256))
                for j,e in enumerate(framedExtrapolations):
                    a[j*256:(1+j)*256,:] = 1 - e
                    a[j*256,:] = 0.5
                    a[(1+j)*256-1,:] = 0.5
                a[0,:] = 0.5
                a[:,0] = 0.5
                a[:,255] = 0.5
                a[256*len(framedExtrapolations)-1,:] = 0.5
                a = 255*a
                # to show the first one
                #a = a[:(256*2),:]
                extrapolationMatrix.append(a)
                saveMatrixAsImage(a,'extrapolations/expert-%d-extrapolation.png'%expertIndex)

            
        
        if not arguments.extrapolate:
            rightEntryOfTable = '''
        \\begin{minipage}{10cm}
        \\begin{verbatim}
%s
        \\end{verbatim}
\\end{minipage}
'''%(parseSketchOutput(result.source).pretty() if result != None and result.source != None else "Solver timeout")
        else:
            rightEntryOfTable = ""
        if extrapolations != [] and arguments.extrapolate:
            #}make the big matrix
            bigMatrix = np.zeros((max([m.shape[0] for m in extrapolationMatrix ]),256*len(extrapolationMatrix)))
            for j,r in enumerate(extrapolationMatrix):
                bigMatrix[0:r.shape[0],256*j:256*(j+1)] = r
            if len(extrapolations) < 10 and False:
                saveMatrixAsImage(bigMatrix,'../TikZpaper/figures/extrapolationMatrix.png')
            else:
                saveMatrixAsImage(bigMatrix,'../TikZpaper/figures/extrapolationMatrixSupplement.png')
            print e
            rightEntryOfTable = '\\includegraphics[width = 5cm]{../TikZ/extrapolations/expert-%d-extrapolation.png}'%expertIndex
        if rightEntryOfTable != "":
            parseImage = '\\includegraphics[width = 5cm]{../TikZ/drawings/expert-%d-parses/0.png}'%expertIndex
            if not os.path.exists('drawings/expert-%d-parses/0.png'%expertIndex):
                parseImage = "Sampled no finished traces."
            latex.append('''
            \\begin{tabular}{lll}
    \\includegraphics[width = 5cm]{../TikZ/drawings/expert-%d.png}&
            %s&
    %s
    \\end{tabular}        
            '''%(expertIndex, parseImage, rightEntryOfTable))
            print

    if arguments.latex:
        latex = '%s'%("\\\\\n".join(latex))
        name = "extrapolations.tex" if arguments.extrapolate else "synthesizerOutputs.tex"
        with open('../TikZpaper/%s'%name,'w') as handle:
            handle.write(latex)
        print "Wrote output to ../TikZpaper/%s"%name

    if arguments.similarity:
        analyzeFeatures(programFeatures)
Exemple #3
0
def viewSynthesisResults(arguments):
    results = pickle.load(open(arguments.name, 'rb'))
    print " [+] Loaded %d synthesis results." % (len(results))

    interestingExtrapolations = [
        7,
        #14,
        17,
        29,
        #35,
        52,
        57,
        63,
        70,
        72,
        88,
        #99]
    ]
    interestingExtrapolations = [
        (16, 12),  #*
        #(17,0),
        (18, 0),  #*
        #(22,0),
        #(23,0),
        #(29,12),
        #(31,27),
        (34, 0),  #*
        #(36,0),
        #(38,12),
        (39, 0),  #*
        #(41,1),
        #(51,1),
        #(52,12),
        #(57,0),
        #(58,0),
        (60, 0),  #*
        #(63,0),
        (66, 2),  #*
        (71, 1),  #*
        #(72,0),
        #(73,0),
        #(74,10),
        #(75,5),
        #(79,0),
        #(85,1),
        (86, 0),  #*
        #(88,0),
        (90, 2),  #*
        #(92,0),
        #(95,8)
    ]

    #interestingExtrapolations = list(range(100))

    latex = []
    extrapolationMatrix = []

    programFeatures = {}

    for expertIndex in list(range(100)):
        f = 'drawings/expert-%d.png' % expertIndex
        parse = getGroundTruthParse(f)
        if parse == None:
            print "No ground truth for %d" % expertIndex
            assert False

        relevantResults = [
            r for r in results if r.job.originalDrawing == f and r.cost != None
        ]
        if relevantResults == []:
            print "No synthesis result for %s" % f
            result = None
        else:
            result = min(relevantResults, key=lambda r: r.cost)
            equallyGoodResults = [
                r for r in relevantResults if r.cost <= result.cost + 1
            ]
            if len(equallyGoodResults) > 1:
                print "Got %d results for %d" % (len(equallyGoodResults),
                                                 expertIndex)

            programs = [ r.program.fixStringParameters().\
                         fixReflections(result.job.parse.canonicalTranslation()).removeDeadCode()
                         for r in equallyGoodResults ]
            gt = result.job.parse.canonicalTranslation()
            badPrograms = [
                p for p in programs
                if p.convertToSequence().canonicalTranslation() != gt
            ]
            if badPrograms:
                print " [-] WARNING: Got %d programs that are inconsistent with ground truth" % (
                    len(badPrograms))
            if False:
                for program in programs:
                    prediction = program.convertToSequence(
                    ).canonicalTranslation()
                    actual = gt
                    if not (prediction == actual):
                        print "FATAL: program does notproduce spec"
                        print "Specification:"
                        print actual
                        print "Program:"
                        print program
                        print program.pretty()
                        print "Program output:"
                        print prediction
                        print set(map(str, prediction.lines))
                        print set(map(str, actual.lines))
                        print set(map(str, actual.lines)) ^ set(
                            map(str, prediction.lines))
                        assert False

        if result == None and arguments.extrapolate:
            print "Synthesis failure for %s" % f
            continue

        print " [+] %s" % f
        print "\t(synthesis time: %s)" % (result.time if result else None)
        print

        if arguments.debug:
            print result.source

        if result != None:
            syntaxTree = result.program.fixStringParameters()
            syntaxTree = syntaxTree.fixReflections(
                result.job.parse.canonicalTranslation())
            print syntaxTree.pretty()
            print syntaxTree.features()
            print syntaxTree.convertToSequence()
            #showImage(fastRender(syntaxTree.convertToSequence()) + loadImage(f)*0.5 + fastRender(result.parse))
            programFeatures[f] = syntaxTree.features()

        if arguments.extrapolate:
            extrapolations = proposeExtrapolations(programs)
            if extrapolations:
                framedExtrapolations = [1 - frameImageNicely(loadImage(f))] + \
                                       [ frameImageNicely(t.draw(adjustCanvasSize = True))
                                         for t in extrapolations ]
                a = 255 * makeImageArray(framedExtrapolations)
                extrapolationMatrix.append(a)
                print "Saving extrapolation column to", 'extrapolations/expert-%d-extrapolation.png' % expertIndex
                saveMatrixAsImage(
                    a,
                    'extrapolations/expert-%d-extrapolation.png' % expertIndex)

        if not arguments.extrapolate:
            rightEntryOfTable = '''
        \\begin{minipage}{10cm}
        \\begin{verbatim}
%s
        \\end{verbatim}
\\end{minipage}
''' % (syntaxTree.pretty() if result != None else "Solver timeout")
        else:
            rightEntryOfTable = ""
        if False and extrapolations != [] and arguments.extrapolate:

            #print e
            rightEntryOfTable = '\\includegraphics[width = 5cm]{../TikZ/extrapolations/expert-%d-extrapolation.png}' % expertIndex
        if rightEntryOfTable != "":
            parseImage = '\\includegraphics[width = 5cm]{../TikZ/drawings/expert-%d-parses/0.png}' % expertIndex
            if not os.path.exists(
                    'drawings/expert-%d-parses/0.png' % expertIndex):
                parseImage = "Sampled no finished traces."
            latex.append('''
            \\begin{tabular}{lll}
    \\includegraphics[width = 5cm]{../TikZ/drawings/expert-%d.png}&
            %s&
    %s
    \\end{tabular}        
            ''' % (expertIndex, parseImage, rightEntryOfTable))
            print

    if arguments.latex:
        latex = '%s' % ("\\\\\n".join(latex))
        name = "extrapolations.tex" if arguments.extrapolate else "synthesizerOutputs.tex"
        with open('../TikZpaper/%s' % name, 'w') as handle:
            handle.write(latex)
        print "Wrote output to ../TikZpaper/%s" % name

    if arguments.similarity:
        analyzeFeatures(programFeatures)

    if arguments.extrapolate:
        #}make the big matrix
        bigMatrix = np.zeros((max([m.shape[0] for m in extrapolationMatrix]),
                              256 * len(extrapolationMatrix)))
        for j, r in enumerate(extrapolationMatrix):
            bigMatrix[0:r.shape[0], 256 * j:256 * (j + 1)] = r
        saveMatrixAsImage(bigMatrix, 'extrapolations/allTheExtrapolations.png')
Exemple #4
0
def viewSynthesisResults(arguments):
    results = pickle.load(open(arguments.name,'rb'))
    print " [+] Loaded %d synthesis results."%(len(results))

    interestingExtrapolations = [7,
                                 #14,
                                 17,
                                 29,
                                 #35,
                                 52,
                                 57,
                                 63,
                                 70,
                                 72,
                                 88,
                                 #99]
                                 ]
    interestingExtrapolations = [(16,12),#*
                                 #(17,0),
                                 (18,0),#*
                                 #(22,0),
                                 #(23,0),
                                 #(29,12),
                                 #(31,27),
                                 (34,0),#*
                                 #(36,0),
                                 #(38,12),
                                 (39,0),#*
                                 #(41,1),
                                 #(51,1),
                                 #(52,12),
                                 #(57,0),
                                 #(58,0),
                                 (60,0),#*
                                 #(63,0),
                                 (66,2),#*
                                 (71,1),#*
                                 #(72,0),
        #(73,0),
                                 #(74,10),
                                 #(75,5),
                                 #(79,0),
                                 #(85,1),
                                 (86,0),#*
                                 #(88,0),
                                 (90,2),#*
                                 #(92,0),
                                 #(95,8)
    ]
    
    #interestingExtrapolations = list(range(100))
                                 

    latex = []
    extrapolationMatrix = []

    programFeatures = {}

    for expertIndex in list(range(100)):# + [101]:
        if arguments.extrapolate:
            if not any([ e == expertIndex or isinstance(e,tuple) and e[0] == expertIndex
                         for e in interestingExtrapolations ]):
                continue
            
        
        f = 'drawings/expert-%d.png'%expertIndex
        parse = getGroundTruthParse(f)
        if parse == None:
            print "No ground truth for %d"%expertIndex
            assert False
            
        relevantResults = [ r for r in results if r.job.originalDrawing == f and r.cost != None ]
        if relevantResults == []:
            print "No synthesis result for %s"%f
            result = None
        else:
            result = min(relevantResults, key = lambda r: r.cost)
            equallyGoodResults = [ r for r in relevantResults if r.cost <= result.cost + 1 ]
            if len(equallyGoodResults) > 1:
                print "Got %d results for %d"%(len(equallyGoodResults),expertIndex)

            programs = [ r.program.fixReflections(result.job.parse.canonicalTranslation()).removeDeadCode()
                         for r in equallyGoodResults ]

        if result == None and arguments.extrapolate:
            print "Synthesis failure for %s"%f
            continue

        print " [+] %s"%f
        print "\t(synthesis time: %s)"%(result.time if result else None)
        print

        if arguments.debug:
            print result.source

        if result != None:
            syntaxTree = result.program
            syntaxTree = syntaxTree.fixReflections(result.job.parse.canonicalTranslation())
            print syntaxTree
            print syntaxTree.features()
            print syntaxTree.convertToPython()
            print syntaxTree.convertToSequence()
            #showImage(fastRender(syntaxTree.convertToSequence()) + loadImage(f)*0.5 + fastRender(result.parse))
            programFeatures[f] = syntaxTree.features()

        extrapolations = []
        if arguments.extrapolate:
            syntaxTree = syntaxTree.explode()
            trace = syntaxTree.convertToSequence().removeDuplicates()
            print "original trace:"
            print trace
            originalUndesirability = parse.undesirabilityVector()
            print "original undesirability",originalUndesirability

            framedExtrapolations = []
            extrapolationGenerators = [ program.explode().extrapolations() for program in programs ]
            for e in interleaveGenerators(extrapolationGenerators):
                t = e.convertToSequence().removeDuplicates()
                newUndesirability = t.undesirabilityVector()
                if (newUndesirability > originalUndesirability).sum() > 0: continue
                if t.canonicalTranslation() == trace.canonicalTranslation(): continue
                if any([t.canonicalTranslation() == o.canonicalTranslation() for o in extrapolations ]): continue
                extrapolations.append(t)

                framedExtrapolations.append(1 - frameImageNicely(t.draw(adjustCanvasSize = True)))

                if len(framedExtrapolations) > 30:
                    break
                
            if framedExtrapolations != []:
                for crap in interestingExtrapolations:
                    if isinstance(crap,tuple) and crap[0] == expertIndex:
                        print "Just taking the %d extrapolation..."%(crap[1])
                        framedExtrapolations = [framedExtrapolations[crap[1]]]
                        break
                
                        
                if arguments.debug:
                    framedExtrapolations = [loadImage(f), syntaxTree.convertToSequence().draw()] + framedExtrapolations
                else:
                    framedExtrapolations = [frameImageNicely(loadImage(f))] + framedExtrapolations
                a = np.zeros((256*len(framedExtrapolations),256))
                for j,e in enumerate(framedExtrapolations):
                    a[j*256:(1+j)*256,:] = 1 - e
                    a[j*256,:] = 0.5
                    a[(1+j)*256-1,:] = 0.5
                a[0,:] = 0.5
                a[:,0] = 0.5
                a[:,255] = 0.5
                a[256*len(framedExtrapolations)-1,:] = 0.5
                a = 255*a
                # to show the first one
                #a = a[:(256*2),:]
                extrapolationMatrix.append(a)
                print "Saving extrapolation column to",'extrapolations/expert-%d-extrapolation.png'%expertIndex
                saveMatrixAsImage(a,'extrapolations/expert-%d-extrapolation.png'%expertIndex)

            
        
        if not arguments.extrapolate:
            rightEntryOfTable = '''
        \\begin{minipage}{10cm}
        \\begin{verbatim}
%s
        \\end{verbatim}
\\end{minipage}
'''%(syntaxTree.pretty() if result != None else "Solver timeout")
        else:
            rightEntryOfTable = ""
        if False and extrapolations != [] and arguments.extrapolate:
            
            #print e
            rightEntryOfTable = '\\includegraphics[width = 5cm]{../TikZ/extrapolations/expert-%d-extrapolation.png}'%expertIndex
        if rightEntryOfTable != "":
            parseImage = '\\includegraphics[width = 5cm]{../TikZ/drawings/expert-%d-parses/0.png}'%expertIndex
            if not os.path.exists('drawings/expert-%d-parses/0.png'%expertIndex):
                parseImage = "Sampled no finished traces."
            latex.append('''
            \\begin{tabular}{lll}
    \\includegraphics[width = 5cm]{../TikZ/drawings/expert-%d.png}&
            %s&
    %s
    \\end{tabular}        
            '''%(expertIndex, parseImage, rightEntryOfTable))
            print

    if arguments.latex:
        latex = '%s'%("\\\\\n".join(latex))
        name = "extrapolations.tex" if arguments.extrapolate else "synthesizerOutputs.tex"
        with open('../TikZpaper/%s'%name,'w') as handle:
            handle.write(latex)
        print "Wrote output to ../TikZpaper/%s"%name

    if arguments.similarity:
        analyzeFeatures(programFeatures)

    if arguments.extrapolate:
        #}make the big matrix
        bigMatrix = np.zeros((max([m.shape[0] for m in extrapolationMatrix ]),256*len(extrapolationMatrix)))
        for j,r in enumerate(extrapolationMatrix):
            bigMatrix[0:r.shape[0],256*j:256*(j+1)] = r
        saveMatrixAsImage(bigMatrix,'extrapolations/allTheExtrapolations.png')