def expertSynthesisJobs(k):
jobs = []
for j in range(100):
originalDrawing = 'drawings/expert-%d.png'%j
particleDirectory = 'drawings/expert-%d-parses'%j
if not os.path.exists(originalDrawing) or not os.path.exists(particleDirectory):
continue
newJobs = []
for p in loadTopParticles(particleDirectory, k):
newJobs.append(SynthesisJob(p.sequence(), originalDrawing, usePrior = not arguments.noPrior))
# but we don't care about synthesizing if there wasn't a ground truth in them
if any([ newJob.parse == getGroundTruthParse(originalDrawing) for newJob in newJobs ]):
jobs += newJobs
return jobs
def makePolicyTrainingData():
jobs = [
SynthesisJob(getGroundTruthParse(f),
f,
usePrior=True,
maximumDepth=d,
canLoop=l,
canReflect=r,
incremental=i) for j in range(100)
for f in ['drawings/expert-%d.png' % j] for d in [1, 2, 3]
for i in [True, False] for l in [True, False] for r in [True, False]
]
print " [+] Constructed %d job objects for the purpose of training a policy" % (
len(jobs))
results = parallelExecute(jobs)
fn = 'policyTrainingData.p'
with open(fn, 'wb') as handle:
pickle.dump(results, handle)
print " [+] Dumped results to %s." % fn
def rankUsingPrograms():
results = pickle.load(open(arguments.name, 'rb'))
print " [+] Loaded %d synthesis results from %s." % (len(results),
arguments.name)
def getProgramForParse(sequence):
for r in results:
if sequence == r.parse and r.usedPrior():
return r
return None
def featuresOfParticle(p):
r = getProgramForParse(p.sequence())
if r != None and r.cost != None and r.source != None:
programFeatures = mergeDictionaries({'failure': 0.0},
parseSketchOutput(
r.source).features())
else:
programFeatures = {'failure': 1.0}
parseFeatures = {
'distance': p.distance[0] + p.distance[1],
'logPrior': p.sequence().logPrior(),
'logLikelihood': p.logLikelihood
}
return mergeDictionaries(parseFeatures, programFeatures)
k = arguments.learnToRank
topParticles = [
loadTopParticles('drawings/expert-%d-parses' % j, k)
for j in range(100)
]
learningProblems = []
for j, ps in enumerate(topParticles):
gt = getGroundTruthParse('drawings/expert-%d.png' % j)
positives = []
negatives = []
for p in ps:
if p.sequence() == gt: positives.append(p)
else: negatives.append(p)
if positives != [] and negatives != []:
learningProblems.append(
(map(featuresOfParticle,
positives), map(featuresOfParticle, negatives)))
featureIndices = list(
set([
f for pn in learningProblems for exs in pn for ex in exs
for f in ex.keys()
]))
def dictionaryToVector(featureMap):
return [featureMap.get(f, 0.0) for f in featureIndices]
learningProblems = [(map(dictionaryToVector,
positives), map(dictionaryToVector, negatives))
for positives, negatives in learningProblems]
parameters = learnToRank(learningProblems)
for f, p in zip(featureIndices, parameters):
print f, p
# showcases where it succeeds
programAccuracy = 0
oldAccuracy = 0
for j, tp in enumerate(topParticles):
if tp == []: continue
gt = getGroundTruthParse('drawings/expert-%d.png' % j)
# the_top_particles_according_to_the_learned_weights
featureVectors = np.array(
[dictionaryToVector(featuresOfParticle(p)) for p in tp])
particleScores = featureVectors.dot(parameters)
bestParticleUsingPrograms = max(zip(particleScores.tolist(), tp))[1]
programPredictionCorrect = False
if bestParticleUsingPrograms.sequence() == gt:
print "Prediction using the program is correct."
programPredictionCorrect = True
programAccuracy += 1
else:
print "Prediction using the program is incorrect."
oldPredictionCorrect = tp[0].sequence() == gt
print "Was the old prediction correct?", oldPredictionCorrect
oldAccuracy += int(oldPredictionCorrect)
visualization = np.zeros((256, 256 * 3))
visualization[:, :256] = 1 - frameImageNicely(
loadImage('drawings/expert-%d.png' % j))
visualization[:, 256:(256 * 2)] = frameImageNicely(
fastRender(tp[0].sequence()))
visualization[:, (256 * 2):(256 * 3)] = frameImageNicely(
fastRender(bestParticleUsingPrograms.sequence()))
visualization[:, 256] = 0.5
visualization[:, 256 * 2] = 0.5
visualization = 255 * visualization
if not oldPredictionCorrect and programPredictionCorrect:
fp = "../TikZpaper/figures/programSuccess%d.png" % j
print "Great success! see %s" % fp
saveMatrixAsImage(visualization, fp)
if oldPredictionCorrect and not programPredictionCorrect:
print "Minor setback!"
print particleScores
print programAccuracy, "vs", oldAccuracy
def viewSynthesisResults(arguments):
results = pickle.load(open(arguments.name, 'rb'))
print " [+] Loaded %d synthesis results." % (len(results))
interestingExtrapolations = [
7,
#14,
17,
29,
#35,
52,
57,
63,
70,
72,
88,
#99]
]
interestingExtrapolations = [
(16, 12), #*
#(17,0),
(18, 0), #*
#(22,0),
#(23,0),
#(29,12),
#(31,27),
(34, 0), #*
#(36,0),
#(38,12),
(39, 0), #*
#(41,1),
#(51,1),
#(52,12),
#(57,0),
#(58,0),
(60, 0), #*
#(63,0),
(66, 2), #*
(71, 1), #*
#(72,0),
#(73,0),
#(74,10),
#(75,5),
#(79,0),
#(85,1),
(86, 0), #*
#(88,0),
(90, 2), #*
#(92,0),
#(95,8)
]
#interestingExtrapolations = list(range(100))
latex = []
extrapolationMatrix = []
programFeatures = {}
for expertIndex in list(range(100)):
f = 'drawings/expert-%d.png' % expertIndex
parse = getGroundTruthParse(f)
if parse == None:
print "No ground truth for %d" % expertIndex
assert False
relevantResults = [
r for r in results if r.job.originalDrawing == f and r.cost != None
]
if relevantResults == []:
print "No synthesis result for %s" % f
result = None
else:
result = min(relevantResults, key=lambda r: r.cost)
equallyGoodResults = [
r for r in relevantResults if r.cost <= result.cost + 1
]
if len(equallyGoodResults) > 1:
print "Got %d results for %d" % (len(equallyGoodResults),
expertIndex)
programs = [ r.program.fixStringParameters().\
fixReflections(result.job.parse.canonicalTranslation()).removeDeadCode()
for r in equallyGoodResults ]
gt = result.job.parse.canonicalTranslation()
badPrograms = [
p for p in programs
if p.convertToSequence().canonicalTranslation() != gt
]
if badPrograms:
print " [-] WARNING: Got %d programs that are inconsistent with ground truth" % (
len(badPrograms))
if False:
for program in programs:
prediction = program.convertToSequence(
).canonicalTranslation()
actual = gt
if not (prediction == actual):
print "FATAL: program does notproduce spec"
print "Specification:"
print actual
print "Program:"
print program
print program.pretty()
print "Program output:"
print prediction
print set(map(str, prediction.lines))
print set(map(str, actual.lines))
print set(map(str, actual.lines)) ^ set(
map(str, prediction.lines))
assert False
if result == None and arguments.extrapolate:
print "Synthesis failure for %s" % f
continue
print " [+] %s" % f
print "\t(synthesis time: %s)" % (result.time if result else None)
print
if arguments.debug:
print result.source
if result != None:
syntaxTree = result.program.fixStringParameters()
syntaxTree = syntaxTree.fixReflections(
result.job.parse.canonicalTranslation())
print syntaxTree.pretty()
print syntaxTree.features()
print syntaxTree.convertToSequence()
#showImage(fastRender(syntaxTree.convertToSequence()) + loadImage(f)*0.5 + fastRender(result.parse))
programFeatures[f] = syntaxTree.features()
if arguments.extrapolate:
extrapolations = proposeExtrapolations(programs)
if extrapolations:
framedExtrapolations = [1 - frameImageNicely(loadImage(f))] + \
[ frameImageNicely(t.draw(adjustCanvasSize = True))
for t in extrapolations ]
a = 255 * makeImageArray(framedExtrapolations)
extrapolationMatrix.append(a)
print "Saving extrapolation column to", 'extrapolations/expert-%d-extrapolation.png' % expertIndex
saveMatrixAsImage(
a,
'extrapolations/expert-%d-extrapolation.png' % expertIndex)
if not arguments.extrapolate:
rightEntryOfTable = '''
\\begin{minipage}{10cm}
\\begin{verbatim}
%s
\\end{verbatim}
\\end{minipage}
''' % (syntaxTree.pretty() if result != None else "Solver timeout")
else:
rightEntryOfTable = ""
if False and extrapolations != [] and arguments.extrapolate:
#print e
rightEntryOfTable = '\\includegraphics[width = 5cm]{../TikZ/extrapolations/expert-%d-extrapolation.png}' % expertIndex
if rightEntryOfTable != "":
parseImage = '\\includegraphics[width = 5cm]{../TikZ/drawings/expert-%d-parses/0.png}' % expertIndex
if not os.path.exists(
'drawings/expert-%d-parses/0.png' % expertIndex):
parseImage = "Sampled no finished traces."
latex.append('''
\\begin{tabular}{lll}
\\includegraphics[width = 5cm]{../TikZ/drawings/expert-%d.png}&
%s&
%s
\\end{tabular}
''' % (expertIndex, parseImage, rightEntryOfTable))
print
if arguments.latex:
latex = '%s' % ("\\\\\n".join(latex))
name = "extrapolations.tex" if arguments.extrapolate else "synthesizerOutputs.tex"
with open('../TikZpaper/%s' % name, 'w') as handle:
handle.write(latex)
print "Wrote output to ../TikZpaper/%s" % name
if arguments.similarity:
analyzeFeatures(programFeatures)
if arguments.extrapolate:
#}make the big matrix
bigMatrix = np.zeros((max([m.shape[0] for m in extrapolationMatrix]),
256 * len(extrapolationMatrix)))
for j, r in enumerate(extrapolationMatrix):
bigMatrix[0:r.shape[0], 256 * j:256 * (j + 1)] = r
saveMatrixAsImage(bigMatrix, 'extrapolations/allTheExtrapolations.png')
def viewSynthesisResults(arguments):
results = pickle.load(open(arguments.name,'rb'))
print " [+] Loaded %d synthesis results."%(len(results))
interestingExtrapolations = [7,
#14,
17,
29,
#35,
52,
57,
63,
70,
72,
88,
#99]
]
interestingExtrapolations = [(14,0),
(23,0),
(31,12),
(35,0),
(38,0),
(39,10),
(58,11),
(75,0),
(93,0),
(99,0)]
interestingExtrapolations = list(range(100))
latex = []
extrapolationMatrix = []
programFeatures = {}
for expertIndex in range(100):
if arguments.extrapolate:
if not any([ e == expertIndex or isinstance(e,tuple) and e[0] == expertIndex
for e in interestingExtrapolations ]):
continue
f = 'drawings/expert-%d.png'%expertIndex
parse = getGroundTruthParse(f)
if parse == None:
print "No ground truth for %d"%expertIndex
assert False
parts = set(map(str,parse.lines))
result = None
for r in results:
if isinstance(r,SynthesisResult) and r.originalDrawing == f:
if set(map(str,r.parse.lines)) == parts and r.usedPrior() == (not arguments.noPrior):
result = r
break
if expertIndex == 38: result = icingResult
if result == None:
print "No synthesis result for %s"%f
if arguments.extrapolate: continue
if result.source == None:
print "Synthesis failure for %s"%f
if arguments.extrapolate: continue
print " [+] %s"%f
print "\t(synthesis time: %f)"%(result.time)
print
if arguments.debug:
print result.source
if result != None and result.source != None:
syntaxTree = parseSketchOutput(result.source)
syntaxTree = syntaxTree.fixReflections(result.parse.canonicalTranslation())
print syntaxTree
print syntaxTree.features()
print syntaxTree.convertToPython()
print syntaxTree.convertToSequence()
#showImage(fastRender(syntaxTree.convertToSequence()) + loadImage(f)*0.5 + fastRender(result.parse))
programFeatures[f] = syntaxTree.features()
extrapolations = []
if arguments.extrapolate:
syntaxTree = syntaxTree.explode()
trace = syntaxTree.convertToSequence()
print trace
originalHasCollisions = result.parse.hasCollisions()
print "COLLISIONS",originalHasCollisions
framedExtrapolations = []
for e in syntaxTree.extrapolations():
t = e.convertToSequence()
if not originalHasCollisions and t.removeDuplicates().hasCollisions(): continue
if t == trace: continue
if any([t == o for o in extrapolations ]): continue
extrapolations.append(t)
framedExtrapolations.append(1 - frameImageNicely(1 - t.framedRendering(result.parse)))
if len(framedExtrapolations) > 20:
break
if framedExtrapolations != []:
for crap in interestingExtrapolations:
if isinstance(crap,tuple) and crap[0] == expertIndex:
framedExtrapolations = [framedExtrapolations[crap[1]]]
break
if arguments.debug:
framedExtrapolations = [loadImage(f), fastRender(syntaxTree.convertToSequence())] + framedExtrapolations
else:
framedExtrapolations = [frameImageNicely(loadImage(f))] + framedExtrapolations
a = np.zeros((256*len(framedExtrapolations),256))
for j,e in enumerate(framedExtrapolations):
a[j*256:(1+j)*256,:] = 1 - e
a[j*256,:] = 0.5
a[(1+j)*256-1,:] = 0.5
a[0,:] = 0.5
a[:,0] = 0.5
a[:,255] = 0.5
a[256*len(framedExtrapolations)-1,:] = 0.5
a = 255*a
# to show the first one
#a = a[:(256*2),:]
extrapolationMatrix.append(a)
saveMatrixAsImage(a,'extrapolations/expert-%d-extrapolation.png'%expertIndex)
if not arguments.extrapolate:
rightEntryOfTable = '''
\\begin{minipage}{10cm}
\\begin{verbatim}
%s
\\end{verbatim}
\\end{minipage}
'''%(parseSketchOutput(result.source).pretty() if result != None and result.source != None else "Solver timeout")
else:
rightEntryOfTable = ""
if extrapolations != [] and arguments.extrapolate:
#}make the big matrix
bigMatrix = np.zeros((max([m.shape[0] for m in extrapolationMatrix ]),256*len(extrapolationMatrix)))
for j,r in enumerate(extrapolationMatrix):
bigMatrix[0:r.shape[0],256*j:256*(j+1)] = r
if len(extrapolations) < 10 and False:
saveMatrixAsImage(bigMatrix,'../TikZpaper/figures/extrapolationMatrix.png')
else:
saveMatrixAsImage(bigMatrix,'../TikZpaper/figures/extrapolationMatrixSupplement.png')
print e
rightEntryOfTable = '\\includegraphics[width = 5cm]{../TikZ/extrapolations/expert-%d-extrapolation.png}'%expertIndex
if rightEntryOfTable != "":
parseImage = '\\includegraphics[width = 5cm]{../TikZ/drawings/expert-%d-parses/0.png}'%expertIndex
if not os.path.exists('drawings/expert-%d-parses/0.png'%expertIndex):
parseImage = "Sampled no finished traces."
latex.append('''
\\begin{tabular}{lll}
\\includegraphics[width = 5cm]{../TikZ/drawings/expert-%d.png}&
%s&
%s
\\end{tabular}
'''%(expertIndex, parseImage, rightEntryOfTable))
print
if arguments.latex:
latex = '%s'%("\\\\\n".join(latex))
name = "extrapolations.tex" if arguments.extrapolate else "synthesizerOutputs.tex"
with open('../TikZpaper/%s'%name,'w') as handle:
handle.write(latex)
print "Wrote output to ../TikZpaper/%s"%name
if arguments.similarity:
analyzeFeatures(programFeatures)
class SynthesisResult():
def __init__(self, parse, time = None, source = None, cost = None, originalDrawing = None, usePrior = True):
self.usePrior = usePrior
self.originalDrawing = originalDrawing
self.parse = parse
self.time = time
self.source = source
self.cost = cost
def __str__(self):
return self.source
def usedPrior(self):
if hasattr(self,'usePrior'): return self.usePrior
return True
icingResult = SynthesisResult(getGroundTruthParse('drawings/expert-38.png'),
source = icingModelOutput,
cost = 0,
originalDrawing = 'drawings/expert-38.png')
class SynthesisJob():
def __init__(self, parse, originalDrawing, usePrior = True):
self.parse = parse
self.originalDrawing = originalDrawing
self.usePrior = usePrior
def __str__(self):
return "SynthesisJob(%s) using the prior? %s:\n%s"%(self.originalDrawing,self.usePrior,str(self.parse))
def execute(self):
startTime = time.time()
result = synthesizeProgram(self.parse,self.usePrior)
elapsedTime = time.time() - startTime
def viewSynthesisResults(arguments):
results = pickle.load(open(arguments.name,'rb'))
print " [+] Loaded %d synthesis results."%(len(results))
interestingExtrapolations = [7,
#14,
17,
29,
#35,
52,
57,
63,
70,
72,
88,
#99]
]
interestingExtrapolations = [(16,12),#*
#(17,0),
(18,0),#*
#(22,0),
#(23,0),
#(29,12),
#(31,27),
(34,0),#*
#(36,0),
#(38,12),
(39,0),#*
#(41,1),
#(51,1),
#(52,12),
#(57,0),
#(58,0),
(60,0),#*
#(63,0),
(66,2),#*
(71,1),#*
#(72,0),
#(73,0),
#(74,10),
#(75,5),
#(79,0),
#(85,1),
(86,0),#*
#(88,0),
(90,2),#*
#(92,0),
#(95,8)
]
#interestingExtrapolations = list(range(100))
latex = []
extrapolationMatrix = []
programFeatures = {}
for expertIndex in list(range(100)):# + [101]:
if arguments.extrapolate:
if not any([ e == expertIndex or isinstance(e,tuple) and e[0] == expertIndex
for e in interestingExtrapolations ]):
continue
f = 'drawings/expert-%d.png'%expertIndex
parse = getGroundTruthParse(f)
if parse == None:
print "No ground truth for %d"%expertIndex
assert False
relevantResults = [ r for r in results if r.job.originalDrawing == f and r.cost != None ]
if relevantResults == []:
print "No synthesis result for %s"%f
result = None
else:
result = min(relevantResults, key = lambda r: r.cost)
equallyGoodResults = [ r for r in relevantResults if r.cost <= result.cost + 1 ]
if len(equallyGoodResults) > 1:
print "Got %d results for %d"%(len(equallyGoodResults),expertIndex)
programs = [ r.program.fixReflections(result.job.parse.canonicalTranslation()).removeDeadCode()
for r in equallyGoodResults ]
if result == None and arguments.extrapolate:
print "Synthesis failure for %s"%f
continue
print " [+] %s"%f
print "\t(synthesis time: %s)"%(result.time if result else None)
print
if arguments.debug:
print result.source
if result != None:
syntaxTree = result.program
syntaxTree = syntaxTree.fixReflections(result.job.parse.canonicalTranslation())
print syntaxTree
print syntaxTree.features()
print syntaxTree.convertToPython()
print syntaxTree.convertToSequence()
#showImage(fastRender(syntaxTree.convertToSequence()) + loadImage(f)*0.5 + fastRender(result.parse))
programFeatures[f] = syntaxTree.features()
extrapolations = []
if arguments.extrapolate:
syntaxTree = syntaxTree.explode()
trace = syntaxTree.convertToSequence().removeDuplicates()
print "original trace:"
print trace
originalUndesirability = parse.undesirabilityVector()
print "original undesirability",originalUndesirability
framedExtrapolations = []
extrapolationGenerators = [ program.explode().extrapolations() for program in programs ]
for e in interleaveGenerators(extrapolationGenerators):
t = e.convertToSequence().removeDuplicates()
newUndesirability = t.undesirabilityVector()
if (newUndesirability > originalUndesirability).sum() > 0: continue
if t.canonicalTranslation() == trace.canonicalTranslation(): continue
if any([t.canonicalTranslation() == o.canonicalTranslation() for o in extrapolations ]): continue
extrapolations.append(t)
framedExtrapolations.append(1 - frameImageNicely(t.draw(adjustCanvasSize = True)))
if len(framedExtrapolations) > 30:
break
if framedExtrapolations != []:
for crap in interestingExtrapolations:
if isinstance(crap,tuple) and crap[0] == expertIndex:
print "Just taking the %d extrapolation..."%(crap[1])
framedExtrapolations = [framedExtrapolations[crap[1]]]
break
if arguments.debug:
framedExtrapolations = [loadImage(f), syntaxTree.convertToSequence().draw()] + framedExtrapolations
else:
framedExtrapolations = [frameImageNicely(loadImage(f))] + framedExtrapolations
a = np.zeros((256*len(framedExtrapolations),256))
for j,e in enumerate(framedExtrapolations):
a[j*256:(1+j)*256,:] = 1 - e
a[j*256,:] = 0.5
a[(1+j)*256-1,:] = 0.5
a[0,:] = 0.5
a[:,0] = 0.5
a[:,255] = 0.5
a[256*len(framedExtrapolations)-1,:] = 0.5
a = 255*a
# to show the first one
#a = a[:(256*2),:]
extrapolationMatrix.append(a)
print "Saving extrapolation column to",'extrapolations/expert-%d-extrapolation.png'%expertIndex
saveMatrixAsImage(a,'extrapolations/expert-%d-extrapolation.png'%expertIndex)
if not arguments.extrapolate:
rightEntryOfTable = '''
\\begin{minipage}{10cm}
\\begin{verbatim}
%s
\\end{verbatim}
\\end{minipage}
'''%(syntaxTree.pretty() if result != None else "Solver timeout")
else:
rightEntryOfTable = ""
if False and extrapolations != [] and arguments.extrapolate:
#print e
rightEntryOfTable = '\\includegraphics[width = 5cm]{../TikZ/extrapolations/expert-%d-extrapolation.png}'%expertIndex
if rightEntryOfTable != "":
parseImage = '\\includegraphics[width = 5cm]{../TikZ/drawings/expert-%d-parses/0.png}'%expertIndex
if not os.path.exists('drawings/expert-%d-parses/0.png'%expertIndex):
parseImage = "Sampled no finished traces."
latex.append('''
\\begin{tabular}{lll}
\\includegraphics[width = 5cm]{../TikZ/drawings/expert-%d.png}&
%s&
%s
\\end{tabular}
'''%(expertIndex, parseImage, rightEntryOfTable))
print
if arguments.latex:
latex = '%s'%("\\\\\n".join(latex))
name = "extrapolations.tex" if arguments.extrapolate else "synthesizerOutputs.tex"
with open('../TikZpaper/%s'%name,'w') as handle:
handle.write(latex)
print "Wrote output to ../TikZpaper/%s"%name
if arguments.similarity:
analyzeFeatures(programFeatures)
if arguments.extrapolate:
#}make the big matrix
bigMatrix = np.zeros((max([m.shape[0] for m in extrapolationMatrix ]),256*len(extrapolationMatrix)))
for j,r in enumerate(extrapolationMatrix):
bigMatrix[0:r.shape[0],256*j:256*(j+1)] = r
saveMatrixAsImage(bigMatrix,'extrapolations/allTheExtrapolations.png')