def load_rb_dc_model_from_path(path,
max_length,
max_index,
improved_dc_grammar,
cuda=True):
basegrammar = Grammar.fromProductions(
RobustFillProductions(max_length, max_index))
if improved_dc_grammar:
specExtractor = RobustFillLearnedFeatureExtractor(printable[:-4],
hidden=128,
use_cuda=cuda)
vocab = robustfill_vocab(basegrammar)
sketchExtractor = SketchFeatureExtractor(vocab,
hidden=128,
use_cuda=cuda)
extractor = HoleSpecificFeatureExtractor(specExtractor,
sketchExtractor,
hidden=128,
use_cuda=cuda)
dcModel = ImprovedRecognitionModel(extractor,
basegrammar,
hidden=[128],
cuda=cuda,
contextual=False)
else:
extractor = RobustFillLearnedFeatureExtractor(
printable[:-4],
hidden=128) # probably want to make it much deeper ....
dcModel = DeepcoderRecognitionModel(
extractor, basegrammar, hidden=[128],
cuda=True) # probably want to make it much deeper ....
dcModel.load_state_dict(torch.load(path))
return dcModel
def load_rb_dc_model_from_path(path, max_length, max_index):
basegrammar = Grammar.fromProductions(
RobustFillProductions(max_length, max_index))
extractor = RobustFillLearnedFeatureExtractor(
printable[:-4],
hidden=128) # probably want to make it much deeper ....
dcModel = DeepcoderRecognitionModel(
extractor, basegrammar, hidden=[128],
cuda=True) # probably want to make it much deeper ....
dcModel.load_state_dict(torch.load(path))
return dcModel
import time
from program import ParseFailure, Context
from grammar import NoCandidates, Grammar
from utilities import timing, callCompiled
from collections import namedtuple
from itertools import islice, zip_longest
from functools import reduce
from RobustFillPrimitives import robustFillPrimitives, RobustFillProductions
from util.pypy_util import alternate
#A nasty hack!!!
max_len = 25
max_index = 4
#_ = robustFillPrimitives(max_len=max_len, max_index=max_index)
basegrammar = Grammar.fromProductions(RobustFillProductions(
max_len, max_index))
SketchTup = namedtuple("SketchTup", ['sketch', 'g'])
RobustFillResult = namedtuple(
"RobustFillResult",
["sketch", "prog", "hit", "n_checked", "time", "g_hit"])
def test_program_on_IO(e,
IO,
n_rec_examples,
generalization=False,
noise=False):
examples = IO if generalization else IO[:n_rec_examples]
if noise:
}[args.sample_fn]
batchsize = args.batchsize
max_length = args.max_length
if args.train_data != 'NA':
train_datas = args.train_data
assert False
if args.use_dc_grammar == 'NA':
use_dc_grammar = False
dc_grammar_path = None
else:
use_dc_grammar = True
dc_model_path = args.use_dc_grammar
basegrammar = Grammar.fromProductions(
RobustFillProductions(args.max_length, args.max_index))
vocab = robustfill_vocab(basegrammar)
if __name__ == "__main__":
print("Loading model", flush=True)
try:
if args.new: raise FileNotFoundError
elif args.load_trained_model:
model = torch.load(args.load_trained_model_path)
print("loading saved trained model, continuing training")
else:
model = torch.load(args.load_pretrained_model_path)
print('found saved model, loaded pretrained model (without holes)')
except FileNotFoundError:
print("no saved model, creating new one")
weights = [sample_fn(w) for w in weights]
#normalize_weights
w_sum = sum(w for w in weights)
weights = [w / w_sum for w in weights]
prog_reward_probs = list(zip(progs, rewards, weights))
if k > 1:
x = random.choices(prog_reward_probs, weights=weights, k=1)
return x[0] #outputs prog, prob
else:
return prog_reward_probs[0] #outputs prog, prob
if __name__ == '__main__':
g = Grammar.fromProductions(RobustFillProductions())
print(len(g))
request = tprogram
p = g.sample(request)
print("request:", request)
print("program:")
print(prettyProgram(p))
s = 'abcdefg'
e = p.evaluate([])
print("prog applied to", s)
print(e(s))
print("flattened_program:")
flat = flatten_program(p)
print(flat)
pr = parseprogram(flat, request)
print(prettyProgram(pr))
# with open('rb_all_tasks.p', 'wb') as savefile:
# pickle.dump(tasks, savefile)
# print('saved rb challenge tasks')
def loadTestTasks(path='rb_test_tasks.p'):
print("data file:", path)
with open(path, 'rb') as datafile:
tasks = pickle.load(datafile)
return tasks
if __name__ == '__main__':
import time
g = Grammar.fromProductions(RobustFillProductions(max_len=50, max_index=4))
d = sample_datum(g=g,
N=4,
V=50,
L=10,
compute_sketches=True,
top_k_sketches=100,
inv_temp=1.0,
reward_fn=None,
sample_fn=None,
dc_model=None)
print(d.p)
for i, o in d.IO:
print("example")
print(i)
print(o)
import time
from collections import OrderedDict
#from util import enumerate_reg, Hole
from grammar import Grammar, NoCandidates
from deepcoderPrimitives import deepcoderProductions, flatten_program
from program import Application, Hole, Primitive, Index, Abstraction, ParseFailure
import math
from type import Context, arrow, tint, tlist, tbool, UnificationFailure
productions = deepcoderProductions(
) # TODO - figure out good production probs ...
grammar = Grammar.fromProductions(productions, logVariable=0.0) # TODO
def tokenize_for_robustfill(IOs):
"""
tokenizes a batch of IOs
"""
newIOs = []
for examples in IOs:
tokenized = []
for xs, y in examples:
if isinstance(y, list):
y = ["LIST_START"] + y + ["LIST_END"]
else:
y = [y]
serializedInputs = []
sys.path.append(os.path.abspath('./ec'))
from grammar import Grammar
from deepcoderPrimitives import deepcoderProductions, flatten_program
#from program import Application, Hole
#import math
from type import Context, arrow, tint, tlist, tbool, UnificationFailure
from program import prettyProgram
from train.main_supervised_deepcoder import parseprogram, make_holey_deepcoder
#g = Grammar.uniform(deepcoderPrimitives())
g = Grammar.fromProductions(
deepcoderProductions(),
logVariable=.9) #TODO - find correct grammar weights
request = arrow(tlist(tint), tint, tint)
p = g.sample(request)
sketch = make_holey_deepcoder(p, 10, g, request)
print("request:", request)
print("program:")
print(prettyProgram(p))
print("flattened_program:")
flat = flatten_program(p)
print(flat)
prog = parseprogram(flat, request)
from itertools import zip_longest, chain
from functools import reduce
import torch
from program_synthesis.algolisp.dataset import dataset
from program_synthesis.algolisp import arguments
from util.algolisp_pypy_util import test_program_on_IO
from program_synthesis.algolisp.dataset import executor
#I want a list of namedTuples
#Datum = namedtuple('Datum', ['tp', 'p', 'pseq', 'IO', 'sketch', 'sketchseq'])
basegrammar = Grammar.fromProductions(algolispProductions()) # Fix this
with open('basegrammar.p', 'rb') as file:
basegrammar = pickle.load(file)
#reweighted basegrammar:
# class FrontierEntry(object):
# def __init__(
# self,
# program,
# _=None,
# logPrior=None,
# logLikelihood=None,
# logPosterior=None):
# self.logPosterior = logPrior + logLikelihood if logPosterior is None else logPosterior
# self.program = program
l.__repr__()
if not l == "\t":
print("l is not tab")
assert False, f"uncaught item: {l}"
#elif l == variable name:
# assert False
return recurse(tree)
if __name__ == '__main__':
from grammar import Grammar
from program import Program
from algolispPrimitives import algolispProductions
g = Grammar.fromProductions(algolispProductions(), logVariable=.9)
#p=Program.parse("(lambda (fn_call filter (list_add_symbol (lambda1_call == (list_add_symbol 1 (list_init_symbol (fn_call mod ( list_add_symbol 2 (list_init_symbol arg1)) ))) ) (list_init_symbol $0)) )")
p = Program.parse(
"(fn_call filter (list_add_symbol (lambda1_call eq (list_add_symbol (symbol_constant 1) (list_init_symbol (fn_call mod ( list_add_symbol (symbol_constant 2) (list_init_symbol (symbol_constant arg1))) ))) ) (list_init_symbol (symbol_constant a))))"
)
print(p)
#tree = p.evaluate(["a"])
tree = p.evaluate([])
print(tree)
prog = tree_to_prog(tree)
print(prog)
from collections import OrderedDict
#from util import enumerate_reg, Hole
import re
from grammar import Grammar, NoCandidates
#from deepcoderPrimitives import deepcoderProductions, flatten_program
from RobustFillPrimitives import RobustFillProductions, flatten_program, tprogram, Constraint_prop, delimiters
from program import Application, Hole, Primitive, Index, Abstraction, ParseFailure, prettyProgram
import math
from type import Context, arrow, UnificationFailure
productions = RobustFillProductions(
) # TODO - figure out good production probs ...
basegrammar = Grammar.fromProductions(productions, logVariable=0.0) # TODO
def robustfill_vocab(grammar):
return [prim.name for prim in grammar.primitives] + ['<HOLE>'] # TODO
class timing(object):
def __init__(self, message):
self.message = message
def __enter__(self):
self.start = time.time()
return self
def __exit__(self, type, value, traceback):
num_inputs = string.count('lambda')
string = string.replace('lambda', '')
string = string.replace('(', '')
string = string.replace(')', '')
#remove '_fn' (optional)
for i in range(num_inputs):
string = string.replace('$' + str(num_inputs - i - 1),
'input_' + str(i))
string = string.split(' ')
string = list(filter(lambda x: x is not '', string))
return string
if __name__ == "__main__":
#g = Grammar.uniform(deepcoderPrimitives())
g = Grammar.fromProductions(deepcoderProductions(), logVariable=.9)
request = arrow(tlist(tint), tint, tint)
p = g.sample(request)
print("request:", request)
print("program:")
print(prettyProgram(p))
print("flattened_program:")
flat = flatten_program(p)
print(flat)
#robustfill output = names from productions + input_0-2 or 3
# # with open("/home/ellisk/om/ec/experimentOutputs/list_aic=1.0_arity=3_ET=1800_expandFrontier=2.0_it=4_likelihoodModel=all-or-nothing_MF=5_baseline=False_pc=10.0_L=1.0_K=5_rec=False.pickle", "rb") as handle:
# # b = pickle.load(handle).grammars[-1]
# # print b
