def makeNewTasks(include_only=None):
#load new data:
taskfile = "./csv_filtered_all_background_novel.p"
with open(taskfile, 'rb') as handle:
data = dill.load(handle)
tasklist = data['background'] #a list of indices
if include_only:
regextasks = [
Task("Data column no. " + str(i), arrow(tpregex, tpregex),
[((), example) for example in task['train']])
for i, task in enumerate(tasklist) if i in include_only
]
else:
regextasks = [
Task("Data column no. " + str(i), arrow(tpregex, tpregex),
[((), example) for example in task['train']])
for i, task in enumerate(tasklist)
]
#for i in train_list:
# regextasks[i].mustTrain = True
return regextasks
def regexHeldOutExamples(task, include_only=None):
#load new data:
global REGEXTASKS
if REGEXTASKS is None:
taskfile = "./csv_filtered_all_background_novel.p"
with open(taskfile, 'rb') as handle:
data = dill.load(handle)
tasklist = data['background'] #a list of indices
if include_only:
regextasks = [
Task("Data column no. " + str(i), arrow(tpregex, tpregex),
[((), example) for example in _task['test']])
for i, _task in enumerate(tasklist) if i in include_only
]
else:
regextasks = [
Task("Data column no. " + str(i), arrow(tpregex, tpregex),
[((), example) for example in _task['test']])
for i, _task in enumerate(tasklist)
]
#for i in train_list:
# regextasks[i].mustTrain = True
REGEXTASKS = {t.name: t.examples for t in regextasks}
fullTask = REGEXTASKS[task.name]
return fullTask
def easyWordsPrimitives():
return [
Primitive("string_" + i, tpregex, pregex.String(i))
for i in printable[10:62] if i not in disallowed_list
] + [
Primitive("r_d", tpregex, pregex.d),
Primitive("r_s", tpregex, pregex.s),
#Primitive("r_w", tpregex, pregex.w),
Primitive("r_l", tpregex, pregex.l),
Primitive("r_u", tpregex, pregex.u),
Primitive("r_kleene", arrow(tpregex, tpregex), _kleene),
Primitive("r_plus", arrow(tpregex, tpregex), _plus),
Primitive("r_maybe", arrow(tpregex, tpregex), _maybe),
Primitive("r_alt", arrow(tpregex, tpregex, tpregex), _alt),
Primitive("r_concat", arrow(tpregex, tpregex, tpregex), _concat),
]
def convertType(t, ecTypes):
"""Converts PyCCG basic and complex types -> EC Types."""
if type(t) is BasicType:
return ecTypes[t.name]
elif type(t) is ComplexType:
return ec_type.arrow(convertType(t.first, ecTypes),
convertType(t.second, ecTypes))
def makeTask(name, f):
xs = [x / 100. for x in range(-500, 500)]
maximum = 10
N = 50
inputs = []
outputs = []
for x in xs:
try:
y = f(x)
except BaseException:
continue
if abs(y) < maximum:
inputs.append(float(x))
outputs.append(float(y))
if len(inputs) >= N:
ex = list(zip(inputs, outputs))
ex = ex[::int(len(ex) / N)][:N]
t = DifferentiableTask(name,
arrow(treal, treal),
[((x, ), y) for x, y in ex],
BIC=1.,
restarts=360,
steps=50,
likelihoodThreshold=-0.05,
temperature=0.1,
maxParameters=6,
loss=squaredErrorLoss)
t.f = f
return t
return None
def makeTestdata(synth=True, challenge=False):
raise NotImplementedError
tasks = []
if synth:
tasks = makeTasks()
if challenge:
challenge_tasks, _ = loadPBETasks()
tasks = tasks + challenge_tasks
tasklist = []
for task in tasks:
if task.stringConstants == [] and task.request == arrow(
tlist(tcharacter), tlist(tcharacter)):
IO = tuple((''.join(x[0]), ''.join(y)) for x, y in task.examples)
program = None
pseq = None
sketch, sketchseq, reward, sketchprob = None, None, None, None
tp = tprogram
tasklist.append(
Datum(tp, program, pseq, IO, sketch, sketchseq, reward,
sketchprob))
return tasklist
def dreamFromGrammar(g, directory, N=100):
if isinstance(g, Grammar):
programs = [
p for _ in range(N)
for p in [g.sample(arrow(turtle, turtle), maximumDepth=20)]
if p is not None
]
else:
programs = g
drawLogo(*programs,
pretty=False,
smoothPretty=False,
resolution=512,
filenames=[f"{directory}/{n}.png" for n in range(len(programs))],
timeout=1)
drawLogo(*programs,
pretty=True,
smoothPretty=False,
resolution=512,
filenames=[
f"{directory}/{n}_pretty.png" for n in range(len(programs))
],
timeout=1)
drawLogo(*programs,
pretty=False,
smoothPretty=True,
resolution=512,
filenames=[
f"{directory}/{n}_smooth_pretty.png"
for n in range(len(programs))
],
timeout=1)
for n, p in enumerate(programs):
with open(f"{directory}/{n}.dream", "w") as handle:
handle.write(str(p))
def makeOldTasks():
# a series of tasks
taskfile = './data_filtered.json'
#task_list = pickle.load(open(taskfile, 'rb'))
with open('./data_filtered.json') as f:
file_contents = f.read()
task_list = json.loads(file_contents)
# if I were to just dump all of them:
regextasks = [
Task("Luke data column no." + str(i), arrow(tpregex, tpregex),
[((), example) for example in task_list[i]])
for i in range(len(task_list))
]
""" regextasks = [
Task("length bool", arrow(none,tstr),
[((l,), len(l))
for _ in range(10)
for l in [[flip() for _ in range(randint(0,10)) ]] ]),
Task("length int", arrow(none,tstr),
[((l,), len(l))
for _ in range(10)
for l in [randomList()] ]),
]
"""
return regextasks # some list of tasks
def sample_datum(g=basegrammar,
N=5,
compute_sketches=False,
top_k_sketches=100,
inv_temp=1.0,
reward_fn=None,
sample_fn=None,
dc_model=None,
use_timeout=False,
continuation=False):
# find tp
if continuation:
tp = arrow(tpregex, tpregex)
else:
tp = tpregex
#sample a program:
#with timing("sample program"):
program = sample_program(g, tp)
# find IO
#with timing("sample IO:"):
IO = generate_IO_examples(program,
num_examples=N,
continuation=continuation)
if IO is None: return None
IO = tuple(IO)
# TODO
# find pseq
pseq = tuple(flatten_program(program, continuation=continuation)) #TODO
if compute_sketches:
# find sketch
# TODO - improved dc_grammar [ ]
# TODO - contextual_grammar [ ]
# TODO - put grammar inference inside make_holey
grammar = g if not dc_model else dc_model.infer_grammar(IO)
#with timing("make_holey"):
sketch, reward, sketchprob = make_holey_regex(
program,
top_k_sketches,
grammar,
tp,
inv_temp=inv_temp,
reward_fn=reward_fn,
sample_fn=sample_fn,
use_timeout=use_timeout) #TODO
# find sketchseq
sketchseq = tuple(flatten_program(sketch, continuation=continuation))
else:
sketch, sketchseq, reward, sketchprob = None, None, None, None
return Datum(tp, program, pseq, IO, sketch, sketchseq, reward, sketchprob)
def makeNumberTasks():
#load new data:
taskfile = "./regex_data_csv_900.p"
with open(taskfile, 'rb') as handle:
data = dill.load(handle)
tasklist = data[0] #a list of indices
#match_col(data[0],'\d*\.\d*')
raw_decimals = [
121, 122, 163, 164, 165, 170, 172, 173, 175, 178, 218, 228, 230, 231,
252, 253, 254, 258, 259, 305, 320, 330, 334, 340, 348, 350, 351, 352,
353, 355, 357, 358, 361, 363, 364, 371, 380, 382, 409, 410, 411, 447,
448, 449, 450, 458, 469, 471, 533, 562, 564
]
decimals_pos_neg_dollar = [
3, 4, 5, 6, 7, 13, 16, 24, 27, 28, 29, 30, 31, 32, 33, 34, 37, 38, 39,
40, 53, 54, 55, 57, 58, 60, 61, 63, 64, 65, 66, 68, 69, 70, 71, 73, 74,
77, 78, 80, 81, 103, 104, 105, 106, 107, 109, 110, 111, 112, 113, 114,
115, 116, 117, 118, 119, 121, 122, 123, 124, 125, 126, 128, 129, 131,
132, 134, 135, 139, 146, 153, 154, 155, 156, 157, 158, 159, 160, 161,
162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175,
176, 177, 178, 180, 181, 182, 183, 184, 185, 186, 193, 194, 195, 204,
205, 207, 209, 210, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221,
223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 249, 250, 251, 252,
253, 254, 255, 256, 258, 259, 260, 261, 263, 266, 267, 270, 271, 272,
277, 299, 301, 302, 305, 306, 307, 309, 312, 313, 315, 319, 320, 324,
326, 327, 330, 334, 340, 348, 350, 351, 352, 353, 354, 355, 356, 357,
358, 361, 362, 363, 364, 368, 371, 373, 377, 380, 382, 400, 401, 402,
403, 405, 406, 409, 410, 411, 413, 435, 439, 446, 447, 448, 449, 450,
451, 452, 453, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466,
469, 470, 471, 477, 498, 500, 502, 503, 507, 512, 518, 519, 520, 532,
533, 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 564, 565, 572,
577
]
#match_col(data[0],'(\d*,?\d*)+')
commas = []
#match_col(data[0],'(\d*,?\d*)+')
commas_and_all = []
#full_list = test_list + train_list
train_list = []
full_list = decimals_pos_neg_dollar
regextasks = [
Task("Data column no. " + str(i), arrow(tpregex, tpregex),
[((), example) for example in task])
for i, task in enumerate(tasklist) if i in full_list
]
for i in train_list:
regextasks[i].mustTrain = True
return regextasks
def task(self):
request = arrow(*[tbool for _ in range(self.numberOfInputs + 1)])
features = Circuit.extractFeatures(list(self.signature))
return Task(self.name,
request,
self.examples,
features=features,
cache=True)
def basePrimitives():
return [Primitive("string_" + i, tpregex, pregex.String(i)) for i in printable[:-4] if i not in disallowed_list
] + [
Primitive("string_" + name, tpregex, pregex.String(char)) for char, name in disallowed
] + [
Primitive("r_dot", tpregex, pregex.dot),
Primitive("r_d", tpregex, pregex.d),
Primitive("r_s", tpregex, pregex.s),
Primitive("r_w", tpregex, pregex.w),
Primitive("r_l", tpregex, pregex.l),
Primitive("r_u", tpregex, pregex.u),
Primitive("r_kleene", arrow(tpregex, tpregex), _kleene),
Primitive("r_plus", arrow(tpregex, tpregex), _plus),
Primitive("r_maybe", arrow(tpregex, tpregex), _maybe),
Primitive("r_alt", arrow(tpregex, tpregex, tpregex), _alt),
Primitive("r_concat", arrow(tpregex, tpregex, tpregex), _concat),
]
def makeWordTasks():
#load new data:
taskfile = "./regex_data_csv_900.p"
with open(taskfile, 'rb') as handle:
data = dill.load(handle)
tasklist = data[0] #a list of indices
all_upper = [0, 2, 8, 9, 10, 11, 12, 17, 18, 19, 20, 22]
all_lower = [1]
# match_col(data[0],'\\u(\l+)')
one_capital_lower_plus = [
144, 200, 241, 242, 247, 296, 390, 392, 444, 445, 481, 483, 485, 489,
493, 542, 549, 550, 581
]
#match_col(data[0],'(\l ?)+')
lower_with_maybe_spaces = [
1, 42, 47, 99, 100, 102, 201, 246, 248, 293, 294, 345, 437, 545, 590
]
#match_col(data[0],'(\\u\l+ ?)+')
capital_then_lower_maybe_spaces = [
144, 200, 241, 242, 247, 296, 390, 392, 395, 438, 444, 445, 481, 483,
484, 485, 487, 489, 493, 494, 542, 546, 549, 550, 578, 581, 582, 588,
591, 624, 629
]
#match_col(data[0],'(\\u+ ?)+')
all_caps_spaces = [
0, 2, 8, 9, 10, 11, 12, 17, 18, 19, 20, 22, 25, 26, 35, 36, 43, 45, 46,
49, 50, 52, 56, 59, 87, 89, 95, 101, 140, 147, 148, 149, 199, 332, 336,
397, 491, 492, 495, 580, 610
]
#one_capital_and_lower = [566, 550, 549, 542, 505, 493, 494, 489, 488, 485, 483, 481, 445, 444, 438, 296, 241, 242, 200, ]
#all_lower_with_a_space = [545]
#all_lower_maybe_space = [534]
#one_capital_lower_maybe_spaces = [259, 262, 263, 264]
#full_list = test_list + train_list
train_list = []
full_list = all_upper + all_lower + one_capital_lower_plus + lower_with_maybe_spaces + capital_then_lower_maybe_spaces + all_caps_spaces
regextasks = [
Task("Data column no. " + str(i), arrow(tpregex, tpregex),
[((), example) for example in task])
for i, task in enumerate(tasklist) if i in full_list
]
for i in train_list:
regextasks[i].mustTrain = True
return regextasks
def matchEmpericalNoLetterPrimitives(corpus):
return lambda: [Primitive("empty_string", tpregex, pregex.String(""))] + [
Primitive("string_" + i, tpregex, pregex.String(i))
for i in printable[:-4]
if i not in disallowed_list + list(printable[10:62])
] + [
Primitive("string_" + name, tpregex, pregex.String(char))
for char, name in disallowed
] + [
Primitive("r_dot", tpregex, emp_dot_no_letter(corpus)),
Primitive("r_d", tpregex, emp_d(corpus)),
Primitive("r_s", tpregex, pregex.s),
Primitive("r_kleene", arrow(tpregex, tpregex), _kleene),
#Primitive("r_plus", arrow(tpregex, tpregex), _plus),
#Primitive("r_maybe", arrow(tpregex, tpregex), _maybe),
Primitive("r_alt", arrow(tpregex, tpregex, tpregex), _alt),
Primitive("r_concat", arrow(tpregex, tpregex, tpregex), _concat),
]
def manualLogoTask(name,
expression,
proto=False,
needToTrain=False,
supervise=False):
p = parseLogo(expression)
from logoPrimitives import primitives
from grammar import Grammar
g = Grammar.uniform(primitives, continuationType=turtle)
gp = Grammar.uniform(primitives)
try:
l = g.logLikelihood(arrow(turtle, turtle), p)
lp = gp.logLikelihood(arrow(turtle, turtle), p)
assert l >= lp
eprint(name, -l, "nats")
except:
eprint("WARNING: could not calculate likelihood of manual logo", p)
attempts = 0
while True:
[output, highresolution] = drawLogo(p, p, resolution=[28, 128])
if output == "timeout" or highresolution == "timeout":
attempts += 1
else:
break
if attempts > 0:
eprint(
f"WARNING: Took {attempts} attempts to render task {name} within timeout"
)
shape = list(map(int, output))
highresolution = list(map(float, highresolution))
t = Task(name, arrow(turtle, turtle), [(([0]), shape)])
t.mustTrain = needToTrain
t.proto = proto
t.specialTask = ("LOGO", {"proto": proto})
t.highresolution = highresolution
if supervise:
t.supervisedSolution = p
return t
def genericType(t):
if t.name == "real": return treal
elif t.name == "positive": return treal
elif t.name == "vector": return tlist(treal)
elif t.name == "list": return tlist(genericType(t.arguments[0]))
elif t.isArrow():
return arrow(genericType(t.arguments[0]),
genericType(t.arguments[1]))
else:
assert False, "could not make type generic: %s" % t
def convertFunction(f, ecTypes, namespace):
"""Converts a typed PyCCG function -> EC Primitive."""
ecArgs = [convertType(t, ecTypes) for t in f.arg_types]
ecReturn = convertType(f.return_type, ecTypes)
ecFunction = ec_program.Primitive(f.name + namespace,
ec_type.arrow(*ecArgs, ecReturn),
curry(f.defn))
return ecFunction
def tasksOfPrograms(self, ps, types):
images = drawLogo(*ps, resolution=128)
if len(ps) == 1: images = [images]
tasks = []
for i in images:
if isinstance(i, str): tasks.append(None)
else:
t = Task("Helm", arrow(turtle, turtle), [])
t.highresolution = i
tasks.append(t)
return tasks
def argumentChoices(t):
if t == turtle:
return [Index(0)]
elif t == arrow(turtle, turtle):
return subprograms
elif t == tint:
return specialNumbers.get(str(p), numbers)
elif t == tangle:
return specialAngles.get(str(p), angles)
elif t == tlength:
return specialDistances.get(str(p), distances)
else:
return []
def extractTasks(dataset):
for i, function in enumerate(dataset):
name = "deep-coder #{}".format(i)
examples = [((x["input"],), x["output"]) for x in function["examples"]]
try:
input_type = guess_type([i for (i,), _ in examples])
output_type = guess_type([o for _, o in examples])
except ValueError:
continue
program_type = arrow(input_type, output_type)
features = list_features(examples)
cache = all(hashable(x) for x in examples)
yield Task(name, program_type, examples, features=features, cache=cache)
def recurse_list(l, target_tp):
#base case
x = recurse(l[0])
x = convert_to_tp(x, target_tp)
e = convert_to_tp(x, tlist(target_tp))
for exp in l[1:]:
x = recurse(exp)
if x.infer() != target_tp:
x = convert_to_tp(x, target_tp) # maybe always convert?
request = arrow(target_tp, tlist(target_tp), tlist(target_tp))
list_converter = [x for x in Primitive.GLOBALS.values() if x.tp==request][0] # TODO
e = Application(Application(list_converter, x), e)
return e
def no_length():
"""this is the primitives without length because one of the reviewers wanted this"""
return [p for p in bootstrapTarget() if p.name != "length"] + [
Primitive("*", arrow(tint, tint, tint), _multiplication),
Primitive("mod", arrow(tint, tint, tint), _mod),
Primitive("gt?", arrow(tint, tint, tbool), _gt),
Primitive("eq?", arrow(tint, tint, tbool), _eq),
Primitive("is-prime", arrow(tint, tbool), _isPrime),
Primitive("is-square", arrow(tint, tbool), _isSquare),
]
def bootstrapTarget_extra():
"""This is the bootstrap target plus list domain specific stuff"""
return bootstrapTarget() + [
Primitive("*", arrow(tint, tint, tint), _multiplication),
Primitive("mod", arrow(tint, tint, tint), _mod),
Primitive("gt?", arrow(tint, tint, tbool), _gt),
Primitive("eq?", arrow(tint, tint, tbool), _eq),
Primitive("is-prime", arrow(tint, tbool), _isPrime),
Primitive("is-square", arrow(tint, tbool), _isSquare),
]
def test_ec_task_as_pyccg_scene():
instruction = 'go to the diamond'
scene = process_scene([SIMPLE_SCENE])
goal = (2,9)
puddleworldTypes, puddleworldPrimitives = convertOntology(ec_ontology)
ec_task = Task(name=instruction,
request=arrow(puddleworldTypes['model'], puddleworldTypes['action']),
examples=[([scene], tuple(goal))],
features=instruction)
converted_instruction, converted_model, converted_goal = ecTaskAsPyCCGUpdate(ec_task, ontology)
eq_(converted_instruction, instruction.split())
eq_(converted_model.scene, scene)
eq_(converted_goal, goal)
def McCarthyPrimitives():
"These are < primitives provided by 1959 lisp as introduced by McCarthy"
return [
Primitive("empty", tlist(t0), []),
Primitive("cons", arrow(t0, tlist(t0), tlist(t0)), _cons),
Primitive("car", arrow(tlist(t0), t0), _car),
Primitive("cdr", arrow(tlist(t0), tlist(t0)), _cdr),
Primitive("empty?", arrow(tlist(t0), tbool), _isEmpty),
#Primitive("unfold", arrow(t0, arrow(t0,t1), arrow(t0,t0), arrow(t0,tbool), tlist(t1)), _isEmpty),
#Primitive("1+", arrow(tint,tint),None),
# Primitive("range", arrow(tint, tlist(tint)), range),
# Primitive("map", arrow(arrow(t0, t1), tlist(t0), tlist(t1)), _map),
# Primitive("index", arrow(tint,tlist(t0),t0),None),
# Primitive("length", arrow(tlist(t0),tint),None),
primitiveRecursion1,
#primitiveRecursion2,
Primitive("gt?", arrow(tint, tint, tbool), _gt),
Primitive("if", arrow(tbool, t0, t0, t0), _if),
Primitive("eq?", arrow(tint, tint, tbool), _eq),
Primitive("+", arrow(tint, tint, tint), _addition),
Primitive("-", arrow(tint, tint, tint), _subtraction),
] + [Primitive(str(j), tint, j) for j in range(2)]
def makeLongTasks():
#load new data:
taskfile = "./regex_data_csv_900.p"
with open(taskfile, 'rb') as handle:
data = dill.load(handle)
tasklist = data[0] #a list of indices
regextasks = [
Task("Data column no. " + str(i), arrow(tpregex, tpregex),
[((), example) for example in task])
for i, task in enumerate(tasklist)
]
return regextasks
def algolispPrimitives():
return [
Primitive("fn_call", arrow(tfunction, tlist(tsymbol), tsymbol), _fn_call),
Primitive("lambda1_call", arrow(tfunction, tlist(tsymbol), tsymbol), lambda f: lambda sx: ["lambda1", [f] + sx] if type(sx)==list else ["lambda1", [f] + [sx]] ),
Primitive("lambda2_call", arrow(tfunction, tlist(tsymbol), tsymbol), lambda f: lambda sx: ["lambda2", [f] + sx] if type(sx)==list else ["lambda2", [f] + [sx]] ),
#symbol converters:
# SYMBOL = constant | argument | function_call | function | lambda
Primitive("symbol_constant", arrow(tconstant, tsymbol), lambda x: x),
Primitive("symbol_function", arrow(tfunction, tsymbol), lambda x: x),
#list converters
Primitive('list_init_symbol', arrow(tsymbol, tlist(tsymbol)), lambda symbol: [symbol] ),
Primitive('list_add_symbol', arrow(tsymbol, tlist(tsymbol), tlist(tsymbol)), lambda symbol: lambda symbols: symbols + [symbol] if type(symbols) == list else [symbols] + [symbol])
] + [
#functions:
Primitive(ec_name, tfunction, algo_name) for algo_name, ec_name in fn_lookup.items()
] + [
#Constants
Primitive(ec_name, tconstant, algo_name) for algo_name, ec_name in const_lookup.items()
]
def makeHandPickedTasks():
#load new data:
taskfile = "./regex_data_csv_900.p"
with open(taskfile, 'rb') as handle:
data = dill.load(handle)
tasklist = data[0] #a list of indices
full_list = list(range(199)) + \
[209,218,222,223,224,225,226] + \
list(range(222,233)) + \
[235,237,238,239,243,244,245,252,253,254,255,257,258,259,260,261,264,265,269,272,274] + \
list(range(275,291)) + \
[295,297,300,303,304,305,306,310,311,312,314,315,316,320,321,323,327,329,330,333,334,335,337,338,339,340,341,342,343,344] + \
list(range(348,359)) + \
[361,369,373,379,380,382,387,403,405,407,408] + \
list(range(409,417)) + \
list(range(418,437)) + \
list(range(440,444)) + \
list(range(446,452)) + \
list(range(456,460)) + \
list(range(466,472)) + \
[503,504]
regextasks = [
Task("Data column no. " + str(i), arrow(tpregex, tpregex),
[((), example) for example in task])
for i, task in enumerate(tasklist) if i in full_list
]
#for i in train_list:
# regextasks[i].mustTrain = True
return regextasks
def convert_examples_to_datum(self):
# Get the type first:
funtype = self.examples[0].typeof()
ioexamples = []
for example in self.examples:
inputs = example.inputs
outputs = example.outputs[0]
non_int = False
for inp in inputs:
if type(inp) != type(1):
non_int = True
# This is all integers, so transform it into a list.
if not non_int:
print("Warning: converting type for SketchAdapt to avoid int->int")
inputs = (list(example.inputs),)
print(inputs)
funtype = arrow(tlist(tint), self.examples[0].outtype)
ioexamples.append((inputs, outputs))
ioexamples = tuple(ioexamples)
datum = Datum(funtype, None, None, ioexamples, None, None, None, None)
return [datum]
def convert_source_to_datum(source, N=5, V=512, L=10, compute_sketches=False, top_k_sketches=20, inv_temp=1.0, reward_fn=None, sample_fn=None, dc_model=None, use_timeout=False):
source = source.replace(' | ', '\n')
dc_program = compile(source, V=V, L=L)
if dc_program is None:
return None
# find IO
IO = tuple(generate_IO_examples(dc_program, N=N, L=L, V=V))
# find tp
ins = [tint if inp == int else tlist(tint) for inp in dc_program.ins]
if dc_program.out == int:
out = tint
else:
assert dc_program.out==[int]
out = tlist(tint)
tp = arrow( *(ins+[out]) )
# find program p
pseq = tuple(convert_dc_program_to_ec(dc_program, tp))
# find pseq
p = parseprogram(pseq, tp) # TODO: use correct grammar, and
if compute_sketches:
# find sketch
grammar = basegrammar if not dc_model else dc_model.infer_grammar(IO) #This line needs to change
sketch, reward, sketchprob = make_holey_deepcoder(p, top_k_sketches, grammar, tp, inv_temp=inv_temp, reward_fn=reward_fn, sample_fn=sample_fn, use_timeout=use_timeout) #TODO
# find sketchseq
sketchseq = tuple(flatten_program(sketch))
else:
sketch, sketchseq, reward, sketchprob = None, None, None, None
return Datum(tp, p, pseq, IO, sketch, sketchseq, reward, sketchprob)