def regexHeldOutExamples(task, include_only=None):
#load new data:
global REGEXTASKS
if REGEXTASKS is None:
taskfile = os.path.join(get_data_dir(),
"csv_filtered_all_background_novel.p")
with open(taskfile, 'rb') as handle:
data = dill.load(handle)
tasklist = data['background'] #a list of indices
if include_only:
regextasks = [
Task("Data column no. " + str(i), arrow(tpregex, tpregex),
[((), example) for example in _task['test']])
for i, _task in enumerate(tasklist) if i in include_only
]
else:
regextasks = [
Task("Data column no. " + str(i), arrow(tpregex, tpregex),
[((), example) for example in _task['test']])
for i, _task in enumerate(tasklist)
]
#for i in train_list:
# regextasks[i].mustTrain = True
REGEXTASKS = {t.name: t.examples for t in regextasks}
fullTask = REGEXTASKS[task.name]
return fullTask
def bonusListProblems():
# Taken from https://www.ijcai.org/Proceedings/75/Papers/037.pdf
# These problems might be a lot easier if we do not use numbers
def randomList(lb=None, ub=None):
if lb is None:
lb = 2
if ub is None:
ub = 5
return [randint(0, 5) for _ in range(randint(lb, ub))]
bonus = [
Task(
"pair reverse", arrow(tlist(tint), tlist(tint)),
[((x,), [x[j + (1 if j % 2 == 0 else -1)]
for j in range(len(x))])
for _ in range(5)
for x in [randomList(10, 10)]]
),
Task(
"duplicate each element", arrow(tlist(tint), tlist(tint)),
[((x,), [a for z in x for a in [z] * 2])
for _ in range(5)
for x in [randomList(4, 6)]]
),
Task(
"reverse duplicate each element", arrow(tlist(tint), tlist(tint)),
[((x,), [a for z in reversed(x) for a in [z] * 2])]
),
]
return bonus
def makeNewTasks(include_only=None):
#load new data:
taskfile = os.path.join(get_data_dir(),
"csv_filtered_all_background_novel.p")
with open(taskfile, 'rb') as handle:
data = dill.load(handle)
tasklist = data['background'] #a list of indices
if include_only:
regextasks = [
Task("Data column no. " + str(i), arrow(tpregex, tpregex),
[((), example) for example in task['train']])
for i, task in enumerate(tasklist) if i in include_only
]
else:
regextasks = [
Task("Data column no. " + str(i), arrow(tpregex, tpregex),
[((), example) for example in task['train']])
for i, task in enumerate(tasklist)
]
#for i in train_list:
# regextasks[i].mustTrain = True
return regextasks
def make_list_bootstrap_tasks():
seed(42)
def suffixes(l):
if l == []:
return []
else:
return [l[1:]] + suffixes(l[1:])
def flip():
return random() > 0.5
def randomSuffix():
return [randint(0, 9) for _ in range(randint(1, 4))]
def randomList(minimum=1, minimumLength=2, maximumLength=2):
return [
randint(minimum, 9)
for _ in range(randint(minimumLength, maximumLength))
]
# def randomListOfLists():
# return [randomSuffix() for _ in range(randint(2, 4))]
# def randomListOfLists_bool(l=None):
# if l is None:
# l = randint(4, 7)
# return [randomBooleanList() for _ in range(l)]
# def randomBooleanList():
# return [flip() for _ in range(randint(4, 7))]
# Reliably learned in under a minute; always triggers learning of length
# primitive
operatorBootstrap = [
Task("add", arrow(tint, tint, tint), [((l[0], l[1]), l[0] + l[1])
for _ in range(50)
for l in [randomList()]]),
Task("minus", arrow(tint, tint, tint), [((l[0], l[1]), l[0] - l[1])
for _ in range(50)
for l in [randomList()]]),
Task("multiply", arrow(tint, tint, tint), [((l[0], l[1]), l[0] * l[1])
for _ in range(50)
for l in [randomList()]])
# Task ("divide", arrow(tint, tint, tint),
# [((l[0],l[1]), l[0] / l[1])
# for _ in range (50)
# for l in [randomList()]])
]
return operatorBootstrap
def makeOldTasks():
# a series of tasks
taskfile = os.path.join(get_data_dir(), 'data_filtered.json')
#task_list = pickle.load(open(taskfile, 'rb'))
with open(taskfile) as f:
file_contents = f.read()
task_list = json.loads(file_contents)
# if I were to just dump all of them:
regextasks = [
Task("Luke data column no." + str(i), arrow(tpregex, tpregex),
[((), example) for example in task_list[i]])
for i in range(len(task_list))
]
""" regextasks = [
Task("length bool", arrow(none,tstr),
[((l,), len(l))
for _ in range(10)
for l in [[flip() for _ in range(randint(0,10)) ]] ]),
Task("length int", arrow(none,tstr),
[((l,), len(l))
for _ in range(10)
for l in [randomList()] ]),
]
"""
return regextasks # some list of tasks
def make_list_task(name, examples, **params):
input_type = guess_type([i for (i,), _ in examples])
output_type = guess_type([o for _, o in examples])
# We can internally handle lists of bools.
# We explicitly create these by modifying existing routines.
if name.startswith("identify"):
boolexamples = [((i,), list(map(bool, o))) for (i,), o in examples]
yield from make_list_task("bool-" + name, boolexamples, **params)
# for now, we'll stick with the boolean-only tasks and not have a copy
# for integers.
return
program_type = arrow(input_type, output_type)
cache = all(hashable(x) for x in examples)
if params:
eq_params = ["{}={}".format(k, v) for k, v in params.items()]
if len(eq_params) == 1:
ext = eq_params[0]
elif len(eq_params) == 2:
ext = "{} and {}".format(*eq_params)
else:
ext = ", ".join(eq_params[:-1])
ext = "{}, and {}".format(ext, eq_params[-1])
name += " with " + ext
yield Task(name, program_type, examples, cache=cache)
def retrieveJSONTasks(filename, features=False):
"""
For JSON of the form:
{"name": str,
"type": {"input" : bool|int|list-of-bool|list-of-int,
"output": bool|int|list-of-bool|list-of-int},
"examples": [{"i": data, "o": data}]}
"""
with open(filename, "r") as f:
loaded = json.load(f)
TP = {
"bool": tbool,
"int": tint,
"list-of-bool": tlist(tbool),
"list-of-int": tlist(tint),
}
return [
Task(
item["name"],
arrow(TP[item["type"]["input"]], TP[item["type"]["output"]]),
[((ex["i"], ), ex["o"]) for ex in item["examples"]],
features=(None if not features else list_features(
[((ex["i"], ), ex["o"]) for ex in item["examples"]])),
cache=False,
) for item in loaded
]
def makeNumberTasks():
#load new data:
taskfile = os.path.join(get_data_dir(), "regex_data_csv_900.p")
with open(taskfile, 'rb') as handle:
data = dill.load(handle)
tasklist = data[0] #a list of indices
#match_col(data[0],'\d*\.\d*')
raw_decimals = [
121, 122, 163, 164, 165, 170, 172, 173, 175, 178, 218, 228, 230, 231,
252, 253, 254, 258, 259, 305, 320, 330, 334, 340, 348, 350, 351, 352,
353, 355, 357, 358, 361, 363, 364, 371, 380, 382, 409, 410, 411, 447,
448, 449, 450, 458, 469, 471, 533, 562, 564
]
decimals_pos_neg_dollar = [
3, 4, 5, 6, 7, 13, 16, 24, 27, 28, 29, 30, 31, 32, 33, 34, 37, 38, 39,
40, 53, 54, 55, 57, 58, 60, 61, 63, 64, 65, 66, 68, 69, 70, 71, 73, 74,
77, 78, 80, 81, 103, 104, 105, 106, 107, 109, 110, 111, 112, 113, 114,
115, 116, 117, 118, 119, 121, 122, 123, 124, 125, 126, 128, 129, 131,
132, 134, 135, 139, 146, 153, 154, 155, 156, 157, 158, 159, 160, 161,
162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175,
176, 177, 178, 180, 181, 182, 183, 184, 185, 186, 193, 194, 195, 204,
205, 207, 209, 210, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221,
223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 249, 250, 251, 252,
253, 254, 255, 256, 258, 259, 260, 261, 263, 266, 267, 270, 271, 272,
277, 299, 301, 302, 305, 306, 307, 309, 312, 313, 315, 319, 320, 324,
326, 327, 330, 334, 340, 348, 350, 351, 352, 353, 354, 355, 356, 357,
358, 361, 362, 363, 364, 368, 371, 373, 377, 380, 382, 400, 401, 402,
403, 405, 406, 409, 410, 411, 413, 435, 439, 446, 447, 448, 449, 450,
451, 452, 453, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466,
469, 470, 471, 477, 498, 500, 502, 503, 507, 512, 518, 519, 520, 532,
533, 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 564, 565, 572,
577
]
#match_col(data[0],'(\d*,?\d*)+')
commas = []
#match_col(data[0],'(\d*,?\d*)+')
commas_and_all = []
#full_list = test_list + train_list
train_list = []
full_list = decimals_pos_neg_dollar
regextasks = [
Task("Data column no. " + str(i), arrow(tpregex, tpregex),
[((), example) for example in task])
for i, task in enumerate(tasklist) if i in full_list
]
for i in train_list:
regextasks[i].mustTrain = True
return regextasks
def makeWordTasks():
#load new data:
taskfile = os.path.join(get_data_dir(), "regex_data_csv_900.p")
with open(taskfile, 'rb') as handle:
data = dill.load(handle)
tasklist = data[0] #a list of indices
all_upper = [0, 2, 8, 9, 10, 11, 12, 17, 18, 19, 20, 22]
all_lower = [1]
# match_col(data[0],'\\u(\l+)')
one_capital_lower_plus = [
144, 200, 241, 242, 247, 296, 390, 392, 444, 445, 481, 483, 485, 489,
493, 542, 549, 550, 581
]
#match_col(data[0],'(\l ?)+')
lower_with_maybe_spaces = [
1, 42, 47, 99, 100, 102, 201, 246, 248, 293, 294, 345, 437, 545, 590
]
#match_col(data[0],'(\\u\l+ ?)+')
capital_then_lower_maybe_spaces = [
144, 200, 241, 242, 247, 296, 390, 392, 395, 438, 444, 445, 481, 483,
484, 485, 487, 489, 493, 494, 542, 546, 549, 550, 578, 581, 582, 588,
591, 624, 629
]
#match_col(data[0],'(\\u+ ?)+')
all_caps_spaces = [
0, 2, 8, 9, 10, 11, 12, 17, 18, 19, 20, 22, 25, 26, 35, 36, 43, 45, 46,
49, 50, 52, 56, 59, 87, 89, 95, 101, 140, 147, 148, 149, 199, 332, 336,
397, 491, 492, 495, 580, 610
]
#one_capital_and_lower = [566, 550, 549, 542, 505, 493, 494, 489, 488, 485, 483, 481, 445, 444, 438, 296, 241, 242, 200, ]
#all_lower_with_a_space = [545]
#all_lower_maybe_space = [534]
#one_capital_lower_maybe_spaces = [259, 262, 263, 264]
#full_list = test_list + train_list
train_list = []
full_list = all_upper + all_lower + one_capital_lower_plus + lower_with_maybe_spaces + capital_then_lower_maybe_spaces + all_caps_spaces
regextasks = [
Task("Data column no. " + str(i), arrow(tpregex, tpregex),
[((), example) for example in task])
for i, task in enumerate(tasklist) if i in full_list
]
for i in train_list:
regextasks[i].mustTrain = True
return regextasks
def tasksOfPrograms(self, ps, types):
images = drawLogo(*ps, resolution=128)
if len(ps) == 1: images = [images]
tasks = []
for i in images:
if isinstance(i, str): tasks.append(None)
else:
t = Task("Helm", arrow(turtle, turtle), [])
t.highresolution = i
tasks.append(t)
return tasks
def manualLogoTask(name,
expression,
proto=False,
needToTrain=False,
supervise=False,
lambdaCalculus=False):
p = Program.parse(expression) if lambdaCalculus else parseLogo(expression)
from dreamcoder.domains.logo.logoPrimitives import primitives
from dreamcoder.grammar import Grammar
g = Grammar.uniform(primitives, continuationType=turtle)
gp = Grammar.uniform(primitives)
try:
l = g.logLikelihood(arrow(turtle, turtle), p)
lp = gp.logLikelihood(arrow(turtle, turtle), p)
assert l >= lp
eprint(name, -l, "nats")
except:
eprint("WARNING: could not calculate likelihood of manual logo", p)
attempts = 0
while True:
[output, highresolution] = drawLogo(p,
p,
resolution=[28, 128],
cost=True)
if output == "timeout" or highresolution == "timeout":
attempts += 1
else:
break
if attempts > 0:
eprint(
f"WARNING: Took {attempts} attempts to render task {name} within timeout"
)
cost = output[1]
output = output[0]
assert highresolution[1] == cost
highresolution = highresolution[0]
shape = list(map(int, output))
highresolution = list(map(float, highresolution))
t = Task(name, arrow(turtle, turtle), [(([0]), shape)])
t.mustTrain = needToTrain
t.proto = proto
t.specialTask = ("LOGO", {"proto": proto})
t.specialTask[1]["cost"] = cost * 1.05
t.highresolution = highresolution
if supervise:
t.supervisedSolution = p
return t
def make_task(self):
if self.solved or self.total_examples == 0:
return None
task_type = arrow(*([tint] * (self.arity + 1)))
examples = []
n_examples = min(self.total_examples, 100)
# examples = random.choices([e for e, _ in self.examples], weights=[p for _, p in self.examples], k=n_examples)
for e, p in self.examples:
examples.extend([e] * int(round(p * n_examples)))
examples = examples[:n_examples]
return Task(str(self.idx), task_type, examples)
def induceGrammar(*args, **kwargs):
if sum(not f.empty for f in args[1]) == 0:
eprint("No nonempty frontiers, exiting grammar induction early.")
return args[0], args[1]
backend = kwargs.pop("backend", "pypy")
if 'pypy' in backend:
# pypy might not like some of the imports needed for the primitives
# but the primitive values are irrelevant for compression
# therefore strip them out and then replace them once we are done
# ditto for task data
g0,frontiers = args[0].strip_primitive_values(), \
[front.strip_primitive_values() for front in args[1]]
original_tasks = {f.task.name: f.task for f in frontiers}
frontiers = [Frontier(f.entries, Task(f.task.name,f.task.request,[]))
for f in frontiers ]
args = [g0,frontiers]
with timing("Induced a grammar"):
if backend == "pypy":
g, newFrontiers = callCompiled(pypyInduce, *args, **kwargs)
elif backend == "rust":
g, newFrontiers = rustInduce(*args, **kwargs)
elif backend == "vs":
g, newFrontiers = rustInduce(*args, vs=True, **kwargs)
elif backend == "pypy_vs":
kwargs.pop('iteration')
kwargs.pop('topk_use_only_likelihood')
fn = '/tmp/vs.pickle'
with open(fn, 'wb') as handle:
pickle.dump((args, kwargs), handle)
eprint("For debugging purposes, the version space compression invocation has been saved to", fn)
g, newFrontiers = callCompiled(induceGrammar_Beta, *args, **kwargs)
elif backend == "ocaml":
kwargs.pop('iteration')
kwargs.pop('topk_use_only_likelihood')
kwargs['topI'] = 300
kwargs['bs'] = 1000000
g, newFrontiers = ocamlInduce(*args, **kwargs)
elif backend == "memorize":
g, newFrontiers = memorizeInduce(*args, **kwargs)
else:
assert False, "unknown compressor"
if 'pypy' in backend:
g, newFrontiers = g.unstrip_primitive_values(), \
[front.unstrip_primitive_values() for front in newFrontiers]
newFrontiers = [Frontier(f.entries, original_tasks[f.task.name])
for f in newFrontiers]
return g, newFrontiers
def train(self, tasks, steps=400):
# list of list of features for each example in each task
optimizer = torch.optim.Adam(self.parameters())
with timing("Trained discriminator"):
losses = []
for i in range(steps):
self.zero_grad()
if random.random() <= self.trainingSuccessRatio:
# success
t = random.choice(tasks)
features = [
self.extract.featuresOfTask(
Task(t.name, t.request, [ex], t.features))
for ex in t.examples
]
loss = (self(features) - 1.0)**2
else:
# fail
t1, t2 = random.sample(tasks, 2)
features1 = [
self.extract.featuresOfTask(
Task(t1.name, t1.request, [ex], t1.features))
for ex in t1.examples[:len(t1.examples) / 2]
]
features2 = [
self.extract.featuresOfTask(
Task(t2.name, t2.request, [ex], t2.features))
for ex in t2.examples[len(t2.examples) / 2:]
]
features = features1 + features2
loss = self(features)**2
loss.backward()
optimizer.step()
losses.append(loss.data[0])
if not i % 50:
eprint("Discriminator Epoch", i, "Loss",
sum(losses) / len(losses))
gc.collect()
def dummy(program, logLikelihood=0., logPrior=0., tp=None):
"""Creates a dummy frontier containing just this program"""
if not tp:
tp = program.infer().negateVariables()
t = Task(
"<dummy %d: %s>" % (Frontier.DUMMYFRONTIERCOUNTER, str(program)),
tp, [])
f = Frontier([
FrontierEntry(program=program,
logLikelihood=logLikelihood,
logPrior=logPrior)
],
task=t)
Frontier.DUMMYFRONTIERCOUNTER += 1
return f
def get():
results = [p.get() for p in promises]
frontiers = []
with timing("(Helmholtz enumeration) Decoded json into frontiers"):
for request, result in zip(requests, results):
response = json.loads(result.decode("utf-8"))
for b, entry in enumerate(response):
frontiers.append(
Frontier([
FrontierEntry(program=Program.parse(p),
logPrior=entry["ll"],
logLikelihood=0.)
for p in entry["programs"]
],
task=Task(str(b), request, [])))
eprint("Total number of Helmholtz frontiers:", len(frontiers))
return frontiers
def makeLongTasks():
#load new data:
taskfile = os.path.join(get_data_dir(), "regex_data_csv_900.p")
with open(taskfile, 'rb') as handle:
data = dill.load(handle)
tasklist = data[0] #a list of indices
regextasks = [
Task("Data column no. " + str(i), arrow(tpregex, tpregex),
[((), example) for example in task])
for i, task in enumerate(tasklist)
]
return regextasks
def make_task(self):
min_examples = 30 if self.arity is not None and self.arity > 0 else 10
min_examples = min_examples if not self.fewshot else 0
max_examples = 100
examples = self.examples
if len(examples) < min_examples or self.solved or None in [
x[1] for x in examples
]:
return None
task_type = arrow(*([tint] * (self.arity + 1)))
if len(examples) > max_examples:
wrong_examples = [e for e, r in zip(examples, self.res) if not r]
right_examples = [e for e, r in zip(examples, self.res) if r]
right_examples = random.choices(right_examples,
k=max_examples -
len(wrong_examples))
examples = wrong_examples + right_examples
examples = random.sample(examples, k=max_examples)
return Task(str(self.idx), task_type, examples)
def makeHandPickedTasks():
#load new data:
taskfile = os.path.join(get_data_dir(), "regex_data_csv_900.p")
with open(taskfile, 'rb') as handle:
data = dill.load(handle)
tasklist = data[0] #a list of indices
full_list = list(range(199)) + \
[209,218,222,223,224,225,226] + \
list(range(222,233)) + \
[235,237,238,239,243,244,245,252,253,254,255,257,258,259,260,261,264,265,269,272,274] + \
list(range(275,291)) + \
[295,297,300,303,304,305,306,310,311,312,314,315,316,320,321,323,327,329,330,333,334,335,337,338,339,340,341,342,343,344] + \
list(range(348,359)) + \
[361,369,373,379,380,382,387,403,405,407,408] + \
list(range(409,417)) + \
list(range(418,437)) + \
list(range(440,444)) + \
list(range(446,452)) + \
list(range(456,460)) + \
list(range(466,472)) + \
[503,504]
regextasks = [
Task("Data column no. " + str(i), arrow(tpregex, tpregex),
[((), example) for example in task])
for i, task in enumerate(tasklist) if i in full_list
]
#for i in train_list:
# regextasks[i].mustTrain = True
return regextasks
def main(args):
"""
Takes the return value of the `commandlineArguments()` function as input and
trains/tests the model on manipulating sequences of numbers.
"""
random.seed(args.pop("random_seed"))
dataset = args.pop("dataset")
tasks = {
"Lucas-old":
lambda: retrieveJSONTasks("data/list_tasks.json") + sortBootstrap(),
"bootstrap":
make_list_bootstrap_tasks,
"sorting":
sortBootstrap,
"Lucas-depth1":
lambda: retrieveJSONTasks("data/list_tasks2.json")[:105],
"Lucas-depth2":
lambda: retrieveJSONTasks("data/list_tasks2.json")[:4928],
"Lucas-depth3":
lambda: retrieveJSONTasks("data/list_tasks2.json"),
}[dataset]()
maxTasks = args.pop("maxTasks")
if maxTasks and len(tasks) > maxTasks:
necessaryTasks = [] # maxTasks will not consider these
if dataset.startswith("Lucas2.0") and dataset != "Lucas2.0-depth1":
necessaryTasks = tasks[:105]
eprint("Unwilling to handle {} tasks, truncating..".format(len(tasks)))
random.shuffle(tasks)
del tasks[maxTasks:]
tasks = necessaryTasks + tasks
if dataset.startswith("Lucas"):
# extra tasks for filter
tasks.extend([
Task("remove empty lists",
arrow(tlist(tlist(tbool)), tlist(tlist(tbool))),
[((ls, ), list(filter(lambda l: len(l) > 0, ls)))
for _ in range(15) for ls in [[[
random.random() < 0.5
for _ in range(random.randint(0, 3))
] for _ in range(4)]]]),
Task("keep squares", arrow(tlist(tint), tlist(tint)), [
((xs, ), list(filter(lambda x: int(math.sqrt(x))**2 == x, xs)))
for _ in range(15) for xs in [[
random.choice([0, 1, 4, 9, 16, 25])
if random.random() < 0.5 else random.randint(0, 9)
for _ in range(7)
]]
]),
Task("keep primes", arrow(tlist(tint), tlist(tint)), [
((xs, ),
list(
filter(
lambda x: x in
{2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37}, xs)))
for _ in range(15) for xs in [[
random.choice([2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37])
if random.random() < 0.5 else random.randint(0, 9)
for _ in range(7)
]]
]),
])
for i in range(4):
tasks.extend([
Task("keep eq %s" % i, arrow(tlist(tint), tlist(tint)),
[((xs, ), list(filter(lambda x: x == i, xs)))
for _ in range(15)
for xs in [[random.randint(0, 6) for _ in range(5)]]]),
Task("remove eq %s" % i, arrow(tlist(tint), tlist(tint)),
[((xs, ), list(filter(lambda x: x != i, xs)))
for _ in range(15)
for xs in [[random.randint(0, 6) for _ in range(5)]]]),
Task("keep gt %s" % i, arrow(tlist(tint), tlist(tint)),
[((xs, ), list(filter(lambda x: x > i, xs)))
for _ in range(15)
for xs in [[random.randint(0, 6) for _ in range(5)]]]),
Task("remove gt %s" % i, arrow(tlist(tint), tlist(tint)),
[((xs, ), list(filter(lambda x: not x > i, xs)))
for _ in range(15)
for xs in [[random.randint(0, 6) for _ in range(5)]]])
])
def isIdentityTask(t):
return all(len(xs) == 1 and xs[0] == y for xs, y in t.examples)
eprint("Removed", sum(isIdentityTask(t) for t in tasks),
"tasks that were just the identity function")
tasks = [t for t in tasks if not isIdentityTask(t)]
prims = {
"base": basePrimitives,
"McCarthy": McCarthyPrimitives,
"common": bootstrapTarget_extra,
"noLength": no_length,
"rich": primitives
}[args.pop("primitives")]()
haveLength = not args.pop("noLength")
haveMap = not args.pop("noMap")
haveUnfold = not args.pop("noUnfold")
eprint(f"Including map as a primitive? {haveMap}")
eprint(f"Including length as a primitive? {haveLength}")
eprint(f"Including unfold as a primitive? {haveUnfold}")
baseGrammar = Grammar.uniform([p
for p in prims
if (p.name != "map" or haveMap) and \
(p.name != "unfold" or haveUnfold) and \
(p.name != "length" or haveLength)])
extractor = {
"learned": LearnedFeatureExtractor,
}[args.pop("extractor")]
extractor.H = args.pop("hidden")
timestamp = datetime.datetime.now().isoformat()
outputDirectory = "experimentOutputs/list/%s" % timestamp
os.system("mkdir -p %s" % outputDirectory)
args.update({
"featureExtractor": extractor,
"outputPrefix": "%s/list" % outputDirectory,
"evaluationTimeout": 0.0005,
})
eprint("Got {} list tasks".format(len(tasks)))
split = args.pop("split")
if split:
train_some = defaultdict(list)
for t in tasks:
necessary = train_necessary(t)
if not necessary:
continue
if necessary == "some":
train_some[t.name.split()[0]].append(t)
else:
t.mustTrain = True
for k in sorted(train_some):
ts = train_some[k]
random.shuffle(ts)
ts.pop().mustTrain = True
test, train = testTrainSplit(tasks, split)
if True:
test = [t for t in test if t.name not in EASYLISTTASKS]
eprint("Alotted {} tasks for training and {} for testing".format(
len(train), len(test)))
else:
train = tasks
test = []
explorationCompression(baseGrammar, train, testingTasks=test, **args)
def sortBootstrap():
# These tasks have as their goal the learning of (1) filter, and
# (2) sort, which uses filter.
def flip(): return random() > 0.5
def randomList(lb=None, ub=None):
if lb is None:
lb = 2
if ub is None:
ub = 5
return [randint(0, 10) for _ in range(randint(lb, ub))]
def randomBooleanList():
return [flip() for _ in range(randint(4, 7))]
def removeDuplicates(l):
if len(l) == 0: return l
return [l[0]] + removeDuplicates([ z for z in l if z != l[0] ])
filterBootstrap = [
# Task("remove empty lists",
# arrow(tlist(tlist(tbool)), tlist(tlist(tbool))),
# [((ls,), [l for l in ls if len(l) > 0])
# for _ in range(10)
# for ls in [[[flip() for _ in range(randint(0, 3))]
# for _ in range(4)]]]),
# Task("remove non 0s",
# arrow(tlist(tint), tlist(tint)),
# [((xs,), filter(lambda x: x == 0, xs))
# for _ in range(10)
# for xs in [[ randint(0,3) for _ in range(5) ]] ]),
Task("remove 0s",
arrow(tlist(tint), tlist(tint)),
[((xs,), [x for x in xs if x != 0])
for _ in range(10)
for xs in [[randint(0, 3) for _ in range(5)]]]),
# Task("remove primes",
# arrow(tlist(tint), tlist(tint)),
# [((xs,), [x for x in xs if not (x in {2,3,5,7,11,13,17,19,23})])
# for _ in range(10)
# for xs in [[randint(0, 20) for _ in range(7)]]]),
Task("remove squares",
arrow(tlist(tint), tlist(tint)),
[((xs,), [x for x in xs if not (int(x**0.5)**2 == x)])
for _ in range(10)
for xs in [[randint(0, 20) for _ in range(7)]]]),
Task("remove > 1",
arrow(tlist(tint), tlist(tint)),
[((xs,), [x for x in xs if not (x > 1)])
for _ in range(10)
for xs in [[randint(0, 5) for _ in range(7)]]]),
]
# Needed for selection sort
minimumBootstrap = [
Task("min2", arrow(tint,tint,tint),
[((x,y),min(x,y))
for x in range(4)
for y in range(4) ]),
Task("minimum of list", arrow(tlist(tint),tint),
[((l,),min(l))
for _ in range(15)
for l in [randomList()] ])
]
appendBootstrap = [
Task("append bool", arrow(tlist(tbool), tlist(tbool), tlist(tbool)),
[((x, y), x + y)
for _ in range(10)
for [x, y] in [[randomBooleanList(), randomBooleanList()]]]),
Task("append int", arrow(tlist(tint), tlist(tint), tlist(tint)),
[((x, y), x + y)
for _ in range(10)
for [x, y] in [[randomList(), randomList()]]])
]
insertionBootstrap = [
Task("filter greater than or equal", arrow(tint,tlist(tint),tlist(tint)),
[((x,l), [y for y in l if y >= x ])
for _ in range(15)
for x in [randint(0,5)]
for l in [randomList()] ]),
Task("filter less than", arrow(tint,tlist(tint),tlist(tint)),
[((x,l), [y for y in l if y < x ])
for _ in range(15)
for x in [randint(0,5)]
for l in [randomList()] ]),
Task("insert into sorted list (I)", arrow(tint,tlist(tint),tlist(tint)),
[((x,l), [y for y in l if y < x ] + [x] + [y for y in l if y >= x ])
for _ in range(15)
for x in [randint(0,5)]
for _l in [randomList()]
for l in [sorted(_l)] ]),
Task("insert into sorted list (II)", arrow(tint,tlist(tint),tlist(tint)),
[((x,l), [y for y in l if y < x ] + [x] + [y for y in l if y >= x ])
for _ in range(15)
for x in [randint(0,5)]
for l in [randomList()] ])
]
sortTask = [
Task("sort-and-deduplicate", arrow(tlist(tint),tlist(tint)),
[((l,),list(sorted(l)))
for _ in range(15)
for l in [removeDuplicates(randomList())]
])]
slowSort = [
Task("+1 maximum list", arrow(tlist(tint), tint),
[((l,),max(l) + 1)
for _ in range(15)
for l in [randomList()] ]),
Task("range +1 maximum list", arrow(tlist(tint), tlist(tint)),
[((l,),list(range(max(l) + 1)))
for _ in range(15)
for l in [randomList()] ]),
]
tasks = sortTask + slowSort
for t in tasks: t.mustTrain = True
return tasks
def make_list_bootstrap_tasks():
seed(42)
def suffixes(l):
if l == []:
return []
else:
return [l[1:]] + suffixes(l[1:])
def randomList(minValue=0, maxValue=20, len=1):
return [randint(minValue, maxValue) for _ in range(int(len))]
def generate_divide_examples ():
i = 0
lists = []
lists.append(((4,8), 0))
lists.append(((3,7), 0))
lists.append(((5,28), 0))
lists.append(((4,47), 0))
lists.append(((9,6), 1))
lists.append(((8,7), 1))
lists.append(((5,3), 1))
lists.append(((24,16), 1))
lists.append(((49,27), 1))
lists.append(((25,23), 1))
while i < 20:
a = randomList(len=n_sample*(1-noise))[0]
b = randomList(minValue=1, len=n_sample*(1-noise))[0]
if a // b >= 2 and ((a, b), _div(a,b)) not in lists:
lists.append(((a, b), _div(a,b)))
i += 1
return lists
def generate_noise(n):
return list(zip(zip(randomList(len=n), randomList(len=n)), randomList(len=n)))
# Functions to be learnt
def _add(x, y): return x + y
def _minus(x, y): return max(0, x - y)
# def _minus(x, y): return x - y
def _mul(x, y): return x * y
def _div(x, y): return x // y
n_sample = 20
noise = 0
operatorBootstrap = [
Task ("add", arrow(tint, tint, tint),
generate_noise(n_sample*noise) +
[((a, b), _add(a,b)) for a, b in zip(randomList(len=n_sample*(1-noise)), randomList(len=n_sample*(1-noise)))]),
Task ("minus", arrow(tint, tint, tint),
generate_noise(n_sample*noise) +
[((a, b), _minus(a,b)) for a, b in zip(randomList(len=n_sample*(1-noise)), randomList(len=n_sample*(1-noise)))]),
Task ("multiply", arrow(tint, tint, tint),
generate_noise(n_sample*noise) +
[((a, b), _mul(a,b)) for a, b in zip(randomList(len=n_sample*(1-noise)), randomList(len=n_sample*(1-noise)))]),
# Task ("divide", arrow(tint, tint, tint),
# generate_noise(n_sample*noise) +
# [((a, b), _div(a,b)) for a, b in zip(randomList(len=n_sample*(1-noise)), randomList(minValue=1, len=n_sample*(1-noise)))]),
Task ("divide", arrow(tint, tint, tint),
generate_noise(n_sample*noise) +
generate_divide_examples()),
]
return operatorBootstrap
test, _ = testTrainSplit(tasks, 100)
random.seed(42)
random.shuffle(test)
test = test[:100]
g = Grammar.uniform(
[real, real_division, real_addition, real_multiplication])
fe = FeatureExtractor([])
BATCHSIZE = 64
elif arguments.domain == "list":
BATCHSIZE = 16
tasks = retrieveJSONTasks("data/list_tasks.json") + sortBootstrap()
tasks.extend([
Task("remove empty lists",
arrow(tlist(tlist(tbool)), tlist(tlist(tbool))),
[((ls, ), list(filter(lambda l: len(l) > 0, ls)))
for _ in range(15) for ls in [[[
random.random() < 0.5
for _ in range(random.randint(0, 3))
] for _ in range(4)]]]),
Task("keep squares", arrow(tlist(tint), tlist(tint)), [
((xs, ), list(filter(lambda x: int(math.sqrt(x))**2 == x, xs)))
for _ in range(15) for xs in [[
random.choice([0, 1, 4, 9, 16, 25])
if random.random() < 0.5 else random.randint(0, 9)
for _ in range(7)
]]
]),
Task("keep primes", arrow(tlist(tint), tlist(tint)), [
((xs, ),
list(
filter(
def make_list_bootstrap_tasks():
seed(42)
def suffixes(l):
if l == []:
return []
else:
return [l[1:]] + suffixes(l[1:])
def flip(): return random() > 0.5
def randomSuffix():
return [randint(0, 9) for _ in range(randint(1, 4))]
def randomList(minimum=0, minimumLength=4, maximumLength=6):
return [randint(minimum, 9) for _ in range(randint(minimumLength, maximumLength))]
def randomListOfLists():
return [randomSuffix() for _ in range(randint(2, 4))]
def randomListOfLists_bool(l=None):
if l is None:
l = randint(4, 7)
return [randomBooleanList() for _ in range(l)]
def randomBooleanList():
return [flip() for _ in range(randint(4, 7))]
# Reliably learned in under a minute; always triggers learning of length
# primitive
lengthBootstrap = [
# Task("length bool", arrow(tlist(tbool), tint),
# [((l,), len(l))
# for _ in range(10)
# for l in [[flip() for _ in range(randint(0, 10))]]]),
Task("length int", arrow(tlist(tint), tint),
[((l,), len(l))
for _ in range(10)
for l in [randomList()]]),
Task("map length", arrow(tlist(tlist(tint)), tlist(tint)),
[((xss,), [len(xs) for xs in xss])
for _ in range(10)
for xss in [randomListOfLists()] ])
]
# Encourages learning of unfolding
unfoldBootstrap = [
Task("countdown", arrow(tint, tlist(tint)),
[((n,), list(range(n + 1, 1, -1)))
for n in range(10)]),
Task("weird count", arrow(tint, tlist(tint)),
[((n,), list(range(-n,0,-1)))
for n in range(-10,0) ]),
Task("take every other", arrow(tlist(tint),tlist(tint)),
[((l,), [x for j,x in enumerate(l) if j%2 == 0])
for _ in range(9)
for l in [ [randint(0, 9) for _ in range(randint(1,4)*2)] ] ] + [(([],),[])]),
# Task("stutter every other", arrow(tlist(tint),tlist(tint)),
# [((l,), [l[int(j/2)] for j in range(len(l)) ])
# for _ in range(10)
# for l in [ [randint(0, 9) for _ in range(randint(1,4)*2)] ] ]),
# Task("take until 3 reached", arrow(tlist(tint),tlist(tint)),
# [((p + [3] + s,),p)
# for _ in range(10)
# for p in [ [z for z in randomList()[:5] if z != 3 ]]
# for s in [randomList()] ]),
Task("drop last element", arrow(tlist(tint),tlist(tint)),
[((l,), l[:-1])
for _ in range(10)
for l in [ [randint(0, 9) for _ in range(randint(2,5))] ] ]),
# Task("suffixes", arrow(tlist(tint), tlist(tlist(tint))),
# [((l,), suffixes(l))
# for _ in range(10)
# for l in [randomList()]]),
Task("range", arrow(tint, tlist(tint)),
[((n,), list(range(n)))
for n in range(10)]),
Task("range inclusive", arrow(tint, tlist(tint)),
[((n,), list(range(n + 1)))
for n in range(10)]),
# Task("range inclusive+1", arrow(tint, tlist(tint)),
# [((n,), list(range(n + 2)))
# for n in range(10)]),
# Task("range exclusive", arrow(tint, tlist(tint)),
# [((n,), list(range(n - 1)))
# for n in range(2, 11)]),
# Task("range length", arrow(tlist(tint),tlist(tint)),
# [((l,),list(range(len(l))))
# for _ in range(10)
# for l in [randomList()] ])
]
# Encourages learning how to treat a list as an array
arrayBootstrap = [
Task("index int", arrow(tint, tlist(tint), tint),
[((n, l), l[n])
for n in range(10)
for l in [[randint(0, 9) for _ in range(randint(n + 1, n + 5))]]]),
# Task("last n", arrow(tint, tlist(tint), tlist(tint)),
# [((n, l), l[-n:])
# for n in range(10)
# for l in [[randint(0, 9) for _ in range(randint(n + 1, n + 5))]]]),
Task("1-index int", arrow(tint, tlist(tint), tint),
[((n, l), l[n - 1])
for n in range(1,11)
for l in [[randint(0, 9) for _ in range(randint(n + 1, n + 4))]]])
# Task("index bool", arrow(tint, tlist(tbool), tbool),
# [((n, l), l[n])
# for n in range(10)
# for l in [[flip() for _ in range(randint(n + 1, n + 5))]]])
]
# Teaches how to slice lists, not sure if we really need this though
sliceBootstrap = [
Task("take bool", arrow(tint, tlist(tbool), tlist(tbool)),
[((n, l), l[:n])
for n in range(10)
for l in [[flip() for _ in range(randint(n, n + 5))]]]),
Task("drop bool", arrow(tint, tlist(tbool), tlist(tbool)),
[((n, l), l[n:])
for n in range(10)
for l in [[flip() for _ in range(randint(n, n + 5))]]]),
Task("take int", arrow(tint, tlist(tint), tlist(tint)),
[((n, l), l[:n])
for n in range(10)
for l in [[randint(0, 9) for _ in range(randint(n, n + 5))]]]),
Task("drop int", arrow(tint, tlist(tint), tlist(tint)),
[((n, l), l[n:])
for n in range(10)
for l in [[randint(0, 9) for _ in range(randint(n, n + 5))]]]),
]
# learning to fold
foldBootstrap = [
Task("stutter", arrow(tlist(tint),tlist(tint)),
[((l,), [z for x in l for z in [x,x] ])
for _ in range(10)
for l in [randomList()] ]),
Task("sum", arrow(tlist(tint), tint),
[((l,), sum(l))
for _ in range(10)
for l in [randomList()]]),
# Task("difference", arrow(tlist(tint), tint),
# [((l,), reduce(lambda x, y: y - x, reversed(l), 1))
# for _ in range(10)
# for l in [randomList()[:4]]]),
# Task("append bool", arrow(tlist(tbool), tlist(tbool), tlist(tbool)),
# [((x, y), x + y)
# for _ in range(10)
# for [x, y] in [[randomBooleanList(), randomBooleanList()]]]),
Task("append constant 0", arrow(tlist(tint),tlist(tint)),
[((l,),l + [0])
for _ in range(10)
for l in [randomList()] ]),
]
# learning to map
mapBootstrap = [
Task("map double", arrow(tlist(tint), tlist(tint)),
[((l,), list(map(lambda n: n * 2, l)))
for _ in range(10)
for l in [randomList()]]),
Task("map increment", arrow(tlist(tint),tlist(tint)),
[((l,),list(map(lambda n: n+1, l)))
for _ in range(10)
for l in [randomList()] ]),
Task("map negation", arrow(tlist(tint),tlist(tint)),
[((l,),list(map(lambda n: 0-n, l)))
for _ in range(10)
for l in [randomList()] ]),
# Task("map car", arrow(tlist(tlist(tint)), tlist(tint)),
# [((l,), [n[0] for n in l])
# for _ in4 range(10)
# for l in [randomListOfLists()]]),
# Task("map cdr", arrow(tlist(tlist(tbool)),tlist(tlist(tbool))),
# [((l,),map(lambda n: n[1:],l))
# for _ in range(10)
# for l in [randomListOfLists_bool()]]),
# Task("map empty?", arrow(tlist(tlist(tint)), tlist(tboolean)),
# [((l,), [n == [] for n in l])
# for _ in range(10)
# for l in [[[] if flip() else randomList() for _ in range(randint(1, 5))]]]),
# Task("map eq 0?", arrow(tlist(tint),tlist(tboolean)),
# [((l,),map(lambda n: 0 == n,l))
# for _ in range(10)
# for l in [[ randint(0,3) for _ in range(randint(4,7)) ]] ])
]
difficultMaps = [
Task("map quadruple", arrow(tlist(tint), tlist(tint)),
[((l,), list(map(lambda n: n * 4, l)))
for _ in range(10)
for l in [randomList()]]),
Task("map add 3", arrow(tlist(tint),tlist(tint)),
[((l,),list(map(lambda n: n+3, l)))
for _ in range(10)
for l in [randomList()] ]),
]
# Learning to zip lists together
zipBootstrap = [
Task("zip plus", arrow(tlist(tint),tlist(tint),tlist(tint)),
[((l1,l2),list(map(lambda x,y: x+y,l1,l2)))
for _ in range(10)
for l1 in [randomList(minimumLength=2, maximumLength=4)]
for l2 in [[ randint(0,9) for _ in range(len(l1)) ]]]),
Task("zip minus", arrow(tlist(tint),tlist(tint),tlist(tint)),
[((l1,l2),list(map(lambda x,y: x-y,l1,l2)))
for _ in range(10)
for l1 in [randomList(minimumLength=2, maximumLength=4)]
for l2 in [[ randint(0,9) for _ in range(len(l1)) ]]]),
# Task("zip eq?", arrow(tlist(tint), tlist(tint), tlist(tbool)),
# [((l1, l2), list(map(lambda x, y: x == y, l1, l2)))
# for _ in range(10)
# for l1 in [[randint(0, 3) for _ in range(randint(4, 7))]]
# for l2 in [[randint(0, 3) for _ in range(len(l1))]]]),
# Task("zip cons", arrow(tlist(tbool), tlist(tlist(tbool)), tlist(tlist(tbool))),
# [((l1, l2), list(map(lambda x, y: [x] + y, l1, l2)))
# for _ in range(10)
# for l1 in [randomBooleanList()]
# for l2 in [randomListOfLists_bool(l=len(l1))]]),
# Task("zip cons", arrow(tlist(tint),tlist(tlist(tint)),tlist(tlist(tint))),
# [((l1,l2),list(map(lambda x,y: [x]+y,l1,l2)))
# for _ in range(10)
# for l1 in [randomList()]
# for l2 in [[ randomList() for _ in range(len(l1)) ]]]),
]
# Learning to filter
filterBootstrap = [
# Task("remove empty lists",
# arrow(tlist(tlist(tbool)), tlist(tlist(tbool))),
# [((ls,), [l for l in ls if len(l) > 0])
# for _ in range(10)
# for ls in [[[flip() for _ in range(randint(0, 3))]
# for _ in range(4)]]])
# Task("remove non 0s",
# arrow(tlist(tint), tlist(tint)),
# [((xs,), filter(lambda x: x == 0, xs))
# for _ in range(10)
# for xs in [[ randint(0,3) for _ in range(5) ]] ]),
Task("remove 0s",
arrow(tlist(tint), tlist(tint)),
[((xs,), [x for x in xs if x != 0])
for _ in range(10)
for xs in [[randint(0, 3) for _ in range(5)]]]),
Task("remove non-positives",
arrow(tlist(tint), tlist(tint)),
[((xs,), [x for x in xs if not (x > 1)])
for _ in range(10)
for xs in [[randint(0, 3) for _ in range(5)]]]),
]
return lengthBootstrap + filterBootstrap + \
unfoldBootstrap + arrayBootstrap + foldBootstrap + mapBootstrap + zipBootstrap
def make_list_bootstrap_tasks():
seed(42)
def suffixes(l):
if l == []:
return []
else:
return [l[1:]] + suffixes(l[1:])
def randomList(minValue=0, maxValue=9, len=1):
return [randint(minValue, maxValue) for _ in range(int(len))]
def generate_noise(n, arity=2):
n = int(n)
xs = []
for _ in range(arity):
xs.append(randomList(len=n))
y = randomList(len=n)
if arity == 0:
return [((), s) for s in y]
return list(zip(zip(*xs), y))
def generate_examples(arity,
fn,
total=1,
noise_level=0,
min_value=0,
max_value=9):
n = total * noise_level
xs = list(
zip(*[
randomList(len=n, minValue=min_value, maxValue=max_value)
for i in range(arity)
]))
y = randomList(len=n, minValue=min_value, maxValue=max_value)
noise = list(zip(xs, y))
n = total * (1 - noise_level)
xs = list(
zip(*[
randomList(len=n, minValue=min_value, maxValue=max_value)
for i in range(arity)
]))
correct = [(x, fn(*x)) for x in xs]
correct = [x for x in correct if x[1] is not None]
data = list(set(noise + correct))
return data
# Functions to be learnt
def _add(x, y):
return x + y
def _minus(x, y):
return max(0, x - y)
# def _minus(x, y): return x - y
def _mul(x, y):
return x * y
def _div(x, y):
return x // y if y != 0 else None
# def _div(x, y): return math.ceil(x / y)
_fac = lambda x: math.factorial(x) if x <= 20 else None
n_sample = 20
noise = 0.
operatorBootstrap = []
operatorBootstrap = [
# Task ("add", arrow(tint, tint, tint),
# generate_noise(n_sample*noise) +
# [((a, b), _add(a,b)) for a, b in zip(randomList(len=n_sample*(1-noise)), randomList(len=n_sample*(1-noise)))]),
# Task ("minus", arrow(tint, tint, tint),
# generate_noise(n_sample*noise) +
# [((a, b), _minus(a,b)) for a, b in zip(randomList(len=n_sample*(1-noise)), randomList(len=n_sample*(1-noise)))]),
# Task ("multiply", arrow(tint, tint, tint),
# generate_noise(n_sample*noise) +
# [((a, b), _mul(a,b)) for a, b in zip(randomList(len=n_sample*(1-noise)), randomList(len=n_sample*(1-noise)))]),
# Task ("divide", arrow(tint, tint, tint),
# generate_noise(n_sample*noise) +
# [((a, b), _div(a,b)) for a, b in zip(randomList(len=n_sample*(1-noise)), randomList(minValue=1, len=n_sample*(1-noise)))]),
Task("divide", arrow(tint, tint, tint),
generate_examples(2, _div, total=n_sample, max_value=20)),
# Task ("factorial", arrow(tint, tint),
# generate_noise(n_sample*noise, arity=1) +
# [((a,), _fac(a)) for a in randomList(len=n_sample*(1-noise))]),
]
# # Add counting task
# from collections import Counter
# for i in range(10):
# task = Task(str(i), arrow(tint),
# generate_noise(n_sample*noise, arity=0) + [((), i)] * int(n_sample- n_sample * noise))
# ys = [y for _, y in task.examples]
# print(i, Counter(ys).most_common(3))
# operatorBootstrap.append(task)
import json
# tasks = json.load(open('../../outputs/tasks.json'))
# for i, t in enumerate(tasks):
# task = Task(str(i), arrow(tint, tint, tint), t)
# operatorBootstrap.append(task)
# task = Task(str(len(tasks)), arrow(tint, tint, tint), [x for t in tasks for x in t])
# operatorBootstrap.append(task)
# examples = {((5, 2), 3): 3, ((6, 3), 3): 3, ((9, 3), 6): 3, ((6, 2), 6): 2, ((1, 9), 0): 2, ((7, 4), 3): 2, ((2, 3), 1): 2, ((9, 2), 8): 2, ((7, 9), 35): 2, ((7, 6), 48): 2, ((3, 4), 1): 2, ((4, 4), 3): 2, ((1, 4), 0): 1, ((2, 4), 56): 1, ((7, 1), 6): 1, ((4, 3), 3): 1, ((5, 9), 4): 1, ((8, 6), 2): 1, ((9, 6), 8): 1, ((9, 8), 1): 1, ((3, 8), 2): 1, ((5, 4), 1): 1, ((7, 2), 6): 1, ((3, 9), 1): 1, ((3, 3), 30): 1, ((6, 6), 64): 1, ((6, 5), 1): 1, ((5, 3), 5): 1, ((6, 4), 2): 1, ((4, 7), 0): 1, ((5, 7), 1): 1, ((2, 9), 1): 1, ((8, 2), 8): 1, ((4, 9), 1): 1, ((7, 3), 35): 1, ((8, 4), 2): 1, ((3, 2), 3): 1, ((9, 9), 3): 1, ((2, 1), 1): 1, ((6, 8), 54): 1, ((3, 1), 2): 1, ((6, 0), 1): 1, ((1, 5), 10): 1, ((3, 0), 4): 1, ((1, 7), 10): 1}
# examples = [[e]*count for e, count in examples.items()]
# examples = [y for x in examples for y in x]
# task = Task("11", arrow(tint, tint, tint), examples)
# operatorBootstrap.append(task)
# examples = {((1, 7), 7): 9, ((1, 4), 4): 9, ((7, 3), 21): 8, ((1, 2), 2): 7, ((7, 9), 63): 7, ((8, 9), 72): 7, ((3, 9), 27): 7, ((1, 9), 9): 6, ((0, 4), 4): 6, ((1, 3), 3): 6, ((1, 8), 8): 5, ((1, 1), 1): 5, ((4, 9), 36): 5, ((2, 9), 18): 5, ((1, 6), 6): 5, ((5, 4), 20): 5, ((5, 2), 10): 5, ((0, 1), 1): 4, ((5, 9), 45): 4, ((6, 7), 42): 3, ((1, 5), 5): 3, ((3, 3), 9): 3, ((9, 5), 45): 3, ((6, 9), 54): 3, ((9, 9), 81): 3, ((7, 8), 56): 3, ((9, 8), 72): 2, ((3, 7), 21): 2, ((8, 8), 64): 2, ((3, 6), 18): 2, ((3, 5), 15): 2, ((9, 2), 18): 2, ((4, 5), 20): 2, ((0, 5), 5): 2, ((0, 7), 7): 2, ((9, 4), 36): 2, ((6, 6), 36): 2, ((9, 3), 27): 2, ((7, 5), 35): 2, ((2, 5), 14): 1, ((7, 6), 48): 1, ((8, 5), 40): 1, ((2, 1), 7): 1, ((3, 4), 12): 1, ((0, 3), 3): 1, ((9, 7), 63): 1, ((9, 6), 54): 1, ((0, 8), 8): 1, ((5, 6), 18): 1, ((6, 2), 6): 1, ((9, 0), 27): 1, ((1, 0), 0): 1, ((3, 1), 6): 1, ((8, 1), 20): 1, ((4, 6), 16): 1, ((8, 6), 30): 1, ((4, 7), 49): 1, ((6, 5), 30): 1, ((6, 8), 16): 1, ((0, 9), 9): 1, ((0, 6), 6): 1, ((8, 2), 4): 1, ((2, 8), 4): 1, ((7, 7), 49): 1, ((3, 8), 64): 1, ((3, 2), 6): 1}
# examples = [[e]*count for e, count in examples.items()]
# examples = [y for x in examples for y in x]
# task = Task("12", arrow(tint, tint, tint), examples)
# operatorBootstrap.append(task)
# examples = {((6, 4), 24): 8, ((8, 3), 24): 7, ((8, 7), 56): 6, ((6, 8), 48): 6, ((4, 2), 8): 5, ((5, 2), 8): 1, ((4, 7), 32): 1, ((0, 6), 0): 1, ((4, 3), 24): 1, ((4, 0), 8): 1}
# examples = [[e]*count for e, count in examples.items()]
# examples = [y for x in examples for y in x]
# task = Task("14", arrow(tint, tint, tint), examples)
# operatorBootstrap.append(task)
return operatorBootstrap