Exemplo n.º 1
0
def regexHeldOutExamples(task, include_only=None):

    #load new data:
    global REGEXTASKS
    if REGEXTASKS is None:
        taskfile = os.path.join(get_data_dir(),
                                "csv_filtered_all_background_novel.p")

        with open(taskfile, 'rb') as handle:
            data = dill.load(handle)

        tasklist = data['background']  #a list of indices

        if include_only:
            regextasks = [
                Task("Data column no. " + str(i), arrow(tpregex, tpregex),
                     [((), example) for example in _task['test']])
                for i, _task in enumerate(tasklist) if i in include_only
            ]
        else:
            regextasks = [
                Task("Data column no. " + str(i), arrow(tpregex, tpregex),
                     [((), example) for example in _task['test']])
                for i, _task in enumerate(tasklist)
            ]

        #for i in train_list:
        #    regextasks[i].mustTrain = True

        REGEXTASKS = {t.name: t.examples for t in regextasks}
    fullTask = REGEXTASKS[task.name]
    return fullTask
Exemplo n.º 2
0
def bonusListProblems():
    # Taken from https://www.ijcai.org/Proceedings/75/Papers/037.pdf
    # These problems might be a lot easier if we do not use numbers
    def randomList(lb=None, ub=None):
        if lb is None:
            lb = 2
        if ub is None:
            ub = 5
        return [randint(0, 5) for _ in range(randint(lb, ub))]

    bonus = [
        Task(
            "pair reverse", arrow(tlist(tint), tlist(tint)),
            [((x,), [x[j + (1 if j % 2 == 0 else -1)]
                     for j in range(len(x))])
             for _ in range(5)
             for x in [randomList(10, 10)]]
        ),
        Task(
            "duplicate each element", arrow(tlist(tint), tlist(tint)),
            [((x,), [a for z in x for a in [z] * 2])
             for _ in range(5)
             for x in [randomList(4, 6)]]
        ),
        Task(
            "reverse duplicate each element", arrow(tlist(tint), tlist(tint)),
            [((x,), [a for z in reversed(x) for a in [z] * 2])]
        ),
    ]
    return bonus
Exemplo n.º 3
0
def makeNewTasks(include_only=None):

    #load new data:

    taskfile = os.path.join(get_data_dir(),
                            "csv_filtered_all_background_novel.p")

    with open(taskfile, 'rb') as handle:
        data = dill.load(handle)

    tasklist = data['background']  #a list of indices

    if include_only:
        regextasks = [
            Task("Data column no. " + str(i), arrow(tpregex, tpregex),
                 [((), example) for example in task['train']])
            for i, task in enumerate(tasklist) if i in include_only
        ]
    else:
        regextasks = [
            Task("Data column no. " + str(i), arrow(tpregex, tpregex),
                 [((), example) for example in task['train']])
            for i, task in enumerate(tasklist)
        ]

    #for i in train_list:
    #    regextasks[i].mustTrain = True

    return regextasks
Exemplo n.º 4
0
def make_list_bootstrap_tasks():
    seed(42)

    def suffixes(l):
        if l == []:
            return []
        else:
            return [l[1:]] + suffixes(l[1:])

    def flip():
        return random() > 0.5

    def randomSuffix():
        return [randint(0, 9) for _ in range(randint(1, 4))]

    def randomList(minimum=1, minimumLength=2, maximumLength=2):
        return [
            randint(minimum, 9)
            for _ in range(randint(minimumLength, maximumLength))
        ]

    # def randomListOfLists():
    #     return [randomSuffix() for _ in range(randint(2, 4))]

    # def randomListOfLists_bool(l=None):
    #     if l is None:
    #         l = randint(4, 7)
    #     return [randomBooleanList() for _ in range(l)]

    # def randomBooleanList():
    #     return [flip() for _ in range(randint(4, 7))]

    # Reliably learned in under a minute; always triggers learning of length
    # primitive

    operatorBootstrap = [
        Task("add", arrow(tint, tint, tint), [((l[0], l[1]), l[0] + l[1])
                                              for _ in range(50)
                                              for l in [randomList()]]),
        Task("minus", arrow(tint, tint, tint), [((l[0], l[1]), l[0] - l[1])
                                                for _ in range(50)
                                                for l in [randomList()]]),
        Task("multiply", arrow(tint, tint, tint), [((l[0], l[1]), l[0] * l[1])
                                                   for _ in range(50)
                                                   for l in [randomList()]])
        # Task ("divide", arrow(tint, tint, tint),
        #      [((l[0],l[1]), l[0] / l[1])
        #      for _ in range (50)
        #      for l in [randomList()]])
    ]

    return operatorBootstrap
Exemplo n.º 5
0
def makeOldTasks():
    # a series of tasks

    taskfile = os.path.join(get_data_dir(), 'data_filtered.json')
    #task_list = pickle.load(open(taskfile, 'rb'))

    with open(taskfile) as f:
        file_contents = f.read()
    task_list = json.loads(file_contents)

    # if I were to just dump all of them:
    regextasks = [
        Task("Luke data column no." + str(i), arrow(tpregex, tpregex),
             [((), example) for example in task_list[i]])
        for i in range(len(task_list))
    ]
    """ regextasks = [
        Task("length bool", arrow(none,tstr),
             [((l,), len(l))
              for _ in range(10)
              for l in [[flip() for _ in range(randint(0,10)) ]] ]),
        Task("length int", arrow(none,tstr),
             [((l,), len(l))
              for _ in range(10)
              for l in [randomList()] ]),
    ]
  """
    return regextasks  # some list of tasks
Exemplo n.º 6
0
def make_list_task(name, examples, **params):
    input_type = guess_type([i for (i,), _ in examples])
    output_type = guess_type([o for _, o in examples])

    # We can internally handle lists of bools.
    # We explicitly create these by modifying existing routines.
    if name.startswith("identify"):
        boolexamples = [((i,), list(map(bool, o))) for (i,), o in examples]
        yield from make_list_task("bool-" + name, boolexamples, **params)
        # for now, we'll stick with the boolean-only tasks and not have a copy
        # for integers.
        return

    program_type = arrow(input_type, output_type)
    cache = all(hashable(x) for x in examples)

    if params:
        eq_params = ["{}={}".format(k, v) for k, v in params.items()]
        if len(eq_params) == 1:
            ext = eq_params[0]
        elif len(eq_params) == 2:
            ext = "{} and {}".format(*eq_params)
        else:
            ext = ", ".join(eq_params[:-1])
            ext = "{}, and {}".format(ext, eq_params[-1])
        name += " with " + ext

    yield Task(name, program_type, examples, cache=cache)
Exemplo n.º 7
0
def retrieveJSONTasks(filename, features=False):
    """
    For JSON of the form:
        {"name": str,
         "type": {"input" : bool|int|list-of-bool|list-of-int,
                  "output": bool|int|list-of-bool|list-of-int},
         "examples": [{"i": data, "o": data}]}
    """
    with open(filename, "r") as f:
        loaded = json.load(f)
    TP = {
        "bool": tbool,
        "int": tint,
        "list-of-bool": tlist(tbool),
        "list-of-int": tlist(tint),
    }
    return [
        Task(
            item["name"],
            arrow(TP[item["type"]["input"]], TP[item["type"]["output"]]),
            [((ex["i"], ), ex["o"]) for ex in item["examples"]],
            features=(None if not features else list_features(
                [((ex["i"], ), ex["o"]) for ex in item["examples"]])),
            cache=False,
        ) for item in loaded
    ]
Exemplo n.º 8
0
def makeNumberTasks():

    #load new data:

    taskfile = os.path.join(get_data_dir(), "regex_data_csv_900.p")

    with open(taskfile, 'rb') as handle:
        data = dill.load(handle)

    tasklist = data[0]  #a list of indices

    #match_col(data[0],'\d*\.\d*')
    raw_decimals = [
        121, 122, 163, 164, 165, 170, 172, 173, 175, 178, 218, 228, 230, 231,
        252, 253, 254, 258, 259, 305, 320, 330, 334, 340, 348, 350, 351, 352,
        353, 355, 357, 358, 361, 363, 364, 371, 380, 382, 409, 410, 411, 447,
        448, 449, 450, 458, 469, 471, 533, 562, 564
    ]

    decimals_pos_neg_dollar = [
        3, 4, 5, 6, 7, 13, 16, 24, 27, 28, 29, 30, 31, 32, 33, 34, 37, 38, 39,
        40, 53, 54, 55, 57, 58, 60, 61, 63, 64, 65, 66, 68, 69, 70, 71, 73, 74,
        77, 78, 80, 81, 103, 104, 105, 106, 107, 109, 110, 111, 112, 113, 114,
        115, 116, 117, 118, 119, 121, 122, 123, 124, 125, 126, 128, 129, 131,
        132, 134, 135, 139, 146, 153, 154, 155, 156, 157, 158, 159, 160, 161,
        162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175,
        176, 177, 178, 180, 181, 182, 183, 184, 185, 186, 193, 194, 195, 204,
        205, 207, 209, 210, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221,
        223, 224, 225, 226, 227, 228, 229, 230, 231, 232, 249, 250, 251, 252,
        253, 254, 255, 256, 258, 259, 260, 261, 263, 266, 267, 270, 271, 272,
        277, 299, 301, 302, 305, 306, 307, 309, 312, 313, 315, 319, 320, 324,
        326, 327, 330, 334, 340, 348, 350, 351, 352, 353, 354, 355, 356, 357,
        358, 361, 362, 363, 364, 368, 371, 373, 377, 380, 382, 400, 401, 402,
        403, 405, 406, 409, 410, 411, 413, 435, 439, 446, 447, 448, 449, 450,
        451, 452, 453, 456, 457, 458, 459, 460, 461, 462, 463, 464, 465, 466,
        469, 470, 471, 477, 498, 500, 502, 503, 507, 512, 518, 519, 520, 532,
        533, 553, 554, 555, 556, 557, 558, 559, 560, 561, 562, 564, 565, 572,
        577
    ]

    #match_col(data[0],'(\d*,?\d*)+')
    commas = []
    #match_col(data[0],'(\d*,?\d*)+')
    commas_and_all = []

    #full_list = test_list + train_list
    train_list = []
    full_list = decimals_pos_neg_dollar

    regextasks = [
        Task("Data column no. " + str(i), arrow(tpregex, tpregex),
             [((), example) for example in task])
        for i, task in enumerate(tasklist) if i in full_list
    ]

    for i in train_list:
        regextasks[i].mustTrain = True

    return regextasks
Exemplo n.º 9
0
def makeWordTasks():

    #load new data:

    taskfile = os.path.join(get_data_dir(), "regex_data_csv_900.p")

    with open(taskfile, 'rb') as handle:
        data = dill.load(handle)

    tasklist = data[0]  #a list of indices

    all_upper = [0, 2, 8, 9, 10, 11, 12, 17, 18, 19, 20, 22]
    all_lower = [1]

    # match_col(data[0],'\\u(\l+)')
    one_capital_lower_plus = [
        144, 200, 241, 242, 247, 296, 390, 392, 444, 445, 481, 483, 485, 489,
        493, 542, 549, 550, 581
    ]

    #match_col(data[0],'(\l ?)+')
    lower_with_maybe_spaces = [
        1, 42, 47, 99, 100, 102, 201, 246, 248, 293, 294, 345, 437, 545, 590
    ]

    #match_col(data[0],'(\\u\l+ ?)+')
    capital_then_lower_maybe_spaces = [
        144, 200, 241, 242, 247, 296, 390, 392, 395, 438, 444, 445, 481, 483,
        484, 485, 487, 489, 493, 494, 542, 546, 549, 550, 578, 581, 582, 588,
        591, 624, 629
    ]

    #match_col(data[0],'(\\u+ ?)+')
    all_caps_spaces = [
        0, 2, 8, 9, 10, 11, 12, 17, 18, 19, 20, 22, 25, 26, 35, 36, 43, 45, 46,
        49, 50, 52, 56, 59, 87, 89, 95, 101, 140, 147, 148, 149, 199, 332, 336,
        397, 491, 492, 495, 580, 610
    ]

    #one_capital_and_lower = [566, 550, 549, 542, 505, 493, 494, 489, 488, 485, 483, 481, 445, 444, 438, 296, 241, 242, 200, ]
    #all_lower_with_a_space = [545]
    #all_lower_maybe_space = [534]
    #one_capital_lower_maybe_spaces = [259, 262, 263, 264]

    #full_list = test_list + train_list
    train_list = []
    full_list = all_upper + all_lower + one_capital_lower_plus + lower_with_maybe_spaces + capital_then_lower_maybe_spaces + all_caps_spaces

    regextasks = [
        Task("Data column no. " + str(i), arrow(tpregex, tpregex),
             [((), example) for example in task])
        for i, task in enumerate(tasklist) if i in full_list
    ]

    for i in train_list:
        regextasks[i].mustTrain = True

    return regextasks
Exemplo n.º 10
0
 def tasksOfPrograms(self, ps, types):
     images = drawLogo(*ps, resolution=128)
     if len(ps) == 1: images = [images]
     tasks = []
     for i in images:
         if isinstance(i, str): tasks.append(None)
         else:
             t = Task("Helm", arrow(turtle, turtle), [])
             t.highresolution = i
             tasks.append(t)
     return tasks
Exemplo n.º 11
0
def manualLogoTask(name,
                   expression,
                   proto=False,
                   needToTrain=False,
                   supervise=False,
                   lambdaCalculus=False):
    p = Program.parse(expression) if lambdaCalculus else parseLogo(expression)
    from dreamcoder.domains.logo.logoPrimitives import primitives
    from dreamcoder.grammar import Grammar
    g = Grammar.uniform(primitives, continuationType=turtle)
    gp = Grammar.uniform(primitives)
    try:
        l = g.logLikelihood(arrow(turtle, turtle), p)
        lp = gp.logLikelihood(arrow(turtle, turtle), p)
        assert l >= lp
        eprint(name, -l, "nats")

    except:
        eprint("WARNING: could not calculate likelihood of manual logo", p)

    attempts = 0
    while True:
        [output, highresolution] = drawLogo(p,
                                            p,
                                            resolution=[28, 128],
                                            cost=True)
        if output == "timeout" or highresolution == "timeout":
            attempts += 1
        else:
            break
    if attempts > 0:
        eprint(
            f"WARNING: Took {attempts} attempts to render task {name} within timeout"
        )

    cost = output[1]
    output = output[0]
    assert highresolution[1] == cost
    highresolution = highresolution[0]

    shape = list(map(int, output))
    highresolution = list(map(float, highresolution))
    t = Task(name, arrow(turtle, turtle), [(([0]), shape)])
    t.mustTrain = needToTrain
    t.proto = proto
    t.specialTask = ("LOGO", {"proto": proto})
    t.specialTask[1]["cost"] = cost * 1.05

    t.highresolution = highresolution

    if supervise:
        t.supervisedSolution = p

    return t
Exemplo n.º 12
0
    def make_task(self):
        if self.solved or self.total_examples == 0:
            return None
        task_type = arrow(*([tint] * (self.arity + 1)))

        examples = []
        n_examples = min(self.total_examples, 100)
        # examples = random.choices([e for e, _ in self.examples], weights=[p for _, p in self.examples], k=n_examples)
        for e, p in self.examples:
            examples.extend([e] * int(round(p * n_examples)))
        examples = examples[:n_examples]
        return Task(str(self.idx), task_type, examples)
Exemplo n.º 13
0
def induceGrammar(*args, **kwargs):
    if sum(not f.empty for f in args[1]) == 0:
        eprint("No nonempty frontiers, exiting grammar induction early.")
        return args[0], args[1]
    backend = kwargs.pop("backend", "pypy")
    if 'pypy' in backend:
        # pypy might not like some of the imports needed for the primitives
        # but the primitive values are irrelevant for compression
        # therefore strip them out and then replace them once we are done
        # ditto for task data
        g0,frontiers = args[0].strip_primitive_values(), \
                       [front.strip_primitive_values() for front in args[1]]
        original_tasks = {f.task.name: f.task for f in frontiers}
        frontiers = [Frontier(f.entries, Task(f.task.name,f.task.request,[]))
                     for f in frontiers ]
        args = [g0,frontiers]

    
    with timing("Induced a grammar"):
        if backend == "pypy":
            g, newFrontiers = callCompiled(pypyInduce, *args, **kwargs)
        elif backend == "rust":
            g, newFrontiers = rustInduce(*args, **kwargs)
        elif backend == "vs":
            g, newFrontiers = rustInduce(*args, vs=True, **kwargs)
        elif backend == "pypy_vs":
            kwargs.pop('iteration')
            kwargs.pop('topk_use_only_likelihood')
            fn = '/tmp/vs.pickle'
            with open(fn, 'wb') as handle:
                pickle.dump((args, kwargs), handle)
            eprint("For debugging purposes, the version space compression invocation has been saved to", fn)
            g, newFrontiers = callCompiled(induceGrammar_Beta, *args, **kwargs)
        elif backend == "ocaml":
            kwargs.pop('iteration')
            kwargs.pop('topk_use_only_likelihood')
            kwargs['topI'] = 300
            kwargs['bs'] = 1000000
            g, newFrontiers = ocamlInduce(*args, **kwargs)
        elif backend == "memorize":
            g, newFrontiers = memorizeInduce(*args, **kwargs)
        else:
            assert False, "unknown compressor"

    if 'pypy' in backend:
        g, newFrontiers = g.unstrip_primitive_values(), \
                          [front.unstrip_primitive_values() for front in newFrontiers]
        newFrontiers = [Frontier(f.entries, original_tasks[f.task.name])
                        for f in newFrontiers] 
        

    return g, newFrontiers
Exemplo n.º 14
0
        def train(self, tasks, steps=400):
            # list of list of features for each example in each task
            optimizer = torch.optim.Adam(self.parameters())
            with timing("Trained discriminator"):
                losses = []
                for i in range(steps):
                    self.zero_grad()
                    if random.random() <= self.trainingSuccessRatio:
                        # success
                        t = random.choice(tasks)
                        features = [
                            self.extract.featuresOfTask(
                                Task(t.name, t.request, [ex], t.features))
                            for ex in t.examples
                        ]
                        loss = (self(features) - 1.0)**2
                    else:
                        # fail
                        t1, t2 = random.sample(tasks, 2)
                        features1 = [
                            self.extract.featuresOfTask(
                                Task(t1.name, t1.request, [ex], t1.features))
                            for ex in t1.examples[:len(t1.examples) / 2]
                        ]
                        features2 = [
                            self.extract.featuresOfTask(
                                Task(t2.name, t2.request, [ex], t2.features))
                            for ex in t2.examples[len(t2.examples) / 2:]
                        ]
                        features = features1 + features2
                        loss = self(features)**2

                    loss.backward()
                    optimizer.step()
                    losses.append(loss.data[0])
                    if not i % 50:
                        eprint("Discriminator Epoch", i, "Loss",
                               sum(losses) / len(losses))
                        gc.collect()
Exemplo n.º 15
0
    def dummy(program, logLikelihood=0., logPrior=0., tp=None):
        """Creates a dummy frontier containing just this program"""
        if not tp:
            tp = program.infer().negateVariables()

        t = Task(
            "<dummy %d: %s>" % (Frontier.DUMMYFRONTIERCOUNTER, str(program)),
            tp, [])
        f = Frontier([
            FrontierEntry(program=program,
                          logLikelihood=logLikelihood,
                          logPrior=logPrior)
        ],
                     task=t)
        Frontier.DUMMYFRONTIERCOUNTER += 1
        return f
Exemplo n.º 16
0
 def get():
     results = [p.get() for p in promises]
     frontiers = []
     with timing("(Helmholtz enumeration) Decoded json into frontiers"):
         for request, result in zip(requests, results):
             response = json.loads(result.decode("utf-8"))
             for b, entry in enumerate(response):
                 frontiers.append(
                     Frontier([
                         FrontierEntry(program=Program.parse(p),
                                       logPrior=entry["ll"],
                                       logLikelihood=0.)
                         for p in entry["programs"]
                     ],
                              task=Task(str(b), request, [])))
     eprint("Total number of Helmholtz frontiers:", len(frontiers))
     return frontiers
Exemplo n.º 17
0
def makeLongTasks():

    #load new data:

    taskfile = os.path.join(get_data_dir(), "regex_data_csv_900.p")

    with open(taskfile, 'rb') as handle:
        data = dill.load(handle)

    tasklist = data[0]  #a list of indices

    regextasks = [
        Task("Data column no. " + str(i), arrow(tpregex, tpregex),
             [((), example) for example in task])
        for i, task in enumerate(tasklist)
    ]

    return regextasks
Exemplo n.º 18
0
 def make_task(self):
     min_examples = 30 if self.arity is not None and self.arity > 0 else 10
     min_examples = min_examples if not self.fewshot else 0
     max_examples = 100
     examples = self.examples
     if len(examples) < min_examples or self.solved or None in [
             x[1] for x in examples
     ]:
         return None
     task_type = arrow(*([tint] * (self.arity + 1)))
     if len(examples) > max_examples:
         wrong_examples = [e for e, r in zip(examples, self.res) if not r]
         right_examples = [e for e, r in zip(examples, self.res) if r]
         right_examples = random.choices(right_examples,
                                         k=max_examples -
                                         len(wrong_examples))
         examples = wrong_examples + right_examples
         examples = random.sample(examples, k=max_examples)
     return Task(str(self.idx), task_type, examples)
Exemplo n.º 19
0
def makeHandPickedTasks():

    #load new data:

    taskfile = os.path.join(get_data_dir(), "regex_data_csv_900.p")

    with open(taskfile, 'rb') as handle:
        data = dill.load(handle)

    tasklist = data[0]  #a list of indices


    full_list = list(range(199)) + \
    [209,218,222,223,224,225,226] + \
    list(range(222,233)) + \
    [235,237,238,239,243,244,245,252,253,254,255,257,258,259,260,261,264,265,269,272,274] + \
    list(range(275,291)) + \
    [295,297,300,303,304,305,306,310,311,312,314,315,316,320,321,323,327,329,330,333,334,335,337,338,339,340,341,342,343,344] + \
    list(range(348,359)) + \
    [361,369,373,379,380,382,387,403,405,407,408] + \
    list(range(409,417)) + \
    list(range(418,437)) + \
    list(range(440,444)) + \
    list(range(446,452)) + \
    list(range(456,460)) + \
    list(range(466,472)) + \
    [503,504]

    regextasks = [
        Task("Data column no. " + str(i), arrow(tpregex, tpregex),
             [((), example) for example in task])
        for i, task in enumerate(tasklist) if i in full_list
    ]

    #for i in train_list:
    #    regextasks[i].mustTrain = True

    return regextasks
Exemplo n.º 20
0
def main(args):
    """
    Takes the return value of the `commandlineArguments()` function as input and
    trains/tests the model on manipulating sequences of numbers.
    """
    random.seed(args.pop("random_seed"))

    dataset = args.pop("dataset")
    tasks = {
        "Lucas-old":
        lambda: retrieveJSONTasks("data/list_tasks.json") + sortBootstrap(),
        "bootstrap":
        make_list_bootstrap_tasks,
        "sorting":
        sortBootstrap,
        "Lucas-depth1":
        lambda: retrieveJSONTasks("data/list_tasks2.json")[:105],
        "Lucas-depth2":
        lambda: retrieveJSONTasks("data/list_tasks2.json")[:4928],
        "Lucas-depth3":
        lambda: retrieveJSONTasks("data/list_tasks2.json"),
    }[dataset]()

    maxTasks = args.pop("maxTasks")
    if maxTasks and len(tasks) > maxTasks:
        necessaryTasks = []  # maxTasks will not consider these
        if dataset.startswith("Lucas2.0") and dataset != "Lucas2.0-depth1":
            necessaryTasks = tasks[:105]

        eprint("Unwilling to handle {} tasks, truncating..".format(len(tasks)))
        random.shuffle(tasks)
        del tasks[maxTasks:]
        tasks = necessaryTasks + tasks

    if dataset.startswith("Lucas"):
        # extra tasks for filter
        tasks.extend([
            Task("remove empty lists",
                 arrow(tlist(tlist(tbool)), tlist(tlist(tbool))),
                 [((ls, ), list(filter(lambda l: len(l) > 0, ls)))
                  for _ in range(15) for ls in [[[
                      random.random() < 0.5
                      for _ in range(random.randint(0, 3))
                  ] for _ in range(4)]]]),
            Task("keep squares", arrow(tlist(tint), tlist(tint)), [
                ((xs, ), list(filter(lambda x: int(math.sqrt(x))**2 == x, xs)))
                for _ in range(15) for xs in [[
                    random.choice([0, 1, 4, 9, 16, 25])
                    if random.random() < 0.5 else random.randint(0, 9)
                    for _ in range(7)
                ]]
            ]),
            Task("keep primes", arrow(tlist(tint), tlist(tint)), [
                ((xs, ),
                 list(
                     filter(
                         lambda x: x in
                         {2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37}, xs)))
                for _ in range(15) for xs in [[
                    random.choice([2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37])
                    if random.random() < 0.5 else random.randint(0, 9)
                    for _ in range(7)
                ]]
            ]),
        ])
        for i in range(4):
            tasks.extend([
                Task("keep eq %s" % i, arrow(tlist(tint), tlist(tint)),
                     [((xs, ), list(filter(lambda x: x == i, xs)))
                      for _ in range(15)
                      for xs in [[random.randint(0, 6) for _ in range(5)]]]),
                Task("remove eq %s" % i, arrow(tlist(tint), tlist(tint)),
                     [((xs, ), list(filter(lambda x: x != i, xs)))
                      for _ in range(15)
                      for xs in [[random.randint(0, 6) for _ in range(5)]]]),
                Task("keep gt %s" % i, arrow(tlist(tint), tlist(tint)),
                     [((xs, ), list(filter(lambda x: x > i, xs)))
                      for _ in range(15)
                      for xs in [[random.randint(0, 6) for _ in range(5)]]]),
                Task("remove gt %s" % i, arrow(tlist(tint), tlist(tint)),
                     [((xs, ), list(filter(lambda x: not x > i, xs)))
                      for _ in range(15)
                      for xs in [[random.randint(0, 6) for _ in range(5)]]])
            ])

    def isIdentityTask(t):
        return all(len(xs) == 1 and xs[0] == y for xs, y in t.examples)

    eprint("Removed", sum(isIdentityTask(t) for t in tasks),
           "tasks that were just the identity function")
    tasks = [t for t in tasks if not isIdentityTask(t)]

    prims = {
        "base": basePrimitives,
        "McCarthy": McCarthyPrimitives,
        "common": bootstrapTarget_extra,
        "noLength": no_length,
        "rich": primitives
    }[args.pop("primitives")]()
    haveLength = not args.pop("noLength")
    haveMap = not args.pop("noMap")
    haveUnfold = not args.pop("noUnfold")
    eprint(f"Including map as a primitive? {haveMap}")
    eprint(f"Including length as a primitive? {haveLength}")
    eprint(f"Including unfold as a primitive? {haveUnfold}")
    baseGrammar = Grammar.uniform([p
                                   for p in prims
                                   if (p.name != "map" or haveMap) and \
                                   (p.name != "unfold" or haveUnfold) and \
                                   (p.name != "length" or haveLength)])

    extractor = {
        "learned": LearnedFeatureExtractor,
    }[args.pop("extractor")]
    extractor.H = args.pop("hidden")

    timestamp = datetime.datetime.now().isoformat()
    outputDirectory = "experimentOutputs/list/%s" % timestamp
    os.system("mkdir -p %s" % outputDirectory)

    args.update({
        "featureExtractor": extractor,
        "outputPrefix": "%s/list" % outputDirectory,
        "evaluationTimeout": 0.0005,
    })

    eprint("Got {} list tasks".format(len(tasks)))
    split = args.pop("split")
    if split:
        train_some = defaultdict(list)
        for t in tasks:
            necessary = train_necessary(t)
            if not necessary:
                continue
            if necessary == "some":
                train_some[t.name.split()[0]].append(t)
            else:
                t.mustTrain = True
        for k in sorted(train_some):
            ts = train_some[k]
            random.shuffle(ts)
            ts.pop().mustTrain = True

        test, train = testTrainSplit(tasks, split)
        if True:
            test = [t for t in test if t.name not in EASYLISTTASKS]

        eprint("Alotted {} tasks for training and {} for testing".format(
            len(train), len(test)))
    else:
        train = tasks
        test = []

    explorationCompression(baseGrammar, train, testingTasks=test, **args)
Exemplo n.º 21
0
def sortBootstrap():
    # These tasks have as their goal the learning of (1) filter, and
    # (2) sort, which uses filter.
    def flip(): return random() > 0.5
    def randomList(lb=None, ub=None):
        if lb is None:
            lb = 2
        if ub is None:
            ub = 5
        return [randint(0, 10) for _ in range(randint(lb, ub))]
    def randomBooleanList():
        return [flip() for _ in range(randint(4, 7))]
    def removeDuplicates(l):
        if len(l) == 0: return l
        return [l[0]] + removeDuplicates([ z for z in l if z != l[0] ])
    
    filterBootstrap = [
        # Task("remove empty lists",
        #      arrow(tlist(tlist(tbool)), tlist(tlist(tbool))),
        #      [((ls,), [l for l in ls if len(l) > 0])
        #       for _ in range(10)
        #       for ls in [[[flip() for _ in range(randint(0, 3))]
        #                   for _ in range(4)]]]),
        # Task("remove non 0s",
        #      arrow(tlist(tint), tlist(tint)),
        #      [((xs,), filter(lambda x: x == 0, xs))
        #       for _ in range(10)
        #       for xs in [[ randint(0,3) for _ in range(5) ]] ]),
        Task("remove 0s",
             arrow(tlist(tint), tlist(tint)),
             [((xs,), [x for x in xs if x != 0])
              for _ in range(10)
              for xs in [[randint(0, 3) for _ in range(5)]]]),
        # Task("remove primes",
        #      arrow(tlist(tint), tlist(tint)),
        #      [((xs,), [x for x in xs if not (x in {2,3,5,7,11,13,17,19,23})])
        #       for _ in range(10)
        #       for xs in [[randint(0, 20) for _ in range(7)]]]),
        Task("remove squares",
             arrow(tlist(tint), tlist(tint)),
             [((xs,), [x for x in xs if not (int(x**0.5)**2 == x)])
              for _ in range(10)
              for xs in [[randint(0, 20) for _ in range(7)]]]),
        Task("remove > 1",
             arrow(tlist(tint), tlist(tint)),
             [((xs,), [x for x in xs if not (x > 1)])
              for _ in range(10)
              for xs in [[randint(0, 5) for _ in range(7)]]]),
    ]

    # Needed for selection sort
    minimumBootstrap = [
        Task("min2", arrow(tint,tint,tint),
             [((x,y),min(x,y))
              for x in range(4)
              for y in range(4) ]),
        Task("minimum of list", arrow(tlist(tint),tint),
             [((l,),min(l))
              for _ in range(15) 
              for l in [randomList()] ])
    ]

    appendBootstrap = [
        Task("append bool", arrow(tlist(tbool), tlist(tbool), tlist(tbool)),
             [((x, y), x + y)
              for _ in range(10)
              for [x, y] in [[randomBooleanList(), randomBooleanList()]]]),
        Task("append int", arrow(tlist(tint), tlist(tint), tlist(tint)),
             [((x, y), x + y)
              for _ in range(10)
              for [x, y] in [[randomList(), randomList()]]])
    ]

    insertionBootstrap = [
        Task("filter greater than or equal", arrow(tint,tlist(tint),tlist(tint)),
             [((x,l), [y for y in l if y >= x ])
              for _ in range(15) 
              for x in [randint(0,5)]
              for l in [randomList()] ]),
        Task("filter less than", arrow(tint,tlist(tint),tlist(tint)),
             [((x,l), [y for y in l if y < x ])
              for _ in range(15) 
              for x in [randint(0,5)]
              for l in [randomList()] ]),
        Task("insert into sorted list (I)", arrow(tint,tlist(tint),tlist(tint)),
             [((x,l), [y for y in l if y < x ] + [x] + [y for y in l if y >= x ])
              for _ in range(15) 
              for x in [randint(0,5)]
              for _l in [randomList()]
              for l in [sorted(_l)] ]),
        Task("insert into sorted list (II)", arrow(tint,tlist(tint),tlist(tint)),
             [((x,l), [y for y in l if y < x ] + [x] + [y for y in l if y >= x ])
              for _ in range(15) 
              for x in [randint(0,5)]
              for l in [randomList()] ])
    ]


    sortTask = [
        Task("sort-and-deduplicate", arrow(tlist(tint),tlist(tint)),
             [((l,),list(sorted(l)))
              for _ in range(15)
              for l in [removeDuplicates(randomList())]
             ])]

    slowSort = [
        Task("+1 maximum list", arrow(tlist(tint), tint),
             [((l,),max(l) + 1)
              for _ in range(15)
              for l in [randomList()] ]),
        Task("range +1 maximum list", arrow(tlist(tint), tlist(tint)),
             [((l,),list(range(max(l) + 1)))
              for _ in range(15)
              for l in [randomList()] ]),
        ]
        

    tasks = sortTask + slowSort
    for t in tasks: t.mustTrain = True
    return tasks
Exemplo n.º 22
0
def make_list_bootstrap_tasks():
    seed(42)

    def suffixes(l):
        if l == []:
            return []
        else:
            return [l[1:]] + suffixes(l[1:])

    def randomList(minValue=0, maxValue=20, len=1):
        return [randint(minValue, maxValue) for _ in range(int(len))]

    def generate_divide_examples ():
        i = 0
        lists = []
        lists.append(((4,8), 0))
        lists.append(((3,7), 0))
        lists.append(((5,28), 0))
        lists.append(((4,47), 0))
        lists.append(((9,6), 1))
        lists.append(((8,7), 1))
        lists.append(((5,3), 1))
        lists.append(((24,16), 1))
        lists.append(((49,27), 1))
        lists.append(((25,23), 1))
        while i < 20:
            a = randomList(len=n_sample*(1-noise))[0]
            b = randomList(minValue=1, len=n_sample*(1-noise))[0]

            if a // b >= 2 and ((a, b), _div(a,b)) not in lists:
                lists.append(((a, b), _div(a,b)))  
                i += 1
        

        return lists

    def generate_noise(n):
        return list(zip(zip(randomList(len=n), randomList(len=n)), randomList(len=n)))

    # Functions to be learnt
    def _add(x, y): return x + y
    def _minus(x, y): return max(0, x - y)
    # def _minus(x, y): return x - y
    def _mul(x, y): return x * y
    def _div(x, y): return x // y

    n_sample = 20
    noise = 0

    operatorBootstrap = [
        Task ("add", arrow(tint, tint, tint),
             generate_noise(n_sample*noise) + 
             [((a, b), _add(a,b)) for a, b in zip(randomList(len=n_sample*(1-noise)), randomList(len=n_sample*(1-noise)))]),
        Task ("minus", arrow(tint, tint, tint),
             generate_noise(n_sample*noise) + 
             [((a, b), _minus(a,b)) for a, b in zip(randomList(len=n_sample*(1-noise)), randomList(len=n_sample*(1-noise)))]),
        Task ("multiply", arrow(tint, tint, tint),
             generate_noise(n_sample*noise) + 
             [((a, b), _mul(a,b)) for a, b in zip(randomList(len=n_sample*(1-noise)), randomList(len=n_sample*(1-noise)))]),
        # Task ("divide", arrow(tint, tint, tint),
        #      generate_noise(n_sample*noise) + 
        #      [((a, b), _div(a,b)) for a, b in zip(randomList(len=n_sample*(1-noise)), randomList(minValue=1, len=n_sample*(1-noise)))]),
        Task ("divide", arrow(tint, tint, tint),
             generate_noise(n_sample*noise) + 
             generate_divide_examples()),
    ]

    return operatorBootstrap
Exemplo n.º 23
0
     test, _ = testTrainSplit(tasks, 100)
     random.seed(42)
     random.shuffle(test)
     test = test[:100]
     g = Grammar.uniform(
         [real, real_division, real_addition, real_multiplication])
     fe = FeatureExtractor([])
     BATCHSIZE = 64
 elif arguments.domain == "list":
     BATCHSIZE = 16
     tasks = retrieveJSONTasks("data/list_tasks.json") + sortBootstrap()
     tasks.extend([
         Task("remove empty lists",
              arrow(tlist(tlist(tbool)), tlist(tlist(tbool))),
              [((ls, ), list(filter(lambda l: len(l) > 0, ls)))
               for _ in range(15) for ls in [[[
                   random.random() < 0.5
                   for _ in range(random.randint(0, 3))
               ] for _ in range(4)]]]),
         Task("keep squares", arrow(tlist(tint), tlist(tint)), [
             ((xs, ), list(filter(lambda x: int(math.sqrt(x))**2 == x, xs)))
             for _ in range(15) for xs in [[
                 random.choice([0, 1, 4, 9, 16, 25])
                 if random.random() < 0.5 else random.randint(0, 9)
                 for _ in range(7)
             ]]
         ]),
         Task("keep primes", arrow(tlist(tint), tlist(tint)), [
             ((xs, ),
              list(
                  filter(
Exemplo n.º 24
0
def make_list_bootstrap_tasks():
    seed(42)

    def suffixes(l):
        if l == []:
            return []
        else:
            return [l[1:]] + suffixes(l[1:])

    def flip(): return random() > 0.5

    def randomSuffix():
        return [randint(0, 9) for _ in range(randint(1, 4))]

    def randomList(minimum=0, minimumLength=4, maximumLength=6):
        return [randint(minimum, 9) for _ in range(randint(minimumLength, maximumLength))]

    def randomListOfLists():
        return [randomSuffix() for _ in range(randint(2, 4))]

    def randomListOfLists_bool(l=None):
        if l is None:
            l = randint(4, 7)
        return [randomBooleanList() for _ in range(l)]

    def randomBooleanList():
        return [flip() for _ in range(randint(4, 7))]

    # Reliably learned in under a minute; always triggers learning of length
    # primitive
    lengthBootstrap = [
        # Task("length bool", arrow(tlist(tbool), tint),
        #      [((l,), len(l))
        #       for _ in range(10)
        #       for l in [[flip() for _ in range(randint(0, 10))]]]),
        Task("length int", arrow(tlist(tint), tint),
             [((l,), len(l))
              for _ in range(10)
              for l in [randomList()]]),
        Task("map length", arrow(tlist(tlist(tint)), tlist(tint)),
             [((xss,), [len(xs) for xs in xss])
              for _ in range(10)
              for xss in [randomListOfLists()] ])
    ]

    # Encourages learning of unfolding
    unfoldBootstrap = [
        Task("countdown", arrow(tint, tlist(tint)),
             [((n,), list(range(n + 1, 1, -1)))
              for n in range(10)]),
        Task("weird count", arrow(tint, tlist(tint)),
             [((n,), list(range(-n,0,-1)))
              for n in range(-10,0) ]),
        Task("take every other", arrow(tlist(tint),tlist(tint)),
             [((l,), [x for j,x in enumerate(l) if j%2 == 0])
              for _ in range(9)
              for l in [ [randint(0, 9) for _ in range(randint(1,4)*2)] ] ] + [(([],),[])]),
        # Task("stutter every other", arrow(tlist(tint),tlist(tint)),
        #      [((l,), [l[int(j/2)] for j in range(len(l)) ])
        #       for _ in range(10)
        #       for l in [ [randint(0, 9) for _ in range(randint(1,4)*2)] ] ]),
        # Task("take until 3 reached", arrow(tlist(tint),tlist(tint)),
        #      [((p + [3] + s,),p)
        #       for _ in range(10)
        #       for p in [ [z for z in randomList()[:5] if z != 3 ]]
        #       for s in [randomList()] ]),
        Task("drop last element", arrow(tlist(tint),tlist(tint)),
             [((l,), l[:-1])
              for _ in range(10)
              for l in [ [randint(0, 9) for _ in range(randint(2,5))] ] ]),
        # Task("suffixes", arrow(tlist(tint), tlist(tlist(tint))),
        #      [((l,), suffixes(l))
        #       for _ in range(10)
        #       for l in [randomList()]]),
        Task("range", arrow(tint, tlist(tint)),
             [((n,), list(range(n)))
              for n in range(10)]),
        Task("range inclusive", arrow(tint, tlist(tint)),
             [((n,), list(range(n + 1)))
              for n in range(10)]),
        # Task("range inclusive+1", arrow(tint, tlist(tint)),
        #      [((n,), list(range(n + 2)))
        #       for n in range(10)]),
        # Task("range exclusive", arrow(tint, tlist(tint)),
        #      [((n,), list(range(n - 1)))
        #       for n in range(2, 11)]),
        # Task("range length", arrow(tlist(tint),tlist(tint)),
        #      [((l,),list(range(len(l))))
        #       for _ in range(10)
        #       for l in [randomList()] ])
    ]

    # Encourages learning how to treat a list as an array
    arrayBootstrap = [
        Task("index int", arrow(tint, tlist(tint), tint),
             [((n, l), l[n])
              for n in range(10)
              for l in [[randint(0, 9) for _ in range(randint(n + 1, n + 5))]]]),
        # Task("last n", arrow(tint, tlist(tint), tlist(tint)),
        #      [((n, l), l[-n:])
        #       for n in range(10)
        #       for l in [[randint(0, 9) for _ in range(randint(n + 1, n + 5))]]]),
        Task("1-index int", arrow(tint, tlist(tint), tint),
             [((n, l), l[n - 1])
              for n in range(1,11)
              for l in [[randint(0, 9) for _ in range(randint(n + 1, n + 4))]]])
        
        # Task("index bool", arrow(tint, tlist(tbool), tbool),
        #      [((n, l), l[n])
        #       for n in range(10)
        #       for l in [[flip() for _ in range(randint(n + 1, n + 5))]]])
    ]

    # Teaches how to slice lists, not sure if we really need this though
    sliceBootstrap = [
        Task("take bool", arrow(tint, tlist(tbool), tlist(tbool)),
             [((n, l), l[:n])
              for n in range(10)
              for l in [[flip() for _ in range(randint(n, n + 5))]]]),
        Task("drop bool", arrow(tint, tlist(tbool), tlist(tbool)),
             [((n, l), l[n:])
              for n in range(10)
              for l in [[flip() for _ in range(randint(n, n + 5))]]]),

        Task("take int", arrow(tint, tlist(tint), tlist(tint)),
             [((n, l), l[:n])
              for n in range(10)
              for l in [[randint(0, 9) for _ in range(randint(n, n + 5))]]]),
        Task("drop int", arrow(tint, tlist(tint), tlist(tint)),
             [((n, l), l[n:])
              for n in range(10)
              for l in [[randint(0, 9) for _ in range(randint(n, n + 5))]]]),

    ]

    # learning to fold
    foldBootstrap = [
        Task("stutter", arrow(tlist(tint),tlist(tint)),
             [((l,), [z for x in l for z in [x,x] ])
              for _ in range(10)
              for l in [randomList()] ]),
        Task("sum", arrow(tlist(tint), tint),
             [((l,), sum(l))
              for _ in range(10)
              for l in [randomList()]]),
        # Task("difference", arrow(tlist(tint), tint),
        #      [((l,), reduce(lambda x, y: y - x, reversed(l), 1))
        #       for _ in range(10)
        #       for l in [randomList()[:4]]]),
        # Task("append bool", arrow(tlist(tbool), tlist(tbool), tlist(tbool)),
        #      [((x, y), x + y)
        #       for _ in range(10)
        #       for [x, y] in [[randomBooleanList(), randomBooleanList()]]]),
        Task("append constant 0", arrow(tlist(tint),tlist(tint)),
             [((l,),l + [0])
              for _ in range(10)
              for l in [randomList()] ]),
    ]

    # learning to map
    mapBootstrap = [
        Task("map double", arrow(tlist(tint), tlist(tint)),
             [((l,), list(map(lambda n: n * 2, l)))
              for _ in range(10)
              for l in [randomList()]]),
        Task("map increment", arrow(tlist(tint),tlist(tint)),
             [((l,),list(map(lambda n: n+1, l)))
              for _ in range(10)
              for l in [randomList()] ]),
        Task("map negation", arrow(tlist(tint),tlist(tint)),
             [((l,),list(map(lambda n: 0-n, l)))
              for _ in range(10)
              for l in [randomList()] ]),
        # Task("map car", arrow(tlist(tlist(tint)), tlist(tint)),
        #      [((l,), [n[0] for n in l])
        #       for _ in4 range(10)
        #       for l in [randomListOfLists()]]),
        # Task("map cdr", arrow(tlist(tlist(tbool)),tlist(tlist(tbool))),
        #      [((l,),map(lambda n: n[1:],l))
        #       for _ in range(10)
        #       for l in [randomListOfLists_bool()]]),
        # Task("map empty?", arrow(tlist(tlist(tint)), tlist(tboolean)),
        #      [((l,), [n == [] for n in l])
        #       for _ in range(10)
        #       for l in [[[] if flip() else randomList() for _ in range(randint(1, 5))]]]),

        # Task("map eq 0?", arrow(tlist(tint),tlist(tboolean)),
        #      [((l,),map(lambda n: 0 == n,l))
        #       for _ in range(10)
        #       for l in [[ randint(0,3) for _ in range(randint(4,7)) ]] ])

    ]
    difficultMaps = [
                Task("map quadruple", arrow(tlist(tint), tlist(tint)),
             [((l,), list(map(lambda n: n * 4, l)))
              for _ in range(10)
              for l in [randomList()]]),
        Task("map add 3", arrow(tlist(tint),tlist(tint)),
             [((l,),list(map(lambda n: n+3, l)))
              for _ in range(10)
              for l in [randomList()] ]),

        ]

    # Learning to zip lists together
    zipBootstrap = [
        Task("zip plus", arrow(tlist(tint),tlist(tint),tlist(tint)),
             [((l1,l2),list(map(lambda x,y: x+y,l1,l2)))
              for _ in range(10)
              for l1 in [randomList(minimumLength=2, maximumLength=4)]
              for l2 in [[ randint(0,9) for _ in range(len(l1)) ]]]),
        Task("zip minus", arrow(tlist(tint),tlist(tint),tlist(tint)),
             [((l1,l2),list(map(lambda x,y: x-y,l1,l2)))
              for _ in range(10)
              for l1 in [randomList(minimumLength=2, maximumLength=4)]
              for l2 in [[ randint(0,9) for _ in range(len(l1)) ]]]),
        # Task("zip eq?", arrow(tlist(tint), tlist(tint), tlist(tbool)),
        #      [((l1, l2), list(map(lambda x, y: x == y, l1, l2)))
        #       for _ in range(10)
        #       for l1 in [[randint(0, 3) for _ in range(randint(4, 7))]]
        #       for l2 in [[randint(0, 3) for _ in range(len(l1))]]]),
        # Task("zip cons", arrow(tlist(tbool), tlist(tlist(tbool)), tlist(tlist(tbool))),
        #      [((l1, l2), list(map(lambda x, y: [x] + y, l1, l2)))
        #       for _ in range(10)
        #       for l1 in [randomBooleanList()]
        #       for l2 in [randomListOfLists_bool(l=len(l1))]]),
        # Task("zip cons", arrow(tlist(tint),tlist(tlist(tint)),tlist(tlist(tint))),
        #      [((l1,l2),list(map(lambda x,y: [x]+y,l1,l2)))
        #       for _ in range(10)
        #       for l1 in [randomList()]
        #       for l2 in [[ randomList() for _ in range(len(l1)) ]]]),
    ]

    # Learning to filter
    filterBootstrap = [
        # Task("remove empty lists",
        #      arrow(tlist(tlist(tbool)), tlist(tlist(tbool))),
        #      [((ls,), [l for l in ls if len(l) > 0])
        #       for _ in range(10)
        #       for ls in [[[flip() for _ in range(randint(0, 3))]
        #                   for _ in range(4)]]])
        # Task("remove non 0s",
        #      arrow(tlist(tint), tlist(tint)),
        #      [((xs,), filter(lambda x: x == 0, xs))
        #       for _ in range(10)
        #       for xs in [[ randint(0,3) for _ in range(5) ]] ]),
        Task("remove 0s",
             arrow(tlist(tint), tlist(tint)),
             [((xs,), [x for x in xs if x != 0])
              for _ in range(10)
              for xs in [[randint(0, 3) for _ in range(5)]]]),
        Task("remove non-positives",
             arrow(tlist(tint), tlist(tint)),
             [((xs,), [x for x in xs if not (x > 1)])
              for _ in range(10)
              for xs in [[randint(0, 3) for _ in range(5)]]]),
    ]

    return lengthBootstrap + filterBootstrap + \
        unfoldBootstrap + arrayBootstrap + foldBootstrap + mapBootstrap + zipBootstrap
Exemplo n.º 25
0
def make_list_bootstrap_tasks():
    seed(42)

    def suffixes(l):
        if l == []:
            return []
        else:
            return [l[1:]] + suffixes(l[1:])

    def randomList(minValue=0, maxValue=9, len=1):
        return [randint(minValue, maxValue) for _ in range(int(len))]

    def generate_noise(n, arity=2):
        n = int(n)
        xs = []
        for _ in range(arity):
            xs.append(randomList(len=n))
        y = randomList(len=n)
        if arity == 0:
            return [((), s) for s in y]
        return list(zip(zip(*xs), y))

    def generate_examples(arity,
                          fn,
                          total=1,
                          noise_level=0,
                          min_value=0,
                          max_value=9):
        n = total * noise_level
        xs = list(
            zip(*[
                randomList(len=n, minValue=min_value, maxValue=max_value)
                for i in range(arity)
            ]))
        y = randomList(len=n, minValue=min_value, maxValue=max_value)
        noise = list(zip(xs, y))
        n = total * (1 - noise_level)
        xs = list(
            zip(*[
                randomList(len=n, minValue=min_value, maxValue=max_value)
                for i in range(arity)
            ]))
        correct = [(x, fn(*x)) for x in xs]
        correct = [x for x in correct if x[1] is not None]
        data = list(set(noise + correct))
        return data

    # Functions to be learnt
    def _add(x, y):
        return x + y

    def _minus(x, y):
        return max(0, x - y)

    # def _minus(x, y): return x - y
    def _mul(x, y):
        return x * y

    def _div(x, y):
        return x // y if y != 0 else None

    # def _div(x, y): return math.ceil(x / y)
    _fac = lambda x: math.factorial(x) if x <= 20 else None

    n_sample = 20
    noise = 0.
    operatorBootstrap = []

    operatorBootstrap = [
        # Task ("add", arrow(tint, tint, tint),
        #      generate_noise(n_sample*noise) +
        #      [((a, b), _add(a,b)) for a, b in zip(randomList(len=n_sample*(1-noise)), randomList(len=n_sample*(1-noise)))]),
        # Task ("minus", arrow(tint, tint, tint),
        #      generate_noise(n_sample*noise) +
        #      [((a, b), _minus(a,b)) for a, b in zip(randomList(len=n_sample*(1-noise)), randomList(len=n_sample*(1-noise)))]),
        # Task ("multiply", arrow(tint, tint, tint),
        #      generate_noise(n_sample*noise) +
        #      [((a, b), _mul(a,b)) for a, b in zip(randomList(len=n_sample*(1-noise)), randomList(len=n_sample*(1-noise)))]),
        # Task ("divide", arrow(tint, tint, tint),
        #      generate_noise(n_sample*noise) +
        #      [((a, b), _div(a,b)) for a, b in zip(randomList(len=n_sample*(1-noise)), randomList(minValue=1, len=n_sample*(1-noise)))]),
        Task("divide", arrow(tint, tint, tint),
             generate_examples(2, _div, total=n_sample, max_value=20)),
        # Task ("factorial", arrow(tint, tint),
        #      generate_noise(n_sample*noise, arity=1) +
        #      [((a,), _fac(a)) for a in randomList(len=n_sample*(1-noise))]),
    ]

    # # Add counting task
    # from collections import Counter
    # for i in range(10):
    #     task = Task(str(i), arrow(tint),
    #          generate_noise(n_sample*noise, arity=0) + [((), i)] * int(n_sample- n_sample * noise))
    #     ys = [y for _, y in task.examples]
    #     print(i, Counter(ys).most_common(3))
    #     operatorBootstrap.append(task)

    import json
    # tasks = json.load(open('../../outputs/tasks.json'))
    # for i, t in enumerate(tasks):
    #     task = Task(str(i), arrow(tint, tint, tint), t)
    #     operatorBootstrap.append(task)
    # task = Task(str(len(tasks)), arrow(tint, tint, tint), [x for t in tasks for x in t])
    # operatorBootstrap.append(task)

    # examples = {((5, 2), 3): 3, ((6, 3), 3): 3, ((9, 3), 6): 3, ((6, 2), 6): 2, ((1, 9), 0): 2, ((7, 4), 3): 2, ((2, 3), 1): 2, ((9, 2), 8): 2, ((7, 9), 35): 2, ((7, 6), 48): 2, ((3, 4), 1): 2, ((4, 4), 3): 2, ((1, 4), 0): 1, ((2, 4), 56): 1, ((7, 1), 6): 1, ((4, 3), 3): 1, ((5, 9), 4): 1, ((8, 6), 2): 1, ((9, 6), 8): 1, ((9, 8), 1): 1, ((3, 8), 2): 1, ((5, 4), 1): 1, ((7, 2), 6): 1, ((3, 9), 1): 1, ((3, 3), 30): 1, ((6, 6), 64): 1, ((6, 5), 1): 1, ((5, 3), 5): 1, ((6, 4), 2): 1, ((4, 7), 0): 1, ((5, 7), 1): 1, ((2, 9), 1): 1, ((8, 2), 8): 1, ((4, 9), 1): 1, ((7, 3), 35): 1, ((8, 4), 2): 1, ((3, 2), 3): 1, ((9, 9), 3): 1, ((2, 1), 1): 1, ((6, 8), 54): 1, ((3, 1), 2): 1, ((6, 0), 1): 1, ((1, 5), 10): 1, ((3, 0), 4): 1, ((1, 7), 10): 1}
    # examples = [[e]*count for e, count in examples.items()]
    # examples = [y for x in examples for y in x]
    # task = Task("11", arrow(tint, tint, tint), examples)
    # operatorBootstrap.append(task)

    # examples = {((1, 7), 7): 9, ((1, 4), 4): 9, ((7, 3), 21): 8, ((1, 2), 2): 7, ((7, 9), 63): 7, ((8, 9), 72): 7, ((3, 9), 27): 7, ((1, 9), 9): 6, ((0, 4), 4): 6, ((1, 3), 3): 6, ((1, 8), 8): 5, ((1, 1), 1): 5, ((4, 9), 36): 5, ((2, 9), 18): 5, ((1, 6), 6): 5, ((5, 4), 20): 5, ((5, 2), 10): 5, ((0, 1), 1): 4, ((5, 9), 45): 4, ((6, 7), 42): 3, ((1, 5), 5): 3, ((3, 3), 9): 3, ((9, 5), 45): 3, ((6, 9), 54): 3, ((9, 9), 81): 3, ((7, 8), 56): 3, ((9, 8), 72): 2, ((3, 7), 21): 2, ((8, 8), 64): 2, ((3, 6), 18): 2, ((3, 5), 15): 2, ((9, 2), 18): 2, ((4, 5), 20): 2, ((0, 5), 5): 2, ((0, 7), 7): 2, ((9, 4), 36): 2, ((6, 6), 36): 2, ((9, 3), 27): 2, ((7, 5), 35): 2, ((2, 5), 14): 1, ((7, 6), 48): 1, ((8, 5), 40): 1, ((2, 1), 7): 1, ((3, 4), 12): 1, ((0, 3), 3): 1, ((9, 7), 63): 1, ((9, 6), 54): 1, ((0, 8), 8): 1, ((5, 6), 18): 1, ((6, 2), 6): 1, ((9, 0), 27): 1, ((1, 0), 0): 1, ((3, 1), 6): 1, ((8, 1), 20): 1, ((4, 6), 16): 1, ((8, 6), 30): 1, ((4, 7), 49): 1, ((6, 5), 30): 1, ((6, 8), 16): 1, ((0, 9), 9): 1, ((0, 6), 6): 1, ((8, 2), 4): 1, ((2, 8), 4): 1, ((7, 7), 49): 1, ((3, 8), 64): 1, ((3, 2), 6): 1}
    # examples = [[e]*count for e, count in examples.items()]
    # examples = [y for x in examples for y in x]
    # task = Task("12", arrow(tint, tint, tint), examples)
    # operatorBootstrap.append(task)

    # examples = {((6, 4), 24): 8, ((8, 3), 24): 7, ((8, 7), 56): 6, ((6, 8), 48): 6, ((4, 2), 8): 5, ((5, 2), 8): 1, ((4, 7), 32): 1, ((0, 6), 0): 1, ((4, 3), 24): 1, ((4, 0), 8): 1}
    # examples = [[e]*count for e, count in examples.items()]
    # examples = [y for x in examples for y in x]
    # task = Task("14", arrow(tint, tint, tint), examples)
    # operatorBootstrap.append(task)

    return operatorBootstrap