def getCutOffHits(result, cutOff): """Return a list of hit percentages; currently only testing tasks supported""" from dreamcoder.likelihoodModel import add_cutoff_values from bin.examineFrontier import testingRegexLikelihood from dreamcoder.domains.regex.groundtruthRegexes import badRegexTasks tasks = [t for t in result.getTestingTasks() if t.name not in badRegexTasks] add_cutoff_values(tasks, cutOff) learningCurve = [] while True: iteration = len(learningCurve) print(f"Calculating hit tasks for iteration {iteration}. We do this once per iteration and once per checkpoint so will take a while :(") hs = 0 for ti,t in enumerate(tasks): if iteration >= len(result.frontiersOverTime[t]): assert ti == 0 return learningCurve frontier = result.frontiersOverTime[t][iteration] if len(frontier) == 0: continue frontier = frontier.normalize() bestLikelihood = frontier.bestPosterior if cutOff == "gt": if arguments.testingLikelihood: p = bestLikelihood.program hs += int(testingRegexLikelihood(t, p) >= t.gt_test - 0.001) else: hs += int(bestLikelihood.logLikelihood >= t.gt - 0.001) elif cutOff == "unigram" or cutOff == "bigram": assert False, "why are you not using a ground truth cut off" if bestLikelihood >= t.ll_cutoff: hs += 1 else: assert False learningCurve.append(100.*hs/len(tasks))
def main(args): """ Takes the return value of the `commandlineArguments()` function as input and trains/tests the model on regular expressions. """ #for dreaming #parse use_ll_cutoff use_ll_cutoff = args.pop('use_ll_cutoff') if not use_ll_cutoff is False: #if use_ll_cutoff is a list of strings, then train_ll_cutoff and train_ll_cutoff #will be tuples of that string followed by the actual model if len(use_ll_cutoff) == 1: train_ll_cutoff = use_ll_cutoff[0] # make_cutoff_model(use_ll_cutoff[0], tasks)) test_ll_cutoff = use_ll_cutoff[0] # make_cutoff_model(use_ll_cutoff[0], tasks)) else: assert len(use_ll_cutoff) == 2 train_ll_cutoff = use_ll_cutoff[0] #make_cutoff_model(use_ll_cutoff[0], tasks)) test_ll_cutoff = use_ll_cutoff[1] #make_cutoff_model(use_ll_cutoff[1], tasks)) else: train_ll_cutoff = None test_ll_cutoff = None regexTasks = {"old": makeOldTasks, "short": makeShortTasks, "long": makeLongTasks, "words": makeWordTasks, "number": makeNumberTasks, "handpicked": makeHandPickedTasks, "new": makeNewTasks, "newNumber": makeNewNumberTasks }[args.pop("tasks")] tasks = regexTasks() # TODO eprint("Generated", len(tasks), "tasks") maxTasks = args.pop("maxTasks") if len(tasks) > maxTasks: eprint("Unwilling to handle {} tasks, truncating..".format(len(tasks))) seed = 42 # previously this was hardcoded and never changed random.seed(seed) random.shuffle(tasks) del tasks[maxTasks:] maxExamples = args.pop("maxExamples") split = args.pop("split") test, train = testTrainSplit(tasks, split) eprint("Split tasks into %d/%d test/train" % (len(test), len(train))) test = add_cutoff_values(test, test_ll_cutoff) train = add_cutoff_values(train, train_ll_cutoff) eprint("added cutoff values to tasks, train: ", train_ll_cutoff, ", test:", test_ll_cutoff ) if args.pop("use_str_const"): assert args["primitives"] == "strConst" or args["primitives"] == "reduced" ConstantInstantiateVisitor.SINGLE = \ ConstantInstantiateVisitor() test = add_string_constants(test) train = add_string_constants(train) eprint("added string constants to test and train") for task in test + train: if len(task.examples) > maxExamples: task.examples = task.examples[:maxExamples] task.specialTask = ("regex", {"cutoff": task.ll_cutoff, "str_const": task.str_const}) task.examples = [(xs, [y for y in ys ]) for xs,ys in task.examples ] task.maxParameters = 1 # from list stuff primtype = args.pop("primitives") prims = {"base": basePrimitives, "alt1": altPrimitives, "alt2": alt2Primitives, "easyWords": easyWordsPrimitives, "concat": concatPrimitives, "reduced": reducedConcatPrimitives, "strConst": strConstConcatPrimitives }[primtype] extractor = { "learned": LearnedFeatureExtractor, "json": MyJSONFeatureExtractor }[args.pop("extractor")] extractor.H = args.pop("hidden") #stardecay = args.stardecay #stardecay = args.pop('stardecay') #decaystr = 'd' + str(stardecay) import datetime timestamp = datetime.datetime.now().isoformat() outputDirectory = "experimentOutputs/regex/%s"%timestamp os.system("mkdir -p %s"%outputDirectory) args.update({ "featureExtractor": extractor, "outputPrefix": "%s/regex"%(outputDirectory), "evaluationTimeout": 0.005, "topk_use_only_likelihood": True, "maximumFrontier": 10, "compressor": "ocaml" }) #### # use the #prim_list = prims(stardecay) prim_list = prims() specials = ["r_kleene", "r_plus", "r_maybe", "r_alt", "r_concat"] n_base_prim = len(prim_list) - len(specials) productions = [ (math.log(0.5 / float(n_base_prim)), prim) if prim.name not in specials else ( math.log(0.10), prim) for prim in prim_list] baseGrammar = Grammar.fromProductions(productions, continuationType=tpregex) #baseGrammar = Grammar.uniform(prims()) #for i in range(100): # eprint(baseGrammar.sample(tpregex)) #eprint(baseGrammar) #explore test_stuff = args.pop("debug") if test_stuff: eprint(baseGrammar) eprint("sampled programs from prior:") for i in range(100): #100 eprint(baseGrammar.sample(test[0].request,maximumDepth=1000)) eprint("""half the probability mass is on higher-order primitives. Therefore half of enumerated programs should have more than one node. However, we do not observe this. Instead we see a very small fraction of programs have more than one node. So something seems to be wrong with grammar.sample. Furthermore: observe the large print statement above. This prints the candidates for sampleDistribution in grammar.sample. the first element of each tuple is the probability passed into sampleDistribution. Half of the probability mass should be on the functions, but instead they are equally weighted with the constants. If you look at the grammar above, this is an error!!!! """) assert False del args["likelihoodModel"] explorationCompression(baseGrammar, train, testingTasks = test, **args)
"[0123456789]", "\\d" ).replace("[abcdefghijklmnopqrstuvwxyz]", "\\l").replace( "[0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~ \t]", ".") if __name__ == "__main__": print("started:", flush=True) with open(checkpoint_file, 'rb') as file: checkpoint = pickle.load(file) tasks = checkpoint.testSearchTime.keys() #recognitionTaskMetrics.keys() from dreamcoder.likelihoodModel import add_cutoff_values tasks = add_cutoff_values(tasks, "gt") #could be "unigram" or "bigram" print("TESTING ONLY:") #print loop? posteriorHits = 0 likelihoodHits = 0 posteriorHits_test = 0 likelihoodHits_test = 0 marginalHits = 0 marginalHits_test = 0 totalTasks = 0 for task in tasks: #if task.name in badRegexTasks: continue try: