def _try_data(dataset="cfq/mcd1", validfrac=0.1, bertname="vanilla", recompute=False, gpu=-1): trainds, validds, testds, fldic, inpdic, reorderer = load_ds( dataset=dataset, validfrac=validfrac, bertname=bertname, recompute=recompute) reorderer = Reorderer(inpD=inpdic, outD=fldic) ex1 = trainds[0] ex2 = trainds[0] print(inpdic.tostr(ex1[0])) print(inpdic.tostr(ex2[0])) print(fldic.tostr(ex1[1])) print(fldic.tostr(ex2[1])) a = 0 c = 0 tt = q.ticktock("trydata") tt.tick("trying data") for j in range(min(len(testds), 100000)): print(j, c) for _ in range(1): ex1 = trainds[j] ex2 = trainds[j] equal = reorderer.are_equal_trees( taglisp_to_tree(fldic.tostr(ex1[1])), taglisp_to_tree(fldic.tostr(ex2[1]))) print(equal) if equal is False: c += 1 print(c) tt.tock()
def tensor_to_trees(x, vocab: Vocab): xstrs = [ vocab.tostr(x[i]).replace("@START@", "") for i in range(len(x)) ] xstrs = [re.sub("::\d+", "", xstr) for xstr in xstrs] trees = [] for xstr in xstrs: # drop everything after @END@, if present xstr = xstr.split("@END@") xstr = xstr[0] # add an opening parentheses if not there xstr = xstr.strip() if len(xstr) == 0 or xstr[0] != "(": xstr = "(" + xstr # balance closing parentheses parenthese_imbalance = xstr.count("(") - xstr.count(")") xstr = xstr + ")" * max(0, parenthese_imbalance ) # append missing closing parentheses xstr = "(" * -min( 0, parenthese_imbalance ) + xstr # prepend missing opening parentheses try: tree = taglisp_to_tree(xstr) if isinstance( tree, tuple) and len(tree) == 2 and tree[0] is None: tree = None except Exception as e: tree = None trees.append(tree) return trees
def __call__(self, x): _x = x x, y = x recons = taglisp_to_tree(y) y = self._reorder_rec(recons) y = tree_to_taglisp(y) if self.reassign_ents_vars: ve = list(self.validentities) self.rnd.shuffle(ve) entmap = list(zip(sorted(ve), ve)) vv = list(self.validvariables) self.rnd.shuffle(vv) varmap = list(zip(sorted(vv), vv)) mapper = dict(entmap + varmap) xmapped = [ mapper[xe.lower()].upper() if xe.lower() in mapper else xe for xe in x.split() ] ymapped = [mapper[ye] if ye in mapper else ye for ye in y.split()] x = " ".join(xmapped) y = " ".join(ymapped) return (x, y)