def build_feed_dict(self, batch_data):
def assign(node, args):
args[0][args[1]] = node
is_leaf = []
node_word_indices = []
labels = []
# print(self.config.max_tree_height)
for _, atree in enumerate(batch_data):
nodes_list = [None] * self.config.max_tree_height
root = atree.root
tree.traverse(root, assign, (nodes_list, 1))
is_leaf.append([0 if node is None or not node.isLeaf else 1 for node in nodes_list])
node_word_indices.append([self.vocab.encode(node.word) if node is not None and node.word else -1 for node in nodes_list])
labels.append([node.label if node is not None else -1 for node in nodes_list])
# print(labels)
feed_dict = {
self.cons_placeholder: 1,
# Using int32 instead of bool for by-passing errors occuring during the construction of the function's gradient
self.is_leaf_placeholder: is_leaf,
self.node_word_indices_placeholder: node_word_indices,
self.labels_placeholder: labels
}
return feed_dict
def traverse(t,gpe):
global lo
if gpe != "":
return
else:
try:
t.label()
except AttributeError:
pass
else:
# Now we know that t.node is defined
if t.label()=="GPE":
is_gpe = True
else:
is_gpe = False
for child in t:
if is_gpe:
if gpe == "":
gpe = child[0]
else:
gpe = gpe + " "
gpe = gpe + child[0]
lo = gpe
else:
traverse(child, gpe)
return
def build_feed_dict(self, atree):
size = pow(2, atree.max_depth + 1)
nodes_list = [None] * size
root = atree.root
def assign(node, args):
args[0][args[1]] = node
tree.traverse(root, assign, (nodes_list, 1))
feed_dict = {
self.cons_placeholder:
1,
# Using int32 instead of bool for by-passing errors occuring during the construction of the function's gradient
self.is_leaf_placeholder: [
0 if node is None or not node.isLeaf else 1
for node in nodes_list
],
self.node_word_indices_placeholder: [
self.vocab.encode(node.word)
if node is not None and node.word else -1
for node in nodes_list
],
self.labels_placeholder:
[node.label if node is not None else -1 for node in nodes_list]
}
return feed_dict
def _check_matching_structures(output_tree, bound_tree):
"""Replace all bounds/arrays with True, then compare pytrees."""
output_struct = tree.traverse(
lambda x: True
if isinstance(x, jnp.ndarray) else None, output_tree)
bound_struct = tree.traverse(
lambda x: True
if isinstance(x, bound_propagation.Bound) else None,
bound_tree)
tree.assert_same_structure(output_struct, bound_struct)
def transform(self,
ac_data: ActionConnectorDataType) -> ActionConnectorDataType:
assert isinstance(
ac_data.output,
tuple), "Action connector requires PolicyOutputType data."
actions, states, fetches = ac_data.output
tree.traverse(make_action_immutable, actions, top_down=False)
return ActionConnectorDataType(
ac_data.env_id,
ac_data.agent_id,
(actions, states, fetches),
)
def UpdateSplits(all_splits, tree, taxa_index):
"""
Adds the splits in tree to the list all_splits
Args:
all_splits - list of splits
tree - Tree object
taxa_index - dictionary from taxa to their index
taxa_set - a set object of the taxa in taxa_index
Returns:
None
Requirements:
- The taxa in tree should be identical to the taxa in taxa_index
"""
nRoot = len(tree.get_children())
if nRoot == 2:
# don't double count at root. If there are two children then must only do once
for node in tree.traverse("postorder"):
if node.is_leaf():
if node.up.is_root():
nRoot -= 1
if nRoot == 0: continue
s = BitVector(taxa_index, node.name)
node.add_feature('split_down', s)
all_splits.append((s.Canonical(), node.dist))
elif node.is_root():
continue
else:
if node.up.is_root():
nRoot -= 1
if nRoot == 0: continue
s = BitVector(taxa_index)
s.AddList([ch.split_down for ch in node.get_children()])
node.add_feature('split_down', s)
all_splits.append((s.Canonical(), node.dist))
else:
for node in tree.traverse("postorder"):
if node.is_leaf():
s = BitVector(taxa_index, node.name)
node.add_feature('split_down', s)
all_splits.append((s.Canonical(), node.dist))
elif node.is_root():
continue
else:
s = BitVector(taxa_index)
s.AddList([ch.split_down for ch in node.get_children()])
node.add_feature('split_down', s)
all_splits.append((s.Canonical(), node.dist))
def UpdateSplits(all_splits, tree, taxa_index):
"""
Adds the splits in tree to the list all_splits
Args:
all_splits - list of splits
tree - Tree object
taxa_index - dictionary from taxa to their index
taxa_set - a set object of the taxa in taxa_index
Returns:
None
Requirements:
- The taxa in tree should be identical to the taxa in taxa_index
"""
nRoot = len(tree.get_children())
if nRoot == 2:
# don't double count at root. If there are two children then must only do once
for node in tree.traverse("postorder"):
if node.is_leaf():
if node.up.is_root():
nRoot -= 1
if nRoot == 0: continue
s = BitVector(taxa_index, node.name)
node.add_feature('split_down', s)
all_splits.append((s.Canonical(), node.dist))
elif node.is_root():
continue
else:
if node.up.is_root():
nRoot -= 1
if nRoot == 0: continue
s = BitVector(taxa_index)
s.AddList([ch.split_down for ch in node.get_children()])
node.add_feature('split_down', s)
all_splits.append((s.Canonical(), node.dist))
else:
for node in tree.traverse("postorder"):
if node.is_leaf():
s = BitVector(taxa_index, node.name)
node.add_feature('split_down', s)
all_splits.append((s.Canonical(), node.dist))
elif node.is_root():
continue
else:
s = BitVector(taxa_index)
s.AddList([ch.split_down for ch in node.get_children()])
node.add_feature('split_down', s)
all_splits.append((s.Canonical(), node.dist))
def testTraverse(self, test_type):
self._init_testdata(test_type)
visited = []
tree.traverse(visited.append, self.dcls_with_map, top_down=False)
self.assertLen(visited, self.dcls_tree_size)
visited_without_dicts = []
def visit_without_dicts(x):
visited_without_dicts.append(x)
return 'X' if isinstance(x, dict) else None
tree.traverse(visit_without_dicts, self.dcls_with_map, top_down=True)
self.assertLen(visited_without_dicts, self.dcls_tree_size_no_dicts)
def _yield_sorted_items(iterable):
"""Yield (key, value) pairs for `iterable` in a deterministic order.
For Sequences, the key will be an int, the array index of a value.
For Mappings, the key will be the dictionary key.
For objects (e.g. namedtuples), the key will be the attribute name.
In all cases, the keys will be iterated in sorted order.
Args:
iterable: an iterable.
Yields:
The iterable's (key, value) pairs, in order of sorted keys.
"""
if not is_nested(iterable):
raise ValueError(f'{iterable} is not an iterable')
top_structure = dm_tree.traverse(
lambda x: None # pylint: disable=g-long-lambda
if x is iterable else False,
iterable)
for p, v in dm_tree.flatten_with_path_up_to(top_structure, iterable):
yield p[0], v
def testTraverseListsToTuples(self):
structure = [(1, 2), [3], {"a": [4]}]
self.assertEqual(((1, 2), (3, ), {
"a": (4, )
}),
tree.traverse(lambda x: tuple(x)
if isinstance(x, list) else x,
structure,
top_down=False))
def testTraverseEarlyTermination(self):
structure = [(1, [2]), [3, (4, 5, 6)]]
visited = []
def visit(x):
visited.append(x)
return "X" if isinstance(x, tuple) and len(x) > 2 else None
output = tree.traverse(visit, structure)
self.assertEqual([(1, [2]), [3, "X"]], output)
self.assertEqual([[(1, [2]), [3, (4, 5, 6)]],
(1, [2]), 1, [2], 2, [3, (4, 5, 6)], 3, (4, 5, 6)],
visited)
def _fetch_sheet(self):
def _cache_entire_tree(wks, el, passthru):
lookup = passthru['lookup']
parent = passthru['parent']
slug = el['row'][tree.C_SLUG]
category = el['row'][tree.C_PATH].split('/')[0]
tag = '{}:{}'.format(category, slug)
el['parent'] = parent
el['slug'] = slug
el['category'] = category
el['tag'] = tag
lookup[tag] = el
return {'lookup': lookup, 'parent': tag}
sheet_lookup = {}
for wks in tree.get_worksheets():
cell_matrix = wks.get_all_values(returnas='matrix')
tree.traverse(tree.generate_tree(wks, quiet=True),
fn=_cache_entire_tree,
arg={
'lookup': sheet_lookup,
'parent': None
})
return sheet_lookup
def randomize_folder_size(tree,minfrac,maxfrac):
for node in tree.traverse():
if node.parent is None:
node.lbound = 0.0
node.rbound = 1.0
node.frac = np.random.uniform(minfrac,maxfrac)
else:
if min(node.elements) == min(node.parent.elements):
#left of the tree
node.lbound = node.parent.lbound
node.rbound = node.parent.lbound + node.parent.frac*(node.parent.rbound - node.parent.lbound)
node.frac = np.random.uniform(minfrac,maxfrac)
else:
node.lbound = node.parent.lbound + node.parent.frac*(node.parent.rbound - node.parent.lbound)
node.rbound = node.parent.rbound
node.frac = np.random.uniform(minfrac,maxfrac)
return tree
def get_traverse_shallow_structure(traverse_fn, structure):
"""Generates a shallow structure from a `traverse_fn` and `structure`.
`traverse_fn` must accept any possible subtree of `structure` and return
a depth=1 structure containing `True` or `False` values, describing which
of the top-level subtrees may be traversed. It may also
return scalar `True` or `False` 'traversal is OK / not OK for all subtrees.'
Examples are available in the unit tests (nest_test.py).
Args:
traverse_fn: Function taking a substructure and returning either a scalar
`bool` (whether to traverse that substructure or not) or a depth=1 shallow
structure of the same type, describing which parts of the substructure to
traverse.
structure: The structure to traverse.
Returns:
A shallow structure containing python bools, which can be passed to
`map_up_to` and `flatten_up_to`.
Raises:
TypeError: if `traverse_fn` returns a sequence for a non-sequence input,
or a structure with depth higher than 1 for a sequence input,
or if any leaf values in the returned structure or scalar are not type
`bool`.
"""
def outer_traverse_fn(subtree):
res = traverse_fn(subtree)
if is_nested(res):
def inner_traverse_fn(do_traverse, subtree):
if do_traverse:
return dm_tree.traverse(outer_traverse_fn, subtree)
else:
return _FALSE_SENTINEL
return dm_tree.map_structure_up_to(res, inner_traverse_fn, res, subtree)
else:
return None if res else _FALSE_SENTINEL
return map_structure(lambda x: False if x is _FALSE_SENTINEL else True,
dm_tree.traverse(outer_traverse_fn, structure))
def run(args=None):
usage = "usage : %prog [options]"
parser = optparse.OptionParser(usage=usage)
parser.add_option("--test",action="store_true",dest="test",default=False)
# Optimizer
parser.add_option("--minibatch",dest="minibatch",type="int",default=30)
parser.add_option("--optimizer",dest="optimizer",type="string",
default="adagrad")
parser.add_option("--epochs",dest="epochs",type="int",default=50)
parser.add_option("--step",dest="step",type="float",default=1e-2)
parser.add_option("--middleDim",dest="middleDim",type="int",default=10)
parser.add_option("--outputDim",dest="outputDim",type="int",default=5)
parser.add_option("--wvecDim",dest="wvecDim",type="int",default=30)
parser.add_option("--outFile",dest="outFile",type="string",
default="models/test.bin")
parser.add_option("--inFile",dest="inFile",type="string",
default="models/test.bin")
parser.add_option("--data",dest="data",type="string",default="train")
parser.add_option("--model",dest="model",type="string",default="RNN")
(opts, args) = parser.parse_args(args)
# make this false if you dont care about your accuracies per epoch, makes things faster!
evaluate_accuracy_while_training = True
# Testing
if opts.test:
test(opts.inFile, opts.data, opts.model)
return
print "Loading data..."
train_accuracies = []
dev_accuracies = []
# load training data
trees = tr.load_trees(TRAIN_DATA_FILE)
opts.numWords = len(tr.load_word_to_index_map())
if (opts.model=='RNTN'):
nn = RNTN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch)
elif(opts.model=='RNN'):
nn = RNN(opts.wvecDim,opts.outputDim,opts.numWords,opts.minibatch)
elif(opts.model=='RNN2'):
nn = RNN2(opts.wvecDim,opts.middleDim,opts.outputDim,opts.numWords,opts.minibatch)
else:
raise '%s is not a valid neural network so far only RNTN, RNN, RNN2' % opts.model
nn.initParams()
sgd = optimizer.SGD(nn, alpha=opts.step, minibatch=opts.minibatch, optimizer=opts.optimizer)
dev_trees = tr.load_trees(DEV_DATA_FILE)
for e in range(opts.epochs):
start = time.time()
print "Running epoch %d" % e
sgd.run(trees)
end = time.time()
print "Time per epoch : %f" % (end-start)
# save the net to the output file
#f = open(opts.outFile, 'wb')
#pickle.dump(opts, f, -1)
#pickle.dump(sgd.costt, f, -1)
#pickle.dump(nn.stack, f, -1)
#np.save(f, nn.stack)
#f.close()
joblib.dump(opts, opts.outFile + "_opts")
joblib.dump(sgd.costt, opts.outFile + "_cost")
joblib.dump(nn.stack, opts.outFile + "_stack")
if evaluate_accuracy_while_training:
print "testing on training set..."
train_accuracies.append(test(opts.outFile, "train", opts.model, trees))
print "testing on dev set..."
dev_accuracies.append(test(opts.outFile, "dev", opts.model, dev_trees))
# clear the fprop flags in trees and dev_trees
for tree in trees:
tr.traverse(tree.root, func=tr.clear_fprop)
for tree in dev_trees:
tr.traverse(tree.root, func=tr.clear_fprop)
print "fprop in trees cleared"
if False: # don't do this for now
#if evaluate_accuracy_while_training:
#print train_accuracies
#print dev_accuracies
# Plot train/dev_accuracies here
x = range(opts.epochs)
figure(figsize=(6,4))
plot(x, train_accuracies, color='b', marker='o', linestyle='-', label="training")
plot(x, dev_accuracies, color='g', marker='o', linestyle='-', label="dev")
title("Accuracy vs num epochs.")
xlabel("Epochs")
ylabel("Accuracy")
#ylim(ymin=0, ymax=max(1.1*max(train_accuracies),3*min(train_accuracies)))
legend()
savefig("train_dev_acc.png")
def inner_traverse_fn(do_traverse, subtree):
if do_traverse:
return dm_tree.traverse(outer_traverse_fn, subtree)
else:
return _FALSE_SENTINEL
n5 = node.Node(1)
n5.payload = 'Loki'
n6 = node.Node(level)
n6.payload = 'Bergen'
# start a tree
tree = tree.Tree(node.maxLevel)
#print(tree)
print ('top free level = ' + str(tree.isFree(level)))
print("Adding n3")
if not tree.addNode(tree.topNode, n3):
print("exiting")
tree.traverse()
sys.exit()
print ('top free level = ' + str(tree.isFree(level)))
print("Adding n4")
if not tree.addNode(tree.topNode, n4):
print("exiting")
tree.traverse()
sys.exit()
print ('top free level = ' + str(tree.isFree(level)))
print("Adding n2")
if not tree.addNode(tree.topNode, n2):
print("exiting")
node2 = tree.treenode(state2)
node.subnodes.append(node1)
node.subnodes.append(node2)
else:
return
for n in node.subnodes:
shakeit(n)
possibility = 0
def printdata(data):
global possibility
possibility = possibility + 1
print "a_points:", data.a_points
print "b_points:", data.b_points
winner = "a" if data.a_points[-1] == 10 else "b"
print "winner is:", winner
if __name__ == "__main__":
global possibility
initstate = state.state()
rootnode = tree.treenode(initstate)
shakeit(rootnode)
tree.traverse(rootnode, printdata)
print "total possibility:", possibility
print "total instances:", state.state.instancecounter
def checkIntersection(self): tn = tree.traverse(self,set()) tn.root = True tree.unfold(tn) tn.rotateToFlat() return tn.checkIntersection()
def _convert_lists_to_tuples(structure: Any) -> Any: list_to_tuple_fn = lambda s: tuple(s) if isinstance(s, list) else s # Traverse depth-first, bottom-up return tree.traverse(list_to_tuple_fn, structure, top_down=False)
def checkIntersection(self):
tn = tree.traverse(self, set())
tn.root = True
tree.unfold(tn)
tn.rotateToFlat()
return tn.checkIntersection()
def main():
xprint("Hello!")
# Get current working directory and append exercise input file to it
cur_path = os.getcwd()
part = os.path.split(cur_path)[0]
# Get input file name and input file
user_in = sys.argv[1]
path = part + '/ex/' + user_in
# Initialize internal representation of ADF and its size
myfun.size = 0
myfun.arguments = []
# Read input file
with open(path, 'r') as c:
contents = c.readlines()
for line in contents:
if line[0] == 's':
a = Argument()
a.name = line[2:len(line) - 3]
a.sym = sympy.symbols(f"{a.name}")
a.dex = myfun.size
myfun.size += 1
myfun.arguments.append(a)
elif line[0:2] == 'ac':
for a in myfun.arguments:
if a.name == line[3:line.find(',')]:
a.ac = rewrite(line[line.find(',') + 1:(len(line) -
3)])
else:
print("Something's wrong with your input file.")
if myfun.pc:
print('Arguments in ADF:')
myfun.print_full_args(myfun.arguments)
print("-------------------")
# Read initial claim and choice of algorithm from command line
initial_claim = myfun.make_one(sys.argv[3], sys.argv[2])
alg = int(sys.argv[4])
xprint(f"v_0 = {initial_claim}")
# Start game
a_prime = myfun.check_info(initial_claim, myfun.size * 'u')[0]
n = tree.Root(initial_claim)
k = 0 # depth
winner = ''
# Get one set of minimal satisfiable interpretations (one for each a in A')
def get_m():
n.num, n.msats = msat_fun.find_new(n.data, a_prime, alg)
i = random.choice(range(n.num))
forward.msat = {}
for a in a_prime:
forward.msat[f'{a.name}'] = n.msats[f'{a.name}'][i]
n.msats[f'{a.name}'].remove(forward.msat[f'{a.name}'])
n.num -= 1
get_m()
# Get delta(v_0, mSAT_A') and put it as the first child node
update = forward.forward_step(initial_claim, a_prime)
n.add_child(update)
n = n.children[0]
k += 1
# Play the discussion game
while True:
xprint(f"v_{k} = {n.data}")
a_prime, contra, found = myfun.check_info(n.data, n.parent.data)
if contra:
# Check if the current node represents the initial claim
if type(n.parent) is tree.Root:
xprint(
"Initial claim already gives a contradiction, P loses game"
)
break
# Apply the backward move
xprint("Contradiction found, will apply backward move")
found_msat = False
# Loop until either a new mSAT is found, or we've backtracked to the initial claim
while not found_msat and type(n) is not tree.Root:
n = n.parent
k -= 1
xprint(f"\t Backtracked to v_{k} = {n.data}")
# v_u = {u} is not in the tree, so v_0.parent does not exist
if type(n) is tree.Root:
par = len(n.data) * 'u'
else:
par = n.parent.data
a_prime = myfun.check_info(v=n.data, oldv=par)[0]
# Try to an unused mSAT, if the list is empty they've all been used
if n.msats[f'{a_prime[0].name}']:
i = random.choice(range(n.num))
try_msat = {}
for a in a_prime:
try_msat[f'{a.name}'] = n.msats[f'{a.name}'][i]
n.msats[f'{a.name}'].remove(try_msat[f'{a.name}'])
n.num -= 1
found_msat = True
if not found_msat:
# We've backtracked to initial claim without finding an unused mSAT
xprint("P loses game")
break
else:
# Another mSAT is found, use it for the forward move and go to the new node
xprint("\t Found another msat!")
n.i += 1
forward.msat = try_msat
update = forward.forward_step(n.data, a_prime)
n.add_child(update)
n = n.children[n.i]
k += 1
elif found:
winner = n.data
xprint("Agreement found! P wins the game.")
break
else:
# Forward move
xprint(
"No contradiction or agreement found, will apply forward move")
get_m()
update = forward.forward_step(n.data, a_prime)
n.add_child(update)
n = n.children[0]
k += 1
# End of the game: print search tree, winning interpretation if it exists, and YES/NO
if myfun.pc:
print("-------------------")
print("Search tree:")
while type(n) is not tree.Root:
n = n.parent
tree.traverse(n, 0)
print("-------------------")
if winner != '':
string = ''
for j, w in enumerate(winner):
if w != 'u':
if string != '':
string = string + ','
else:
string = string + '{'
string = string + myfun.arguments[j].name + '->' + w
print("Interpretation: %s " % string + '}')
print("-------------------")
if winner != '':
print("YES")
else:
print("NO")
def test_make_action_immutable(self):
import gym
from types import MappingProxyType
# Test Box space.
space = gym.spaces.Box(low=-1.0,
high=1.0,
shape=(8, ),
dtype=np.float32)
action = space.sample()
action = make_action_immutable(action)
self.assertFalse(action.flags["WRITEABLE"])
# Test Discrete space.
# Nothing to be tested as sampled actions are integers
# and integers are immutable by nature.
# Test MultiDiscrete space.
space = gym.spaces.MultiDiscrete([3, 3, 3])
action = space.sample()
action = make_action_immutable(action)
self.assertFalse(action.flags["WRITEABLE"])
# Test MultiBinary space.
space = gym.spaces.MultiBinary([2, 2, 2])
action = space.sample()
action = make_action_immutable(action)
self.assertFalse(action.flags["WRITEABLE"])
# Test Tuple space.
space = gym.spaces.Tuple((
gym.spaces.Discrete(2),
gym.spaces.Box(low=-1.0, high=1.0, shape=(8, ), dtype=np.float32),
))
action = space.sample()
action = tree.traverse(make_action_immutable, action, top_down=False)
self.assertFalse(action[1].flags["WRITEABLE"])
# Test Dict space.
space = gym.spaces.Dict({
"a":
gym.spaces.Discrete(2),
"b":
gym.spaces.Box(low=-1.0, high=1.0, shape=(8, ), dtype=np.float32),
"c":
gym.spaces.Tuple((
gym.spaces.Discrete(2),
gym.spaces.Box(low=-1.0,
high=1.0,
shape=(8, ),
dtype=np.float32),
)),
})
action = space.sample()
action = tree.traverse(make_action_immutable, action, top_down=False)
def fail_fun(obj):
obj["a"] = 5
self.assertRaises(TypeError, fail_fun, action)
self.assertFalse(action["b"].flags["WRITEABLE"])
self.assertFalse(action["c"][1].flags["WRITEABLE"])
self.assertTrue(isinstance(action, MappingProxyType))
if user_city_input == "goodbye":
print " [ Goodbye ] "
tenkiba1_1 = "I heard you. Have a nice day! Goodbye."
print tenkiba1_1
print ""
print "* * * * * * * * * * * * * * * * * * * * * * * * *"
tts(tenkiba1_1)
sys.exit()
# break # exit the loop right away
# Step 1-2: parse user_city_input
result = ""
tokens = nltk.word_tokenize(user_city_input)
tagged = nltk.pos_tag(tokens)
chunk = nltk.chunk.ne_chunk(tagged)
traverse(chunk, result)
if lo == "":
location = default_city
else:
location = lo
# Step 2-0: user_ask_input --
tenkiba2_1 = "Got it. So,"
print "--Got it. So,"
tts(tenkiba2_1)
tenkiba2_2 = location
print "[ " + (tenkiba2_2) + " ]"
tts(tenkiba2_2)