objfun_file.write('f=%s;' % make_matlab_min_func(root))
objfun_file.close()
num_vars = len(root.get_all_predicates())
cmd_file = open('do_opt.m', 'w')
cmd_file.write('x0=[%s];\n' % ','.join(['0'] * num_vars))
cmd_file.write('lb=[%s];\n' % ','.join(['0'] * num_vars))
cmd_file.write('[x,fval] = fmincon(@objfun,x0,[],[],[],[],lb);\n')
cmd_file.write('csvwrite(\'result.csv\', x);\n')
cmd_file.close()
def run_fmincon(root):
make_files(root)
#server = 'scs.itd.umich.edu'
output_file = 'result.csv'
matlab_cmd = 'matlab -nojvm -nodisplay -r \'do_opt;quit\' &> /dev/null'
#os.system('scp objfun.m %s:%s/' % (server, d))
#os.system('scp do_opt.m %s:%s/' % (server, d))
#os.system('ssh %s "cd %s; %s"' % (server, d, matlab_cmd))
#os.system('scp %s:%s/%s .' % (server, d, output_file))
assert os.system(matlab_cmd) == 0, "Matlab run failed"
result = [float(s) for s in open(output_file).read().split(',')]
return result
if __name__ == '__main__':
from treegen import TreeGen
tgen = TreeGen()
root = tgen.generate_random()
result = run_fmincon(root)
print result
def test_goal_dist_heuristic():
min_len = 4
max_len = 10
indicators = dict((d, ['I%d' % d]) for d in range(1, max_len))
indicator_set = set(reduce(lambda x,y: x+y, indicators.values()))
tree_gen = TreeGen()
tree_gen.min_branch_len = min_len
tree_gen.max_branch_len = max_len
src_root = tree_gen.generate_indicators(indicators)
src_root.make_graphviz(open('src_tree.dot','w'))
tgt_root = src_root.deep_copy()
# change all predicates in the target to be lowercase version of source
tgt_root.map(state_preds_lower)
tgt_root.make_graphviz(open('tgt_tree.dot','w'))
src_preds = list(src_root.get_all_predicates())
num_preds = len(src_preds)
src_preds.sort()
src_weights = fmincon.run_fmincon(src_root)
# the heuristic for the source
hs = heuristic.Heuristic(dict(zip(src_preds, src_weights)))
#hs = heuristic.Heuristic(indicators)
src_search = AStar(src_root, lambda x: hs(x.preds))
iter = src_search.gen
iterations = 0
try:
while not iter.next():
iterations += 1
except StopIteration:
src_iters = iterations
non_ind_preds = [p for p in src_preds if p not in indicator_set]
data = []
for n_ind, n_rest, mapping in ind_mapping_gen(indicator_set, non_ind_preds, num_preds, 100):
#for n_matches, mapping in reg_mapping_gen(src_preds, 100):
#print "The mapping is ..."
#for s,t in mapping.items():
# print s, t
tgt_preds2weights = {}
for p,w in zip(src_preds, src_weights):
if p in mapping:
# we assign the weight of the source predicate to the target
tgt_preds2weights[mapping[p]] = w
ht = heuristic.Heuristic(tgt_preds2weights)
tgt_search = AStar(tgt_root, lambda x: ht(x.preds))
iter = tgt_search.gen
iterations = 0
try:
while not iter.next():
iterations += 1
except StopIteration:
tgt_iters = iterations
#print 'The target problem takes %d iterations' % tgt_iters
#print 'Solution is', [''.join(s.preds) for s in tgt_search.solution]
data.append((n_ind, n_rest, tgt_iters))
#data.append((n_matches, tgt_iters))
n_preds = len(src_preds)
return (n_preds, src_iters, data)
def test_goal_dist_heuristic():
min_len = 4
max_len = 10
indicators = dict((d, ['I%d' % d]) for d in range(1, max_len))
indicator_set = set(reduce(lambda x, y: x + y, indicators.values()))
tree_gen = TreeGen()
tree_gen.min_branch_len = min_len
tree_gen.max_branch_len = max_len
src_root = tree_gen.generate_indicators(indicators)
src_root.make_graphviz(open('src_tree.dot', 'w'))
tgt_root = src_root.deep_copy()
# change all predicates in the target to be lowercase version of source
tgt_root.map(state_preds_lower)
tgt_root.make_graphviz(open('tgt_tree.dot', 'w'))
src_preds = list(src_root.get_all_predicates())
num_preds = len(src_preds)
src_preds.sort()
src_weights = fmincon.run_fmincon(src_root)
# the heuristic for the source
hs = heuristic.Heuristic(dict(zip(src_preds, src_weights)))
#hs = heuristic.Heuristic(indicators)
src_search = AStar(src_root, lambda x: hs(x.preds))
iter = src_search.gen
iterations = 0
try:
while not iter.next():
iterations += 1
except StopIteration:
src_iters = iterations
non_ind_preds = [p for p in src_preds if p not in indicator_set]
data = []
for n_ind, n_rest, mapping in ind_mapping_gen(indicator_set, non_ind_preds,
num_preds, 100):
#for n_matches, mapping in reg_mapping_gen(src_preds, 100):
#print "The mapping is ..."
#for s,t in mapping.items():
# print s, t
tgt_preds2weights = {}
for p, w in zip(src_preds, src_weights):
if p in mapping:
# we assign the weight of the source predicate to the target
tgt_preds2weights[mapping[p]] = w
ht = heuristic.Heuristic(tgt_preds2weights)
tgt_search = AStar(tgt_root, lambda x: ht(x.preds))
iter = tgt_search.gen
iterations = 0
try:
while not iter.next():
iterations += 1
except StopIteration:
tgt_iters = iterations
#print 'The target problem takes %d iterations' % tgt_iters
#print 'Solution is', [''.join(s.preds) for s in tgt_search.solution]
data.append((n_ind, n_rest, tgt_iters))
#data.append((n_matches, tgt_iters))
n_preds = len(src_preds)
return (n_preds, src_iters, data)
def __init__(self, gen_method='generate_random()'):
self.tree_gen = TreeGen()
self.gen_method = gen_method
self.preserve_preds = set()
class TestEnviron:
def __init__(self, gen_method='generate_random()'):
self.tree_gen = TreeGen()
self.gen_method = gen_method
self.preserve_preds = set()
def gen_tree(self):
return eval('self.tree_gen.%s' % self.gen_method)
def gen_contradict(self):
ind_map = dict((d, ['I%d' % d]) for d in range(0, 100))
indicators = ['I%d' % d for d in range(0, 100)]
src_root, contradict_preds, mapped = self.tree_gen.generate_contradictory_indicators(
ind_map)
src_preds = src_root.get_all_predicates()
src_rs = RuleSet([], src_root.get_all_states(), src_preds)
tgt_root = src_root.deep_copy()
tgt_root.map(state_preds_lower)
tgt_preds = tgt_root.get_all_predicates()
tgt_rs = RuleSet([], tgt_root.get_all_states(), tgt_preds)
# the source rules will always be split on some permutation of the 4
# predicates which are present at every level of the tree. The target
# rules will be split on mapped1 or mapped2, alternating by level
src_partition = set(contradict_preds + indicators)
tgt_partition = set([p.lower() for p in mapped] + indicators)
pdb.set_trace()
#tgt_partition2 = set([p.lower() for p in mapped2] + indicators)
self.__gen_contradict_rec(src_root, src_preds, tgt_root, tgt_preds,
src_partition, tgt_partition, src_rs, tgt_rs)
src_to_preserve = set(indicators) | src_partition | mapped
tgt_to_preserve = set(p.lower() for p in src_to_preserve)
src_rs.minimize_rule_conds(src_to_preserve)
#src_rs.minimize()
tgt_rs.minimize_rule_conds(tgt_to_preserve)
#tgt_rs.minimize()
return src_root, src_rs, tgt_root, tgt_rs
def __gen_contradict_rec(self, src_state, src_preds, tgt_state, tgt_preds,
src_part, tgt_part, src_rs, tgt_rs):
src_max_rules = src_state.make_max_rules(src_preds)
for r in src_max_rules:
if r.is_goal_rule():
src_rs.rules.append(r)
else:
rules = src_rs.split_rule(r, src_part)
if rules is None:
src_rs.rules.append(r)
print 'No Split'
else:
src_rs.rules.extend(rules)
tgt_max_rules = tgt_state.make_max_rules(tgt_preds)
for r in tgt_max_rules:
if r.is_goal_rule():
tgt_rs.rules.append(r)
else:
rules = tgt_rs.split_rule(r, tgt_part)
if rules is None:
tgt_rs.rules.append(r)
print 'No Split'
else:
tgt_rs.rules.extend(rules)
for a in src_state.get_actions():
src_c = src_state.get_child(a)
tgt_c = tgt_state.get_child(a)
if not src_c.is_goal():
assert not tgt_c.is_goal()
self.__gen_contradict_rec(src_c, src_preds, tgt_c, tgt_preds,
src_part, tgt_part, src_rs, tgt_rs)
def gen_src_tgt_split(self, split_prob):
src_root = self.gen_tree()
src_rs = gen_min(src_root, self.tree_gen.predicates,
self.preserve_preds)
tgt_root = src_root.deep_copy()
# change all predicates in the target to be lowercase version of source
tgt_root.map(state_preds_lower)
preds_lower = [p.lower() for p in self.tree_gen.predicates]
tgt_rs = gen_split(tgt_root, preds_lower, split_prob,
set(x.lower() for x in self.preserve_preds))
return src_root, src_rs, tgt_root, tgt_rs
def test_heuristic_gen(self):
root = self.gen_tree()
pvar = pred_variance(root)
max_var = 1.5 * max(pvar.values())
pvar1 = dict((p, max_var - v) for p, v in pvar.items())
pvar1s = sum(pvar1.values())
weights = dict((p, root.num_occurrences(p) * v / pvar1s)
for p, v in pvar1.items())
for p, w in weights.items():
print '%s: %f' % (p, w)
preds = list(root.get_all_predicates())
preds.sort()
weights = fmincon.run_fmincon(root)
h = heuristic.Heuristic(dict(zip(preds, weights)))
root.make_graphviz(open('temp.dot', 'w'))
search = AStar(root, lambda x: h(x.preds))
iter = search.gen
iterations = 0
try:
while not iter.next():
iterations += 1
except StopIteration:
print iterations
print[''.join(s.preds) for s in search.solution]
def test_all(self, nbins, split_prob):
#src_root, src_rs, tgt_root, tgt_rs = self.gen_src_tgt_split(split_prob)
src_root, src_rs, tgt_root, tgt_rs = self.gen_contradict()
src_file = 'source.kif'
tgt_file = 'target.kif'
src_rs.make_kif(src_root.preds, open(src_file, 'w'))
tgt_rs.make_kif(tgt_root.preds, open(tgt_file, 'w'))
src_preds = list(src_root.get_all_predicates())
src_preds.sort()
src_weights = fmincon.run_fmincon(src_root)
# the heuristic for the source
hs = heuristic.Heuristic(dict(zip(src_preds, src_weights)))
src_search = AStar(src_root, lambda x: hs(x.preds))
iter = src_search.gen
iterations = 0
try:
while not iter.next():
iterations += 1
except StopIteration:
src_iters = iterations
print 'The source problem takes %d iterations' % src_iters
print 'Solution is', [''.join(s.preds) for s in src_search.solution]
src_bins = dict((Predicate(p, Sentence.STATE, []), b)
for p, b in make_bins(src_preds, nbins).items())
tgt_bins = dict(
(Predicate(p.get_name().lower(), Sentence.STATE, []), b)
for p, b in src_bins.items())
mapping = rule_mapper2.map_kifs(src_file, tgt_file, src_bins, tgt_bins)
str_mapping = dict(
(s.get_name(), t.get_name()) for s, t in mapping.items())
print "The mapping is ..."
for s, t in str_mapping.items():
print s, t
num_correct = 0
num_correct_inds = 0
for s, t in str_mapping.items():
if s.lower() == t:
num_correct += 1
if s[0] == 'I' and s[-1] in '0123456789':
num_correct_inds += 1
print "Number of correct predicates:", num_correct
tgt_preds = []
tgt_weights = []
for p, w in zip(src_preds, src_weights):
if p in str_mapping:
# we assign the weight of the source predicate to the target
tgt_preds.append(str_mapping[p])
tgt_weights.append(w)
ht = heuristic.Heuristic(dict(zip(tgt_preds, tgt_weights)))
tgt_search = AStar(tgt_root, lambda x: ht(x.preds))
iter = tgt_search.gen
try:
while not iter.next():
iterations += 1
except StopIteration:
tgt_iters = iterations
print 'The target problem takes %d iterations' % tgt_iters
print 'Solution is', [''.join(s.preds) for s in tgt_search.solution]
os.remove(src_file)
os.remove(tgt_file)
ret_labels = [
'len(src_preds)', 'num_correct', 'num_correct_inds', 'src_iters',
'tgt_iters'
]
ret_vals = eval('[%s]' % ','.join(ret_labels))
return (ret_labels, ret_vals)
def __init__(self, gen_method='generate_random()'): self.tree_gen = TreeGen() self.gen_method = gen_method self.preserve_preds = set()
class TestEnviron:
def __init__(self, gen_method='generate_random()'):
self.tree_gen = TreeGen()
self.gen_method = gen_method
self.preserve_preds = set()
def gen_tree(self):
return eval('self.tree_gen.%s' % self.gen_method)
def gen_contradict(self):
ind_map = dict((d, ['I%d' % d]) for d in range(0, 100))
indicators = ['I%d' % d for d in range(0, 100)]
src_root, contradict_preds, mapped = self.tree_gen.generate_contradictory_indicators(ind_map)
src_preds = src_root.get_all_predicates()
src_rs = RuleSet([], src_root.get_all_states(), src_preds)
tgt_root = src_root.deep_copy()
tgt_root.map(state_preds_lower)
tgt_preds = tgt_root.get_all_predicates()
tgt_rs = RuleSet([], tgt_root.get_all_states(), tgt_preds)
# the source rules will always be split on some permutation of the 4
# predicates which are present at every level of the tree. The target
# rules will be split on mapped1 or mapped2, alternating by level
src_partition = set(contradict_preds + indicators)
tgt_partition = set([p.lower() for p in mapped] + indicators)
pdb.set_trace()
#tgt_partition2 = set([p.lower() for p in mapped2] + indicators)
self.__gen_contradict_rec(src_root, src_preds, tgt_root, tgt_preds, src_partition, tgt_partition, src_rs, tgt_rs)
src_to_preserve = set(indicators) | src_partition | mapped
tgt_to_preserve = set(p.lower() for p in src_to_preserve)
src_rs.minimize_rule_conds(src_to_preserve)
#src_rs.minimize()
tgt_rs.minimize_rule_conds(tgt_to_preserve)
#tgt_rs.minimize()
return src_root, src_rs, tgt_root, tgt_rs
def __gen_contradict_rec(self, src_state, src_preds, tgt_state, tgt_preds, src_part, tgt_part, src_rs, tgt_rs):
src_max_rules = src_state.make_max_rules(src_preds)
for r in src_max_rules:
if r.is_goal_rule():
src_rs.rules.append(r)
else:
rules = src_rs.split_rule(r, src_part)
if rules is None:
src_rs.rules.append(r)
print 'No Split'
else:
src_rs.rules.extend(rules)
tgt_max_rules = tgt_state.make_max_rules(tgt_preds)
for r in tgt_max_rules:
if r.is_goal_rule():
tgt_rs.rules.append(r)
else:
rules = tgt_rs.split_rule(r, tgt_part)
if rules is None:
tgt_rs.rules.append(r)
print 'No Split'
else:
tgt_rs.rules.extend(rules)
for a in src_state.get_actions():
src_c = src_state.get_child(a)
tgt_c = tgt_state.get_child(a)
if not src_c.is_goal():
assert not tgt_c.is_goal()
self.__gen_contradict_rec(src_c, src_preds, tgt_c, tgt_preds, src_part, tgt_part, src_rs, tgt_rs)
def gen_src_tgt_split(self, split_prob):
src_root = self.gen_tree()
src_rs = gen_min(src_root, self.tree_gen.predicates, self.preserve_preds)
tgt_root = src_root.deep_copy()
# change all predicates in the target to be lowercase version of source
tgt_root.map(state_preds_lower)
preds_lower = [p.lower() for p in self.tree_gen.predicates]
tgt_rs = gen_split(tgt_root, preds_lower, split_prob, set(x.lower() for x in self.preserve_preds))
return src_root, src_rs, tgt_root, tgt_rs
def test_heuristic_gen(self):
root = self.gen_tree()
pvar = pred_variance(root)
max_var = 1.5 * max(pvar.values())
pvar1 = dict((p, max_var - v) for p, v in pvar.items())
pvar1s = sum(pvar1.values())
weights = dict((p, root.num_occurrences(p) * v / pvar1s) for p, v in pvar1.items())
for p, w in weights.items():
print '%s: %f' % (p, w)
preds = list(root.get_all_predicates())
preds.sort()
weights = fmincon.run_fmincon(root)
h = heuristic.Heuristic(dict(zip(preds,weights)))
root.make_graphviz(open('temp.dot', 'w'))
search = AStar(root, lambda x: h(x.preds))
iter = search.gen
iterations = 0
try:
while not iter.next():
iterations += 1
except StopIteration:
print iterations
print [''.join(s.preds) for s in search.solution]
def test_all(self, nbins, split_prob):
#src_root, src_rs, tgt_root, tgt_rs = self.gen_src_tgt_split(split_prob)
src_root, src_rs, tgt_root, tgt_rs = self.gen_contradict()
src_file = 'source.kif'
tgt_file = 'target.kif'
src_rs.make_kif(src_root.preds, open(src_file,'w'))
tgt_rs.make_kif(tgt_root.preds, open(tgt_file,'w'))
src_preds = list(src_root.get_all_predicates())
src_preds.sort()
src_weights = fmincon.run_fmincon(src_root)
# the heuristic for the source
hs = heuristic.Heuristic(dict(zip(src_preds, src_weights)))
src_search = AStar(src_root, lambda x: hs(x.preds))
iter = src_search.gen
iterations = 0
try:
while not iter.next():
iterations += 1
except StopIteration:
src_iters = iterations
print 'The source problem takes %d iterations' % src_iters
print 'Solution is', [''.join(s.preds) for s in src_search.solution]
src_bins = dict((Predicate(p, Sentence.STATE, []), b) for p,b in make_bins(src_preds, nbins).items())
tgt_bins = dict((Predicate(p.get_name().lower(), Sentence.STATE, []), b) for p, b in src_bins.items())
mapping = rule_mapper2.map_kifs(src_file, tgt_file, src_bins, tgt_bins)
str_mapping = dict((s.get_name(), t.get_name()) for s,t in mapping.items())
print "The mapping is ..."
for s,t in str_mapping.items():
print s, t
num_correct = 0
num_correct_inds = 0
for s, t in str_mapping.items():
if s.lower() == t:
num_correct += 1
if s[0] == 'I' and s[-1] in '0123456789':
num_correct_inds += 1
print "Number of correct predicates:", num_correct
tgt_preds = []
tgt_weights = []
for p,w in zip(src_preds, src_weights):
if p in str_mapping:
# we assign the weight of the source predicate to the target
tgt_preds.append(str_mapping[p])
tgt_weights.append(w)
ht = heuristic.Heuristic(dict(zip(tgt_preds, tgt_weights)))
tgt_search = AStar(tgt_root, lambda x: ht(x.preds))
iter = tgt_search.gen
try:
while not iter.next():
iterations += 1
except StopIteration:
tgt_iters = iterations
print 'The target problem takes %d iterations' % tgt_iters
print 'Solution is', [''.join(s.preds) for s in tgt_search.solution]
os.remove(src_file)
os.remove(tgt_file)
ret_labels = ['len(src_preds)', 'num_correct', 'num_correct_inds', 'src_iters', 'tgt_iters' ]
ret_vals = eval('[%s]' % ','.join(ret_labels))
return (ret_labels, ret_vals)
