class Conversation:
def __init__(self, tweet):
self.root_tweet = tweet
self.conversation_tree = Tree()
self.conversation_tree.create_node(tweet, tweet)
self.depth = int()
self.tweets_id = list()
self.tweets_id.append(tweet)
self.width = int()
def add_replay(self, tweet, parent_tweet):
self.conversation_tree.create_node(tweet, tweet, parent=parent_tweet)
self.tweets_id.append(tweet)
def set_depth(self):
self.depth = self.conversation_tree.depth() + 1
def find_depth(self):
return self.depth
def get_tweets_id(self):
return self.tweets_id
def set_width(self):
self.width = len(self.tweets_id)
def find_width(self):
return self.width
def get_conversation_tree(self):
return self.conversation_tree
class EntityTree(object):
def __init__(self):
self.tree = Tree()
def create_ent_tree(self,
ent_dct,
cls_name_map,
parent=None,
cls_data_map={}):
'''递归，依据遍历标签间层级关系生成树'''
tag = ent_dct['LabelName']
name = cls_name_map.get(tag)
data = cls_data_map.get(tag, 0)
if not name:
print("Error: tag %s not found in cls_name_map!" % tag)
sys.exit(-1)
if DEBUG:
print("# of tree nodes: %d, tree height: %d, # of leaves: %d" %
(len(self.tree), self.tree.depth(), len(self.tree.leaves())))
nd = self.tree.create_node(tag=name, parent=parent, data=data)
if 'Subcategory' in ent_dct.keys():
for dct in ent_dct['Subcategory']:
self.create_ent_tree(dct,
cls_name_map,
parent=nd.identifier,
cls_data_map=cls_data_map)
def render_vocabulary_word(tree: Tree, node: Node) -> str:
assert tree.parent(node.identifier)
assert tree.parent(node.identifier).identifier.lower() == 'vocabulary'
assert node.data
depth = tree.depth(node) + 1
word = node.tag
text = ('*' * depth + ' ' + word + 2 * '\n')
if 'audio' in node.data and node.data['audio']:
text += render_link(node.data['audio'], 'play') + '\n'
if 'book_examples' in node.data:
for example in node.data['book_examples']:
text += render_quote(example)
text += '\n'
if 'definitions' in node.data:
text += 'Definitions\n'
for i, definition in enumerate(node.data['definitions']):
text += str(i + 1) + '. ' + definition['definition']
if 'synonyms' in definition:
text += ' /'
for n, synonym in enumerate(definition['synonyms']):
text += synonym + ', '
if n == 2:
break
text = text[:-2]
text += '/'
text += '\n'
text += '\n'
return text
def sum_orbits(orbital_tree: Tree) -> int: sum_total_orbits = 0 for node in orbital_tree.all_nodes(): sum_total_orbits += orbital_tree.depth(node) return sum_total_orbits
def day7():
nodes = []
node_children = []
tmp_nodes = {}
tree = {}
leafNodes = []
with open('day7.txt') as f:
lines = f.read().splitlines()
for l in lines:
node = getNodeProperties(l)
tmp_nodes.update(node)
if "->" in l:
nodes.append(l.split(' ')[0])
node_children.append(l.split('->')[1])
node_name = l.split(' ')[0]
node_weight = l.split('(')[1].split(')')[0]
#tmp_nodes[l.split(' ')[0]] = {}
else:
node_name = l.split(' ')[0]
node_weight = l.split('(')[1].split(')')[0]
my_tree = Tree()
my_dict_tree = {}
root = findRoot(node_children,nodes)
print("Root",root)
# Determine Towers
print("weight",add_weights(root,tmp_nodes))
check_weights(root,tmp_nodes)
tree_dict = {}
build_tree(tree_dict,root,tmp_nodes,None,my_tree)
my_tree.show()
#print(tree_dict)
print(my_tree.depth())
depth = my_tree.depth()
#print(tree_dict)
add_balance(tree_dict,my_tree,my_tree.root)
#print(tree_dict)
check_balance(tree_dict,my_tree,my_tree.root,False)
class AcquisitionChain(object):
def __init__(self):
self._tree = Tree()
self._root_node = self._tree.create_node("acquisition chain", "root")
self._device_to_node = dict()
def add(self, master, slave):
slave_node = self._tree.get_node(slave)
master_node = self._tree.get_node(master)
if slave_node is not None and isinstance(slave, AcquisitionDevice):
if slave_node.bpointer is not self._root_node and master_node is not slave_node.bpointer:
raise RuntimeError(
"Cannot add acquisition device %s to multiple masters, current master is %s"
% (slave, slave_node._bpointer)
)
else: # user error, multiple add, ignore for now
return
if master_node is None:
master_node = self._tree.create_node(tag=master.name, identifier=master, parent="root")
if slave_node is None:
slave_node = self._tree.create_node(tag=slave.name, identifier=slave, parent=master)
else:
self._tree.move_node(slave_node, master_node)
def _execute(self, func_name):
tasks = list()
prev_level = None
for dev in reversed(list(self._tree.expand_tree(mode=Tree.WIDTH))[1:]):
node = self._tree.get_node(dev)
level = self._tree.depth(node)
if prev_level != level:
gevent.joinall(tasks)
tasks = list()
func = getattr(dev, func_name)
tasks.append(gevent.spawn(func))
gevent.joinall(tasks)
def prepare(self, dm, scan_info):
# self._devices_tree = self._get_devices_tree()
for master in (x for x in self._tree.expand_tree() if isinstance(x, AcquisitionMaster)):
del master.slaves[:]
for dev in self._tree.get_node(master).fpointer:
master.slaves.append(dev)
dm_prepare_task = gevent.spawn(dm.prepare, scan_info, self._tree)
self._execute("_prepare")
dm_prepare_task.join()
def start(self):
self._execute("_start")
for acq_dev in (x for x in self._tree.expand_tree() if isinstance(x, AcquisitionDevice)):
acq_dev.wait_reading()
dispatcher.send("end", acq_dev)
def find_deepest_child(data):
tree = Tree()
root = tree.create_node("root", "root")
tree = build_tree(data=data, tree=tree, parent=root)
res = tree.paths_to_leaves()
depth = tree.depth()
for path in res:
if len(path) == depth + 1:
return path[-1]
break
def render_org_tree(tree: Tree, node: Node, payload='') -> str:
parent = tree.parent(node.identifier)
if parent and parent.identifier.lower() == 'vocabulary':
payload += render_vocabulary_word(tree, node)
else:
depth = tree.depth(node) + 1
payload += ('*' * depth + ' ' + node.tag + '\n')
for child in tree.children(node.identifier):
payload += render_org_tree(tree, tree[child.identifier])
return payload
def conversation_regarding_language(cursor):
conversation_amount = postgres_queries.find_conversation_number(cursor)
conversation_list = list()
depth_dict = dict()
depth_dict_long = dict()
depth_dict_short = dict()
number_of_tweets_dict = dict()
test_i = 0
for i in range(0, conversation_amount + 1, 1):
conversation_tree = Tree()
conversation = postgres_queries.find_conversation(i, cursor)
test_i += len(conversation)
for tweet in conversation:
if tweet[2] is None and tweet[5] is True:
conversation_tree.create_node(tweet[0], tweet[0])
tweets_in_conversation = list()
build_conversation_lang(tweet[0], conversation, conversation_tree, tweets_in_conversation)
depth = conversation_tree.depth() + 1
number_of_tweets = len(conversation_tree.all_nodes())
#short/long
if number_of_tweets >=20:
if depth in depth_dict_long:
depth_dict_long[depth] += 1
else:
depth_dict_long[depth] = 1
else:
if depth in depth_dict_short:
depth_dict_short[depth] += 1
else:
depth_dict_short[depth] = 1
if number_of_tweets in number_of_tweets_dict:
number_of_tweets_dict[number_of_tweets] += 1
else:
number_of_tweets_dict[number_of_tweets] = 1
if depth in depth_dict:
depth_dict[depth] += 1
else:
depth_dict[depth] = 1
# check if conversation_tree is null- dont add
if len(conversation_tree.all_nodes())!=0:
conversation_list.append(conversation_tree)
# number = 0
new_tweet_list_id = list()
for con in conversation_list:
nodes = con.all_nodes()
for node in nodes:
new_tweet_list_id.append(int(node.tag))
# number += len(con.all_nodes())
# print len(new_tweet_list_id)
# for tweet_id in new_tweet_list_id:
# print tweet_id
return new_tweet_list_id, conversation_list
def types_of_conversation():
conversation_amount = postgres_queries.find_annotated_conversation_number()
conversation_list = list()
depth_dict = dict()
depth_dict_long = dict()
depth_dict_short = dict()
number_of_tweets_dict = dict()
for i in range (0, conversation_amount + 1, 1):
conversation_tree = Tree()
converastion = postgres_queries.find_conversation(i)
for tweet in converastion:
if tweet[1] is None:
conversation_tree.create_node(tweet[0], tweet[0])
build_conversation(tweet[0], converastion, conversation_tree)
depth = conversation_tree.depth() + 1
number_of_tweets = len(conversation_tree.all_nodes())
#short/long
if number_of_tweets >=20:
if depth in depth_dict_long:
depth_dict_long[depth] += 1
else:
depth_dict_long[depth] = 1
else:
if depth in depth_dict_short:
depth_dict_short[depth] += 1
else:
depth_dict_short[depth] = 1
if number_of_tweets in number_of_tweets_dict:
number_of_tweets_dict[number_of_tweets] += 1
else:
number_of_tweets_dict[number_of_tweets] = 1
if depth in depth_dict:
depth_dict[depth] += 1
else:
depth_dict[depth] = 1
conversation_list.append(conversation_tree)
#print depth_dict
print 'Depth of a conversation'
for depth, count in depth_dict.iteritems():
print depth, '\t', count
print 'Number of tweets in a conversation'
for number, count in number_of_tweets_dict.iteritems():
print number, '\t', count
print 'Depth of a long conversation'
for depth, count in depth_dict_long.iteritems():
print depth, '\t', count
print 'Depth of a short conversation'
for depth, count in depth_dict_short.iteritems():
print depth, '\t', count
return conversation_list
def get_metadata(node: Node, tree: Tree, *, \
include_count=True,
include_attrs=False,
include_depth=True) -> Dict:
data = {'id': node.identifier, 'tag': node.tag}
if include_count:
data['childrenCount'] = len(tree.children(node.identifier) or [])
if include_depth:
data['depth'] = tree.depth(node)
if include_attrs:
data.update(get_attributes(node))
return data
def conversation_regarding_language(): # with width and depth
conversation_amount = postgres_queries.find_annotated_conversation_number()
conversation_list = list()
depth_dict = dict()
depth_dict_long = dict()
depth_dict_short = dict()
number_of_tweets_dict = dict()
test_i = 0
for i in range(0, conversation_amount + 1, 1):
conversation_tree = Tree()
converastion = postgres_queries.find_conversation(i)
test_i += len(converastion)
for tweet in converastion:
if tweet[1] is None and tweet[5] is True:
conversation_tree.create_node(tweet[0], tweet[0])
build_conversation_lang(tweet[0], converastion, conversation_tree)
depth = conversation_tree.depth() + 1
number_of_tweets = len(conversation_tree.all_nodes())
#short/long
if number_of_tweets >=20:
if depth in depth_dict_long:
depth_dict_long[depth] += 1
else:
depth_dict_long[depth] = 1
else:
if depth in depth_dict_short:
depth_dict_short[depth] += 1
else:
depth_dict_short[depth] = 1
if number_of_tweets in number_of_tweets_dict:
number_of_tweets_dict[number_of_tweets] += 1
else:
number_of_tweets_dict[number_of_tweets] = 1
if depth in depth_dict:
depth_dict[depth] += 1
else:
depth_dict[depth] = 1
conversation_list.append(conversation_tree)
number = 0
new_tweet_list_id = list()
for con in conversation_list:
nodes = con.all_nodes()
for node in nodes:
new_tweet_list_id.append(node.tag)
number += len(con.all_nodes())
return new_tweet_list_id, conversation_list
def solve(alpha, beta, k):
# Maain initializers
solutions = []
tree = Tree()
# Parents starts with root value aka empty string.
# We need to ensure the depth does not exceed k due to complexity issues.
parents = ['']
tree.create_node(tag='', identifier='')
while len(parents) > 0:
item = parents.pop()
for i in range(0, len(alpha)):
# Ensure we do not exceed the depth for a single node.
if tree.depth(item) == k:
break
split = item.split()
newAlpha = ""
newBeta = ""
# Try to retrieve the parent node string and append.
# Otherwise node is root.
if len(split) > 1:
newAlpha = split[0] + alpha[i]
newBeta = split[1] + beta[i]
else:
newAlpha = alpha[i]
newBeta = beta[i]
# Check to see if the node is valid.
# The node will be valid if there is a prefix.
if is_valid(newAlpha, newBeta):
node = newAlpha + ' ' + newBeta
# Check to see if node is a solution.
if is_solution(newAlpha, newBeta):
solutions.append(node)
# Create tree node since it is valid.
tree.create_node(tag=node, identifier=node, parent=item)
parents.append(node)
# Will leave only valid solutions.
return solutions
def generatePool(N, maxDepth, tree, rule_table, debug=False):
# Collecting all non-terminal nodes
nodeIDs = []
for nodeID in tree.expand_tree(mode=Tree.DEPTH):
nodeType = tree[nodeID].data
if isinstance(nodeType, PiRL.DataStructures.Token.NonTerminal):
nodeIDs.append(nodeID)
neighbours = []
for _ in range(N):
# Making a deep copy
newTree = Tree(tree=tree, deep=True)
# Selecting a random non-terminal to be replaced
nodeToReplaceID = random.choice(nodeIDs)
# Generating a new subtree
newSubTree = getProgTree(newTree[nodeToReplaceID].data, maxDepth - newTree.depth(nodeToReplaceID))
# Replacing subtree
try:
newTree.replace_node(newTree[nodeToReplaceID].predecessor(newTree.identifier), nodeToReplaceID, newSubTree, deep=False)
except Exception as e:
# traceback.print_exc()
if debug:
print("Root node replaced")
newTree = newSubTree
if debug:
print("Generated neighbour:", end=' ')
# newTree.show(data_property='str')
for leaf in newTree.leaves():
print(leaf.data.name, end=' ')
print("\n")
neighbours.append(newTree)
return neighbours
kraken_data['num_reads_clade'] = int(line[1])
kraken_data['num_reads_taxon'] = int(line[2])
kraken_data['rank_code'] = line[3]
kraken_data['ncbi_taxonomy_id'] = line[4]
kraken_data['name'] = line[5]
kraken_data['depth'] = calculate_depth(kraken_data['name'])
kraken_data['name'] = kraken_data['name'].lstrip(' ')
if kraken_data['name'] == "unclassified":
unclassified = Node(tag = kraken_data['name'],
identifier = kraken_data['ncbi_taxonomy_id'],
data = kraken_data)
elif kraken_data['name'] == "root":
add_node(tree, None, kraken_data)
previous = tree.get_node(kraken_data['ncbi_taxonomy_id'])
elif kraken_data['depth'] > tree.depth(previous):
add_node(tree, previous, kraken_data)
previous = tree.get_node(kraken_data['ncbi_taxonomy_id'])
elif kraken_data['depth'] == tree.depth(previous):
add_node(tree, tree.parent(previous.identifier), kraken_data)
previous = tree.get_node(kraken_data['ncbi_taxonomy_id'])
elif kraken_data['depth'] < tree.depth(previous):
previous_search = previous
while(tree.depth(previous_search) > kraken_data['depth']):
previous_search = tree.parent(previous_search.identifier)
add_node(tree, tree.parent(previous_search.identifier), kraken_data)
previous = tree.get_node(kraken_data['ncbi_taxonomy_id'])
tree_dict = tree.to_dict(with_data=True)
def transform(tree_dict):
def depth_cost(G: nx.Graph, T: tl.Tree):
return T.depth()
class MonteCarlo:
N_THREADS = 1
PERCENTILE = 100
def __init__(self, engine=None, hero=None):
# self.last_ev = 0
# self.rolling_10 = deque(maxlen=10)
# self.rolling_40 = deque(maxlen=40)
self.ev_history = {}
self.time_start = None
self.duration = None
self.queue = None
self.leaf_path = None
if not engine:
# logger.info('engine not given, loading from file...')
self.engine_checksum = None
self.load_engine(hero)
else:
# logger.info('engine given')
self.init(engine, hero)
@property
def current_actions(self):
return [(c.data['action'], c.data['ev'], c.data['traversed'])
for c in self.tree.children(self.tree.root)]
def is_time_left(self):
return time.time() - self.time_start < self.duration
@retrace.retry(on_exception=(EOFError, KeyError), interval=0.1, limit=None)
def load_engine(self, hero):
with shelve.open(Engine.FILE) as shlv:
if shlv['hash'] != self.engine_checksum:
# logger.info('loading engine from file...')
self.engine_checksum = shlv['hash']
self.init(shlv['engine'], hero)
def init(self, engine, hero):
# logger.info('init state')
self.engine = engine
self.hero = hero or self.engine.q[0][0]
self.hero_pocket = self.engine.data[self.hero]['hand']
for s in self.engine.data:
self.ev_history[s] = deque(maxlen=50)
# logger.info('HERO is at seat {} with {}'.format(self.hero, self.hero_pocket))
self.watched = False
self.init_tree()
def init_tree(self):
"""create the tree. Add a root; available action will add the first level of children"""
# self.traversed_ceiling = 1
self.tree = Tree()
root = self.tree.create_node('root', identifier='root', data={'traversed': 0, 'ev': 0, 'stats': 1, 'cum_stats': 1})
# # logger.info('tree:\n{}'.format(self.tree.show()))
# input('new tree')
def watch(self):
"""Runs when engine file changes. Just kicks off run for 3s sprints"""
# logger.info('Monte Carlo watching every {}s...'.format(self.timeout))
while True:
# loads new engine file if checksum changed
self.load_engine()
# do not analyze if game finished
if self.engine.phase in [self.engine.PHASE_SHOWDOWN, self.engine.PHASE_GG]:
if not self.watched:
# logger.error('game is finished')
self.watched = True
time.sleep(3)
continue
# do not analyze if hero does not have pocket
if self.hero_pocket in [['__', '__'], [' ', ' ']]:
if not self.watched:
# logger.error('hero does not have a pocket')
self.watched = True
time.sleep(0.5)
continue
# do not analyze if hero is not to play
if self.hero != self.engine.q[0][0]:
if not self.watched:
# logger.error('hero is not to act')
self.watched = True
time.sleep(0.5)
continue
if self.is_complete:
if not self.watched:
# logger.error('mc is complete')
self.watched = True
time.sleep(2)
continue
# run a few sims
# logger.debug('running now with timeout {}'.format(self.timeout))
self.run()
self.timeout += 0.1
def run(self, duration):
"""Run simulations
For x:
- clone engine
- start at root
-- iterate and find next unprocessed node
-- action engine to that node parent
-- process that node
- keep processing
- with return EV
Levelling:
extremely huge iterations when many players. So
do the most probably actions only till all done.
Handling close action approximations:
"""
# logger.info('Monte Carlo started')
total_traversions_start = sum(a[2] for a in self.current_actions)
# cannot run if engine in showdown or gg
if self.engine.phase in [self.engine.PHASE_SHOWDOWN, self.engine.PHASE_GG]:
logger.warning('cannot run mc with no actions')
return
self.duration = duration
self.time_start = time.time()
self.queue = PriorityQueue()
# threads = []
# for _ in range(self.N_THREADS):
# t = MCWorker(self)
# # t.start()
# threads.append(t)
# self.traversed_focus = 0
leaves = self.tree.paths_to_leaves()
# logger.debug('leaves from tree: {}'.format(len(leaves)))
# leaves.sort(key=lambda lp: len(lp) + sum(int(lpn.split('_')[0]) for lpn in lp), reverse=True)
# # logger.debug('{} leaves are now sorted by formula'.format(len(leaves)))
# logger.debug('{}'.format(json.dumps(leaves[:3], indent=4, default=str)))
# leaves.sort(key=len)
# logger.debug('{} leaves are now sorted by length'.format(len(leaves)))
# logger.debug('{}'.format(json.dumps(leaves[:3], indent=4, default=str)))
# leaves.sort(key=lambda lp: int(lp[-1][:3]), reverse=True)
# logger.debug('{} leaves are now sorted by rank'.format(len(leaves)))
# logger.error(json.dumps(leaves, indent=4, default=str))
# input('>>')
for leaf_path in leaves:
node = self.tree[leaf_path[-1]]
item = (
1 - node.data['cum_stats'],
leaf_path,
)
self.queue.put(item)
# for t in threads:
# t.start()
#
# for t in threads:
# t.join()
# if t.error:
# raise Exception().with_traceback(t.error[2])
while self.is_time_left() and not self.queue.empty():
priority, self.leaf_path = self.queue.get_nowait()
self.run_item(self.leaf_path)
if self.queue.empty():
logger.info(f'Everything was processed in queue!')
total_traversions_end = sum(a[2] for a in self.current_actions)
if total_traversions_end <= total_traversions_start:
logger.warning(f'No new traversion added to {total_traversions_start}')
def run_item(self, path):
# logger.debug('running this path: {}'.format(path))
e = deepcopy(self.engine)
e.mc = True
"""To calculate the investment for the loss EV, the total amounts used till end is required. Cannot
use final player balance on engine as that might have winnings allocated to it by the engine. Instead
the difference from all the new matched bets from the current matched bets will be used.
Need to add current contrib
"""
e.matched_start = e.data[self.hero]['matched'] + e.data[self.hero]['contrib']
# logger.info('hero starting with matched = {} from {} + {}'.format(
# e.matched_start, e.data[self.hero]['matched'], e.data[self.hero]['contrib']))
# self.tree.show()
self.fast_forward(e, path)
# logger.info('{}'.format('-' * 200))
# input('check item')
def show_best_action(self):
"""Calculates best action on root"""
# logger.error("\n\n")
sum_traversed = 0
delta = 0
max_ev = float('-inf')
action = None
amount = None
for nid in self.tree[self.tree.root].fpointer:
child = self.tree[nid]
# logger.debug('{} {}'.format(child.tag, child.data))
dat = child.data
sum_traversed += dat['traversed']
# logger.error('{} @{} => {}'.format(dat['action'], dat['traversed'], round(dat['ev'], 4)))
# delta += abs(1 - (self.convergence.get(dat['action'], 1) / dat['ev'] if dat['ev'] else 1))
# self.convergence[dat['action']] = dat['ev']
if dat['ev'] > max_ev:
max_ev = dat['ev']
action = dat['action']
if action.startswith('bet') or action.startswith('raise') or action.startswith('allin'):
amount = dat['amount']
best_action = '{}{}'.format(action, ' with {}'.format(amount) if amount else '')
# self.convergence['deq'].append(round(delta, 1))
self.convergence['deq'].append(best_action)
# # logger.error('deq: {}'.format(list(self.convergence['deq'])))
# logger.error('')
# logger.error('Timeout: {}'.format(round(self.timeout, 1)))
# logger.error('Traversed: {}'.format(sum_traversed))
deq_cnts = Counter(list(self.convergence['deq']))
# # logger.error('deq: {}'.format(deq_cnts.most_common()))
# logger.error('{}% for {}'.format(
# 100 * sum(dq == deq_list[-1] for dq in deq_list[:-1]) // (len(deq_list) - 1)
# 100 * (deq_cnts.most_common()[0][1] - deq_cnts.most_common()[1][1]) // self.convergence_size
# if len(deq_cnts) > 1 else 100 * len(self.convergence['deq']) // self.convergence_size,
# deq_cnts.most_common()[0][0]
# ))
def fast_forward(self, e, path):
"""Do actions on engine till the leaf is reached. Need to do available_actions before
every DO
First check if the leave is already processed, then skip this path. When the leaf is reached
then process from that node.
Remember to send through only the first letter for the action.
Then update the nodes from this leaf back up the tree
"""
# logger.info('Fast forwarding {} nodes'.format(len(path)))
if len(path) == 1:
# logger.info('processing root for first time')
self.process_node(e, self.tree[path[0]])
return
leaf_node = self.tree[path[-1]]
# logger.debug('checking if last node has been processed:')
# logger.debug('last node leaf {} has node data {}'.format(leaf_node.tag, leaf_node.data))
if leaf_node.data['traversed']:
# logger.info('This leaf node ({}) above focus level {}'.format(leaf_node.tag, self.traversed_focus))
# can happen as all actions are added, but then one was chosen to continue on
# and that path for that action wasn't removed from the queue
return
for nid in path[1:]:
node = self.tree[nid]
# logger.debug('fast forwarding action for node {}'.format(node.tag))
e.available_actions()
cmd = [node.data['action'][0]]
if 'amount' in node.data:
cmd.append(node.data['amount'])
# logger.debug('Adding bet value of {}'.format(node.data['amount']))
# logger.debug('Executing path action {} for {}'.format(cmd, node.tag))
# logger.debug('Executing path action {} with data {}'.format(cmd, node.data))
e.do(cmd)
if node.is_leaf():
# logger.debug('{} is a leaf node, processing next...'.format(node.tag))
self.process_node(e, node)
logger.info('nodes processed, now updating nodes that were fast forwarded')
for processed_nid in reversed(path[1:]):
processed_node = self.tree[processed_nid]
self.update_node(processed_node)
self.ev_history[self.engine.s].append(sum(a[1] for a in self.current_actions))
def process_node(self, e, n):
"""Process node
Get actions available for node
Pick action to traverse with UCT
Process action selected
Return EV
"""
# logger.info('processing node {} with data {}'.format(n.tag, n.data))
# this node is the hero folding (to prevent this being processed as leaf)
# was created with other children (but not most probable at that time to be proc as child)
# if hero folding, then make this node a leaf node with fold eq
# exiting before adding children alleviates the need to remove the immediately again thereafter
# bug: cannot use engine.q as it already rotated after taking action getting here
if not n.is_root() and n.data['action'] == 'fold' and self.hero == n.data['seat']:
winnings, losses = self.net(e)
result = {
'ev': losses,
'traversed': 1,
}
# logger.info('hero has folded this node given: {}'.format(result))
n.data.update(result)
# logger.info('node data after fold: {}'.format(n.data))
return
# add the children of the node
if not n.fpointer:
self.add_actions(e, n)
# this node is a leaf (no more actions to take!)
# either the game finished and we have winner and pot
# or we have to use pokereval.winners
if n.is_leaf():
# logger.info('node {} is the final action in the game'.format(n.tag))
# winner given (easy resolution)
if e.winner:
# logger.debug('engine gave winner {}'.format(e.winner))
winnings, losses = self.net(e)
ev = winnings if self.hero in e.winner else losses
# else if the winner is unknown
# then calculate winners and use
# percentage of hero as amt
else:
if 'in' not in e.data[self.hero]['status']:
# hero fold is handled before in method
# and thus for equities calc it is just 0
# logger.debug('Hero {} is not in game'.format(self.hero))
ev = 0
else:
winnings, losses = self.net(e)
equities = PE.showdown_equities(e)
# equities = self.get_showdown_equities(e)
ev_pos = winnings * equities[self.hero]
# logger.debug('ev_pos = {} from winnings {} * eq {}'.format(ev_pos, winnings, equities[self.hero]))
ev_neg = losses * (1 - equities[self.hero])
# logger.debug('ev_neg = {} from losses {} * -eq {}'.format(ev_neg, losses, (1 - equities[self.hero])))
ev = ev_pos + ev_neg
logger.info('Net EV: {} from {} + {}'.format(ev, ev_pos, ev_neg))
result = {
'ev': ev,
'traversed': 1,
}
# logger.info('{} leaf has result {}'.format(n.tag, result))
n.data.update(result)
return
# node is all good (not leaf (has children) and not hero folding)
# get child actions and process most probable action
a_node = self.most_probable_action(n)
action = a_node.data['action']
# logger.info('taking next child node action {}'.format(action))
# if it is hero and he folds,
# it is not necessarily an immediate ZERO equity
# since my previous contrib needs to be added to the pot (i.e. contribs after starting mc)
# i.e. make this a leaf node implicitly
# no child nodes to remove for fold
if action == 'fold' and self.hero == a_node.data['seat']:
winnings, losses = self.net(e)
result = {
'ev': losses,
'traversed': 1,
}
# logger.info('hero has folded the child node selected: {}'.format(result))
a_node.data.update(result)
# logger.info('a_node data after: {}'.format(a_node.data))
# else we must process the node
else:
# logger.info('taking action {} and processing that node'.format(action))
cmd = [action[0]]
if 'amount' in a_node.data:
cmd.append(a_node.data['amount'])
# logger.debug('Adding bet value of {}'.format(a_node.data['amount']))
e.do(cmd)
self.process_node(e, a_node)
# action node has been processed, now update node
self.update_node(n)
def update_node(self, node):
"""Update the node's data
If leaf, then it was already calculated during processing, and now
do not change it: the ev is the ev
Minimax applied, hero pick best and foe picks min after p
Traversed will stay the traversed_focus level for leaves, but for parent nodes
the traversed will be the number of leaves reached from that node.
"""
is_hero = node.data.get('seat') == self.hero
# logger.debug('is hero? {}'.format(is_hero))
# it will traverse back up to the root
# root can be skipped
if node.is_root():
# input('hero {} node data {}'.format(self.hero, node.data.get('seat')))
# if is_hero:
# self.rolling_10.append(abs(self.last_ev))
# self.rolling_40.append(abs(self.last_ev))
# logger.debug('Added {} ev to collection'.format(self.last_ev))
# input('Added {} ev to collection'.format(self.last_ev))
# logger.debug('reached the root')
# self.update_ev_change()
return
# fast forwarding will send here, just ignore node if leaf
if node.is_leaf():
# logger.debug('not updating {}: it is final game result (no leaf nodes)'.format(node.tag))
# logger.debug('not updating {}: final data {}'.format(node.tag, node.data))
return
depth = self.tree.depth(node)
# logger.info('updating node {} at depth {}'.format(node.tag, depth))
# logger.info('node has {} before update'.format(node.data))
if not len(node.fpointer):
# logger.error('node {} with {} as no children...'.format(node.tag, node.data))
raise Exception('not necessary to process leaves')
# logger.debug('extracting data from {} children nodes...'.format(len(node.fpointer)))
n_ev = float('-inf') if is_hero else 0
n_traversed = 0
for child_nid in node.fpointer:
child_node = self.tree[child_nid]
# logger.debug('child node {} has {}'.format(child_node.tag, child_node.data))
dat = child_node.data
if not dat['traversed']:
# logger.debug('skipping untraversed {}'.format(child_node.tag))
continue
# get max for hero
if is_hero:
# todo is this +ev dampening necessary
# todo this should be fixed when setting for hand range
# equities = PE.showdown_equities(self.engine)
# n_ev = max(n_ev, dat['ev'] * equities.get(self.hero, 0))
n_ev = max(n_ev, dat['ev'])
# get min for foe
else:
# ev_adj = dat['ev'] * dat['stats']
# logger.debug('foe min between {} and {}'.format(n_ev, ev_adj))
# n_ev = min(n_ev, ev_adj)
n_ev += dat['ev'] * dat['stats'] / dat['divider']
n_traversed += dat['traversed']
# logger.debug('added {} traversed: now have {} so far'.format(dat['traversed'], n_traversed))
self.last_ev = node.data['ev'] - n_ev
node.data.update({
'ev': n_ev,
'traversed': n_traversed,
})
# logger.info('now node has {} ev~{} after {}'.format(node.tag, round(n_ev, 3), n_traversed))
if not node.data['traversed']:
raise Exception('node cannot be untraversed')
def net(self, e):
"""Stored the balance at the start of sim.
Now calculate difference as player total matched contrib.
Winnings will be less initial starting contrib.
"""
e.gather_the_money()
p = e.players[self.hero]
d = e.data[self.hero]
matched_diff = d['matched'] - e.matched_start
# logger.debug('matched diff = {} from {} - {}'.format(matched_diff, d['matched'], e.matched_start))
winnings = int(e.pot - matched_diff)
# logger.debug('winnings diff = {} from pot {} less matched {}'.format(winnings, e.pot, matched_diff))
losses = int(-matched_diff)
# logger.info('Winnings = {} and losses = {}'.format(winnings, losses))
return winnings, losses
def most_probable_action(self, parent):
"""All nodes will be processed once at least but it will never happen. Just return
the most probable node for most accurate play. Using stats fields on data
There should not be any untraversed nodes. So first get untraversed, then sort
and pop first one"""
# logger.info('getting most probable action after {}'.format(parent.tag))
children = self.tree.children(parent.identifier)
children = [c for c in children if not c.data['traversed']]
if not children:
raise MonteCarloError('Cannot choose most probable action when all nodes are traversed')
children.sort(key=lambda c: c.data['stats'], reverse=True)
child = children[0]
# logger.debug('{} is untraversed, returning that node for actioning'.format(child.tag))
self.leaf_path.append(child.identifier)
return child
def add_actions(self, e, parent):
"""Add actions available to this node
If in GG phase then no actions possible, ever.
Remove 'hand'
Bets:
- preflop are 2-4x BB
- postflop are 40-100% pot
Raise:
- always double
Allin:
- only on river
- if out of money then converted to allin
Scale non-fold probabilities even though it should not have an effect.
"""
# logger.info('adding actions to {}'.format(parent.tag))
actions = e.available_actions()
s, p = e.q[0]
d = e.data[s]
balance_left = p['balance'] - d['contrib']
if not actions:
# logger.warn('no actions to add to node')
return
if 'gg' in actions:
# logger.debug('no actions available, got gg')
return
actions.remove('hand')
# remove fold if player can check
if 'check' in actions:
actions.remove('fold')
# # logger.debug('removed fold when check available')
# remove fold for hero
# if s == self.hero and 'fold' in actions:
# actions.remove('fold')
# # logger.debug('removed fold from hero')
# remove raise if player has already been aggressive
if 'raise' in actions and any(pa['action'] in 'br' for pa in d[e.phase]):
actions.remove('raise')
# # logger.debug('removed raise as player has already been aggressive')
# remove allin, but add it later with final stats (if increased from bet/raised)
if 'allin' in actions:
actions.remove('allin')
# logger.debug('removed allin by default')
# load stats (codes with counts)
stats = ES.player_stats(e, s)
max_contrib = max(pd['contrib'] for pd in e.data.values())
# contrib_short = max_contrib - d['contrib']
# allin needs to be the doc count
# where bets and raises result in allin, add those prob dists to this
# that will give proper probability
go_allin = stats['actions'].get('a', 0)
# # logger.info('filtered actions: {}'.format(actions))
# ev 0 instead of none because of root node sum when not all traversed it gives error
action_nodes = []
for a in actions:
node_data = {
'stats': stats['actions'].get(ACTIONS_TO_ABBR[a], 0.01),
'divider': 1,
'action': a,
'phase': e.phase,
'seat': s,
'name': p['name'],
'traversed': 0,
'ev': 0,
}
if a in ['bet', 'raise']:
btps_and_amts = []
total_pot = sum(pd['contrib'] for pd in e.data.values()) + e.pot
# for preflop only do 2x and 3x
if e.phase == e.PHASE_PREFLOP:
btps_and_amts.append(('double', e.bb_amt * 2))
btps_and_amts.append(('triple', e.bb_amt * 3))
# else do half and full pots
else:
btps_and_amts.append(('half_pot', total_pot * 0.50))
btps_and_amts.append(('full_pot', total_pot * 1.00))
# round bets up to a BB
# btps_and_amts = [(btp, -(amt // -e.bb_amt) * e.bb_amt)
# for btp, amt in btps_and_amts]
betting_info = []
amts_seen = []
for btp, amt in btps_and_amts:
if amt in amts_seen:
# logger.debug('already using {}, skipping duplicate'.format(amt))
continue
if a == 'bet' and amt < e.bb_amt:
# logger.debug('bet cannot be less than BB {}'.format(e.bb_amt))
continue
if a == 'raise' and amt < (max_contrib * 2):
# logger.debug('raise cannot be less than 2x contrib of {}'.format(max_contrib * 2))
continue
betting_info.append((btp, amt))
amts_seen.append(amt)
# change raises that cause allin
betting_info_final = []
for btp, amt in betting_info:
# if amt is more than player balance, it is an allin
if amt >= balance_left:
go_allin += node_data['stats'] / len(betting_info)
else:
betting_info_final.append((btp, amt))
# all good, can have this bet as option
for btp, amt in betting_info_final:
node_data_copy = deepcopy(node_data)
node_data_copy['divider'] = len(betting_info_final)
node_data_copy['action'] = f'{a}_{btp}'
node_data_copy['amount'] = amt
action_nodes.append(node_data_copy)
else:
action_nodes.append(node_data)
# allin will have doc counts (from stat, maybe from bets, maybe from raise)
if go_allin:
node_data = {
'stats': go_allin,
'divider': 1,
'action': 'allin',
'phase': e.phase,
'seat': s,
'name': p['name'],
'traversed': 0,
'ev': 0,
'amount': balance_left,
}
action_nodes.append(node_data)
# logger.debug('added allin to actions with stat {}'.format(node_data['stats']))
# scale the stats (it is currently term counts aka histogram) and it is required to be
# a probability distribution (p~1)
# Also, certain actions like fold can be removed, and the total stats is not 1
total_stats = sum(an['stats'] / an['divider'] for an in action_nodes)
for action_node in action_nodes:
action_node['stats'] = max(0.01, action_node['stats'] / action_node['divider'] / total_stats)
action_node['cum_stats'] = parent.data['cum_stats'] * action_node['stats']
node_tag = f'{action_node["action"]}_{s}_{e.phase}'
identifier = f'{node_tag}_{str(uuid.uuid4())[:8]}'
self.tree.create_node(identifier=identifier, tag=node_tag, parent=parent.identifier, data=action_node)
# logger.debug('new {} for {} with data {}'.format(node_tag, s, action_node))
item = (
1 - action_node['cum_stats'],
self.leaf_path + [identifier]
)
self.queue.put(item)
# logger.debug('new {} for {} with data {}'.format(node_tag, s, action_node))
# logger.info('{} node actions added'.format(len(action_nodes)))
def analyze_tree(self):
"""Analyze tree to inspect best action from ev"""
# self.tree.show()
# check all finished paths
for path in self.tree.paths_to_leaves():
# skip untraversed end
last_node = self.tree[path[-1]]
if not last_node.data['traversed']:
logger.debug('skipping untraversed endpoint {}'.format(last_node.tag))
continue
# show all actions
for nid in path:
node = self.tree[nid]
d = node.data
logger.info('Node: {} ev={}'.format(node.tag, d['ev']))
0/0
input('$ check tree')
def get_showdown_equities(self, e):
"""instead of using pokereval, use hs from se"""
hss = {}
for s, d in e.data.items():
if 'in' in d['status']:
hss[s] = ES.showdown_hs(e, s, percentile=self.PERCENTILE)
# calculate for hero
if self.hero in hss:
d = e.data[self.hero]
hss[self.hero] = PE.hand_strength(d['hand'], e.board, e.rivals)
# normalize
total = sum(hs for hs in hss.values())
equities = {s: hs / total for s, hs in hss.items()}
return equities
class search_tree(object):
def __init__(self, query_id, depth, carpe_diem):
self.tree = Tree()
self.tree.create_node(identifier='query_' + query_id, data=node())
root_node = self.tree.get_node('query_' + query_id)
root_node.data.num = 1.0
self.node_map = {}
self.count = 0.0
self.carpe_diem = carpe_diem
self.max_depth = depth
self.expand(self.tree.get_node(self.tree.root))
def expand(self, leaf_node):
doc_list = leaf_node.data.doc
p_doc_id, p_pred = self.carpe_diem.policy(self.tree.root, doc_list)
for doc in p_doc_id:
self.node_map[' '.join(doc_list + [doc])] = len(self.node_map)
new_node = node()
new_node.doc = doc_list + [doc]
new_node.p = p_pred[p_doc_id.index(doc)]
self.tree.create_node(identifier=self.node_map[' '.join(
new_node.doc)],
data=new_node,
parent=leaf_node.identifier)
def update(self, node_list, value):
for node_id in node_list:
tmp_node = self.tree.get_node(node_id)
tmp_node.data.Q = (tmp_node.data.Q * tmp_node.data.num +
value) / (tmp_node.data.num + 1)
tmp_node.data.num += 1
def search(self, start_node_id):
tmp_node = self.tree.get_node(start_node_id)
has_visit_num = tmp_node.data.num - 1
self.count = has_visit_num
if int(self.carpe_diem.search_time - has_visit_num) > 0:
start_node_search_time = int(self.carpe_diem.search_time -
has_visit_num)
else:
start_node_search_time = 0
for time in range(start_node_search_time):
search_list = [start_node_id]
tmp_node = self.tree.get_node(start_node_id)
while not tmp_node.is_leaf():
max_score = float("-inf")
max_id = -1
for child_id in tmp_node.fpointer:
child_node = self.tree.get_node(child_id)
score = self.carpe_diem.beta * child_node.data.p * (
(tmp_node.data.num - 1)**0.5 /
(1 + child_node.data.num))
score += child_node.data.Q
if score > max_score:
max_id = child_id
max_score = score
search_list.append(max_id)
tmp_node = self.tree.get_node(max_id)
query_id_mcts = self.tree.root.split('_')[1]
if self.tree.depth(tmp_node) == self.max_depth:
listPermutation = copy.deepcopy(
self.carpe_diem.dictQueryPermutaion[query_id_mcts]
['permutation'])
idealScore = self.carpe_diem.alphaDCG(0.5, query_id_mcts,
listPermutation,
self.max_depth)
v = self.carpe_diem.alphaDCG(0.5, query_id_mcts,
tmp_node.data.doc, self.max_depth)
v = v / idealScore
else:
v = self.carpe_diem.value_function(self.tree.root,
tmp_node.data.doc)
self.update(search_list, v)
self.count += 1
if tmp_node.is_leaf() and (self.tree.depth(tmp_node) <
self.max_depth):
self.expand(tmp_node)
###########
if time % 100 == 0:
tmp_policy = self.get_policy(start_node_id)
print tmp_policy.values()
print sum(tmp_policy.values())
print time
def take_action(self, start_node_id):
tmp_node = self.tree.get_node(start_node_id)
max_time = -1
prob = {}
for child_id in tmp_node.fpointer:
child_node = self.tree.get_node(child_id)
prob[child_node.data.doc[-1]] = child_node.data.num / self.count
if child_node.data.num > max_time:
max_time = child_node.data.num
select_doc = child_node.data.doc[-1]
select_doc_node_id = child_node.identifier
return prob, select_doc, select_doc_node_id
def get_policy(self, start_node_id):
tmp_node = self.tree.get_node(start_node_id)
max_time = -1
prob = {}
for child_id in tmp_node.fpointer:
child_node = self.tree.get_node(child_id)
if self.count == 0:
prob[child_node.data.doc[-1]] = 0.0
else:
prob[
child_node.data.doc[-1]] = child_node.data.num / self.count
return prob
class BasicTree:
def __init__(self, vehsInfo):
self.tree = Tree()
self.root = self.tree.create_node("Root", "root") # root node
self.vehsInfo = vehsInfo
self.vehList = list(vehsInfo.keys())
self.i = 1
def _build(self, currentNode, vehList):
'''
:param vehList: A dict, keys is the set of vehicles, value is a tuple which represents (lane, position)
:param currentNode: The current node in the tree
:return: None
'''
s = [currentNode.tag.find(vid) for vid in vehList] # the quit contidion in recursion
if (np.array(s) >= 0).all():
return
for vehId in vehList:
if vehId not in currentNode.tag:
if currentNode.is_root:
prefix = currentNode.tag.replace("Root", "")
else:
prefix = currentNode.tag
self.tree.create_node(prefix + vehId + "-", prefix + vehId, parent=currentNode)
for node in self.tree.all_nodes():
if node.is_leaf():
self._build(currentNode=node, vehList=vehList)
def _prune(self):
laneId = [value[0] for value in self.vehsInfo.values()]
sortedList = []
for i in list(set(laneId)):
lane_info = {k: v[1] for k, v in self.vehsInfo.items() if v[0] == i}
# Vehicles in front are at the front of the lane
sortedList.append([vid[0] for vid in sorted(lane_info.items(), key=itemgetter(1), reverse=True)])
pruneList = [sublist for sublist in sortedList if len(sublist) > 1]
for subList in pruneList:
for index in range(1, len(subList)):
# first, prune th subtree which begin with illegal vehicle id
self.tree.remove_subtree(subList[index])
# second, delete the nodes which match the illegal pattern
pattern = subList[index] + ".*" + subList[0]
for node in self.tree.all_nodes():
if re.search(pattern, node.tag):
try:
self.tree.remove_node(node.identifier)
except:
pass
def build(self):
self._build(self.root, self.vehList)
self._prune()
def show(self):
self.tree.show()
def _leaves(self):
'''
:return: All the plan for vehicle passing currently.
'''
all_nodes = self.tree.all_nodes()
return [node for node in all_nodes if node.is_leaf()]
def legal_orders(self):
leaves = self._leaves()
orders = []
for pattern in leaves:
# upToRight.1-leftToBelow.18-belowToRight.2-belowToRight.3-
tmp = pattern.tag.split("-")
try:
tmp.remove('')
except:
pass
if len(tmp) == self.tree.depth():
orders.append(tmp)
return orders
class RRT:
def __init__(self,
current,
desir,
step_size,
kenamatic,
world_size,
obstacle=0):
self.jobs = []
self.size = world_size
# Desired start and end point
self.current = current
self.desir = desir
# A matrix that holds all dots from both starts
# The trees that arrange the matrix
self.tree = Tree()
# Did we finish the search
self.NOF = Value("i", 0)
self.statos = Value("i", 0)
# self.winindex = Value("i", 0)
self.winindex = Manager().list()
self.badindex = Manager().list()
self.badpoint = Manager().list()
self.pool = np.float32(np.zeros((1, 12)))
self.pool_index = np.zeros(1)
self.num_pool = 9
# The number of dots already inserted into the tree
self.NOP = 1
# Step size
self.step = step_size
# Number of dimensions
# self.NOD = len(current.T)
self.t = time.time()
self.obstacle = obstacle
# The kinematics of the arms
self.kin = kenamatic
# Initial position of the arm base
print("open new RRT model")
def goliniar(self, point_a, point_b):
mid_all_point = self.midpoint(point_a[0], point_b[0])
i = 0
dis = 1
loops = len(mid_all_point)
#print((self.obstacle_gpu.dtype),"68")
while i < loops:
ans = self.kin.configure_check1(mid_all_point[i].reshape(1, 12),
self.obstacle_gpu)
if ans == 0:
return i - dis, mid_all_point
else:
dis = max(int(ans**2 / 4000), 1)
i = i + dis
return loops - 1, mid_all_point
def goliniarpros(self, pointstart, pointend, index, winindex, badindex,
badpoint):
print("tread {} start".format(index))
secsed = True
allpoints = self.midpoint(pointstart, pointend)
i = 0
loops = len(allpoints)
while i < loops:
ans = self.kin.configure_check(allpoints[i].reshape(1, 12),
self.obstacle)
if not ans:
secsed = False
if (i > 8):
badpoint.append(allpoints[i - 8])
badindex.append(index)
break
else:
i = i + max(int(ans**2 / 3000), 1)
if (self.statos.value):
secsed = False
break
if secsed:
self.statos.value = 1
winindex.append(index)
print("---tread {} secsed ---".format(index))
#print((allpoints.dtype),"107")
print("tread {} die ".format(index))
self.NOF.value -= 1
def get_direction(self, tree, index):
tree.show()
#allrote = self.current
if not index == 0:
root = tree[index].predecessor(tree._identifier)
good_route = tree[index].data
while (not root == 0 or root == None):
point = np.array(tree[root].data)
good_route = np.append(point, good_route, axis=0)
root = tree[root].predecessor(tree._identifier)
allrote = self.midpoint(self.desir[0], good_route[-1])
allrote = np.append(allrote[:-1],
self.midpoint(allrote[-1], good_route[0]),
axis=0)
for i in range(1, len(good_route)):
allrote = np.append(allrote[:-1],
self.midpoint(good_route[i - 1],
good_route[i]),
axis=0)
allrote = np.append(allrote[:-1],
self.midpoint(allrote[-1],
self.current[0]),
axis=0)
else:
allrote = self.midpoint(self.desir[0], self.current[0])
# =============================================================================
# good_route = np.append(self.current, self.desir, axis=0)
# for i in range(1, len(good_route)):
# allrote = np.append(allrote, self.midpoint(good_route[i - 1], good_route[i]), axis=0)
# =============================================================================
# allrote = allrote[1:, :]
return np.flip(allrote, axis=0)
# Extracting the path we found from the tree
def get_direction1(self, tree, index):
#tree.show()
allrote = self.current
if not index == 0:
root = tree[index].predecessor(tree._identifier)
good_route = tree[index].data
while (not root == 0 or root == None):
point = np.array(tree[root].data)
good_route = np.append(point, good_route, axis=0)
root = tree[root].predecessor(tree._identifier)
for i in range(1, len(good_route)):
allrote = np.append(allrote[:-1],
self.midpoint(good_route[i - 1],
good_route[i]),
axis=0)
allrote = np.append(allrote[1:],
self.midpoint(self.all_point[index],
self.desir[0]),
axis=0)
allrote = np.append(self.midpoint(allrote[0], allrote[1]),
allrote[1:, :],
axis=0)
allrote = np.append(self.midpoint(self.current[0], allrote[0]),
allrote,
axis=0)
else:
good_route = np.append(self.current, self.desir, axis=0)
for i in range(1, len(good_route)):
allrote = np.append(allrote,
self.midpoint(good_route[i - 1],
good_route[i]),
axis=0)
return allrote
def get_direction2(self, tree, index):
#tree.show()
allrote = self.current
if not index == 0:
root = tree[index].predecessor(tree._identifier)
good_route = tree[index].data
while (not root == 0 or root == None):
point = np.array(tree[root].data)
good_route = np.append(point, good_route, axis=0)
root = tree[root].predecessor(tree._identifier)
for i in range(1, len(good_route)):
allrote = np.append(allrote,
self.all_point[i].reshape(1, 12),
axis=0)
allrote = np.append(allrote, self.desir, axis=0)
print(tree.depth(index))
return allrote
# Gets a vector and normalizes it to a desired step size
def get_normelize_vec(self, vec):
return np.float32((vec / np.sqrt(np.sum(pow(vec, 2)))) * self.step)
def midpoint(self, startp, endp):
return np.linspace(startp,
endp,
int(np.max(np.abs(startp - endp) + 1)),
dtype=np.float32)
# Gets a vector point and returns the geometrically closest vector point
def min_distance(self, newpoint):
return np.argmin(
np.sum(pow(np.subtract(newpoint, self.all_point), 2), axis=1))
def dis_2point(self, point_A, point_B):
return np.min(np.sqrt(np.sum(pow(np.subtract(point_A, point_B), 2))))
# Adds a point into the tree
def add_to_tree(self, index, newpoint):
self.tree.create_node(self.NOP, self.NOP, parent=index, data=newpoint)
self.all_point = np.append(self.all_point, newpoint, axis=0)
self.NOP += 1
print("NOP : ", self.NOP)
def bildwold(self):
NOO = 200
a = 1
p = 50
obs = np.array(
[np.ones(p) * NOO,
np.linspace(-NOO, NOO, num=p),
np.ones(p)]).T
for i in range(1, int(NOO * a)):
obs = np.append(obs,
np.array([
np.ones(p) * NOO,
np.linspace(-NOO, NOO, num=p),
np.ones(p) * (i)
]).T,
axis=0)
obs = np.append(obs,
np.array([
np.ones(p) * -NOO,
np.linspace(-NOO, NOO, num=p),
np.ones(p) * (i)
]).T,
axis=0)
obs = np.append(obs,
np.array([
np.linspace(-NOO, NOO, num=p),
np.ones(p) * NOO,
np.ones(p)
]).T,
axis=0)
for i in range(1, int(NOO * a)):
obs = np.append(obs,
np.array([
np.linspace(-NOO, NOO, num=p),
np.ones(p) * NOO,
np.ones(p) * (i)
]).T,
axis=0)
obs = np.append(obs,
np.array([
np.linspace(-NOO, NOO, num=p),
np.ones(p) * -NOO,
np.ones(p) * (i)
]).T,
axis=0)
NOO = NOO * 3
for i in range(1, int(NOO * 2)):
obs = np.append(obs,
np.array([
np.linspace(-NOO, NOO, num=p),
np.ones(p) * (NOO - i),
np.zeros(p)
]).T,
axis=0)
return np.float32(obs)
def open_prosses(self, srart_pos, end_pos, index):
p = Process(target=self.goliniarpros,
args=(
srart_pos,
end_pos,
index,
self.winindex,
self.badindex,
self.badpoint,
))
self.jobs.append(p)
p.start()
self.NOF.value += 1
# Adds a point in a random direction
def add_point(self, newpoint):
index = self.min_distance(newpoint) #V
newpoint = self.get_normelize_vec(
np.subtract(newpoint,
self.all_point[index])) + self.all_point[index]
godindex, temppoints = self.goliniar(
self.all_point[index].reshape(1, 12), newpoint)
#print((temppoints.dtype),"240")
#print((self.all_point.dtype),"241")
if (godindex > 0):
self.add_to_tree(index, temppoints[godindex].reshape(1, 12))
if (self.NOF.value < self.num_pool):
self.open_prosses(self.all_point[-1], self.desir[0],
self.NOP - 1)
else:
self.pool = np.append(self.pool,
self.all_point[-1].reshape(1, 12),
axis=0)
self.pool_index = np.append(self.pool_index, (self.NOP - 1))
print("number of processes that are currently running : {}".format(
self.NOF.value))
def midserce1(self, allpoint):
e = 1
max = int(len(allpoint) / 2)
min = 0
mid = max
new_Peza = int(max / 4)
old_peza = 0
while (abs(new_Peza - old_peza) > e):
index_a = mid + new_Peza
index_b = mid - new_Peza
good_index, temp_points = self.goliniar(
allpoint[index_a].reshape(1, 12),
allpoint[index_b].reshape(1, 12))
if (good_index + 1 == len(temp_points)):
min = new_Peza
old_peza = new_Peza
new_Peza = int(new_Peza + (max - new_Peza) / 2)
else:
max = new_Peza
old_peza = new_Peza
new_Peza = int(new_Peza - (new_Peza - min) / 2)
print("Peza = ", new_Peza)
return new_Peza, mid
def midserce(self, allpoint, max_val, min_val, start_point):
print("strt : {}\tmax : {}\tmin : {}".format(start_point, max_val,
min_val))
e = 2
new_index = int((max_val + min_val) / 2)
print("new_index = ", new_index)
old_index = 0
while (abs(new_index - old_index) > e):
good_index, temp_points = self.goliniar(
allpoint[start_point].reshape(1, 12),
allpoint[new_index].reshape(1, 12))
if (good_index + 1 == len(temp_points)):
min_val = new_index
old_index = new_index
new_index = int(new_index + (max_val - new_index) / 2)
else:
max_val = new_index
old_index = new_index
new_index = int(new_index - (new_index - min_val) / 2)
if (new_index < min_val):
print("faild")
return old_index, False
print("end : ", new_index)
return new_index, True
def improved_path1(self, allpoint):
print("\n\n\nstart -> 1")
t = time.time()
a = len(allpoint)
max_val = len(allpoint)
min_val = 0
start_point = [0]
new_beast = 0
old_beast = -10
while max_val - new_beast > 5 and new_beast - old_beast > 2:
old_beast = new_beast
new_beast, stat = self.midserce(allpoint, max_val, min_val,
start_point[-1])
start_point.append(new_beast)
min_val = new_beast
optmaize = self.current
j = 0
for i in start_point:
optmaize = np.append(optmaize,
self.midpoint(allpoint[j], allpoint[i]),
axis=0)
j = i
if not stat:
optmaize = np.append(optmaize,
allpoint[start_point[-1]:, :],
axis=0)
else:
optmaize = np.append(optmaize,
self.midpoint(optmaize[-1], self.desir[0]),
axis=0)
allpoint = optmaize
max_val = len(allpoint)
print(-t + time.time())
return allpoint
# =============================================================================
#
#
# def improved_path2(self, allpoint):
# print("\n\n\nstart -> 6")
# t=time.time()
# a=len(allpoint)
# print("a->",a)
#
# max_val=len(allpoint)
# min_val=0
# start_point=[0]
# new_beast=0
# stat=True
# dellist=[]
# while max_val-new_beast>1 :
# old_beast=new_beast
# new_beast,stat=self.midserce1(allpoint,max_val,min_val,start_point[-1])
# start_point.append(new_beast)
# min_val=new_beast
#
# print(start_point,"-----------")
# if len(start_point)>1:
# for i in range(1,len(start_point)):
# for i in (range(start_point[i-1]+1,start_point[i])):
# dellist.append(i)
# print(dellist)
# b=np.delete(allpoint,dellist,0)
# print("b->",len(b))
#
# optmaize=self.current
# for i in range(1,len(b)):
# optmaize=np.append(optmaize,self.midpoint(b[i-1],b[i]),axis=0)
#
# allpoint = optmaize
# for i in range(1,len(allpoint)):
# if (not (np.linalg.norm(allpoint[i-1]-allpoint[i]))):
# print(0)
# max_val=len(allpoint)
# print("old : {} \t new : {}".format(a,len(allpoint)))
# print(-t+time.time())
# return allpoint
# =============================================================================
# =============================================================================
# def improved_path(self, allpoint):
# print("start -> 5")
# t=time.time()
#
# a=len(allpoint)
# beast,mid=self.midserce(allpoint)
#
# if (not beast == 0):
# tempend = allpoint[ (mid + beast-1):,:]
# tempstart = allpoint[:(mid - beast+1),:]
# if(not len(tempend)):
# tempend=allpoint[-1,:].reshape((1,12))
# if(not len(tempstart)):
# tempend=allpoint[0,:].reshape((1,12))
# tempmid = self.midpoint(tempstart[-1,: ], tempend[0,: ])
# temp = np.append(tempstart, tempmid[1:-1], axis=0)
# allpoint = np.append(temp, tempend, axis=0)
#
# print("old : {} \t new : {}".format(a,len(allpoint)))
# print(-t+time.time())
# return allpoint
#
#
# =============================================================================
def get_winindex(self):
print(self.winindex)
print("Extracting the route")
dis = self.tree.depth(self.winindex[0]) * self.step + np.linalg.norm(
self.tree[self.winindex[0]].data - self.desir)
min_val = 99999
for i in self.winindex:
dis = self.tree.depth(i) * self.step + np.linalg.norm(
self.tree[i].data - self.desir)
print(i, "\t", self.tree.depth(i), "\t",
np.linalg.norm(self.tree[i].data - self.desir), '\t', dis)
if (dis < min_val):
min_val = dis
win = i
print(win)
self.winindex = Manager().list()
return win
# The function that activates everything
def let_the_magic_begin(self, ros_fun, isflipt):
self.open_prosses(self.current[0], self.desir[0], self.NOP - 1)
NOL = 0
while (not self.statos.value):
if (not NOL % 10):
self.size += 10
self.size = min(self.size, 360)
newpoint = (np.float32(np.random.rand(
1,
12,
)) * self.kin.limit * self.size - self.kin.offset)
#print((newpoint.dtype),"436")
self.add_point(newpoint)
if (len(self.pool) > 1):
while (len(self.pool) > 1 and self.NOF.value < self.num_pool):
if (self.statos.value):
break
self.open_prosses(self.pool[-1], self.desir[0],
int(self.pool_index[-1]))
self.pool = np.delete(self.pool, -1, 0)
self.pool_index = np.delete(self.pool_index, -1)
if (len(self.badindex)):
for i in range(len(self.badindex)):
self.add_to_tree(self.badindex[0],
self.badpoint[0].reshape(1, 12))
del self.badindex[0]
del self.badpoint[0]
NOL += 1
print("Waiting for all processes to die")
for job in self.jobs:
job.join()
best_point_index = self.get_winindex()
allrote = self.get_direction1(self.tree, best_point_index)
if isflipt:
allrote = np.flip(allrote, axis=0)
tmp = self.desir
self.desir = self.current
self.current = tmp
print(time.time() - self.t)
input("preace any key to improved path\n")
if (best_point_index == 0):
print("no need to improve path")
allrote1 = allrote
#else:
# =============================================================================
# for i in allrote:
# print(self.kin.configure_check1(i.reshape(1,12),self.obstacle_gpu))
# if not (self.kin.configure_check1(i.reshape(1,12),self.obstacle_gpu)):
# print("faild")
# =============================================================================
# =============================================================================
#
# allrotetemp = self.get_direction2(self.tree, best_point_index)
# allrote2=self.improved_path2(allrotetemp)
# print("first : old -> {} \tnew -> {}".format(len(allrote),len(allrote2)))
# allrote3=self.improved_path1(allrote2)
# print("sec : old -> {} \tnew -> {}".format(len(allrote),len(allrote2)))
# =============================================================================
allrote6 = self.improved_path1(allrote)
print("fird : old -> {} \tnew -> {}".format(len(allrote),
len(allrote6)))
#allrote6=self.improved_path(allrote6)
#allrote3=self.improved_path(allrote)
input("preace any key to run the simulation\n")
ros_fun.send_to_arm(allrote6 * np.pi / 180)
input("preace any key to run the simulation\n")
ros_fun.send_to_arm(np.flip(allrote, axis=0) * np.pi / 180)
def run_serce(self, ros_fun, isinvers, invers_pos):
self.obstacle_gpu = th.from_numpy(self.obstacle).float().to('cuda')
if (isinvers):
self.desir = self.kin.invers_arms_cfg(invers_pos)
self.t = time.time()
dis_des = self.kin.configure_check1(self.desir, self.obstacle_gpu)
dis_crr = self.kin.configure_check1(self.current, self.obstacle_gpu)
if dis_des:
if dis_des < dis_crr:
temp = self.desir
self.desir = self.current
self.current = temp
isflipt = True
print("desir = {} \ncurrent={} \nneed to flip".format(
dis_des, dis_crr))
else:
isflipt = False
print("desir = {} \ncurrent={} \nno need to flip".format(
dis_des, dis_crr))
self.tree.create_node(0, 0, data=self.current) # root node
self.all_point = self.current
#print(self.current.dtype)
self.let_the_magic_begin(ros_fun, isflipt)
else:
print("can't go to disre location")
class SentenceTree(object):
def __init__(self):
self.tree = Tree()
self.sentence = []
self.matrix = []
# Builds a tree by backtracking the sentence matrix
def build_tree(self, sentence_matrix):
self.matrix = sentence_matrix
sentence_length = len(sentence_matrix)-1
# Saves the ST's sentence as a list of strings
for i in range(1,sentence_length+1):
self.sentence.append(self.matrix[0][i][0])
# Finds the most probable sentence option
options = sentence_matrix[sentence_length][1]
max_option = [options[x] for x in options if options[x][1]==max([y[1] for y in options.values()])][0]
# Builds the tree
self._nid = sentence_length+1
root = max_option
self.tree.create_node(root[0], self._nid)
self._create_children(root, self._nid) # Call recursive function
# Ensures unique node id in _create_children()
def _nnid(self):
self._nid +=1
return self._nid
# Recursive function which builds the children nodes of a given parse_option and then builds their children
def _create_children(self, parse_option, pid):
if parse_option is None:
return None
else:
# If parse_option has children, extract those
if parse_option[2] is not None:
left_coord = parse_option[2]
right_coord = parse_option[3]
left_child = self.matrix[left_coord[0]][left_coord[1]][left_coord[2]]
right_child = self.matrix[right_coord[0]][right_coord[1]][right_coord[2]]
# Create left child as node (plus extra word node if leaf)
cid = self._nnid()
self.tree.create_node(left_child[0], cid, parent=pid)
if left_child[2] is None: #If left_child is a leaf node, append a word node
nid = left_coord[1]-1
word = self.matrix[left_coord[0]-1][left_coord[1]][0]
word = word.decode('utf-8', "ignore")
self.tree.create_node(word, nid, parent=cid)
else:
self._create_children(left_child, cid) # Create children of left_child
# Create right child as node (plus extra word node if leaf)
cid = self._nnid()
self.tree.create_node(right_child[0], cid, parent=pid)
if right_child[2] is None: #If right_child is a leaf node, append a word node
nid = right_coord[1]-1
word = self.matrix[right_coord[0]-1][right_coord[1]][0]
word = word.decode('utf-8', "ignore")
self.tree.create_node(word, nid, parent=cid)
else:
self._create_children(right_child, cid) # Create children of right_child
# Returns the sentence's sentiment score
def get_sentiment_score(self, sentimentDict, term):
total_score = 0
# negation dictionary
negationList = ["ikke", "ej"]
# Check the term against every sentiment word
n1 = self.sentence.index(term)
for word in sentimentDict:
if term==word: continue # If topic term is an opinion word, ignore.
n2 = self._in_sentence(word)
if n2 is not False:
d = self._get_distance(n1, n2)
if d == 0: score = float(sentimentDict[word])
score = float(sentimentDict[word]) / float(d)
# If SentWord is negated, flip the score derived from it
if self._is_negated(word, negationList):
score = score * -1
print "Term: %s | SentWord: %s | Distance: %s | Score: %s" % (term, word, d,score)
total_score += score
print "Total score:", total_score
return total_score
# Checks whether a word is within a specified threshold distance of a negation word
def _is_negated(self, w, negationList):
negationThreshold = 3
n1 = self._in_sentence(w)
if n1 is None: return False
for nw in negationList:
n2 = self._in_sentence(nw)
if n2 is not None:
if (self._get_distance(n1, n2)) < negationThreshold:
print "negating word", w
return True
return False
# Checks whether word w exists in the ST's sentence
def _in_sentence(self, w):
if w in self.sentence:
return self.sentence.index(w)
return False
# Returns distance between two nodes n1 and n2
def _get_distance(self, n1, n2):
LCA = self._get_LCA(self.tree.root, n1, n2)
distance = self.tree.depth(self.tree.get_node(n1)) + self.tree.depth(self.tree.get_node(n2))
distance = distance - 2 * self.tree.depth(self.tree.get_node(LCA))
return abs(distance)
# Returns lowest common ancestor of two nodes n1 and n2
# Supporting method of _get_distance()
def _get_LCA(self, current_node, n1, n2):
if current_node is None: return None
if current_node == n1 or current_node == n2: return current_node
if len(self.tree.get_node(current_node).fpointer) == 0: return None #if leaf, return None
if len(self.tree.get_node(current_node).fpointer) == 1: #if terminal node, check its single leaf node
return self._get_LCA(self.tree.get_node(current_node).fpointer[0], n1, n2)
if len(self.tree.get_node(current_node).fpointer) == 2:
left = self._get_LCA(self.tree.get_node(current_node).fpointer[0],n1,n2)
right = self._get_LCA(self.tree.get_node(current_node).fpointer[1],n1,n2)
if left is not None and right is not None: return current_node
if left is not None: return left
if right is not None:return right
return None
continue
else:
rootid = nodeid
unique_ids = set(nodeids)
tree.create_node(tag=rootid,identifier=rootid)
while len(unique_ids) != len(tree.all_nodes()):
for rightNode in list(parent_check.keys()):
if tree.get_node(parent_check[rightNode]) is not None and tree.get_node(rightNode) is None:
tree.create_node(tag=rightNode, identifier=rightNode,
parent=tree.get_node(parent_check[rightNode]))
for node in tree.all_nodes():
sum += tree.depth(node)
print(sum)
def intersection(lst1, lst2):
lst3 = [value for value in lst1 if value in lst2]
return lst3
sanIndex = 0
youInex = 0
for path in tree.paths_to_leaves():
if path[len(path) -1] == "SAN":
sanIndex = path[path.index('7LD'):]
elif path[len(path) - 1] == "YOU":
youIndex = path[path.index('7LD'):]
class RRT:
def __init__(self,
current,
desir,
step_size,
kenamatic,
world_size,
obstacle=0):
self.jobs = []
self.size = world_size
# Desired start and end point
self.current = current
self.desir = desir
# A matrix that holds all dots from both starts
self.all_point = self.current
# The trees that arrange the matrix
self.tree = Tree()
self.tree.create_node(0, 0, data=self.current) # root node
# Did we finish the search
self.NOF = Value("i", 0)
self.statos = Value("i", 0)
# self.winindex = Value("i", 0)
self.winindex = Manager().list()
self.badindex = Manager().list()
self.badpoint = Manager().list()
self.pool_point = Manager().list()
self.pool_index = Manager().list()
self.num_pool = 9
# The number of dots already inserted into the tree
self.NOP = 1
# Step size
self.step = step_size
# Number of dimensions
# self.NOD = len(current.T)
self.t = time.time()
self.obstacle = obstacle
# The kinematics of the arms
self.kin = kenamatic
# Initial position of the arm base
print("open new RRT model")
def goliniar(self, point_a, point_b):
mid_all_point = self.midpoint(point_a[0], point_b[0])
i = 0
dis = 1
loops = len(mid_all_point)
while i < loops:
ans = self.kin.configure_check(mid_all_point[i].reshape(1, 12),
self.obstacle)
if ans == 0:
return i - dis, mid_all_point
else:
dis = max(int(ans**2 / 4000), 1)
i = i + dis
return loops - 1, mid_all_point
def goliniarpros(self, a):
steelrun = True
while steelrun:
for i in range(len(self.pool_point)):
secsed = True
allpoints = self.midpoint(self.pool_point[i], self.desir)
i = 0
loops = len(allpoints)
while i < loops:
ans = self.kin.configure_check1(
allpoints[i].reshape(1, 12), self.obstacle_gpu)
if not ans:
secsed = False
if (i > 8):
self.badpoint.append(allpoints[i - 8])
self.badindex.append(self.pool_index[i])
del self.pool_point[i]
del self.pool_index[i]
break
else:
i = i + max(int(ans**2 / 3000), 1)
if secsed:
self.statos.value = 1
self.winindex.append(self.pool_index[i])
steelrun = False
print("tread {} die ".format(self.pool_index[i]))
self.NOF.value -= 1
def get_direction(self, tree, index):
tree.show()
#allrote = self.current
if not index == 0:
root = tree[index].predecessor(tree._identifier)
good_route = tree[index].data
while (not root == 0 or root == None):
point = np.array(tree[root].data)
good_route = np.append(point, good_route, axis=0)
root = tree[root].predecessor(tree._identifier)
allrote = self.midpoint(self.desir[0], good_route[-1])
allrote = np.append(allrote[:-1],
self.midpoint(allrote[-1], good_route[0]),
axis=0)
for i in range(1, len(good_route)):
allrote = np.append(allrote[:-1],
self.midpoint(good_route[i - 1],
good_route[i]),
axis=0)
allrote = np.append(allrote[:-1],
self.midpoint(allrote[-1], self.current[0]),
axis=0)
else:
allrote = self.midpoint(self.desir[0], self.current[0])
# =============================================================================
# good_route = np.append(self.current, self.desir, axis=0)
# for i in range(1, len(good_route)):
# allrote = np.append(allrote, self.midpoint(good_route[i - 1], good_route[i]), axis=0)
# =============================================================================
# allrote = allrote[1:, :]
return np.flip(allrote, axis=0)
# Extracting the path we found from the tree
def get_direction1(self, tree, index):
#tree.show()
allrote = self.current
if not index == 0:
root = tree[index].predecessor(tree._identifier)
good_route = tree[index].data
while (not root == 0 or root == None):
point = np.array(tree[root].data)
good_route = np.append(point, good_route, axis=0)
root = tree[root].predecessor(tree._identifier)
for i in range(1, len(good_route)):
allrote = np.append(allrote[:-1],
self.midpoint(good_route[i - 1],
good_route[i]),
axis=0)
allrote = np.append(allrote[1:],
self.midpoint(self.all_point[index],
self.desir[0]),
axis=0)
allrote = np.append(self.midpoint(allrote[0], allrote[1]),
allrote[1:, :],
axis=0)
allrote = np.append(self.midpoint(self.current[0], allrote[0]),
allrote,
axis=0)
else:
good_route = np.append(self.current, self.desir, axis=0)
for i in range(1, len(good_route)):
allrote = np.append(allrote,
self.midpoint(good_route[i - 1],
good_route[i]),
axis=0)
# allrote = allrote[1:, :]
return allrote
# Gets a vector and normalizes it to a desired step size
def get_normelize_vec(self, vec):
return (vec / np.sqrt(np.sum(pow(vec, 2)))) * self.step
def midpoint(self, startp, endp):
return np.linspace(startp, endp,
int(np.max(np.abs(startp - endp) + 1)))
# Gets a vector point and returns the geometrically closest vector point
def min_distance(self, newpoint, data_points):
return np.argmin(
np.sum(pow(np.subtract(newpoint, data_points), 2), axis=1))
# Adds a point into the tree
def add_to_tree(self, index, newpoint):
self.tree.create_node(self.NOP, self.NOP, parent=index, data=newpoint)
self.all_point = np.append(self.all_point, newpoint, axis=0)
self.NOP += 1
print("NOP : ", self.NOP)
def bildwold(self):
NOO = 200
a = 1
p = 50
obs = np.array(
[np.ones(p) * NOO,
np.linspace(-NOO, NOO, num=p),
np.ones(p)]).T
for i in range(1, int(NOO * a)):
obs = np.append(obs,
np.array([
np.ones(p) * NOO,
np.linspace(-NOO, NOO, num=p),
np.ones(p) * (i)
]).T,
axis=0)
obs = np.append(obs,
np.array([
np.ones(p) * -NOO,
np.linspace(-NOO, NOO, num=p),
np.ones(p) * (i)
]).T,
axis=0)
obs = np.append(obs,
np.array([
np.linspace(-NOO, NOO, num=p),
np.ones(p) * NOO,
np.ones(p)
]).T,
axis=0)
for i in range(1, int(NOO * a)):
obs = np.append(obs,
np.array([
np.linspace(-NOO, NOO, num=p),
np.ones(p) * NOO,
np.ones(p) * (i)
]).T,
axis=0)
obs = np.append(obs,
np.array([
np.linspace(-NOO, NOO, num=p),
np.ones(p) * -NOO,
np.ones(p) * (i)
]).T,
axis=0)
NOO = NOO * 3
for i in range(1, int(NOO * 2)):
obs = np.append(obs,
np.array([
np.linspace(-NOO, NOO, num=p),
np.ones(p) * (NOO - i),
np.zeros(p)
]).T,
axis=0)
return obs
def open_prosses(self):
p = Process(target=self.goliniarpros, args=(self, ))
self.jobs.append(p)
p.start()
self.NOF.value += 1
# Adds a point in a random direction
def add_point(self, newpoint):
index = self.min_distance(newpoint, self.all_point) #V
newpoint = self.get_normelize_vec(
np.subtract(newpoint,
self.all_point[index])) + self.all_point[index]
godindex, temppoints = self.goliniar(
self.all_point[index].reshape(1, 12), newpoint)
if (godindex > 0):
self.add_to_tree(index, temppoints[godindex].reshape(1, 12))
self.pool_point.append(self.all_point[-1])
self.pool_index.append(index)
#print("number of processes that are currently running : {}".format(self.NOF.value))
def improved_path1(self, allpoint):
print("1")
t = time.time()
templen = 1000000
while (templen - len(allpoint) > 10):
a = len(allpoint)
mid = int(len(allpoint) / 2)
Peza = 5
index_a = mid + Peza
index_b = mid - Peza
beast = 0
templen = len(allpoint)
while (index_a <= len(allpoint) - 1 and index_b >= 0):
good_index, temp_points = self.goliniar(
allpoint[index_a].reshape(1, 12),
allpoint[index_b].reshape(1, 12))
if (good_index + 1 == len(temp_points)):
beast = Peza
Peza += 5
index_a = mid + Peza
index_b = mid - Peza
if (not beast == 0):
tempend = allpoint[(mid + beast - 1):, :]
tempstart = allpoint[:(mid - beast + 1), :]
if (not len(tempend)):
tempend = allpoint[-1, :].reshape((1, 12))
if (not len(tempstart)):
tempend = allpoint[0, :].reshape((1, 12))
tempmid = self.midpoint(tempstart[-1, :], tempend[0, :])
temp = np.append(tempstart, tempmid[1:-1], axis=0)
allpoint = np.append(temp, tempend, axis=0)
print("old : {} \t new : {}".format(a, len(allpoint)))
print(-t + time.time())
return allpoint
def improved_path2(self, allpoint):
print("2")
t = time.time()
templen = 1000000
for i in range(1, 4):
mid = int(len(allpoint) / 4 * i)
Peza = 5
index_a = mid + Peza
index_b = mid - Peza
beast = 0
templen = len(allpoint)
while (index_a <= len(allpoint) - 1 and index_b >= 0):
good_index, temp_points = self.goliniar(
allpoint[index_a].reshape(1, 12),
allpoint[index_b].reshape(1, 12))
if (good_index + 1 == len(temp_points)):
beast = Peza
else:
break
Peza += 5
index_a = mid + Peza
index_b = mid - Peza
if (not beast == 0):
tempend = allpoint[(mid + beast - 1):, :]
tempstart = allpoint[:(mid - beast + 1), :]
if (not len(tempend)):
tempend = allpoint[-1, :].reshape((1, 12))
if (not len(tempstart)):
tempend = allpoint[0, :].reshape((1, 12))
tempmid = self.midpoint(tempstart[-1, :], tempend[0, :])
temp = np.append(tempstart, tempmid[1:-1], axis=0)
allpoint = np.append(temp, tempend, axis=0)
# =============================================================================
# i=len(allpoint)-5
# while(i<0):
# good_index, temp_points =self.goliniar(allpoint[i].reshape(1,12),allpoint[-1].reshape(1,12))
# if (good_index+1==len(temp_points)):
# beast = i
# else:
# break
# i -= 5
# if (not beast == 0):
# tempmid = self.midpoint( allpoint[beast-1],allpoint[-1])
# allpoint = np.append(allpoint[(beast-1):],tempmid, axis=0)
# =============================================================================
print(-t + time.time())
return allpoint
def improved_path3(self, allpoint):
print("4")
t = time.time()
templen = 1000000
a = len(allpoint)
i = 5
a = len(allpoint)
while (i < len(allpoint)):
good_index, temp_points = self.goliniar(allpoint[0].reshape(1, 12),
allpoint[i].reshape(1, 12))
if (good_index + 1 == len(temp_points)):
beast = i
else:
break
i += 5
if (not beast == 0):
tempmid = self.midpoint(allpoint[0], allpoint[beast - 1])
allpoint = np.append(tempmid, allpoint[(beast - 1):], axis=0)
i = len(allpoint) - 1
print("old : {} \t new : {}".format(a, len(allpoint)))
a = len(allpoint)
while (i > 0):
good_index, temp_points = self.goliniar(
allpoint[i].reshape(1, 12), allpoint[-1].reshape(1, 12))
if (good_index + 1 == len(temp_points)):
beast = i
else:
break
i -= 5
if (not beast == len(allpoint) - 1):
tempmid = self.midpoint(allpoint[beast - 1], allpoint[-1])
allpoint = np.append(allpoint[:(beast - 1)], tempmid, axis=0)
print("old : {} \t new : {}".format(a, len(allpoint)))
a = len(allpoint)
mid = int(len(allpoint) / 2)
Peza = 5
index_a = mid + Peza
index_b = mid - Peza
beast = 0
templen = len(allpoint)
while (index_a <= len(allpoint) - 1 and index_b >= 0):
good_index, temp_points = self.goliniar(
allpoint[index_a].reshape(1, 12),
allpoint[index_b].reshape(1, 12))
if (good_index + 1 == len(temp_points)):
beast = Peza
Peza += 5
index_a = mid + Peza
index_b = mid - Peza
if (not beast == 0):
tempend = allpoint[(mid + beast - 1):, :]
tempstart = allpoint[:(mid - beast + 1), :]
if (not len(tempend)):
tempend = allpoint[-1, :].reshape((1, 12))
if (not len(tempstart)):
tempend = allpoint[0, :].reshape((1, 12))
tempmid = self.midpoint(tempstart[-1, :], tempend[0, :])
temp = np.append(tempstart, tempmid[1:-1], axis=0)
allpoint = np.append(temp, tempend, axis=0)
print("old : {} \t new : {}".format(a, len(allpoint)))
print(-t + time.time())
return allpoint
def get_winindex(self):
print(self.winindex)
print("Extracting the route")
dis = self.tree.depth(self.winindex[0]) * self.step + np.linalg.norm(
self.tree[self.winindex[0]].data - self.desir)
min_val = 99999
for i in self.winindex:
dis = self.tree.depth(i) * self.step + np.linalg.norm(
self.tree[i].data - self.desir)
print(i, "\t", self.tree.depth(i), "\t",
np.linalg.norm(self.tree[i].data - self.desir), '\t', dis)
if (dis < min_val):
min_val = dis
win = i
print(win)
self.winindex = Manager().list()
return win
# The function that activates everything
def let_the_magic_begin(self, ros_fun):
# self.open_prosses(self.current[0],self.desir[0],self.NOP - 1)
self.pool_point.append(self.current[0])
self.pool_index.append(0)
self.open_prosses()
NOL = 0
while (not self.statos.value):
if (not NOL % 10):
self.size += 10
self.size = min(self.size, 360)
newpoint = (np.random.rand(1, 12) * self.kin.limit * self.size -
self.kin.offset)
self.add_point(newpoint)
if (len(self.badindex)):
for i in range(len(self.badindex)):
self.add_to_tree(self.badindex[0],
self.badpoint[0].reshape(1, 12))
del self.badindex[0]
del self.badpoint[0]
NOL += 1
print("Waiting for all processes to die")
for job in self.jobs:
job.join()
best_point_index = self.get_winindex()
allrote = self.get_direction1(self.tree, best_point_index)
print(time.time() - self.t)
input("preace any key to improved path\n")
if (best_point_index == 0):
print("no need to improve path")
allrote1 = allrote
else:
for i in allrote:
print(
self.kin.configure_check1(i.reshape(1, 12),
self.obstacle_gpu))
if not (self.kin.configure_check1(i.reshape(1, 12),
self.obstacle_gpu)):
print("faild")
# =============================================================================
#
# allrote2=self.improved_path1(allrote)
# for i in allrote2:
# if not (self.kin.configure_check1(i.reshape(1,12),self.obstacle_gpu)):
# print("faild")
#
# allrote3=self.improved_path2(allrote)
# for i in allrote3:
# if not (self.kin.configure_check1(i.reshape(1,12),self.obstacle_gpu)):
# print("faild")
#
# allrote4=self.improved_path3(allrote)
# for i in allrote4:
# if not (self.kin.configure_check1(i.reshape(1,12),self.obstacle_gpu)):
# print(0)
# print("faild")
# =============================================================================
# =============================================================================
#
# # allrote2=self.improved_path(allrote)
# print("old -> : {}".format(len(allrote)))
# print("1 -> : {} imp -> {}".format(len(allrote2),len(allrote)-len(allrote2)))
# print("2 -> : {} imp -> {}".format(len(allrote3),len(allrote)-len(allrote3)))
# print("3 -> : {} imp -> {}".format(len(allrote4),len(allrote)-len(allrote4)))
# =============================================================================
input("preace any key to run the simulation\n")
ros_fun.send_to_arm(allrote * np.pi / 180)
input("preace any key to run the simulation\n")
ros_fun.send_to_arm(np.flip(allrote, axis=0) * np.pi / 180)
def run_serce(self, ros_fun, isinvers, invers_pos):
self.obstacle_gpu = th.from_numpy(self.obstacle).float().to('cuda')
if (isinvers):
self.desir = self.kin.invers_arms_cfg(invers_pos)
self.t = time.time()
if self.kin.configure_check1(self.desir, self.obstacle_gpu):
self.let_the_magic_begin(ros_fun)
else:
print("can't go to disre location")
class SentenceTree(object): def __init__(self): self.tree = Tree() self.tree_logger = Logger() self.sentence = [] # Builds a tree by backtracking the sentence matrix def build_tree(self, sentence_matrix): self.tree_logger.start_timer() self.matrix = sentence_matrix sentence_length = len(sentence_matrix)-1 # Saves the ST's sentence as a list of strings for i in range(1,sentence_length+1): self.sentence.append(self.matrix[0][i][0]) # Check if the sentence resolves to a tree if len(sentence_matrix[sentence_length][1])==0: return None # Find the most probable sentence option maximum = 0 index = None for option in sentence_matrix[sentence_length][1]: if option.probability > maximum: maximum = option.probability index = sentence_matrix[sentence_length][1].index(option) if index is None: return None # Build the tree self._nid = sentence_length+2 root = sentence_matrix[sentence_length][1][index] self.tree.create_node(root.constituent, self._nid) self._create_children(root, self._nid) # Call recursive function self.tree_logger.stop_timer() # Ensures unique node id in _create_children() def _nnid(self): self._nid +=1 return self._nid # Recursive function which builds the children nodes of a given parse_option # and then builds their children def _create_children(self, parse_option, pid): if parse_option is None: return None else: # If parse_option has children, extract those if parse_option.left_coord is not None: left_child = self.matrix[parse_option.left_coord[0]][parse_option.left_coord[1]][parse_option.left_coord[2]] right_child = self.matrix[parse_option.right_coord[0]][parse_option.right_coord[1]][parse_option.right_coord[2]] # Create left child as node (plus extra word node if leaf) cid = self._nnid() self.tree.create_node(left_child.constituent, cid, parent=pid) if left_child.left_coord is None: #If left_child is a leaf node, append a word node nid = parse_option.left_coord[1]-1 word = self.matrix[parse_option.left_coord[0]-1][parse_option.left_coord[1]][0] self.tree.create_node(word, nid, parent=cid) else: self._create_children(left_child, cid) # Create children of left_child # Create right child as node (plus extra word node if leaf) cid = self._nnid() self.tree.create_node(right_child.constituent, cid, parent=pid) if right_child.right_coord is None: #If left_child is a leaf node, append a word node nid = parse_option.right_coord[1]-1 word = self.matrix[parse_option.right_coord[0]-1][parse_option.right_coord[1]][0] self.tree.create_node(word, nid, parent=cid) else: self._create_children(right_child, cid) # Create children of right_child # Returns the sentence's sentiment score def get_sentiment_score(self, sentimentDict, term): total_score = 0 # placeholder dictionaries -TESTING PURPOSES negationList = ["ikke", "liden"] # Check the term against every sentiment word n1 = self.sentence.index(term) for key in sentimentDict: n2 = self._in_sentence(key) if n2 is not False: d = self._get_distance(n1, n2) score = float(sentimentDict[key]) / float(d) # If SentWord is negated, flip the score derived from it if self._is_negated(key, negationList): score = score * -1 print ">>SENTIMENTSCORE: Term: %s | SentWord: %s | Distance: %s | Score: %s" % (term, key, d,score) total_score += score return total_score # Checks whether a word is within a specified threshold distance of a negation word def _is_negated(self, w, negationList): negationThreshold = 3 n1 = self._in_sentence(w) if n1 is None: return False for nw in negationList: n2 = self._in_sentence(nw) if n2 is not None: if (self._get_distance(n1, n2)) < negationThreshold: print "negating word", w return True return False # Checks whether word w exists in the ST's sentence def _in_sentence(self, w): if w in self.sentence: return self.sentence.index(w) return False # Returns distance between two nodes n1 and n2 def _get_distance(self, n1, n2): LCA = self._get_LCA(self.tree.root, n1, n2) distance = self.tree.depth(self.tree.get_node(n1)) + self.tree.depth(self.tree.get_node(n2)) distance = distance - 2 * self.tree.depth(self.tree.get_node(LCA)) return distance-2 # Returns lowest common ancestor of two nodes n1 and n2 # Supporting method of _get_distance() def _get_LCA(self, root, n1, n2): if root is None: return None if root == n1 or root == n2: return root if len(self.tree.get_node(root).fpointer) == 0: return None #if leaf, return None if len(self.tree.get_node(root).fpointer) == 1: #if terminal node, check its single leaf node return self._get_LCA(self.tree.get_node(root).fpointer[0], n1, n2) if len(self.tree.get_node(root).fpointer) == 2: left = self._get_LCA(self.tree.get_node(root).fpointer[0],n1,n2) right = self._get_LCA(self.tree.get_node(root).fpointer[1],n1,n2) if left is not None and right is not None: return root if left is not None: return left if right is not None:return right return None
class AcquisitionChainIter(object):
def __init__(self, acquisition_chain, parallel_prepare=True):
self.__sequence_index = -1
self._parallel_prepare = parallel_prepare
self.__acquisition_chain_ref = weakref.ref(acquisition_chain)
# set all slaves into master
for master in (x for x in acquisition_chain._tree.expand_tree() if isinstance(x, AcquisitionMaster)):
del master.slaves[:]
master.slaves.extend(
acquisition_chain._tree.get_node(master).fpointer)
# create iterators tree
self._tree = Tree()
self._root_node = self._tree.create_node("acquisition chain", "root")
device2iter = dict()
for dev in acquisition_chain._tree.expand_tree():
if not isinstance(dev, (AcquisitionDevice, AcquisitionMaster)):
continue
dev_node = acquisition_chain._tree.get_node(dev)
parent = device2iter.get(dev_node.bpointer, "root")
try:
it = iter(dev)
except TypeError:
one_shot = self.acquisition_chain._device2one_shot_flag.get(
dev, True)
dev_iter = DeviceIterator(dev, one_shot)
else:
dev_iter = DeviceIteratorWrapper(it)
device2iter[dev] = dev_iter
self._tree.create_node(
tag=dev.name, identifier=dev_iter, parent=parent)
@property
def acquisition_chain(self):
return self.__acquisition_chain_ref()
def prepare(self, scan, scan_info):
preset_tasks = list()
if self.__sequence_index == 0:
preset_tasks.extend([gevent.spawn(preset.prepare)
for preset in self.acquisition_chain._presets_list])
scan.prepare(scan_info, self.acquisition_chain._tree)
self._execute(
"_prepare", wait_between_levels=not self._parallel_prepare)
if self.__sequence_index == 0:
gevent.joinall(preset_tasks, raise_error=True)
def start(self):
if self.__sequence_index == 0:
preset_tasks = [gevent.spawn(
preset.start) for preset in self.acquisition_chain._presets_list]
gevent.joinall(preset_tasks, raise_error=True)
self._execute("_start")
def stop(self):
self._execute("stop", master_to_slave=True, wait_all_tasks=True)
preset_tasks = [gevent.spawn(preset.stop)
for preset in self.acquisition_chain._presets_list]
gevent.joinall(preset_tasks) # wait to call all stop on preset
gevent.joinall(preset_tasks, raise_error=True)
def next(self):
self.__sequence_index += 1
gevent.joinall([gevent.spawn(dev_iter.wait_ready) for dev_iter in self._tree.expand_tree()
if dev_iter is not 'root'],
raise_error=True)
try:
if self.__sequence_index:
for dev_iter in self._tree.expand_tree():
if dev_iter is 'root':
continue
dev_iter.next()
except StopIteration: # should we stop all devices?
for acq_dev_iter in (x for x in self._tree.expand_tree() if x is not 'root' and
isinstance(x.device, (AcquisitionDevice, AcquisitionMaster))):
if hasattr(acq_dev_iter, 'wait_reading'):
acq_dev_iter.wait_reading()
dispatcher.send("end", acq_dev_iter.device)
raise
return self
def _execute(self, func_name,
master_to_slave=False, wait_between_levels=True,
wait_all_tasks=False):
tasks = list()
prev_level = None
if master_to_slave:
devs = list(self._tree.expand_tree(mode=Tree.WIDTH))[1:]
else:
devs = reversed(list(self._tree.expand_tree(mode=Tree.WIDTH))[1:])
for dev in devs:
node = self._tree.get_node(dev)
level = self._tree.depth(node)
if wait_between_levels and prev_level != level:
gevent.joinall(tasks, raise_error=True)
tasks = list()
prev_level = level
func = getattr(dev, func_name)
tasks.append(gevent.spawn(func))
# ensure that all tasks are executed
# (i.e: don't raise the first exception on stop)
if wait_all_tasks:
gevent.joinall(tasks)
gevent.joinall(tasks, raise_error=True)
"Select extra knowledge to load mirror infomation")
if filename is not None:
with open(filename, 'r') as load_f:
load_dict = json.load(load_f)
mirror = get_mirror(root.startEA)
procs = get_all_procs()
tree.create_node(fname,
hex(root.startEA),
data=Xref_node(fname, hex(root.startEA), XType.code,
mirror))
if mirror == fname:
add_xrefs(root.startEA, XType.code)
Message("Reference Tree:\n\n")
tree.show(line_type="ascii-em", idhidden=False, data_property='mirror')
Message("Unique references:\n")
for node in tree.all_nodes_itr():
if type(node.data.mirror) is str:
print node.identifier
#hierarchical output
for level in range(1, tree.depth()):
Message("\nLevel %d: %d\n" % (level, tree.size(level)))
for node in tree.all_nodes():
if tree.level(node.identifier) == level and type(
node.data.mirror) is str:
print node.identifier
Message("\n%d subroutines in routine %s need transplanting.\n" %
(Xref_node.xrefTrans - 1, fname))
conn.close()
else:
Warning("No function found at location %x" % here())
class PrePruneTree:
def __init__(self, vehsInfo):
self.tree = Tree()
self.root = self.tree.create_node("Root", "root") # root node
self.vehsInfo = vehsInfo
self.vehList = list(vehsInfo.keys())
self.pruneList = None
@tail_call_optimized
def _build(self, currentNode, vehList):
'''
:param vehList: A dict, keys is the set of vehicles, value is a tuple which represents (lane, position)
:param currentNode: The current node in the tree
'''
s = [currentNode.tag.find(vid) for vid in vehList] # the quit condition in recursion
if (np.array(s) >= 0).all() or self._testIllegel(currentNode.tag):
return
for vehId in vehList:
if vehId not in currentNode.tag:
if currentNode.is_root:
prefix = currentNode.tag.replace("Root", "")
else:
prefix = currentNode.tag
self.tree.create_node(prefix + vehId + "-", prefix + vehId, parent=currentNode)
# self.show()
for node in self.tree.all_nodes():
if node.is_leaf() and not self._testPrePrune(node.tag):
self._build(currentNode=node, vehList=vehList)
def _testIllegel(self, tag):
'''
test whether need to stop recursion
:param tag: Node tag
:return: boolean (if true, the recursion will stop)
'''
# upToRight.1-leftToBelow.18-belowToRight.2-belowToRight.3-
flag = False
tmp = tag.split("-")
try:
tmp.remove('')
except:
pass
if len(tmp) < len(list(self.vehsInfo.keys())) - 1:
return flag
surplusVeh = list(set(self.vehList) - set(tmp))
for veh1 in surplusVeh:
for veh2 in list(set(self.vehsInfo.keys()) - set(tmp)):
for subList in self.pruneList:
if surplusVeh in subList and veh2 in subList:
if subList.index(veh2) > subList.index(veh1):
flag = True
return flag
def _testPrePrune(self, tag):
'''
test whether need to preprune
:param tag:
:return:
'''
flag = False
vehs = tag.split("-")
try:
vehs.remove('')
except:
pass
for veh1 in vehs:
for veh2 in list(set(self.vehsInfo.keys()) - set(vehs)):
for subList in self.pruneList:
if veh1 in subList and veh2 in subList:
if subList.index(veh2) < subList.index(veh1):
flag = True
return flag
def obtainPruneList(self):
laneId = [value[0] for value in self.vehsInfo.values()]
sortedList = []
for i in list(set(laneId)):
lane_info = {k: v[1] for k, v in self.vehsInfo.items() if v[0] == i}
# Vehicles in front are at the front of the lane
sortedList.append([vid[0] for vid in sorted(lane_info.items(), key=itemgetter(1), reverse=True)])
pruneList = [sublist for sublist in sortedList if len(sublist) > 1]
return pruneList
def build(self):
threading.stack_size(20000000)
self.pruneList = self.obtainPruneList()
# start a new thread to generate tree
thread = threading.Thread(target=self._build(self.root, self.vehList))
thread.start()
def show(self):
self.tree.show()
def _leaves(self):
'''
:return: All the plan for vehicle passing currently.
'''
all_nodes = self.tree.all_nodes()
return [node for node in all_nodes if node.is_leaf()]
def legal_orders(self):
leaves = self._leaves()
orders = []
for pattern in leaves:
# upToRight.1-leftToBelow.18-belowToRight.2-belowToRight.3-
tmp = pattern.tag.split("-")
try:
tmp.remove('')
except:
pass
if len(tmp) == self.tree.depth():
orders.append(tmp)
return orders
if __name__ == '__main__':
tree = Tree()
node_id = 0
root_node = (3, 3, 1, node_id)
all_nodes = [root_node]
tree.create_node(tuple_to_string(root_node), root_node)
depth = int(input("Enter the depth: "))
for node in all_nodes:
if node[2] == 1:
temp_node = (node[0] - 1, node[1] - 1, 0, node_id)
place_in_tree(temp_node)
if tree.depth() > depth:
tree.remove_node(temp_node)
break
temp_node = (node[0] - 2, node[1], 0, node_id)
place_in_tree(temp_node)
temp_node = (node[0], node[1] - 2, 0, node_id)
place_in_tree(temp_node)
temp_node = (node[0] - 1, node[1], 0, node_id)
place_in_tree(temp_node)
temp_node = (node[0], node[1] - 1, 0, node_id)
place_in_tree(temp_node)
else:
temp_node = (node[0] + 1, node[1] + 1, 1, node_id)
place_in_tree(temp_node)
if tree.depth() > depth:
tree.remove_node(temp_node)
tree.create_node(tag=json_data[0].get("mid"),
identifier=json_data[0].get("mid"),
data=Info(json_data[0], param=param))
uu = 1000 if len(json_data) >= 1000 else len(json_data)
# print('uu:',uu)
for j in range(1, uu):
try:
tree.create_node(tag=json_data[j].get("mid"),
identifier=json_data[j].get("mid"),
parent=json_data[j].get("parent"),
data=Info(json_data[j], param=param))
except:
pass
# 单颗传播树构建完成
# 对传播树进行简化
print('简化前:tree_depth:', tree.depth(), 'tree_nodes:',
len(tree.all_nodes()))
tree = simplify_tree(tree)
print('***********简化后:tree_depth:', tree.depth(), 'tree_nodes:',
len(tree.all_nodes()))
trees.append(tree)
# print(eid,"---trees simplified")
# avg_tree_size += tree.size()
# avg_trees_size.append(int(avg_tree_size/data_array.shape[0]))
# plt.plot(params, avg_trees_size,'ro-')
# plt.xlabel('alpha')
# plt.ylabel('trees avg size')
# plt.show()
# plt.savefig('./img/img_1.jpg')
# # 提取新闻原文的相关特征,并写入文件
file = open(sys.argv[1], 'r')
line = file.readline()
d = dict()
while (line):
m = line.split(')')
key = m[0]
value = m[1].strip('\n')
if key not in d.keys():
d.setdefault(key, [])
d[key].append(value)
else:
d[key].append(value)
line = file.readline()
tree = Tree()
tree.create_node(identifier='COM')
add_orbits(tree, 'COM', d)
grandparents = []
for node in tree.all_nodes():
if tree.is_ancestor(node.identifier, 'YOU') and tree.is_ancestor(
node.identifier, 'SAN'):
grandparents.append(node.identifier)
common = grandparents[-1:]
m = tree.depth(tree.get_node(common[0]))
you = tree.depth(tree.get_node('YOU'))
san = tree.depth(tree.get_node('SAN'))
print(you - m + san - m - 2)
class search_tree(object):
def __init__(self, mcst, q_id, p_sen_list, max_depth, max_search_time,
beta, m_value, l_passages, ref_answer, vocab):
self.tree = Tree()
self.q_id = q_id
self.tree.create_node(identifier='question_' + str(q_id), data=node())
root_node = self.tree.get_node('question_' + str(q_id))
root_node.data.num = 1.0
self.node_map = {}
self.l_passages = l_passages
self.m_value = m_value
self.ref_answer = ref_answer
self.max_search_time = max_search_time
self.count = 0.0
self.carpe_diem = mcst
self.max_depth = max_depth
self.beta = beta
self.vocab = vocab
self.p_sen_list = p_sen_list
self.expand(self.tree.get_node(self.tree.root))
def expand(self, leaf_node):
#print '---------------------- start expand: -----------------------'
time_tree_start = time.time()
sens_list = leaf_node.data.sen
# print 'word_list:'
# print words_list
sens_id_list = map(eval, sens_list)
p_sen_id_list, p_pred_list = self.carpe_diem.get_policy(
sens_id_list, self.l_passages)
# print 'candidate_id: '
# print np.shape(p_word_id)
# print 'p_pred'
# print np.shape(p_pred)
for sen in p_sen_id_list:
self.node_map[' '.join(sens_list + [str(sen)])] = len(
self.node_map)
new_node = node()
new_node.sen = sens_list + [str(sen)]
#print ('new_node.sen', new_node.sen)
new_node.p = p_pred_list[p_sen_id_list.index(sen)]
self.tree.create_node(identifier=self.node_map[' '.join(
new_node.sen)],
data=new_node,
parent=leaf_node.identifier)
#print ('&&&&&&&&&&&&&&& tree expand time = %3.2f s &&&&&&&&&&&&' % (time.time() - time_tree_start))
def update(self, node_list, value):
#print '----update'
for node_id in node_list:
tmp_node = self.tree.get_node(node_id)
tmp_node.data.Q = (tmp_node.data.Q * tmp_node.data.num +
value) / (tmp_node.data.num + 1)
tmp_node.data.num += 1
def search(self, start_node_id):
#print '----tree search'
tmp_node = self.tree.get_node(start_node_id)
#print tmp_node.data.num
has_visit_num = tmp_node.data.num - 1
self.count = has_visit_num
if int(self.max_search_time - has_visit_num) > 0:
start_node_search_time = int(self.max_search_time - has_visit_num)
else:
start_node_search_time = 0
for tm in range(start_node_search_time):
if tm % 10 == 0:
batch_start_time = time.time()
#print ('search time',tm)
search_list = [start_node_id]
tmp_node = self.tree.get_node(start_node_id)
#print 'search time :'+ str(time)
while not tmp_node.is_leaf():
max_score = float("-inf")
max_id = -1
for child_id in tmp_node.fpointer:
child_node = self.tree.get_node(child_id)
score = self.beta * child_node.data.p * (
(tmp_node.data.num)**0.5 / (1 + child_node.data.num))
#print 'child_node.data.Q: '
#print child_node.data.Q
score += child_node.data.Q
#print 'score: '
#print score
#print '**************'
if score > max_score:
max_id = child_id
max_score = score
search_list.append(max_id)
tmp_node = self.tree.get_node(max_id)
if not tmp_node.data.value == None:
v = tmp_node.data.value
else:
if tmp_node.data.sen[-1] == str(self.l_passages - 1):
pred_answer = tmp_node.data.sen
# print 'search to end pred_answer: '
# print pred_answer
# print 'listSelectedSet'
listSelectedSet_sens = []
listSelectedSet = map(eval, pred_answer)
# print listSelectedSet
for idx in listSelectedSet:
listSelectedSet_sens.append(self.p_sen_list[idx])
# print 'pred_answer '
pred_answer_str = ''
for sen in listSelectedSet_sens:
str123_list = self.carpe_diem.vocab.recover_from_ids(
sen, 0)
for s in str123_list:
pred_answer_str += s
# print 'pred_answer_str: '
# print pred_answer_str
# print 'ref_answer_str: '
# print list2string(self.ref_answer[0]['answers'])
pred_answers = []
pred_answers.append({
'question_id': [self.q_id],
'question_type': [],
'answers': [''.join(pred_answer_str)],
'entity_answers': [[]],
'yesno_answers': []
})
if len(self.ref_answer) > 0:
pred_dict, ref_dict = {}, {}
for pred, ref in zip(pred_answers, self.ref_answer):
question_id = ref['question_id']
if len(ref['answers']) > 0:
pred_dict[question_id] = normalize(
pred['answers'])
ref_dict[question_id] = normalize(
ref['answers'])
# print '========compare in tree======='
# print pred_dict[question_id]
# print '----------------------'
# print ref_dict[question_id]
bleu_rouge = compute_bleu_rouge(pred_dict, ref_dict)
else:
bleu_rouge = None
# print 'last words ++++++++++++++ '
# print bleu_rouge
v = input_v = bleu_rouge['Rouge-L'] * self.m_value['Rouge-L'] \
+ bleu_rouge['Bleu-4'] * self.m_value['Bleu-4'] \
+ bleu_rouge['Bleu-1'] * self.m_value['Bleu-1'] \
+ bleu_rouge['Bleu-3'] * self.m_value['Bleu-3'] \
+ bleu_rouge['Bleu-2'] * self.m_value['Bleu-2']
else:
v = self.carpe_diem.value_function(tmp_node.data.sen)[0][0]
tmp_node.data.value = v
# if tmp_node.data.sen[-1] == str(self.l_passages - 1):
# pred_answer = tmp_node.data.sen
# listSelectedSet_sens = []
# listSelectedSet = map(eval, pred_answer)
# # print listSelectedSet
# for idx in listSelectedSet:
# listSelectedSet_sens.append(self.p_sen_list[idx])
# # print 'pred_answer '
# pred_answer_str = ''
# for sen in listSelectedSet_sens:
# str123_list = self.carpe_diem.vocab.recover_from_ids(sen, 0)
# for s in str123_list:
# pred_answer_str += s
#
# pred_answers = []
#
# pred_answers.append({'question_id': [self.q_id],
# 'question_type': [],
# 'answers': [''.join(pred_answer_str)],
# 'entity_answers': [[]],
# 'yesno_answers': []})
# if len(self.ref_answer) > 0:
# pred_dict, ref_dict = {}, {}
# for pred, ref in zip(pred_answers, self.ref_answer):
# question_id = ref['question_id']
# if len(ref['answers']) > 0:
# pred_dict[question_id] = normalize(pred['answers'])
# ref_dict[question_id] = normalize(ref['answers'])
# bleu_rouge = compute_bleu_rouge(pred_dict, ref_dict)
# else:
# bleu_rouge = None
# v = bleu_rouge['Rouge-L'] * self.m_value['Rouge-L'] \
# + bleu_rouge['Bleu-4'] * self.m_value['Bleu-4'] \
# + bleu_rouge['Bleu-1'] * self.m_value['Bleu-1'] \
# + bleu_rouge['Bleu-3'] * self.m_value['Bleu-3'] \
# + bleu_rouge['Bleu-2'] * self.m_value['Bleu-2']
# else:
# v = self.carpe_diem.value_function(tmp_node.data.sen)[0][0]
self.update(search_list, v)
self.count += 1
if tmp_node.is_leaf() and (
self.tree.depth(tmp_node) < self.max_depth
) and tmp_node.data.sen[-1] != str(self.l_passages - 1):
self.expand(tmp_node)
# if tm %10 == 0:
# print ('==================== search 10 time = %3.2f s ====================' % (time.time() - batch_start_time))
###########
'''
if time % 100 == 0:
tmp_policy = self.get_ppolicy(start_node_id)
print tmp_policy.values()
print sum(tmp_policy.values())
print time
'''
#print tmp_node.data.word
#print '------finish search '
#print '===== finish all search ======'
def search_eval(self, start_node_id):
#print '----tree search'
tmp_node = self.tree.get_node(start_node_id)
#print tmp_node.data.num
has_visit_num = tmp_node.data.num - 1
self.count = has_visit_num
if int(self.max_search_time - has_visit_num) > 0:
start_node_search_time = int(self.max_search_time - has_visit_num)
else:
start_node_search_time = 0
for time in range(start_node_search_time):
#print ('search time',time)
search_list = [start_node_id]
tmp_node = self.tree.get_node(start_node_id)
#print 'search time :'+ str(time)
while not tmp_node.is_leaf():
max_score = float("-inf")
max_id = -1
for child_id in tmp_node.fpointer:
child_node = self.tree.get_node(child_id)
score = self.beta * child_node.data.p * (
(tmp_node.data.num)**0.5 / (1 + child_node.data.num))
#print 'child_node.data.Q: '
#print child_node.data.Q
score += child_node.data.Q
#print 'score: '
#print score
#print '**************'
if score > max_score:
max_id = child_id
max_score = score
search_list.append(max_id)
tmp_node = self.tree.get_node(max_id)
#if tmp_node.data.word[-1] == str(self.l_passages-1):
v = self.carpe_diem.value_function(tmp_node.data.sen)[0][0]
#print 'v: '
#print v
self.update(search_list, v)
self.count += 1
if tmp_node.is_leaf() and (
self.tree.depth(tmp_node) < self.max_depth
) and tmp_node.data.sen[-1] != str(self.l_passages - 1):
self.expand(tmp_node)
###########
'''
if time % 100 == 0:
tmp_policy = self.get_ppolicy(start_node_id)
print tmp_policy.values()
print sum(tmp_policy.values())
print time
'''
#print tmp_node.data.word
#print '------finish search '
#print '===== finish all search ======'
def take_action(self, start_node_id):
#print '----take action: '
tmp_node = self.tree.get_node(start_node_id)
max_time = -1
prob = {}
for child_id in tmp_node.fpointer:
child_node = self.tree.get_node(child_id)
prob[child_node.data.sen[-1]] = child_node.data.num / self.count
if child_node.data.num > max_time:
#print child_node.data.num
#print max_time
#print 'child_node.data.num > max_time'
max_time = child_node.data.num
select_word = child_node.data.sen[-1]
select_word_node_id = child_node.identifier
#else:
#print 'not > max time '
#print select_word
#print select_word_node_id
#print '-----take action end'
return prob, select_word, select_word_node_id
def get_ppolicy(self, start_node_id):
tmp_node = self.tree.get_node(start_node_id)
max_time = -1
prob = {}
for child_id in tmp_node.fpointer:
child_node = self.tree.get_node(child_id)
if self.count == 0:
prob[child_node.data.sen[-1]] = 0.0
else:
prob[
child_node.data.sen[-1]] = child_node.data.num / self.count
return prob
json_data = json.load(load_f)
#构建新闻事件传播树
tree = Tree()
tree.create_node(tag=json_data[0].get("mid"),identifier=json_data[0].get("mid"),data=Info(json_data[0],param = 10))
uu = 1000 if len(json_data) >=1000 else len(json_data)
# print('uu:',uu)
for j in range(1,uu):
try:
tree.create_node(tag=json_data[j].get("mid"),identifier=json_data[j].get("mid"),parent=json_data[j].get("parent"),data=Info(json_data[j]))
except:
pass
#单颗传播树构建完成
#对传播树进行简化
print('简化前:tree_depth:',tree.depth(),'tree_nodes:',len(tree.all_nodes()))
tree = simplify_tree(tree)
print('简化后:tree_depth:', tree.depth(),'tree_nodes:',len(tree.all_nodes()))
trees.append(tree)
# print(eid,"---trees simplified")
#所有新闻事件的传播树构建结束
#3折交叉法
pd_data = pd.read_csv('id_label.txt', sep='\t', header=None)
# wb_data = pd_data.as_matrix()[0:20,]
if len(tree.get_node(root).fpointer) == 2: left = get_LCA(tree.get_node(root).fpointer[0],n1,n2) right = get_LCA(tree.get_node(root).fpointer[1],n1,n2) if left is not None and right is not None: return root if left is not None: return left if right is not None:return right return None LCA = get_LCA(tree.root, s.index(entity), s.index(opinionw)) # Calculate distance between two nodes distance = tree.depth(tree.get_node(s.index(entity))) + tree.depth(tree.get_node(s.index(opinionw))) distance = distance - 2 * tree.depth(tree.get_node(LCA)) print "Looking for words:", entity, opinionw print "LCA is", tree.get_node(LCA).tag print "Distance is", distance def get_distance(word1, word2, tree): pass
class RST_DT:
def load(self, path2file):
self.id_EDUs = []
self.EDU = {}
self.treeNS = Tree()
self.tree = Tree()
# nombre max d'espace pour init id_parents
with open(path2file, "r") as f:
max_space = 0
nb_line = 0
for i, line in enumerate(f):
nb_space = 0
for c in line:
if c == " ":
nb_space += 1
else:
break
if nb_space > max_space:
max_space = nb_space
nb_line += 1
with open(path2file, "r") as f:
id_parents = [0] * max_space
NS_parents = [0] * max_space
for i, line in enumerate(f):
# nombre d'espace détermine le parent
nb_space = 0
for c in line:
if c == " ":
nb_space += 1
else:
break
space = nb_space / 2
id_parents[space] = i
parent = id_parents[space - 1]
reg = "\(([\w\-\[\]]+)|(_!.+!_)" # récupération du contenu
match = re.findall(reg, line)[0]
if match[0] == "":
content = match[1] # feuille EDU
self.id_EDUs.append(i)
# print content
self.EDU[i] = re.findall("_!(.*)!_", content)
else:
content = match[0]
reg2 = "\[(N|S)\]" # récupération NS
match2 = re.findall(reg2, content)
NS_parents[space] = match2 # ['N','S']
# création du noeud
if i == 0:
self.tree.create_node(content, 0)
self.treeNS.create_node("Root", 0)
else:
id_NS = len(self.tree.is_branch(parent)) # 0 ou 1 car arbre binaire
self.tree.create_node(content, i, parent=parent)
self.treeNS.create_node(NS_parents[space - 1][id_NS], i, parent=parent)
def toDEP(self):
###############################
# Etape 1 : construction du head_tree
# parcours en largeur de tree afin de récupérer chaque id_node
# pour chaque profondeur (init à 0) _! sans compter !_ les feuilles (EDUs)
nodes_depth = [-1] * self.tree.size()
for i in xrange(self.tree.size()):
id_nodes = [0]
depth = [999] * self.tree.size()
while id_nodes: # False if empty
id_node = id_nodes.pop(0)
node = self.tree.get_node(id_node)
if node.bpointer != None:
node_parent = self.tree.get_node(node.bpointer)
depth[node.identifier] = depth[node_parent.identifier] + 1
else:
depth[node.identifier] = 0
if id_node == i:
# print 'noeud ',i,' en profondeur', depth[node.identifier]
if node.fpointer:
nodes_depth[i] = depth[i]
break
if node.fpointer:
id_nodes.append(node.fpointer[0])
id_nodes.append(node.fpointer[1])
# print nodes_depth
id_nodes_depth = []
for d in xrange(self.tree.depth()):
id_nodes_depth.append([])
for i in xrange(self.tree.size()):
if nodes_depth[i] == d:
id_nodes_depth[d].append(i)
# print id_nodes_depth
#
# construction du head_tree
head_tree = [-1] * self.treeNS.size()
# pour chaque noeud (non EDU/feuille) en partant de la plus grande profondeur dans l'arbre
for d in range(len(id_nodes_depth) - 1, -1, -1):
for id_node in id_nodes_depth[d]:
node = self.treeNS.get_node(id_node)
node_left = self.treeNS.get_node(node.fpointer[0])
node_right = self.treeNS.get_node(node.fpointer[1])
if node_left.tag == "N":
if head_tree[node_left.identifier] == -1:
identifier = node_left.identifier
else:
identifier = head_tree[node_left.identifier]
else:
if head_tree[node_right.identifier] == -1:
identifier = node_right.identifier
else:
identifier = head_tree[node_right.identifier]
head_tree[id_node] = identifier
# print head_tree
###############################
# Etape 2 : construction du DEP
#
# construction du DEP
# init
# root est le premier noeud de head
# pour chaque EDU son père est le root dans DEP
dep_tree = Tree()
id_root = head_tree[0]
root = self.tree.get_node(id_root)
# dep_tree.create_node(root.tag, root.identifier)
dep_tree.create_node(root.tag, root.identifier)
for id_EDU in xrange(len(head_tree)):
if head_tree[id_EDU] == -1 and id_EDU != id_root:
node = self.tree.get_node(id_EDU)
# dep_tree.create_node(node.tag, node.identifier, parent=id_root)
# dep_tree.create_node(str(id_EDU), node.identifier, parent=id_root)
dep_tree.create_node(node.tag, node.identifier, parent=id_root)
# print '//////////////////////'
# print 'EDU', id_root
# pour chaque EDU
for id_EDU in xrange(len(head_tree)):
if head_tree[id_EDU] == -1 and id_EDU != id_root:
EDU_NS = self.treeNS.get_node(id_EDU)
# print '.......................'
# print 'EDU', id_EDU
# print 'TAG', EDU_NS.tag
if EDU_NS.tag == "N":
# parcours en largeur jusqu'à trouver un S avec un head donc qui soit pas EDU
id_nodes = [EDU_NS.identifier]
visited = [False] * self.treeNS.size()
while id_nodes:
id_node = id_nodes.pop(0)
EDU = self.tree.get_node(id_node)
# print 'visited EDU', EDU.identifier
visited[EDU.identifier] = True
# cas d'arret
head_EDU = head_tree[EDU.identifier] == -1
head_EDU = False
node_tag = self.treeNS.get_node(EDU.identifier).tag
# print ' head_EDU', head_EDU
# print ' node_tag', node_tag
if not head_EDU and node_tag == "S":
break
if EDU.bpointer:
if not visited[EDU.bpointer]:
id_nodes.append(EDU.bpointer)
if EDU.fpointer: # sécurité
if not visited[EDU.fpointer[0]]:
id_nodes.append(EDU.fpointer[0])
if not visited[EDU.fpointer[1]]:
id_nodes.append(EDU.fpointer[1])
# puis ajouter au DEP comme enfant du head du parent du noeud S
id_head = head_tree[EDU.bpointer]
# si parent S
else:
# parcours en largeur des ancêtre jusqu'à trouver un ancêtre avec un head
parent = self.treeNS.get_node(EDU_NS.bpointer)
id_head = head_tree[parent.identifier]
# puis ajouter au DEP comme enfant de ce head
if id_EDU != id_head:
dep_tree.move_node(id_EDU, id_head)
EDU = self.tree.get_node(id_EDU)
# print '---- ajout de',EDU.identifier,' à',id_head
# if id_EDU == id_head:
# dep_tree.show()
return dep_tree
# showDepth(dep_tree, 4)
# dep_tree.show()
# node = dep_tree.
def toString(self):
""" affiche comme la sortie de Hilda """
showDepth(self.tree, 0)
def do_md_section(sysid, levelid, connection_str, output_format, output_file,
compact, doc_handler, dotree):
"""
Given the MD ids, dump the content in the output file and produce the product tree diagrams
:param sysid: MagicDraw subsystem id
:param levelid: MagicDraw level id (package id containing the tree to extract)
:param connection_str: MagicDraw encoded connection string
:param output_format: OutputFormat
:param output_file: File to dump
:param compact: True if the portrait diagrams have to be in compact form
:param doc_handler: the document id
:param sub_name: name of the subsystem
:return: none
"""
global template_path
global productTree
global tree_dict
productTree = Tree()
products = []
tree_dict = {}
# get the information from MagicDraw
mdr = build_md_tree(sysid, levelid, connection_str)
print("\n Product tree depth:", productTree.depth())
nodes = productTree.expand_tree(key=lambda x: x.data.index, sorting=True)
for n in nodes:
products.append(productTree[n].data)
tree_dict[productTree[n].data.id] = productTree[n].data
print(f" Found {{np}} products (including container folders).".format(
np=len(tree_dict)))
envs = Environment(loader=ChoiceLoader([
FileSystemLoader(Config.TEMPLATE_DIRECTORY),
PackageLoader('ptree', 'templates')
]),
lstrip_blocks=True,
trim_blocks=True,
autoescape=None)
# dump a csv file
if dotree:
do_csv(products, output_file)
# create the diagrams tex files
do_trees_diagrams(productTree, output_file, products[0].shortname,
compact)
# define the template
template_path = f"section.{Config.TEMPLATE_LANGUAGE}.jinja2"
else:
# no diagrams are created
# use a simplified template
template_path = f"notree_section.{Config.TEMPLATE_LANGUAGE}.jinja2"
# sort tree dictionary based
mdp = productTree.to_dict(with_data=False)
# get ordered dictionary
template = envs.get_template(template_path)
new_mdpt = dict()
for k0 in mdp:
new_mdpt[k0] = order_tree_level(mdp[k0])
# dump the tex section
metadata = dict()
metadata["template"] = template.filename
text = template.render(metadata=metadata,
mdrev=mdr,
filename=output_file,
doc_handler=doc_handler,
mdt_dict=tree_dict,
mdp=new_mdpt,
mdps=products)
tex_file_name = output_file + ".tex"
file = open(tex_file_name, "w")
print(_as_output_format(text, output_format), file=file)
file.close()
return tree_dict
class TreeT(object):
def __init__(self, max_id=0):
self.tree = Tree()
def from_ptb_to_tree(self, line, max_id=0, leaf_id=1, parent_id=None):
# starts by ['(', 'pos']
pos_tag = line[1]
if parent_id is None:
pos_id = 0
else:
pos_id = max_id
max_id += 1
self.tree.create_node(pos_tag, pos_id, parent_id, TreeData())
parent_id = pos_id
total_offset = 2
if line[2] != '(':
# sub-tree is leaf
# line[0:3] = ['(', 'pos', 'word', ')']
word_tag = line[2]
self.tree.create_node(word_tag, leaf_id, parent_id, TreeData())
return 4, max_id, leaf_id + 1
line = line[2:]
while line[0] != ')':
offset, max_id, leaf_id = self.from_ptb_to_tree(
line, max_id, leaf_id, parent_id)
total_offset += offset
line = line[offset:]
return total_offset + 1, max_id, leaf_id
def add_height(self, tree_dep):
for n in self.tree.all_nodes():
n.data.leaves = []
for leaf in self.tree.leaves():
lid = leaf.identifier
hid = tree_dep[lid]
if hid == self.tree.root:
self.tree[lid].data.height = self.tree.depth(self.tree[lid])
for cid in [
p for p in self.tree.paths_to_leaves() if lid in p
][0]:
self.tree[cid].data.leaves += [lid]
else:
height = -1
cid = lid
cond = True
while cond:
self.tree[cid].data.leaves += [lid]
height += 1
cid = self.tree.parent(cid).identifier
cid_leaves = [l.identifier for l in self.tree.leaves(cid)]
cid_l_dep = [tree_dep[l] for l in cid_leaves if l != lid]
cond = set(cid_l_dep).issubset(set(cid_leaves))
self.tree[lid].data.height = height
x_nodes = [
n.identifier for n in self.tree.all_nodes() if n.data.leaves == []
]
for x_node in x_nodes[::-1]:
min_id = min(self.tree.children(x_node),
key=lambda c: c.data.height)
_lid = min_id.data.leaves[0]
self.tree[_lid].data.height += 1
self.tree[x_node].data.leaves += [_lid]
return True
def _from_tree_to_ptb(self, nid):
nid = self.tree.subtree(nid).root
if self.tree[nid].is_leaf():
return ' (' + self.tree[nid].tag + ' ' + self.tree[
nid].data.word + ')'
res = ' (' + self.tree[nid].tag
for c_nid in sorted(self.tree.children(nid),
key=lambda x: x.identifier):
res += self._from_tree_to_ptb(c_nid.identifier)
return res + ')'
def from_tree_to_ptb(self):
return self._from_tree_to_ptb(self.tree.root)
def from_tag_to_tree(self, tag, word, pos_id=0):
parent_id = None
for tag_nodes in tag:
if tag_nodes[0] in [CL, CR]:
c_side = tag_nodes[0]
_tag_nodes = tag_nodes[1:] if len(tag_nodes) > 1 else ['']
else:
c_side = ''
_tag_nodes = tag_nodes
self.tree.create_node(_tag_nodes[0],
pos_id,
parent=parent_id,
data=TreeData(comb_side=c_side))
parent_id = pos_id
pos_id += 1
for tag_node in _tag_nodes[1:]:
self.tree.create_node(tag_node[1:],
pos_id,
parent=parent_id,
data=TreeData(miss_side=tag_node[0]))
pos_id += 1
for l in self.tree.leaves():
if l.data.miss_side == '':
l.data.word = word
break
return pos_id
@memoize
def is_combine_to(self, side):
return self.tree[self.tree.root].data.comb_side == side
@memoize
def is_combine_right(self):
return self.is_combine_to(CR)
@memoize
def is_combine_left(self):
return self.is_combine_to(CL)
@memoize
def is_complete_tree(self):
return all([n.data.miss_side == '' for n in self.tree.all_nodes()])
@memoize
def get_missing_leaves_to(self, miss_val, side):
return [
l.identifier for l in self.tree.leaves(self.tree.root)
if l.data.miss_side == side and l.tag == miss_val
]
@memoize
def get_missing_leaves_left(self, miss_val):
return self.get_missing_leaves_to(miss_val, L)
@memoize
def get_missing_leaves_right(self, miss_val):
return self.get_missing_leaves_to(miss_val, R)
@memoize
def root_tag(self):
return self.tree[self.tree.root].tag
@memoize
def is_no_missing_leaves(self):
return all(
[l.data.miss_side == '' for l in self.tree.leaves(self.tree.root)])
@memoize
def combine_tree(self, _tree, comb_leaf):
self.tree.paste(comb_leaf, _tree.tree)
self.tree.link_past_node(comb_leaf)
return self
def tree_to_path(self, nid, path):
# Stop condition
if self.tree[nid].is_leaf():
path[nid] = []
return nid, self.tree[nid].data.height
# Recursion
flag = CR
for child in self.tree.children(nid):
cid = child.identifier
leaf_id, height = self.tree_to_path(cid, path)
if (height == 0):
# Reached end of path can add flag
path[leaf_id].insert(0, flag)
# path[leaf_id].append(flag)
if height > 0:
path[leaf_id].insert(0, nid)
# only single child will have height>0
# and its value will be the one that is returned
# to the parent
ret_leaf_id, ret_height = leaf_id, height - 1
# once we reached a height>0, it means that
# this path includes the parent, and thus flag
# direction should flip
flag = CL
return ret_leaf_id, ret_height
def path_to_tags(self, path):
tags = []
for p in path:
_res = []
_p = copy.copy(p)
if _p[0] in [CL, CR]:
_res.append(_p[0])
_p = _p[1:]
while _p[:-1]:
el_p = _p.pop(0)
_res.append(self.tree[el_p].tag)
for c in self.tree.children(el_p):
if c.identifier != _p[0]:
_res.append(R + c.tag if c.identifier > _p[0] else L +
c.tag)
_res.append(self.tree[_p[0]].tag)
tags.append(_res)
return tags
def path_to_words(self, path):
return [self.tree[k].tag for k in path]
def from_tree_to_tag(self):
path = {}
self.tree_to_path(self.tree.root, path)
return {
'tags': self.path_to_tags(path.values()),
'words': self.path_to_words(path.keys())
}
def from_ptb_to_tag(self, line, max_id, depend):
self.from_ptb_to_tree(line, max_id)
self.add_height(depend)
path = {}
self.tree_to_path(self.tree.root, path)
return self.path_to_tags(path.values())
class StepParse:
def __init__(self):
pass
def load_step(self, step_filename):
self.nauo_lines = []
self.prod_def_lines = []
self.prod_def_form_lines = []
self.prod_lines = []
self.filename = os.path.splitext(step_filename)[0]
line_hold = ''
line_type = ''
# Find all search lines
with open(step_filename) as f:
for line in f:
# TH: read pointer of lines as they are read, so if the file has text wrap it will notice and add it to the following lines
index = re.search("#(.*)=", line)
if index:
# TH: if not none then it is the start of a line so read it
# want to hold line until it has checked next line
# if next line is a new indexed line then save previous line
if line_hold:
if line_type == 'nauo':
self.nauo_lines.append(line_hold)
elif line_type == 'prod_def':
self.prod_def_lines.append(line_hold)
elif line_type == 'prod_def_form':
self.prod_def_form_lines.append(line_hold)
elif line_type == 'prod':
self.prod_lines.append(line_hold)
line_hold = ''
line_type = ''
prev_index = True # TH remember previous line had an index
if 'NEXT_ASSEMBLY_USAGE_OCCURRENCE' in line:
line_hold = line.rstrip()
line_type = 'nauo'
elif ('PRODUCT_DEFINITION ' in line
or 'PRODUCT_DEFINITION(' in line):
line_hold = line.rstrip()
line_type = 'prod_def'
elif 'PRODUCT_DEFINITION_FORMATION' in line:
line_hold = line.rstrip()
line_type = 'prod_def_form'
elif ('PRODUCT ' in line or 'PRODUCT(' in line):
line_hold = line.rstrip()
line_type = 'prod'
else:
prev_index = False
#TH: if end of file and previous line was held
if 'ENDSEC;' in line:
if line_hold:
if line_type == 'nauo':
self.nauo_lines.append(line_hold)
elif line_type == 'prod_def':
self.prod_def_lines.append(line_hold)
elif line_type == 'prod_def_form':
self.prod_def_form_lines.append(line_hold)
elif line_type == 'prod':
self.prod_lines.append(line_hold)
line_hold = ''
line_type = ''
else:
#TH: if not end of file
line_hold = line_hold + line.rstrip()
self.nauo_refs = []
self.prod_def_refs = []
self.prod_def_form_refs = []
self.prod_refs = []
# TH: added 'replace(","," ").' to replace ',' with a space to make the spilt easier if there are not spaces inbetween the words'
# Find all (# hashed) line references and product names
# TH: it might be worth finding a different way of extracting data we do want rather than fixes to get rid of the data we don't
for j, el_ in enumerate(self.nauo_lines):
self.nauo_refs.append([
el.rstrip(',')
for el in el_.replace(",", " ").replace("=", " ").split()
if el.startswith('#')
])
for j, el_ in enumerate(self.prod_def_lines):
self.prod_def_refs.append([
el.rstrip(',')
for el in el_.replace(",", " ").replace("=", " ").split()
if el.startswith('#')
])
for j, el_ in enumerate(self.prod_def_form_lines):
self.prod_def_form_refs.append([
el.rstrip(',')
for el in el_.replace(",", " ").replace("=", " ").split()
if el.startswith('#')
])
for j, el_ in enumerate(self.prod_lines):
self.prod_refs.append([
el.strip(',') for el in el_.replace(",", " ").replace(
"(", " ").replace("=", " ").split() if el.startswith('#')
])
self.prod_refs[j].append(el_.split("'")[1])
# Get first two items in each sublist (as third is shape ref)
#
# First item is 'PRODUCT_DEFINITION' ref
# Second item is 'PRODUCT_DEFINITION_FORMATION <etc>' ref
self.prod_all_refs = [el[:2] for el in self.prod_def_refs]
# Match up all references down to level of product name
for j, el_ in enumerate(self.prod_all_refs):
# Add 'PRODUCT_DEFINITION' ref
for i, el in enumerate(self.prod_def_form_refs):
if el[0] == el_[1]:
el_.append(el[1])
break
# Add names from 'PRODUCT_DEFINITION' lines
for i, el in enumerate(self.prod_refs):
if el[0] == el_[2]:
el_.append(el[2])
break
# Find all parent and child relationships (3rd and 2nd item in each sublist)
self.parent_refs = [el[1] for el in self.nauo_refs]
self.child_refs = [el[2] for el in self.nauo_refs]
# Find distinct parts and assemblies via set operations; returns list, so no repetition of items
self.all_type_refs = set(self.child_refs) | set(self.parent_refs)
self.ass_type_refs = set(self.parent_refs)
self.part_type_refs = set(self.child_refs) - set(self.parent_refs)
#TH: find root node
self.root_type_refs = set(self.parent_refs) - set(self.child_refs)
# Create simple parts dictionary (ref + label)
self.part_dict = {el[0]: el[3] for el in self.prod_all_refs}
# self.part_dict_inv = {el[3]:el[0] for el in self.prod_all_refs}
def show_values(self):
# TH: basic testing, if needed these could be spilt up
print(self.nauo_lines)
print(self.prod_def_lines)
print(self.prod_def_form_lines)
print(self.prod_lines)
print(self.nauo_refs)
print(self.prod_def_refs)
print(self.prod_def_form_refs)
print(self.prod_refs)
# HR: "create_dict" replaced by list comprehension elsewhere
#
# def create_dict(self):
#
# # TH: links nauo number with a name and creates dict
# self.part_dict = {}
# for part in self.all_type_refs:
# for sublist in self.prod_def_refs:
# if sublist[0] == part:
# prod_loc = '#' + re.findall('\d+',sublist[1])[0]
# pass
# for sublist in self.prod_def_form_refs:
# if sublist[0] == prod_loc:
# prod_loc = '#' + str(re.findall('\d+',sublist[1])[0])
# pass
# for sublist in self.prod_refs:
# if sublist[0] == prod_loc:
# part_name = sublist[2]
#
# self.part_dict[part] = part_name
def create_tree(self):
#TH: create tree diagram in newick format
#TH: find root node
self.tree = Tree()
#TH: check if there are any parts to make a tree from, if not don't bother
if self.part_dict == {}:
return
root_node_ref = list(self.root_type_refs)[0]
# HR added part reference as data for later use
self.tree.create_node(self.part_dict[root_node_ref],
0,
data={'ref': root_node_ref})
#TH: created root node now fill in next layer
#TH: create dict for tree, as each node needs a unique name
i = [0] # Iterates through nodes
self.tree_dict = {}
self.tree_dict[i[0]] = root_node_ref
def tree_next_layer(self, parent):
root_node = self.tree_dict[i[0]]
for line in self.nauo_refs:
if line[1] == root_node:
i[0] += 1
self.tree_dict[i[0]] = str(line[2])
# HR added part reference as data for later use
self.tree.create_node(self.part_dict[line[2]],
i[0],
parent=parent,
data={'ref': str(line[2])})
tree_next_layer(self, i[0])
tree_next_layer(self, 0)
self.appended = False
self.get_levels()
def get_levels(self):
# Initialise dict and get first level (leaves)
self.levels = {}
self.levels_set_p = set()
self.levels_set_a = set()
self.leaf_ids = [el.identifier for el in self.tree.leaves()]
self.all_ids = [el for el in self.tree.nodes]
self.non_leaf_ids = set(self.all_ids) - set(self.leaf_ids)
self.part_level = 1
def do_level(self, tree_level):
# Get all nodes within this level
node_ids = [
el for el in self.tree.nodes
if self.tree.level(el) == tree_level
]
for el in node_ids:
# If leaf, then n_p = 1 and n_a = 1
if el in self.leaf_ids:
self.levels[el] = {}
self.levels[el]['n_p'] = self.part_level
self.levels[el]['n_a'] = self.part_level
# If assembly, then get all children and sum all parts + assemblies
else:
# Get all children of node and sum levels
child_ids = self.tree.is_branch(el)
child_sum_p = 0
child_sum_a = 0
for el_ in child_ids:
child_sum_p += self.levels[el_]['n_p']
child_sum_a += self.levels[el_]['n_a']
self.levels[el] = {}
self.levels[el]['n_p'] = child_sum_p
self.levels[el]['n_a'] = child_sum_a + 1
self.levels_set_p.add(child_sum_p)
self.levels_set_a.add(child_sum_a + 1)
# Go up through tree levels and populate lattice level dict
for i in range(self.tree.depth(), -1, -1):
do_level(self, i)
self.create_lattice()
self.levels_p_sorted = sorted(list(self.levels_set_p))
self.levels_a_sorted = sorted(list(self.levels_set_a))
# Function to return dictionary of item IDs for each lattice level
def get_levels_inv(list_in, key):
#Initialise
levels_inv = {}
levels_inv[self.part_level] = []
for el in list_in:
levels_inv[el] = []
for k, v in self.levels.items():
levels_inv[v[key]].append(k)
return levels_inv
self.levels_p_inv = get_levels_inv(self.levels_p_sorted, 'n_p')
self.levels_a_inv = get_levels_inv(self.levels_a_sorted, 'n_a')
def get_all_children(self, id_):
ancestors = [el.identifier for el in self.tree.children(id_)]
parents = ancestors
while parents:
children = []
for parent in parents:
children = [el.identifier for el in self.tree.children(parent)]
ancestors.extend(children)
parents = children
return ancestors
def create_lattice(self):
# Create lattice
self.g = nx.DiGraph()
self.default_colour = 'r'
# Get root node and set parent to -1 to maintain data type of "parent"
# Set position to top/middle
node_id = self.tree.root
label_text = self.tree.get_node(node_id).tag
self.g.add_node(node_id,
parent=-1,
label=label_text,
colour=self.default_colour)
# Do nodes from treelib "nodes" dictionary
for key in self.tree.nodes:
# Exclude root
if key != self.tree.root:
parent_id = self.tree.parent(key).identifier
label_text = self.tree.get_node(key).tag
# Node IDs same as for tree
self.g.add_node(key,
parent=parent_id,
label=label_text,
colour=self.default_colour)
# Do edges from nodes
for key in self.tree.nodes:
# Exclude root
if key != self.tree.root:
parent_id = self.tree.parent(key).identifier
self.g.add_edge(key, parent_id)
# Escape if only one node
# HR 6/3/20 QUICK BUG FIX: SINGLE-NODE TREE DOES NOT PLOT
# IMPROVE LATER; SHOULD BE PART OF A GENERAL METHOD
if self.tree.size() == 1:
id_ = [el.identifier for el in self.tree.leaves()]
self.g.nodes[id_[-1]]['pos'] = (0, 0)
return
# Get set of parents of leaf nodes
leaf_parents = set(
[self.tree.parent(el).identifier for el in self.leaf_ids])
# For each leaf_parent, set position of leaf nodes sequentially
i = 0
no_leaves = len(self.tree.leaves())
for el in leaf_parents:
for el_ in self.tree.is_branch(el):
child_ids = [el.identifier for el in self.tree.leaves()]
if el_ in child_ids:
self.g.nodes[el_]['pos'] = ((i / (no_leaves)), 1)
i += 1
# To set plot positions of nodes from lattice levels
# ---
# Traverse upwards from leaves
for el in sorted(list(self.levels_set_a)):
# Get all nodes at that level
node_ids = [k for k, v in self.levels.items() if v['n_a'] == el]
# Get all positions of children of that node
# and set position as mean value of them
for el_ in node_ids:
child_ids = self.tree.is_branch(el_)
pos_sum = 0
for el__ in child_ids:
pos_ = self.g.nodes[el__]['pos'][0]
pos_sum += pos_
pos_sum = pos_sum / len(child_ids)
self.g.nodes[el_]['pos'] = (pos_sum, el)
def print_tree(self):
try:
self.tree.show()
except:
self.create_tree()
self.tree.show()
def tree_to_json(self, save_to_file=False, filename='file', path=''):
#TH: return json format tree, can also save to file
if self.tree.size() != 0:
data = self.tree.to_json()
j = json.loads(data)
if save_to_file == True:
if path:
file_path = os.path.join(path, filename)
else:
file_path = filename
with open(file_path + '.json', 'w') as outfile:
json.dump(j, outfile)
return data
else:
print("no tree to print")
return
def hierarchy(fna_mapping, dist):
pending = list(fna_mapping.keys())
node_id = max(pending) + 1
mapping = bidict.bidict()
cls_dist = []
cls_dist_temp = {}
index = 0
pending.sort()
for i in pending:
mapping[i] = index
index += 1
for i in range(0, len(pending)):
temp1 = []
for j in range(0, i):
temp1.append(cls_dist_temp[(mapping[pending[j]],
mapping[pending[i]])])
for j in range(i, len(pending)):
temp = cal_cls_dist(dist, fna_mapping[pending[i]],
fna_mapping[pending[j]])
temp1.append(temp)
cls_dist_temp[(mapping[pending[i]], mapping[pending[j]])] = temp
cls_dist.append([np.array(temp1)])
cls_dist = np.concatenate(cls_dist)
cls_dist_recls = cls_dist.copy()
mapping_recls = mapping.copy()
tree_relationship = {}
pending = set(pending)
while (len(pending) > 1):
(child_a, child_b) = divmod(np.argmax(cls_dist), cls_dist.shape[1])
temp1 = [
np.concatenate([[cls_dist[child_a]],
[cls_dist[child_b]]]).max(axis=0)
]
cls_dist = np.concatenate([cls_dist, temp1], axis=0)
temp1 = np.append(temp1, -1)
temp1 = np.vstack(temp1)
cls_dist = np.concatenate([cls_dist, temp1], axis=1)
cls_dist = np.delete(cls_dist, [child_a, child_b], axis=0)
cls_dist = np.delete(cls_dist, [child_a, child_b], axis=1)
# change mapping
cluster_a = mapping.inv[child_a]
cluster_b = mapping.inv[child_b] # cluster id
tree_relationship[node_id] = (cluster_a, cluster_b)
del mapping[cluster_a], mapping[cluster_b]
pending.remove(cluster_a)
pending.remove(cluster_b)
pending = sorted(list(pending))
for i in pending:
if (mapping[i] > min([child_a, child_b])
and mapping[i] < max([child_a, child_b])):
mapping[i] -= 1
elif (mapping[i] > max([child_a, child_b])):
mapping[i] -= 2
mapping[node_id] = len(cls_dist) - 1
pending = set(pending)
pending.add(node_id)
node_id += 1
# build tree structure
pending = list(pending)
tree = Tree()
T0 = Node(identifier=pending[0])
tree.add_node(T0)
while (len(pending) > 0):
parent = pending[0]
for i in tree_relationship[parent]:
tree.add_node(Node(identifier=i), parent=parent)
if (i in tree_relationship):
pending.append(i)
pending.remove(parent)
# load depth info
depths = {}
depths_mapping = defaultdict(set)
leaves = set(tree.leaves())
for i in tree.all_nodes():
depths[i] = tree.depth(node=i)
if (i in leaves):
depths_mapping[depths[i]].add(i)
return cls_dist_recls, mapping_recls, tree, depths, depths_mapping
class Blockchain(object):
def __init__(self, genesis):
# TODO: figure out if genesis should be passed in or created here
# self.tinput = tinput
self.blockCount = 0
self.blockchain = Tree()
self.genesis = genesis
self.addGenesisBlock(genesis) #Add the genesis block to chain
def addGenesisBlock(self, genesis):
self.blockchain.create_node("Genesis Block" + " ID: " +
genesis.proofOfWork[:12],
genesis.proofOfWork,
data=genesis)
def printBlockchain(self):
self.blockchain.show()
def addBlock(self, block):
# TODO: run proof of work verification before adding block
# Add block to chain & return true if POW valid
# Else return false
self.blockCount += 1
self.blockchain.create_node("Block " + str(self.blockCount) + " ID: " +
block.proofOfWork[:12],
block.proofOfWork,
parent=block.prevBlockHash,
data=block)
def getGenesisID(self):
return self.blockchain.root
def getLongestChainBlocks(self):
allNodes = self.blockchain.all_nodes()
forkNum = 0 #number of leaves at longest branch
treeDepth = self.blockchain.depth()
longestPathLeaves = [
] #WIll hold leaves with treeDepth depth ie longest branch(es)
for node in allNodes:
currentDepth = self.blockchain.depth(node)
if (currentDepth == treeDepth):
forkNum += 1
longestPathLeaves.append(node)
return forkNum, longestPathLeaves
def blockchainLength(self):
# returns the depth of the tree ie the length of
# the longest chain
return self.blockchain.depth()
def numBlocks(self):
return self.blockchain.size()
def printChain(self, chain):
chain.show(data_property="humanID")
def tailBlocks(self, chain):
leaves = chain.leaves()
print("Num leaves" + str(len(leaves)))
print(leaves)
def checkBlock(self):
# Check the proof work work
# return true if proof of work is valid
# else rerturn false
print("printing block")
def createBlockchainGraph(self, outfilename):
print("creating graph")
self.blockchain.to_graphviz(filename=outfilename + '.gv',
shape=u'box',
graph=u'digraph')
g = Source.from_file(outfilename + '.gv')
g.render()
def createBlockchainImg(self, outfilename):
print("creating graph")
self.blockchain.to_graphviz(filename=outfilename + '.gv',
shape=u'box',
graph=u'digraph')
g = Source.from_file(outfilename + '.png')
g.render()
class AcquisitionChainIter(object):
def __init__(self, acquisition_chain, parallel_prepare=True):
self.__sequence_index = -1
self._parallel_prepare = parallel_prepare
self.__acquisition_chain_ref = weakref.ref(acquisition_chain)
# set all slaves into master
for master in (x for x in acquisition_chain._tree.expand_tree()
if isinstance(x, AcquisitionMaster)):
del master.slaves[:]
master.slaves.extend(
acquisition_chain._tree.get_node(master).fpointer)
# create iterators tree
self._tree = Tree()
self._root_node = self._tree.create_node("acquisition chain", "root")
device2iter = dict()
for dev in acquisition_chain._tree.expand_tree():
if not isinstance(dev, (AcquisitionDevice, AcquisitionMaster)):
continue
dev_node = acquisition_chain._tree.get_node(dev)
parent = device2iter.get(dev_node.bpointer, "root")
try:
it = iter(dev)
except TypeError:
one_shot = self.acquisition_chain._device2one_shot_flag.get(
dev, True)
dev_iter = DeviceIterator(dev, one_shot)
else:
dev_iter = DeviceIteratorWrapper(it)
device2iter[dev] = dev_iter
self._tree.create_node(tag=dev.name,
identifier=dev_iter,
parent=parent)
@property
def acquisition_chain(self):
return self.__acquisition_chain_ref()
def prepare(self, scan, scan_info):
preset_tasks = list()
if self.__sequence_index == 0:
preset_tasks.extend([
gevent.spawn(preset.prepare)
for preset in self.acquisition_chain._presets_list
])
scan.prepare(scan_info, self.acquisition_chain._tree)
self._execute("_prepare",
wait_between_levels=not self._parallel_prepare)
if self.__sequence_index == 0:
gevent.joinall(preset_tasks, raise_error=True)
def start(self):
if self.__sequence_index == 0:
preset_tasks = [
gevent.spawn(preset.start)
for preset in self.acquisition_chain._presets_list
]
gevent.joinall(preset_tasks, raise_error=True)
self._execute("_start")
def stop(self):
self._execute("stop", master_to_slave=True, wait_all_tasks=True)
preset_tasks = [
gevent.spawn(preset.stop)
for preset in self.acquisition_chain._presets_list
]
gevent.joinall(preset_tasks) # wait to call all stop on preset
gevent.joinall(preset_tasks, raise_error=True)
def next(self):
self.__sequence_index += 1
gevent.joinall([
gevent.spawn(dev_iter.wait_ready)
for dev_iter in self._tree.expand_tree() if dev_iter is not 'root'
],
raise_error=True)
try:
if self.__sequence_index:
for dev_iter in self._tree.expand_tree():
if dev_iter is 'root':
continue
dev_iter.next()
except StopIteration: # should we stop all devices?
for acq_dev_iter in (
x for x in self._tree.expand_tree()
if x is not 'root' and isinstance(x.device, (
AcquisitionDevice, AcquisitionMaster))):
if hasattr(acq_dev_iter, 'wait_reading'):
acq_dev_iter.wait_reading()
dispatcher.send("end", acq_dev_iter.device)
raise
return self
def _execute(self,
func_name,
master_to_slave=False,
wait_between_levels=True,
wait_all_tasks=False):
tasks = list()
prev_level = None
if master_to_slave:
devs = list(self._tree.expand_tree(mode=Tree.WIDTH))[1:]
else:
devs = reversed(list(self._tree.expand_tree(mode=Tree.WIDTH))[1:])
for dev in devs:
node = self._tree.get_node(dev)
level = self._tree.depth(node)
if wait_between_levels and prev_level != level:
gevent.joinall(tasks, raise_error=True)
tasks = list()
prev_level = level
func = getattr(dev, func_name)
tasks.append(gevent.spawn(func))
# ensure that all tasks are executed
# (i.e: don't raise the first exception on stop)
if wait_all_tasks:
gevent.joinall(tasks)
gevent.joinall(tasks, raise_error=True)
prev = None
a.md5 = str(a.md5).replace("\\x00", "")[2:-1]
a.path = str(a.path).replace("\\x00", "")[2:-5]
a.path = str(a.path).replace("\\\\", "/")
a.path = str(a.path).replace("\r", "")
a.path = str(a.path).replace("\n", "")
a.clone = str(a.clone).replace("\\x00", "")[2:-5]
a.clone = str(a.clone).replace("\\\\", "/")
a.clone = str(a.clone).replace("\r", "")
a.clone = str(a.clone).replace("\n", "")
master = ""
for folder in a.path.split("/")[0:-1]:
master += folder + "/"
if tree.get_node(master) == None:
tree.create_node(folder, master, parent=prev)
prev = master
folder = a.path.split("/")[-1:][0]
master += folder + "/"
if tree.get_node(master) == None:
tree.create_node(str(a.isUnique), master, parent=prev, data=a)
prev = master
#print(tree.show())
print(tree.depth())
#time.sleep(1)
DFS(tree)
toPrune(tree)
class Route(object):
def __init__(self, universe):
self.route = Tree()
self.universe = universe
self.max_hops = 4
def show(self):
self.route.show()
def asString(self):
return (','.join([self.route[node].tag for node in self.route.expand_tree(mode=Tree.DEPTH)]))
def getRoute(self):
return self.route
def byScore_key(self, s):
return s.score
def findRoute(self, start):
parent = self.universe.findSystem(start)
self.route.create_node(start, start, data=parent)
systems = self.findNextSystems(start, start)
self.buildRoute(systems, start)
return self.route
def buildRoute(self, systems, parent):
for s in systems:
n = s.name
h = 0
if (self.route.contains(n) == False):
self.route.create_node(n, n, parent=parent, data=s)
hop = h + self.route.depth(n)
if (hop < self.max_hops):
sub_systems = self.findNextSystems(parent, n)
self.buildRoute(sub_systems, n)
else:
n = parent + ' --> ' + n
self.route.create_node(n, n, parent=parent, data=s)
def getSystemId(self, name, i=0):
if (self.route.contains(name) == False):
return name
else:
i += 1
n = name + '(' + str(i) + ')'
return self.getSystemId(n)
def findNextSystems(self, parent, start):
systems = []
optimal = self.universe.distances.findOptimalSystems(start)
for s in sorted(set(optimal)):
if (s != parent):
i = self.universe.findSystem(s)
if (i.permit == False):
systems.append(i)
s = sorted(systems, key = self.byScore_key)
return s[:self.max_hops]
# http://xiaming.me/treelib/examples.html
#
# class SystemTree(object):
# def __init__(self):
# self.tree = Tree()
#
# def addNode(self, id, o):
# self.tree.create_node(o, id)
#
# def addChildNode(self, p, id, o):
# self.tree.create_node(o, id, parent=p)
#
# def getNode(self, id):
# return self.tree.subtree(id)
#
# def __repr__(self):
# return self.tree.to_json(with_data=True)
#
#
# t = SystemTree()
# t.addNode('Aerial', 'Aerial')
# t.addChildNode('Aerial', 'Jotun', 'Jotun')
# t.addChildNode('Jotun', 'Rusani', 'Rusani')
# n = t.getNode('Jotun')
# print(n)
# n = t.tree.contains('Invalid')
# print(n)
# t.tree.show()
