class Conversation: def __init__(self, tweet): self.root_tweet = tweet self.conversation_tree = Tree() self.conversation_tree.create_node(tweet, tweet) self.depth = int() self.tweets_id = list() self.tweets_id.append(tweet) self.width = int() def add_replay(self, tweet, parent_tweet): self.conversation_tree.create_node(tweet, tweet, parent=parent_tweet) self.tweets_id.append(tweet) def set_depth(self): self.depth = self.conversation_tree.depth() + 1 def find_depth(self): return self.depth def get_tweets_id(self): return self.tweets_id def set_width(self): self.width = len(self.tweets_id) def find_width(self): return self.width def get_conversation_tree(self): return self.conversation_tree
class EntityTree(object): def __init__(self): self.tree = Tree() def create_ent_tree(self, ent_dct, cls_name_map, parent=None, cls_data_map={}): '''递归,依据遍历标签间层级关系生成树''' tag = ent_dct['LabelName'] name = cls_name_map.get(tag) data = cls_data_map.get(tag, 0) if not name: print("Error: tag %s not found in cls_name_map!" % tag) sys.exit(-1) if DEBUG: print("# of tree nodes: %d, tree height: %d, # of leaves: %d" % (len(self.tree), self.tree.depth(), len(self.tree.leaves()))) nd = self.tree.create_node(tag=name, parent=parent, data=data) if 'Subcategory' in ent_dct.keys(): for dct in ent_dct['Subcategory']: self.create_ent_tree(dct, cls_name_map, parent=nd.identifier, cls_data_map=cls_data_map)
def render_vocabulary_word(tree: Tree, node: Node) -> str: assert tree.parent(node.identifier) assert tree.parent(node.identifier).identifier.lower() == 'vocabulary' assert node.data depth = tree.depth(node) + 1 word = node.tag text = ('*' * depth + ' ' + word + 2 * '\n') if 'audio' in node.data and node.data['audio']: text += render_link(node.data['audio'], 'play') + '\n' if 'book_examples' in node.data: for example in node.data['book_examples']: text += render_quote(example) text += '\n' if 'definitions' in node.data: text += 'Definitions\n' for i, definition in enumerate(node.data['definitions']): text += str(i + 1) + '. ' + definition['definition'] if 'synonyms' in definition: text += ' /' for n, synonym in enumerate(definition['synonyms']): text += synonym + ', ' if n == 2: break text = text[:-2] text += '/' text += '\n' text += '\n' return text
def sum_orbits(orbital_tree: Tree) -> int: sum_total_orbits = 0 for node in orbital_tree.all_nodes(): sum_total_orbits += orbital_tree.depth(node) return sum_total_orbits
def day7(): nodes = [] node_children = [] tmp_nodes = {} tree = {} leafNodes = [] with open('day7.txt') as f: lines = f.read().splitlines() for l in lines: node = getNodeProperties(l) tmp_nodes.update(node) if "->" in l: nodes.append(l.split(' ')[0]) node_children.append(l.split('->')[1]) node_name = l.split(' ')[0] node_weight = l.split('(')[1].split(')')[0] #tmp_nodes[l.split(' ')[0]] = {} else: node_name = l.split(' ')[0] node_weight = l.split('(')[1].split(')')[0] my_tree = Tree() my_dict_tree = {} root = findRoot(node_children,nodes) print("Root",root) # Determine Towers print("weight",add_weights(root,tmp_nodes)) check_weights(root,tmp_nodes) tree_dict = {} build_tree(tree_dict,root,tmp_nodes,None,my_tree) my_tree.show() #print(tree_dict) print(my_tree.depth()) depth = my_tree.depth() #print(tree_dict) add_balance(tree_dict,my_tree,my_tree.root) #print(tree_dict) check_balance(tree_dict,my_tree,my_tree.root,False)
class AcquisitionChain(object): def __init__(self): self._tree = Tree() self._root_node = self._tree.create_node("acquisition chain", "root") self._device_to_node = dict() def add(self, master, slave): slave_node = self._tree.get_node(slave) master_node = self._tree.get_node(master) if slave_node is not None and isinstance(slave, AcquisitionDevice): if slave_node.bpointer is not self._root_node and master_node is not slave_node.bpointer: raise RuntimeError( "Cannot add acquisition device %s to multiple masters, current master is %s" % (slave, slave_node._bpointer) ) else: # user error, multiple add, ignore for now return if master_node is None: master_node = self._tree.create_node(tag=master.name, identifier=master, parent="root") if slave_node is None: slave_node = self._tree.create_node(tag=slave.name, identifier=slave, parent=master) else: self._tree.move_node(slave_node, master_node) def _execute(self, func_name): tasks = list() prev_level = None for dev in reversed(list(self._tree.expand_tree(mode=Tree.WIDTH))[1:]): node = self._tree.get_node(dev) level = self._tree.depth(node) if prev_level != level: gevent.joinall(tasks) tasks = list() func = getattr(dev, func_name) tasks.append(gevent.spawn(func)) gevent.joinall(tasks) def prepare(self, dm, scan_info): # self._devices_tree = self._get_devices_tree() for master in (x for x in self._tree.expand_tree() if isinstance(x, AcquisitionMaster)): del master.slaves[:] for dev in self._tree.get_node(master).fpointer: master.slaves.append(dev) dm_prepare_task = gevent.spawn(dm.prepare, scan_info, self._tree) self._execute("_prepare") dm_prepare_task.join() def start(self): self._execute("_start") for acq_dev in (x for x in self._tree.expand_tree() if isinstance(x, AcquisitionDevice)): acq_dev.wait_reading() dispatcher.send("end", acq_dev)
def find_deepest_child(data): tree = Tree() root = tree.create_node("root", "root") tree = build_tree(data=data, tree=tree, parent=root) res = tree.paths_to_leaves() depth = tree.depth() for path in res: if len(path) == depth + 1: return path[-1] break
def render_org_tree(tree: Tree, node: Node, payload='') -> str: parent = tree.parent(node.identifier) if parent and parent.identifier.lower() == 'vocabulary': payload += render_vocabulary_word(tree, node) else: depth = tree.depth(node) + 1 payload += ('*' * depth + ' ' + node.tag + '\n') for child in tree.children(node.identifier): payload += render_org_tree(tree, tree[child.identifier]) return payload
def conversation_regarding_language(cursor): conversation_amount = postgres_queries.find_conversation_number(cursor) conversation_list = list() depth_dict = dict() depth_dict_long = dict() depth_dict_short = dict() number_of_tweets_dict = dict() test_i = 0 for i in range(0, conversation_amount + 1, 1): conversation_tree = Tree() conversation = postgres_queries.find_conversation(i, cursor) test_i += len(conversation) for tweet in conversation: if tweet[2] is None and tweet[5] is True: conversation_tree.create_node(tweet[0], tweet[0]) tweets_in_conversation = list() build_conversation_lang(tweet[0], conversation, conversation_tree, tweets_in_conversation) depth = conversation_tree.depth() + 1 number_of_tweets = len(conversation_tree.all_nodes()) #short/long if number_of_tweets >=20: if depth in depth_dict_long: depth_dict_long[depth] += 1 else: depth_dict_long[depth] = 1 else: if depth in depth_dict_short: depth_dict_short[depth] += 1 else: depth_dict_short[depth] = 1 if number_of_tweets in number_of_tweets_dict: number_of_tweets_dict[number_of_tweets] += 1 else: number_of_tweets_dict[number_of_tweets] = 1 if depth in depth_dict: depth_dict[depth] += 1 else: depth_dict[depth] = 1 # check if conversation_tree is null- dont add if len(conversation_tree.all_nodes())!=0: conversation_list.append(conversation_tree) # number = 0 new_tweet_list_id = list() for con in conversation_list: nodes = con.all_nodes() for node in nodes: new_tweet_list_id.append(int(node.tag)) # number += len(con.all_nodes()) # print len(new_tweet_list_id) # for tweet_id in new_tweet_list_id: # print tweet_id return new_tweet_list_id, conversation_list
def types_of_conversation(): conversation_amount = postgres_queries.find_annotated_conversation_number() conversation_list = list() depth_dict = dict() depth_dict_long = dict() depth_dict_short = dict() number_of_tweets_dict = dict() for i in range (0, conversation_amount + 1, 1): conversation_tree = Tree() converastion = postgres_queries.find_conversation(i) for tweet in converastion: if tweet[1] is None: conversation_tree.create_node(tweet[0], tweet[0]) build_conversation(tweet[0], converastion, conversation_tree) depth = conversation_tree.depth() + 1 number_of_tweets = len(conversation_tree.all_nodes()) #short/long if number_of_tweets >=20: if depth in depth_dict_long: depth_dict_long[depth] += 1 else: depth_dict_long[depth] = 1 else: if depth in depth_dict_short: depth_dict_short[depth] += 1 else: depth_dict_short[depth] = 1 if number_of_tweets in number_of_tweets_dict: number_of_tweets_dict[number_of_tweets] += 1 else: number_of_tweets_dict[number_of_tweets] = 1 if depth in depth_dict: depth_dict[depth] += 1 else: depth_dict[depth] = 1 conversation_list.append(conversation_tree) #print depth_dict print 'Depth of a conversation' for depth, count in depth_dict.iteritems(): print depth, '\t', count print 'Number of tweets in a conversation' for number, count in number_of_tweets_dict.iteritems(): print number, '\t', count print 'Depth of a long conversation' for depth, count in depth_dict_long.iteritems(): print depth, '\t', count print 'Depth of a short conversation' for depth, count in depth_dict_short.iteritems(): print depth, '\t', count return conversation_list
def get_metadata(node: Node, tree: Tree, *, \ include_count=True, include_attrs=False, include_depth=True) -> Dict: data = {'id': node.identifier, 'tag': node.tag} if include_count: data['childrenCount'] = len(tree.children(node.identifier) or []) if include_depth: data['depth'] = tree.depth(node) if include_attrs: data.update(get_attributes(node)) return data
def conversation_regarding_language(): # with width and depth conversation_amount = postgres_queries.find_annotated_conversation_number() conversation_list = list() depth_dict = dict() depth_dict_long = dict() depth_dict_short = dict() number_of_tweets_dict = dict() test_i = 0 for i in range(0, conversation_amount + 1, 1): conversation_tree = Tree() converastion = postgres_queries.find_conversation(i) test_i += len(converastion) for tweet in converastion: if tweet[1] is None and tweet[5] is True: conversation_tree.create_node(tweet[0], tweet[0]) build_conversation_lang(tweet[0], converastion, conversation_tree) depth = conversation_tree.depth() + 1 number_of_tweets = len(conversation_tree.all_nodes()) #short/long if number_of_tweets >=20: if depth in depth_dict_long: depth_dict_long[depth] += 1 else: depth_dict_long[depth] = 1 else: if depth in depth_dict_short: depth_dict_short[depth] += 1 else: depth_dict_short[depth] = 1 if number_of_tweets in number_of_tweets_dict: number_of_tweets_dict[number_of_tweets] += 1 else: number_of_tweets_dict[number_of_tweets] = 1 if depth in depth_dict: depth_dict[depth] += 1 else: depth_dict[depth] = 1 conversation_list.append(conversation_tree) number = 0 new_tweet_list_id = list() for con in conversation_list: nodes = con.all_nodes() for node in nodes: new_tweet_list_id.append(node.tag) number += len(con.all_nodes()) return new_tweet_list_id, conversation_list
def solve(alpha, beta, k): # Maain initializers solutions = [] tree = Tree() # Parents starts with root value aka empty string. # We need to ensure the depth does not exceed k due to complexity issues. parents = [''] tree.create_node(tag='', identifier='') while len(parents) > 0: item = parents.pop() for i in range(0, len(alpha)): # Ensure we do not exceed the depth for a single node. if tree.depth(item) == k: break split = item.split() newAlpha = "" newBeta = "" # Try to retrieve the parent node string and append. # Otherwise node is root. if len(split) > 1: newAlpha = split[0] + alpha[i] newBeta = split[1] + beta[i] else: newAlpha = alpha[i] newBeta = beta[i] # Check to see if the node is valid. # The node will be valid if there is a prefix. if is_valid(newAlpha, newBeta): node = newAlpha + ' ' + newBeta # Check to see if node is a solution. if is_solution(newAlpha, newBeta): solutions.append(node) # Create tree node since it is valid. tree.create_node(tag=node, identifier=node, parent=item) parents.append(node) # Will leave only valid solutions. return solutions
def generatePool(N, maxDepth, tree, rule_table, debug=False): # Collecting all non-terminal nodes nodeIDs = [] for nodeID in tree.expand_tree(mode=Tree.DEPTH): nodeType = tree[nodeID].data if isinstance(nodeType, PiRL.DataStructures.Token.NonTerminal): nodeIDs.append(nodeID) neighbours = [] for _ in range(N): # Making a deep copy newTree = Tree(tree=tree, deep=True) # Selecting a random non-terminal to be replaced nodeToReplaceID = random.choice(nodeIDs) # Generating a new subtree newSubTree = getProgTree(newTree[nodeToReplaceID].data, maxDepth - newTree.depth(nodeToReplaceID)) # Replacing subtree try: newTree.replace_node(newTree[nodeToReplaceID].predecessor(newTree.identifier), nodeToReplaceID, newSubTree, deep=False) except Exception as e: # traceback.print_exc() if debug: print("Root node replaced") newTree = newSubTree if debug: print("Generated neighbour:", end=' ') # newTree.show(data_property='str') for leaf in newTree.leaves(): print(leaf.data.name, end=' ') print("\n") neighbours.append(newTree) return neighbours
kraken_data['num_reads_clade'] = int(line[1]) kraken_data['num_reads_taxon'] = int(line[2]) kraken_data['rank_code'] = line[3] kraken_data['ncbi_taxonomy_id'] = line[4] kraken_data['name'] = line[5] kraken_data['depth'] = calculate_depth(kraken_data['name']) kraken_data['name'] = kraken_data['name'].lstrip(' ') if kraken_data['name'] == "unclassified": unclassified = Node(tag = kraken_data['name'], identifier = kraken_data['ncbi_taxonomy_id'], data = kraken_data) elif kraken_data['name'] == "root": add_node(tree, None, kraken_data) previous = tree.get_node(kraken_data['ncbi_taxonomy_id']) elif kraken_data['depth'] > tree.depth(previous): add_node(tree, previous, kraken_data) previous = tree.get_node(kraken_data['ncbi_taxonomy_id']) elif kraken_data['depth'] == tree.depth(previous): add_node(tree, tree.parent(previous.identifier), kraken_data) previous = tree.get_node(kraken_data['ncbi_taxonomy_id']) elif kraken_data['depth'] < tree.depth(previous): previous_search = previous while(tree.depth(previous_search) > kraken_data['depth']): previous_search = tree.parent(previous_search.identifier) add_node(tree, tree.parent(previous_search.identifier), kraken_data) previous = tree.get_node(kraken_data['ncbi_taxonomy_id']) tree_dict = tree.to_dict(with_data=True) def transform(tree_dict):
def depth_cost(G: nx.Graph, T: tl.Tree): return T.depth()
class MonteCarlo: N_THREADS = 1 PERCENTILE = 100 def __init__(self, engine=None, hero=None): # self.last_ev = 0 # self.rolling_10 = deque(maxlen=10) # self.rolling_40 = deque(maxlen=40) self.ev_history = {} self.time_start = None self.duration = None self.queue = None self.leaf_path = None if not engine: # logger.info('engine not given, loading from file...') self.engine_checksum = None self.load_engine(hero) else: # logger.info('engine given') self.init(engine, hero) @property def current_actions(self): return [(c.data['action'], c.data['ev'], c.data['traversed']) for c in self.tree.children(self.tree.root)] def is_time_left(self): return time.time() - self.time_start < self.duration @retrace.retry(on_exception=(EOFError, KeyError), interval=0.1, limit=None) def load_engine(self, hero): with shelve.open(Engine.FILE) as shlv: if shlv['hash'] != self.engine_checksum: # logger.info('loading engine from file...') self.engine_checksum = shlv['hash'] self.init(shlv['engine'], hero) def init(self, engine, hero): # logger.info('init state') self.engine = engine self.hero = hero or self.engine.q[0][0] self.hero_pocket = self.engine.data[self.hero]['hand'] for s in self.engine.data: self.ev_history[s] = deque(maxlen=50) # logger.info('HERO is at seat {} with {}'.format(self.hero, self.hero_pocket)) self.watched = False self.init_tree() def init_tree(self): """create the tree. Add a root; available action will add the first level of children""" # self.traversed_ceiling = 1 self.tree = Tree() root = self.tree.create_node('root', identifier='root', data={'traversed': 0, 'ev': 0, 'stats': 1, 'cum_stats': 1}) # # logger.info('tree:\n{}'.format(self.tree.show())) # input('new tree') def watch(self): """Runs when engine file changes. Just kicks off run for 3s sprints""" # logger.info('Monte Carlo watching every {}s...'.format(self.timeout)) while True: # loads new engine file if checksum changed self.load_engine() # do not analyze if game finished if self.engine.phase in [self.engine.PHASE_SHOWDOWN, self.engine.PHASE_GG]: if not self.watched: # logger.error('game is finished') self.watched = True time.sleep(3) continue # do not analyze if hero does not have pocket if self.hero_pocket in [['__', '__'], [' ', ' ']]: if not self.watched: # logger.error('hero does not have a pocket') self.watched = True time.sleep(0.5) continue # do not analyze if hero is not to play if self.hero != self.engine.q[0][0]: if not self.watched: # logger.error('hero is not to act') self.watched = True time.sleep(0.5) continue if self.is_complete: if not self.watched: # logger.error('mc is complete') self.watched = True time.sleep(2) continue # run a few sims # logger.debug('running now with timeout {}'.format(self.timeout)) self.run() self.timeout += 0.1 def run(self, duration): """Run simulations For x: - clone engine - start at root -- iterate and find next unprocessed node -- action engine to that node parent -- process that node - keep processing - with return EV Levelling: extremely huge iterations when many players. So do the most probably actions only till all done. Handling close action approximations: """ # logger.info('Monte Carlo started') total_traversions_start = sum(a[2] for a in self.current_actions) # cannot run if engine in showdown or gg if self.engine.phase in [self.engine.PHASE_SHOWDOWN, self.engine.PHASE_GG]: logger.warning('cannot run mc with no actions') return self.duration = duration self.time_start = time.time() self.queue = PriorityQueue() # threads = [] # for _ in range(self.N_THREADS): # t = MCWorker(self) # # t.start() # threads.append(t) # self.traversed_focus = 0 leaves = self.tree.paths_to_leaves() # logger.debug('leaves from tree: {}'.format(len(leaves))) # leaves.sort(key=lambda lp: len(lp) + sum(int(lpn.split('_')[0]) for lpn in lp), reverse=True) # # logger.debug('{} leaves are now sorted by formula'.format(len(leaves))) # logger.debug('{}'.format(json.dumps(leaves[:3], indent=4, default=str))) # leaves.sort(key=len) # logger.debug('{} leaves are now sorted by length'.format(len(leaves))) # logger.debug('{}'.format(json.dumps(leaves[:3], indent=4, default=str))) # leaves.sort(key=lambda lp: int(lp[-1][:3]), reverse=True) # logger.debug('{} leaves are now sorted by rank'.format(len(leaves))) # logger.error(json.dumps(leaves, indent=4, default=str)) # input('>>') for leaf_path in leaves: node = self.tree[leaf_path[-1]] item = ( 1 - node.data['cum_stats'], leaf_path, ) self.queue.put(item) # for t in threads: # t.start() # # for t in threads: # t.join() # if t.error: # raise Exception().with_traceback(t.error[2]) while self.is_time_left() and not self.queue.empty(): priority, self.leaf_path = self.queue.get_nowait() self.run_item(self.leaf_path) if self.queue.empty(): logger.info(f'Everything was processed in queue!') total_traversions_end = sum(a[2] for a in self.current_actions) if total_traversions_end <= total_traversions_start: logger.warning(f'No new traversion added to {total_traversions_start}') def run_item(self, path): # logger.debug('running this path: {}'.format(path)) e = deepcopy(self.engine) e.mc = True """To calculate the investment for the loss EV, the total amounts used till end is required. Cannot use final player balance on engine as that might have winnings allocated to it by the engine. Instead the difference from all the new matched bets from the current matched bets will be used. Need to add current contrib """ e.matched_start = e.data[self.hero]['matched'] + e.data[self.hero]['contrib'] # logger.info('hero starting with matched = {} from {} + {}'.format( # e.matched_start, e.data[self.hero]['matched'], e.data[self.hero]['contrib'])) # self.tree.show() self.fast_forward(e, path) # logger.info('{}'.format('-' * 200)) # input('check item') def show_best_action(self): """Calculates best action on root""" # logger.error("\n\n") sum_traversed = 0 delta = 0 max_ev = float('-inf') action = None amount = None for nid in self.tree[self.tree.root].fpointer: child = self.tree[nid] # logger.debug('{} {}'.format(child.tag, child.data)) dat = child.data sum_traversed += dat['traversed'] # logger.error('{} @{} => {}'.format(dat['action'], dat['traversed'], round(dat['ev'], 4))) # delta += abs(1 - (self.convergence.get(dat['action'], 1) / dat['ev'] if dat['ev'] else 1)) # self.convergence[dat['action']] = dat['ev'] if dat['ev'] > max_ev: max_ev = dat['ev'] action = dat['action'] if action.startswith('bet') or action.startswith('raise') or action.startswith('allin'): amount = dat['amount'] best_action = '{}{}'.format(action, ' with {}'.format(amount) if amount else '') # self.convergence['deq'].append(round(delta, 1)) self.convergence['deq'].append(best_action) # # logger.error('deq: {}'.format(list(self.convergence['deq']))) # logger.error('') # logger.error('Timeout: {}'.format(round(self.timeout, 1))) # logger.error('Traversed: {}'.format(sum_traversed)) deq_cnts = Counter(list(self.convergence['deq'])) # # logger.error('deq: {}'.format(deq_cnts.most_common())) # logger.error('{}% for {}'.format( # 100 * sum(dq == deq_list[-1] for dq in deq_list[:-1]) // (len(deq_list) - 1) # 100 * (deq_cnts.most_common()[0][1] - deq_cnts.most_common()[1][1]) // self.convergence_size # if len(deq_cnts) > 1 else 100 * len(self.convergence['deq']) // self.convergence_size, # deq_cnts.most_common()[0][0] # )) def fast_forward(self, e, path): """Do actions on engine till the leaf is reached. Need to do available_actions before every DO First check if the leave is already processed, then skip this path. When the leaf is reached then process from that node. Remember to send through only the first letter for the action. Then update the nodes from this leaf back up the tree """ # logger.info('Fast forwarding {} nodes'.format(len(path))) if len(path) == 1: # logger.info('processing root for first time') self.process_node(e, self.tree[path[0]]) return leaf_node = self.tree[path[-1]] # logger.debug('checking if last node has been processed:') # logger.debug('last node leaf {} has node data {}'.format(leaf_node.tag, leaf_node.data)) if leaf_node.data['traversed']: # logger.info('This leaf node ({}) above focus level {}'.format(leaf_node.tag, self.traversed_focus)) # can happen as all actions are added, but then one was chosen to continue on # and that path for that action wasn't removed from the queue return for nid in path[1:]: node = self.tree[nid] # logger.debug('fast forwarding action for node {}'.format(node.tag)) e.available_actions() cmd = [node.data['action'][0]] if 'amount' in node.data: cmd.append(node.data['amount']) # logger.debug('Adding bet value of {}'.format(node.data['amount'])) # logger.debug('Executing path action {} for {}'.format(cmd, node.tag)) # logger.debug('Executing path action {} with data {}'.format(cmd, node.data)) e.do(cmd) if node.is_leaf(): # logger.debug('{} is a leaf node, processing next...'.format(node.tag)) self.process_node(e, node) logger.info('nodes processed, now updating nodes that were fast forwarded') for processed_nid in reversed(path[1:]): processed_node = self.tree[processed_nid] self.update_node(processed_node) self.ev_history[self.engine.s].append(sum(a[1] for a in self.current_actions)) def process_node(self, e, n): """Process node Get actions available for node Pick action to traverse with UCT Process action selected Return EV """ # logger.info('processing node {} with data {}'.format(n.tag, n.data)) # this node is the hero folding (to prevent this being processed as leaf) # was created with other children (but not most probable at that time to be proc as child) # if hero folding, then make this node a leaf node with fold eq # exiting before adding children alleviates the need to remove the immediately again thereafter # bug: cannot use engine.q as it already rotated after taking action getting here if not n.is_root() and n.data['action'] == 'fold' and self.hero == n.data['seat']: winnings, losses = self.net(e) result = { 'ev': losses, 'traversed': 1, } # logger.info('hero has folded this node given: {}'.format(result)) n.data.update(result) # logger.info('node data after fold: {}'.format(n.data)) return # add the children of the node if not n.fpointer: self.add_actions(e, n) # this node is a leaf (no more actions to take!) # either the game finished and we have winner and pot # or we have to use pokereval.winners if n.is_leaf(): # logger.info('node {} is the final action in the game'.format(n.tag)) # winner given (easy resolution) if e.winner: # logger.debug('engine gave winner {}'.format(e.winner)) winnings, losses = self.net(e) ev = winnings if self.hero in e.winner else losses # else if the winner is unknown # then calculate winners and use # percentage of hero as amt else: if 'in' not in e.data[self.hero]['status']: # hero fold is handled before in method # and thus for equities calc it is just 0 # logger.debug('Hero {} is not in game'.format(self.hero)) ev = 0 else: winnings, losses = self.net(e) equities = PE.showdown_equities(e) # equities = self.get_showdown_equities(e) ev_pos = winnings * equities[self.hero] # logger.debug('ev_pos = {} from winnings {} * eq {}'.format(ev_pos, winnings, equities[self.hero])) ev_neg = losses * (1 - equities[self.hero]) # logger.debug('ev_neg = {} from losses {} * -eq {}'.format(ev_neg, losses, (1 - equities[self.hero]))) ev = ev_pos + ev_neg logger.info('Net EV: {} from {} + {}'.format(ev, ev_pos, ev_neg)) result = { 'ev': ev, 'traversed': 1, } # logger.info('{} leaf has result {}'.format(n.tag, result)) n.data.update(result) return # node is all good (not leaf (has children) and not hero folding) # get child actions and process most probable action a_node = self.most_probable_action(n) action = a_node.data['action'] # logger.info('taking next child node action {}'.format(action)) # if it is hero and he folds, # it is not necessarily an immediate ZERO equity # since my previous contrib needs to be added to the pot (i.e. contribs after starting mc) # i.e. make this a leaf node implicitly # no child nodes to remove for fold if action == 'fold' and self.hero == a_node.data['seat']: winnings, losses = self.net(e) result = { 'ev': losses, 'traversed': 1, } # logger.info('hero has folded the child node selected: {}'.format(result)) a_node.data.update(result) # logger.info('a_node data after: {}'.format(a_node.data)) # else we must process the node else: # logger.info('taking action {} and processing that node'.format(action)) cmd = [action[0]] if 'amount' in a_node.data: cmd.append(a_node.data['amount']) # logger.debug('Adding bet value of {}'.format(a_node.data['amount'])) e.do(cmd) self.process_node(e, a_node) # action node has been processed, now update node self.update_node(n) def update_node(self, node): """Update the node's data If leaf, then it was already calculated during processing, and now do not change it: the ev is the ev Minimax applied, hero pick best and foe picks min after p Traversed will stay the traversed_focus level for leaves, but for parent nodes the traversed will be the number of leaves reached from that node. """ is_hero = node.data.get('seat') == self.hero # logger.debug('is hero? {}'.format(is_hero)) # it will traverse back up to the root # root can be skipped if node.is_root(): # input('hero {} node data {}'.format(self.hero, node.data.get('seat'))) # if is_hero: # self.rolling_10.append(abs(self.last_ev)) # self.rolling_40.append(abs(self.last_ev)) # logger.debug('Added {} ev to collection'.format(self.last_ev)) # input('Added {} ev to collection'.format(self.last_ev)) # logger.debug('reached the root') # self.update_ev_change() return # fast forwarding will send here, just ignore node if leaf if node.is_leaf(): # logger.debug('not updating {}: it is final game result (no leaf nodes)'.format(node.tag)) # logger.debug('not updating {}: final data {}'.format(node.tag, node.data)) return depth = self.tree.depth(node) # logger.info('updating node {} at depth {}'.format(node.tag, depth)) # logger.info('node has {} before update'.format(node.data)) if not len(node.fpointer): # logger.error('node {} with {} as no children...'.format(node.tag, node.data)) raise Exception('not necessary to process leaves') # logger.debug('extracting data from {} children nodes...'.format(len(node.fpointer))) n_ev = float('-inf') if is_hero else 0 n_traversed = 0 for child_nid in node.fpointer: child_node = self.tree[child_nid] # logger.debug('child node {} has {}'.format(child_node.tag, child_node.data)) dat = child_node.data if not dat['traversed']: # logger.debug('skipping untraversed {}'.format(child_node.tag)) continue # get max for hero if is_hero: # todo is this +ev dampening necessary # todo this should be fixed when setting for hand range # equities = PE.showdown_equities(self.engine) # n_ev = max(n_ev, dat['ev'] * equities.get(self.hero, 0)) n_ev = max(n_ev, dat['ev']) # get min for foe else: # ev_adj = dat['ev'] * dat['stats'] # logger.debug('foe min between {} and {}'.format(n_ev, ev_adj)) # n_ev = min(n_ev, ev_adj) n_ev += dat['ev'] * dat['stats'] / dat['divider'] n_traversed += dat['traversed'] # logger.debug('added {} traversed: now have {} so far'.format(dat['traversed'], n_traversed)) self.last_ev = node.data['ev'] - n_ev node.data.update({ 'ev': n_ev, 'traversed': n_traversed, }) # logger.info('now node has {} ev~{} after {}'.format(node.tag, round(n_ev, 3), n_traversed)) if not node.data['traversed']: raise Exception('node cannot be untraversed') def net(self, e): """Stored the balance at the start of sim. Now calculate difference as player total matched contrib. Winnings will be less initial starting contrib. """ e.gather_the_money() p = e.players[self.hero] d = e.data[self.hero] matched_diff = d['matched'] - e.matched_start # logger.debug('matched diff = {} from {} - {}'.format(matched_diff, d['matched'], e.matched_start)) winnings = int(e.pot - matched_diff) # logger.debug('winnings diff = {} from pot {} less matched {}'.format(winnings, e.pot, matched_diff)) losses = int(-matched_diff) # logger.info('Winnings = {} and losses = {}'.format(winnings, losses)) return winnings, losses def most_probable_action(self, parent): """All nodes will be processed once at least but it will never happen. Just return the most probable node for most accurate play. Using stats fields on data There should not be any untraversed nodes. So first get untraversed, then sort and pop first one""" # logger.info('getting most probable action after {}'.format(parent.tag)) children = self.tree.children(parent.identifier) children = [c for c in children if not c.data['traversed']] if not children: raise MonteCarloError('Cannot choose most probable action when all nodes are traversed') children.sort(key=lambda c: c.data['stats'], reverse=True) child = children[0] # logger.debug('{} is untraversed, returning that node for actioning'.format(child.tag)) self.leaf_path.append(child.identifier) return child def add_actions(self, e, parent): """Add actions available to this node If in GG phase then no actions possible, ever. Remove 'hand' Bets: - preflop are 2-4x BB - postflop are 40-100% pot Raise: - always double Allin: - only on river - if out of money then converted to allin Scale non-fold probabilities even though it should not have an effect. """ # logger.info('adding actions to {}'.format(parent.tag)) actions = e.available_actions() s, p = e.q[0] d = e.data[s] balance_left = p['balance'] - d['contrib'] if not actions: # logger.warn('no actions to add to node') return if 'gg' in actions: # logger.debug('no actions available, got gg') return actions.remove('hand') # remove fold if player can check if 'check' in actions: actions.remove('fold') # # logger.debug('removed fold when check available') # remove fold for hero # if s == self.hero and 'fold' in actions: # actions.remove('fold') # # logger.debug('removed fold from hero') # remove raise if player has already been aggressive if 'raise' in actions and any(pa['action'] in 'br' for pa in d[e.phase]): actions.remove('raise') # # logger.debug('removed raise as player has already been aggressive') # remove allin, but add it later with final stats (if increased from bet/raised) if 'allin' in actions: actions.remove('allin') # logger.debug('removed allin by default') # load stats (codes with counts) stats = ES.player_stats(e, s) max_contrib = max(pd['contrib'] for pd in e.data.values()) # contrib_short = max_contrib - d['contrib'] # allin needs to be the doc count # where bets and raises result in allin, add those prob dists to this # that will give proper probability go_allin = stats['actions'].get('a', 0) # # logger.info('filtered actions: {}'.format(actions)) # ev 0 instead of none because of root node sum when not all traversed it gives error action_nodes = [] for a in actions: node_data = { 'stats': stats['actions'].get(ACTIONS_TO_ABBR[a], 0.01), 'divider': 1, 'action': a, 'phase': e.phase, 'seat': s, 'name': p['name'], 'traversed': 0, 'ev': 0, } if a in ['bet', 'raise']: btps_and_amts = [] total_pot = sum(pd['contrib'] for pd in e.data.values()) + e.pot # for preflop only do 2x and 3x if e.phase == e.PHASE_PREFLOP: btps_and_amts.append(('double', e.bb_amt * 2)) btps_and_amts.append(('triple', e.bb_amt * 3)) # else do half and full pots else: btps_and_amts.append(('half_pot', total_pot * 0.50)) btps_and_amts.append(('full_pot', total_pot * 1.00)) # round bets up to a BB # btps_and_amts = [(btp, -(amt // -e.bb_amt) * e.bb_amt) # for btp, amt in btps_and_amts] betting_info = [] amts_seen = [] for btp, amt in btps_and_amts: if amt in amts_seen: # logger.debug('already using {}, skipping duplicate'.format(amt)) continue if a == 'bet' and amt < e.bb_amt: # logger.debug('bet cannot be less than BB {}'.format(e.bb_amt)) continue if a == 'raise' and amt < (max_contrib * 2): # logger.debug('raise cannot be less than 2x contrib of {}'.format(max_contrib * 2)) continue betting_info.append((btp, amt)) amts_seen.append(amt) # change raises that cause allin betting_info_final = [] for btp, amt in betting_info: # if amt is more than player balance, it is an allin if amt >= balance_left: go_allin += node_data['stats'] / len(betting_info) else: betting_info_final.append((btp, amt)) # all good, can have this bet as option for btp, amt in betting_info_final: node_data_copy = deepcopy(node_data) node_data_copy['divider'] = len(betting_info_final) node_data_copy['action'] = f'{a}_{btp}' node_data_copy['amount'] = amt action_nodes.append(node_data_copy) else: action_nodes.append(node_data) # allin will have doc counts (from stat, maybe from bets, maybe from raise) if go_allin: node_data = { 'stats': go_allin, 'divider': 1, 'action': 'allin', 'phase': e.phase, 'seat': s, 'name': p['name'], 'traversed': 0, 'ev': 0, 'amount': balance_left, } action_nodes.append(node_data) # logger.debug('added allin to actions with stat {}'.format(node_data['stats'])) # scale the stats (it is currently term counts aka histogram) and it is required to be # a probability distribution (p~1) # Also, certain actions like fold can be removed, and the total stats is not 1 total_stats = sum(an['stats'] / an['divider'] for an in action_nodes) for action_node in action_nodes: action_node['stats'] = max(0.01, action_node['stats'] / action_node['divider'] / total_stats) action_node['cum_stats'] = parent.data['cum_stats'] * action_node['stats'] node_tag = f'{action_node["action"]}_{s}_{e.phase}' identifier = f'{node_tag}_{str(uuid.uuid4())[:8]}' self.tree.create_node(identifier=identifier, tag=node_tag, parent=parent.identifier, data=action_node) # logger.debug('new {} for {} with data {}'.format(node_tag, s, action_node)) item = ( 1 - action_node['cum_stats'], self.leaf_path + [identifier] ) self.queue.put(item) # logger.debug('new {} for {} with data {}'.format(node_tag, s, action_node)) # logger.info('{} node actions added'.format(len(action_nodes))) def analyze_tree(self): """Analyze tree to inspect best action from ev""" # self.tree.show() # check all finished paths for path in self.tree.paths_to_leaves(): # skip untraversed end last_node = self.tree[path[-1]] if not last_node.data['traversed']: logger.debug('skipping untraversed endpoint {}'.format(last_node.tag)) continue # show all actions for nid in path: node = self.tree[nid] d = node.data logger.info('Node: {} ev={}'.format(node.tag, d['ev'])) 0/0 input('$ check tree') def get_showdown_equities(self, e): """instead of using pokereval, use hs from se""" hss = {} for s, d in e.data.items(): if 'in' in d['status']: hss[s] = ES.showdown_hs(e, s, percentile=self.PERCENTILE) # calculate for hero if self.hero in hss: d = e.data[self.hero] hss[self.hero] = PE.hand_strength(d['hand'], e.board, e.rivals) # normalize total = sum(hs for hs in hss.values()) equities = {s: hs / total for s, hs in hss.items()} return equities
class search_tree(object): def __init__(self, query_id, depth, carpe_diem): self.tree = Tree() self.tree.create_node(identifier='query_' + query_id, data=node()) root_node = self.tree.get_node('query_' + query_id) root_node.data.num = 1.0 self.node_map = {} self.count = 0.0 self.carpe_diem = carpe_diem self.max_depth = depth self.expand(self.tree.get_node(self.tree.root)) def expand(self, leaf_node): doc_list = leaf_node.data.doc p_doc_id, p_pred = self.carpe_diem.policy(self.tree.root, doc_list) for doc in p_doc_id: self.node_map[' '.join(doc_list + [doc])] = len(self.node_map) new_node = node() new_node.doc = doc_list + [doc] new_node.p = p_pred[p_doc_id.index(doc)] self.tree.create_node(identifier=self.node_map[' '.join( new_node.doc)], data=new_node, parent=leaf_node.identifier) def update(self, node_list, value): for node_id in node_list: tmp_node = self.tree.get_node(node_id) tmp_node.data.Q = (tmp_node.data.Q * tmp_node.data.num + value) / (tmp_node.data.num + 1) tmp_node.data.num += 1 def search(self, start_node_id): tmp_node = self.tree.get_node(start_node_id) has_visit_num = tmp_node.data.num - 1 self.count = has_visit_num if int(self.carpe_diem.search_time - has_visit_num) > 0: start_node_search_time = int(self.carpe_diem.search_time - has_visit_num) else: start_node_search_time = 0 for time in range(start_node_search_time): search_list = [start_node_id] tmp_node = self.tree.get_node(start_node_id) while not tmp_node.is_leaf(): max_score = float("-inf") max_id = -1 for child_id in tmp_node.fpointer: child_node = self.tree.get_node(child_id) score = self.carpe_diem.beta * child_node.data.p * ( (tmp_node.data.num - 1)**0.5 / (1 + child_node.data.num)) score += child_node.data.Q if score > max_score: max_id = child_id max_score = score search_list.append(max_id) tmp_node = self.tree.get_node(max_id) query_id_mcts = self.tree.root.split('_')[1] if self.tree.depth(tmp_node) == self.max_depth: listPermutation = copy.deepcopy( self.carpe_diem.dictQueryPermutaion[query_id_mcts] ['permutation']) idealScore = self.carpe_diem.alphaDCG(0.5, query_id_mcts, listPermutation, self.max_depth) v = self.carpe_diem.alphaDCG(0.5, query_id_mcts, tmp_node.data.doc, self.max_depth) v = v / idealScore else: v = self.carpe_diem.value_function(self.tree.root, tmp_node.data.doc) self.update(search_list, v) self.count += 1 if tmp_node.is_leaf() and (self.tree.depth(tmp_node) < self.max_depth): self.expand(tmp_node) ########### if time % 100 == 0: tmp_policy = self.get_policy(start_node_id) print tmp_policy.values() print sum(tmp_policy.values()) print time def take_action(self, start_node_id): tmp_node = self.tree.get_node(start_node_id) max_time = -1 prob = {} for child_id in tmp_node.fpointer: child_node = self.tree.get_node(child_id) prob[child_node.data.doc[-1]] = child_node.data.num / self.count if child_node.data.num > max_time: max_time = child_node.data.num select_doc = child_node.data.doc[-1] select_doc_node_id = child_node.identifier return prob, select_doc, select_doc_node_id def get_policy(self, start_node_id): tmp_node = self.tree.get_node(start_node_id) max_time = -1 prob = {} for child_id in tmp_node.fpointer: child_node = self.tree.get_node(child_id) if self.count == 0: prob[child_node.data.doc[-1]] = 0.0 else: prob[ child_node.data.doc[-1]] = child_node.data.num / self.count return prob
class BasicTree: def __init__(self, vehsInfo): self.tree = Tree() self.root = self.tree.create_node("Root", "root") # root node self.vehsInfo = vehsInfo self.vehList = list(vehsInfo.keys()) self.i = 1 def _build(self, currentNode, vehList): ''' :param vehList: A dict, keys is the set of vehicles, value is a tuple which represents (lane, position) :param currentNode: The current node in the tree :return: None ''' s = [currentNode.tag.find(vid) for vid in vehList] # the quit contidion in recursion if (np.array(s) >= 0).all(): return for vehId in vehList: if vehId not in currentNode.tag: if currentNode.is_root: prefix = currentNode.tag.replace("Root", "") else: prefix = currentNode.tag self.tree.create_node(prefix + vehId + "-", prefix + vehId, parent=currentNode) for node in self.tree.all_nodes(): if node.is_leaf(): self._build(currentNode=node, vehList=vehList) def _prune(self): laneId = [value[0] for value in self.vehsInfo.values()] sortedList = [] for i in list(set(laneId)): lane_info = {k: v[1] for k, v in self.vehsInfo.items() if v[0] == i} # Vehicles in front are at the front of the lane sortedList.append([vid[0] for vid in sorted(lane_info.items(), key=itemgetter(1), reverse=True)]) pruneList = [sublist for sublist in sortedList if len(sublist) > 1] for subList in pruneList: for index in range(1, len(subList)): # first, prune th subtree which begin with illegal vehicle id self.tree.remove_subtree(subList[index]) # second, delete the nodes which match the illegal pattern pattern = subList[index] + ".*" + subList[0] for node in self.tree.all_nodes(): if re.search(pattern, node.tag): try: self.tree.remove_node(node.identifier) except: pass def build(self): self._build(self.root, self.vehList) self._prune() def show(self): self.tree.show() def _leaves(self): ''' :return: All the plan for vehicle passing currently. ''' all_nodes = self.tree.all_nodes() return [node for node in all_nodes if node.is_leaf()] def legal_orders(self): leaves = self._leaves() orders = [] for pattern in leaves: # upToRight.1-leftToBelow.18-belowToRight.2-belowToRight.3- tmp = pattern.tag.split("-") try: tmp.remove('') except: pass if len(tmp) == self.tree.depth(): orders.append(tmp) return orders
class RRT: def __init__(self, current, desir, step_size, kenamatic, world_size, obstacle=0): self.jobs = [] self.size = world_size # Desired start and end point self.current = current self.desir = desir # A matrix that holds all dots from both starts # The trees that arrange the matrix self.tree = Tree() # Did we finish the search self.NOF = Value("i", 0) self.statos = Value("i", 0) # self.winindex = Value("i", 0) self.winindex = Manager().list() self.badindex = Manager().list() self.badpoint = Manager().list() self.pool = np.float32(np.zeros((1, 12))) self.pool_index = np.zeros(1) self.num_pool = 9 # The number of dots already inserted into the tree self.NOP = 1 # Step size self.step = step_size # Number of dimensions # self.NOD = len(current.T) self.t = time.time() self.obstacle = obstacle # The kinematics of the arms self.kin = kenamatic # Initial position of the arm base print("open new RRT model") def goliniar(self, point_a, point_b): mid_all_point = self.midpoint(point_a[0], point_b[0]) i = 0 dis = 1 loops = len(mid_all_point) #print((self.obstacle_gpu.dtype),"68") while i < loops: ans = self.kin.configure_check1(mid_all_point[i].reshape(1, 12), self.obstacle_gpu) if ans == 0: return i - dis, mid_all_point else: dis = max(int(ans**2 / 4000), 1) i = i + dis return loops - 1, mid_all_point def goliniarpros(self, pointstart, pointend, index, winindex, badindex, badpoint): print("tread {} start".format(index)) secsed = True allpoints = self.midpoint(pointstart, pointend) i = 0 loops = len(allpoints) while i < loops: ans = self.kin.configure_check(allpoints[i].reshape(1, 12), self.obstacle) if not ans: secsed = False if (i > 8): badpoint.append(allpoints[i - 8]) badindex.append(index) break else: i = i + max(int(ans**2 / 3000), 1) if (self.statos.value): secsed = False break if secsed: self.statos.value = 1 winindex.append(index) print("---tread {} secsed ---".format(index)) #print((allpoints.dtype),"107") print("tread {} die ".format(index)) self.NOF.value -= 1 def get_direction(self, tree, index): tree.show() #allrote = self.current if not index == 0: root = tree[index].predecessor(tree._identifier) good_route = tree[index].data while (not root == 0 or root == None): point = np.array(tree[root].data) good_route = np.append(point, good_route, axis=0) root = tree[root].predecessor(tree._identifier) allrote = self.midpoint(self.desir[0], good_route[-1]) allrote = np.append(allrote[:-1], self.midpoint(allrote[-1], good_route[0]), axis=0) for i in range(1, len(good_route)): allrote = np.append(allrote[:-1], self.midpoint(good_route[i - 1], good_route[i]), axis=0) allrote = np.append(allrote[:-1], self.midpoint(allrote[-1], self.current[0]), axis=0) else: allrote = self.midpoint(self.desir[0], self.current[0]) # ============================================================================= # good_route = np.append(self.current, self.desir, axis=0) # for i in range(1, len(good_route)): # allrote = np.append(allrote, self.midpoint(good_route[i - 1], good_route[i]), axis=0) # ============================================================================= # allrote = allrote[1:, :] return np.flip(allrote, axis=0) # Extracting the path we found from the tree def get_direction1(self, tree, index): #tree.show() allrote = self.current if not index == 0: root = tree[index].predecessor(tree._identifier) good_route = tree[index].data while (not root == 0 or root == None): point = np.array(tree[root].data) good_route = np.append(point, good_route, axis=0) root = tree[root].predecessor(tree._identifier) for i in range(1, len(good_route)): allrote = np.append(allrote[:-1], self.midpoint(good_route[i - 1], good_route[i]), axis=0) allrote = np.append(allrote[1:], self.midpoint(self.all_point[index], self.desir[0]), axis=0) allrote = np.append(self.midpoint(allrote[0], allrote[1]), allrote[1:, :], axis=0) allrote = np.append(self.midpoint(self.current[0], allrote[0]), allrote, axis=0) else: good_route = np.append(self.current, self.desir, axis=0) for i in range(1, len(good_route)): allrote = np.append(allrote, self.midpoint(good_route[i - 1], good_route[i]), axis=0) return allrote def get_direction2(self, tree, index): #tree.show() allrote = self.current if not index == 0: root = tree[index].predecessor(tree._identifier) good_route = tree[index].data while (not root == 0 or root == None): point = np.array(tree[root].data) good_route = np.append(point, good_route, axis=0) root = tree[root].predecessor(tree._identifier) for i in range(1, len(good_route)): allrote = np.append(allrote, self.all_point[i].reshape(1, 12), axis=0) allrote = np.append(allrote, self.desir, axis=0) print(tree.depth(index)) return allrote # Gets a vector and normalizes it to a desired step size def get_normelize_vec(self, vec): return np.float32((vec / np.sqrt(np.sum(pow(vec, 2)))) * self.step) def midpoint(self, startp, endp): return np.linspace(startp, endp, int(np.max(np.abs(startp - endp) + 1)), dtype=np.float32) # Gets a vector point and returns the geometrically closest vector point def min_distance(self, newpoint): return np.argmin( np.sum(pow(np.subtract(newpoint, self.all_point), 2), axis=1)) def dis_2point(self, point_A, point_B): return np.min(np.sqrt(np.sum(pow(np.subtract(point_A, point_B), 2)))) # Adds a point into the tree def add_to_tree(self, index, newpoint): self.tree.create_node(self.NOP, self.NOP, parent=index, data=newpoint) self.all_point = np.append(self.all_point, newpoint, axis=0) self.NOP += 1 print("NOP : ", self.NOP) def bildwold(self): NOO = 200 a = 1 p = 50 obs = np.array( [np.ones(p) * NOO, np.linspace(-NOO, NOO, num=p), np.ones(p)]).T for i in range(1, int(NOO * a)): obs = np.append(obs, np.array([ np.ones(p) * NOO, np.linspace(-NOO, NOO, num=p), np.ones(p) * (i) ]).T, axis=0) obs = np.append(obs, np.array([ np.ones(p) * -NOO, np.linspace(-NOO, NOO, num=p), np.ones(p) * (i) ]).T, axis=0) obs = np.append(obs, np.array([ np.linspace(-NOO, NOO, num=p), np.ones(p) * NOO, np.ones(p) ]).T, axis=0) for i in range(1, int(NOO * a)): obs = np.append(obs, np.array([ np.linspace(-NOO, NOO, num=p), np.ones(p) * NOO, np.ones(p) * (i) ]).T, axis=0) obs = np.append(obs, np.array([ np.linspace(-NOO, NOO, num=p), np.ones(p) * -NOO, np.ones(p) * (i) ]).T, axis=0) NOO = NOO * 3 for i in range(1, int(NOO * 2)): obs = np.append(obs, np.array([ np.linspace(-NOO, NOO, num=p), np.ones(p) * (NOO - i), np.zeros(p) ]).T, axis=0) return np.float32(obs) def open_prosses(self, srart_pos, end_pos, index): p = Process(target=self.goliniarpros, args=( srart_pos, end_pos, index, self.winindex, self.badindex, self.badpoint, )) self.jobs.append(p) p.start() self.NOF.value += 1 # Adds a point in a random direction def add_point(self, newpoint): index = self.min_distance(newpoint) #V newpoint = self.get_normelize_vec( np.subtract(newpoint, self.all_point[index])) + self.all_point[index] godindex, temppoints = self.goliniar( self.all_point[index].reshape(1, 12), newpoint) #print((temppoints.dtype),"240") #print((self.all_point.dtype),"241") if (godindex > 0): self.add_to_tree(index, temppoints[godindex].reshape(1, 12)) if (self.NOF.value < self.num_pool): self.open_prosses(self.all_point[-1], self.desir[0], self.NOP - 1) else: self.pool = np.append(self.pool, self.all_point[-1].reshape(1, 12), axis=0) self.pool_index = np.append(self.pool_index, (self.NOP - 1)) print("number of processes that are currently running : {}".format( self.NOF.value)) def midserce1(self, allpoint): e = 1 max = int(len(allpoint) / 2) min = 0 mid = max new_Peza = int(max / 4) old_peza = 0 while (abs(new_Peza - old_peza) > e): index_a = mid + new_Peza index_b = mid - new_Peza good_index, temp_points = self.goliniar( allpoint[index_a].reshape(1, 12), allpoint[index_b].reshape(1, 12)) if (good_index + 1 == len(temp_points)): min = new_Peza old_peza = new_Peza new_Peza = int(new_Peza + (max - new_Peza) / 2) else: max = new_Peza old_peza = new_Peza new_Peza = int(new_Peza - (new_Peza - min) / 2) print("Peza = ", new_Peza) return new_Peza, mid def midserce(self, allpoint, max_val, min_val, start_point): print("strt : {}\tmax : {}\tmin : {}".format(start_point, max_val, min_val)) e = 2 new_index = int((max_val + min_val) / 2) print("new_index = ", new_index) old_index = 0 while (abs(new_index - old_index) > e): good_index, temp_points = self.goliniar( allpoint[start_point].reshape(1, 12), allpoint[new_index].reshape(1, 12)) if (good_index + 1 == len(temp_points)): min_val = new_index old_index = new_index new_index = int(new_index + (max_val - new_index) / 2) else: max_val = new_index old_index = new_index new_index = int(new_index - (new_index - min_val) / 2) if (new_index < min_val): print("faild") return old_index, False print("end : ", new_index) return new_index, True def improved_path1(self, allpoint): print("\n\n\nstart -> 1") t = time.time() a = len(allpoint) max_val = len(allpoint) min_val = 0 start_point = [0] new_beast = 0 old_beast = -10 while max_val - new_beast > 5 and new_beast - old_beast > 2: old_beast = new_beast new_beast, stat = self.midserce(allpoint, max_val, min_val, start_point[-1]) start_point.append(new_beast) min_val = new_beast optmaize = self.current j = 0 for i in start_point: optmaize = np.append(optmaize, self.midpoint(allpoint[j], allpoint[i]), axis=0) j = i if not stat: optmaize = np.append(optmaize, allpoint[start_point[-1]:, :], axis=0) else: optmaize = np.append(optmaize, self.midpoint(optmaize[-1], self.desir[0]), axis=0) allpoint = optmaize max_val = len(allpoint) print(-t + time.time()) return allpoint # ============================================================================= # # # def improved_path2(self, allpoint): # print("\n\n\nstart -> 6") # t=time.time() # a=len(allpoint) # print("a->",a) # # max_val=len(allpoint) # min_val=0 # start_point=[0] # new_beast=0 # stat=True # dellist=[] # while max_val-new_beast>1 : # old_beast=new_beast # new_beast,stat=self.midserce1(allpoint,max_val,min_val,start_point[-1]) # start_point.append(new_beast) # min_val=new_beast # # print(start_point,"-----------") # if len(start_point)>1: # for i in range(1,len(start_point)): # for i in (range(start_point[i-1]+1,start_point[i])): # dellist.append(i) # print(dellist) # b=np.delete(allpoint,dellist,0) # print("b->",len(b)) # # optmaize=self.current # for i in range(1,len(b)): # optmaize=np.append(optmaize,self.midpoint(b[i-1],b[i]),axis=0) # # allpoint = optmaize # for i in range(1,len(allpoint)): # if (not (np.linalg.norm(allpoint[i-1]-allpoint[i]))): # print(0) # max_val=len(allpoint) # print("old : {} \t new : {}".format(a,len(allpoint))) # print(-t+time.time()) # return allpoint # ============================================================================= # ============================================================================= # def improved_path(self, allpoint): # print("start -> 5") # t=time.time() # # a=len(allpoint) # beast,mid=self.midserce(allpoint) # # if (not beast == 0): # tempend = allpoint[ (mid + beast-1):,:] # tempstart = allpoint[:(mid - beast+1),:] # if(not len(tempend)): # tempend=allpoint[-1,:].reshape((1,12)) # if(not len(tempstart)): # tempend=allpoint[0,:].reshape((1,12)) # tempmid = self.midpoint(tempstart[-1,: ], tempend[0,: ]) # temp = np.append(tempstart, tempmid[1:-1], axis=0) # allpoint = np.append(temp, tempend, axis=0) # # print("old : {} \t new : {}".format(a,len(allpoint))) # print(-t+time.time()) # return allpoint # # # ============================================================================= def get_winindex(self): print(self.winindex) print("Extracting the route") dis = self.tree.depth(self.winindex[0]) * self.step + np.linalg.norm( self.tree[self.winindex[0]].data - self.desir) min_val = 99999 for i in self.winindex: dis = self.tree.depth(i) * self.step + np.linalg.norm( self.tree[i].data - self.desir) print(i, "\t", self.tree.depth(i), "\t", np.linalg.norm(self.tree[i].data - self.desir), '\t', dis) if (dis < min_val): min_val = dis win = i print(win) self.winindex = Manager().list() return win # The function that activates everything def let_the_magic_begin(self, ros_fun, isflipt): self.open_prosses(self.current[0], self.desir[0], self.NOP - 1) NOL = 0 while (not self.statos.value): if (not NOL % 10): self.size += 10 self.size = min(self.size, 360) newpoint = (np.float32(np.random.rand( 1, 12, )) * self.kin.limit * self.size - self.kin.offset) #print((newpoint.dtype),"436") self.add_point(newpoint) if (len(self.pool) > 1): while (len(self.pool) > 1 and self.NOF.value < self.num_pool): if (self.statos.value): break self.open_prosses(self.pool[-1], self.desir[0], int(self.pool_index[-1])) self.pool = np.delete(self.pool, -1, 0) self.pool_index = np.delete(self.pool_index, -1) if (len(self.badindex)): for i in range(len(self.badindex)): self.add_to_tree(self.badindex[0], self.badpoint[0].reshape(1, 12)) del self.badindex[0] del self.badpoint[0] NOL += 1 print("Waiting for all processes to die") for job in self.jobs: job.join() best_point_index = self.get_winindex() allrote = self.get_direction1(self.tree, best_point_index) if isflipt: allrote = np.flip(allrote, axis=0) tmp = self.desir self.desir = self.current self.current = tmp print(time.time() - self.t) input("preace any key to improved path\n") if (best_point_index == 0): print("no need to improve path") allrote1 = allrote #else: # ============================================================================= # for i in allrote: # print(self.kin.configure_check1(i.reshape(1,12),self.obstacle_gpu)) # if not (self.kin.configure_check1(i.reshape(1,12),self.obstacle_gpu)): # print("faild") # ============================================================================= # ============================================================================= # # allrotetemp = self.get_direction2(self.tree, best_point_index) # allrote2=self.improved_path2(allrotetemp) # print("first : old -> {} \tnew -> {}".format(len(allrote),len(allrote2))) # allrote3=self.improved_path1(allrote2) # print("sec : old -> {} \tnew -> {}".format(len(allrote),len(allrote2))) # ============================================================================= allrote6 = self.improved_path1(allrote) print("fird : old -> {} \tnew -> {}".format(len(allrote), len(allrote6))) #allrote6=self.improved_path(allrote6) #allrote3=self.improved_path(allrote) input("preace any key to run the simulation\n") ros_fun.send_to_arm(allrote6 * np.pi / 180) input("preace any key to run the simulation\n") ros_fun.send_to_arm(np.flip(allrote, axis=0) * np.pi / 180) def run_serce(self, ros_fun, isinvers, invers_pos): self.obstacle_gpu = th.from_numpy(self.obstacle).float().to('cuda') if (isinvers): self.desir = self.kin.invers_arms_cfg(invers_pos) self.t = time.time() dis_des = self.kin.configure_check1(self.desir, self.obstacle_gpu) dis_crr = self.kin.configure_check1(self.current, self.obstacle_gpu) if dis_des: if dis_des < dis_crr: temp = self.desir self.desir = self.current self.current = temp isflipt = True print("desir = {} \ncurrent={} \nneed to flip".format( dis_des, dis_crr)) else: isflipt = False print("desir = {} \ncurrent={} \nno need to flip".format( dis_des, dis_crr)) self.tree.create_node(0, 0, data=self.current) # root node self.all_point = self.current #print(self.current.dtype) self.let_the_magic_begin(ros_fun, isflipt) else: print("can't go to disre location")
class SentenceTree(object): def __init__(self): self.tree = Tree() self.sentence = [] self.matrix = [] # Builds a tree by backtracking the sentence matrix def build_tree(self, sentence_matrix): self.matrix = sentence_matrix sentence_length = len(sentence_matrix)-1 # Saves the ST's sentence as a list of strings for i in range(1,sentence_length+1): self.sentence.append(self.matrix[0][i][0]) # Finds the most probable sentence option options = sentence_matrix[sentence_length][1] max_option = [options[x] for x in options if options[x][1]==max([y[1] for y in options.values()])][0] # Builds the tree self._nid = sentence_length+1 root = max_option self.tree.create_node(root[0], self._nid) self._create_children(root, self._nid) # Call recursive function # Ensures unique node id in _create_children() def _nnid(self): self._nid +=1 return self._nid # Recursive function which builds the children nodes of a given parse_option and then builds their children def _create_children(self, parse_option, pid): if parse_option is None: return None else: # If parse_option has children, extract those if parse_option[2] is not None: left_coord = parse_option[2] right_coord = parse_option[3] left_child = self.matrix[left_coord[0]][left_coord[1]][left_coord[2]] right_child = self.matrix[right_coord[0]][right_coord[1]][right_coord[2]] # Create left child as node (plus extra word node if leaf) cid = self._nnid() self.tree.create_node(left_child[0], cid, parent=pid) if left_child[2] is None: #If left_child is a leaf node, append a word node nid = left_coord[1]-1 word = self.matrix[left_coord[0]-1][left_coord[1]][0] word = word.decode('utf-8', "ignore") self.tree.create_node(word, nid, parent=cid) else: self._create_children(left_child, cid) # Create children of left_child # Create right child as node (plus extra word node if leaf) cid = self._nnid() self.tree.create_node(right_child[0], cid, parent=pid) if right_child[2] is None: #If right_child is a leaf node, append a word node nid = right_coord[1]-1 word = self.matrix[right_coord[0]-1][right_coord[1]][0] word = word.decode('utf-8', "ignore") self.tree.create_node(word, nid, parent=cid) else: self._create_children(right_child, cid) # Create children of right_child # Returns the sentence's sentiment score def get_sentiment_score(self, sentimentDict, term): total_score = 0 # negation dictionary negationList = ["ikke", "ej"] # Check the term against every sentiment word n1 = self.sentence.index(term) for word in sentimentDict: if term==word: continue # If topic term is an opinion word, ignore. n2 = self._in_sentence(word) if n2 is not False: d = self._get_distance(n1, n2) if d == 0: score = float(sentimentDict[word]) score = float(sentimentDict[word]) / float(d) # If SentWord is negated, flip the score derived from it if self._is_negated(word, negationList): score = score * -1 print "Term: %s | SentWord: %s | Distance: %s | Score: %s" % (term, word, d,score) total_score += score print "Total score:", total_score return total_score # Checks whether a word is within a specified threshold distance of a negation word def _is_negated(self, w, negationList): negationThreshold = 3 n1 = self._in_sentence(w) if n1 is None: return False for nw in negationList: n2 = self._in_sentence(nw) if n2 is not None: if (self._get_distance(n1, n2)) < negationThreshold: print "negating word", w return True return False # Checks whether word w exists in the ST's sentence def _in_sentence(self, w): if w in self.sentence: return self.sentence.index(w) return False # Returns distance between two nodes n1 and n2 def _get_distance(self, n1, n2): LCA = self._get_LCA(self.tree.root, n1, n2) distance = self.tree.depth(self.tree.get_node(n1)) + self.tree.depth(self.tree.get_node(n2)) distance = distance - 2 * self.tree.depth(self.tree.get_node(LCA)) return abs(distance) # Returns lowest common ancestor of two nodes n1 and n2 # Supporting method of _get_distance() def _get_LCA(self, current_node, n1, n2): if current_node is None: return None if current_node == n1 or current_node == n2: return current_node if len(self.tree.get_node(current_node).fpointer) == 0: return None #if leaf, return None if len(self.tree.get_node(current_node).fpointer) == 1: #if terminal node, check its single leaf node return self._get_LCA(self.tree.get_node(current_node).fpointer[0], n1, n2) if len(self.tree.get_node(current_node).fpointer) == 2: left = self._get_LCA(self.tree.get_node(current_node).fpointer[0],n1,n2) right = self._get_LCA(self.tree.get_node(current_node).fpointer[1],n1,n2) if left is not None and right is not None: return current_node if left is not None: return left if right is not None:return right return None
continue else: rootid = nodeid unique_ids = set(nodeids) tree.create_node(tag=rootid,identifier=rootid) while len(unique_ids) != len(tree.all_nodes()): for rightNode in list(parent_check.keys()): if tree.get_node(parent_check[rightNode]) is not None and tree.get_node(rightNode) is None: tree.create_node(tag=rightNode, identifier=rightNode, parent=tree.get_node(parent_check[rightNode])) for node in tree.all_nodes(): sum += tree.depth(node) print(sum) def intersection(lst1, lst2): lst3 = [value for value in lst1 if value in lst2] return lst3 sanIndex = 0 youInex = 0 for path in tree.paths_to_leaves(): if path[len(path) -1] == "SAN": sanIndex = path[path.index('7LD'):] elif path[len(path) - 1] == "YOU": youIndex = path[path.index('7LD'):]
class RRT: def __init__(self, current, desir, step_size, kenamatic, world_size, obstacle=0): self.jobs = [] self.size = world_size # Desired start and end point self.current = current self.desir = desir # A matrix that holds all dots from both starts self.all_point = self.current # The trees that arrange the matrix self.tree = Tree() self.tree.create_node(0, 0, data=self.current) # root node # Did we finish the search self.NOF = Value("i", 0) self.statos = Value("i", 0) # self.winindex = Value("i", 0) self.winindex = Manager().list() self.badindex = Manager().list() self.badpoint = Manager().list() self.pool_point = Manager().list() self.pool_index = Manager().list() self.num_pool = 9 # The number of dots already inserted into the tree self.NOP = 1 # Step size self.step = step_size # Number of dimensions # self.NOD = len(current.T) self.t = time.time() self.obstacle = obstacle # The kinematics of the arms self.kin = kenamatic # Initial position of the arm base print("open new RRT model") def goliniar(self, point_a, point_b): mid_all_point = self.midpoint(point_a[0], point_b[0]) i = 0 dis = 1 loops = len(mid_all_point) while i < loops: ans = self.kin.configure_check(mid_all_point[i].reshape(1, 12), self.obstacle) if ans == 0: return i - dis, mid_all_point else: dis = max(int(ans**2 / 4000), 1) i = i + dis return loops - 1, mid_all_point def goliniarpros(self, a): steelrun = True while steelrun: for i in range(len(self.pool_point)): secsed = True allpoints = self.midpoint(self.pool_point[i], self.desir) i = 0 loops = len(allpoints) while i < loops: ans = self.kin.configure_check1( allpoints[i].reshape(1, 12), self.obstacle_gpu) if not ans: secsed = False if (i > 8): self.badpoint.append(allpoints[i - 8]) self.badindex.append(self.pool_index[i]) del self.pool_point[i] del self.pool_index[i] break else: i = i + max(int(ans**2 / 3000), 1) if secsed: self.statos.value = 1 self.winindex.append(self.pool_index[i]) steelrun = False print("tread {} die ".format(self.pool_index[i])) self.NOF.value -= 1 def get_direction(self, tree, index): tree.show() #allrote = self.current if not index == 0: root = tree[index].predecessor(tree._identifier) good_route = tree[index].data while (not root == 0 or root == None): point = np.array(tree[root].data) good_route = np.append(point, good_route, axis=0) root = tree[root].predecessor(tree._identifier) allrote = self.midpoint(self.desir[0], good_route[-1]) allrote = np.append(allrote[:-1], self.midpoint(allrote[-1], good_route[0]), axis=0) for i in range(1, len(good_route)): allrote = np.append(allrote[:-1], self.midpoint(good_route[i - 1], good_route[i]), axis=0) allrote = np.append(allrote[:-1], self.midpoint(allrote[-1], self.current[0]), axis=0) else: allrote = self.midpoint(self.desir[0], self.current[0]) # ============================================================================= # good_route = np.append(self.current, self.desir, axis=0) # for i in range(1, len(good_route)): # allrote = np.append(allrote, self.midpoint(good_route[i - 1], good_route[i]), axis=0) # ============================================================================= # allrote = allrote[1:, :] return np.flip(allrote, axis=0) # Extracting the path we found from the tree def get_direction1(self, tree, index): #tree.show() allrote = self.current if not index == 0: root = tree[index].predecessor(tree._identifier) good_route = tree[index].data while (not root == 0 or root == None): point = np.array(tree[root].data) good_route = np.append(point, good_route, axis=0) root = tree[root].predecessor(tree._identifier) for i in range(1, len(good_route)): allrote = np.append(allrote[:-1], self.midpoint(good_route[i - 1], good_route[i]), axis=0) allrote = np.append(allrote[1:], self.midpoint(self.all_point[index], self.desir[0]), axis=0) allrote = np.append(self.midpoint(allrote[0], allrote[1]), allrote[1:, :], axis=0) allrote = np.append(self.midpoint(self.current[0], allrote[0]), allrote, axis=0) else: good_route = np.append(self.current, self.desir, axis=0) for i in range(1, len(good_route)): allrote = np.append(allrote, self.midpoint(good_route[i - 1], good_route[i]), axis=0) # allrote = allrote[1:, :] return allrote # Gets a vector and normalizes it to a desired step size def get_normelize_vec(self, vec): return (vec / np.sqrt(np.sum(pow(vec, 2)))) * self.step def midpoint(self, startp, endp): return np.linspace(startp, endp, int(np.max(np.abs(startp - endp) + 1))) # Gets a vector point and returns the geometrically closest vector point def min_distance(self, newpoint, data_points): return np.argmin( np.sum(pow(np.subtract(newpoint, data_points), 2), axis=1)) # Adds a point into the tree def add_to_tree(self, index, newpoint): self.tree.create_node(self.NOP, self.NOP, parent=index, data=newpoint) self.all_point = np.append(self.all_point, newpoint, axis=0) self.NOP += 1 print("NOP : ", self.NOP) def bildwold(self): NOO = 200 a = 1 p = 50 obs = np.array( [np.ones(p) * NOO, np.linspace(-NOO, NOO, num=p), np.ones(p)]).T for i in range(1, int(NOO * a)): obs = np.append(obs, np.array([ np.ones(p) * NOO, np.linspace(-NOO, NOO, num=p), np.ones(p) * (i) ]).T, axis=0) obs = np.append(obs, np.array([ np.ones(p) * -NOO, np.linspace(-NOO, NOO, num=p), np.ones(p) * (i) ]).T, axis=0) obs = np.append(obs, np.array([ np.linspace(-NOO, NOO, num=p), np.ones(p) * NOO, np.ones(p) ]).T, axis=0) for i in range(1, int(NOO * a)): obs = np.append(obs, np.array([ np.linspace(-NOO, NOO, num=p), np.ones(p) * NOO, np.ones(p) * (i) ]).T, axis=0) obs = np.append(obs, np.array([ np.linspace(-NOO, NOO, num=p), np.ones(p) * -NOO, np.ones(p) * (i) ]).T, axis=0) NOO = NOO * 3 for i in range(1, int(NOO * 2)): obs = np.append(obs, np.array([ np.linspace(-NOO, NOO, num=p), np.ones(p) * (NOO - i), np.zeros(p) ]).T, axis=0) return obs def open_prosses(self): p = Process(target=self.goliniarpros, args=(self, )) self.jobs.append(p) p.start() self.NOF.value += 1 # Adds a point in a random direction def add_point(self, newpoint): index = self.min_distance(newpoint, self.all_point) #V newpoint = self.get_normelize_vec( np.subtract(newpoint, self.all_point[index])) + self.all_point[index] godindex, temppoints = self.goliniar( self.all_point[index].reshape(1, 12), newpoint) if (godindex > 0): self.add_to_tree(index, temppoints[godindex].reshape(1, 12)) self.pool_point.append(self.all_point[-1]) self.pool_index.append(index) #print("number of processes that are currently running : {}".format(self.NOF.value)) def improved_path1(self, allpoint): print("1") t = time.time() templen = 1000000 while (templen - len(allpoint) > 10): a = len(allpoint) mid = int(len(allpoint) / 2) Peza = 5 index_a = mid + Peza index_b = mid - Peza beast = 0 templen = len(allpoint) while (index_a <= len(allpoint) - 1 and index_b >= 0): good_index, temp_points = self.goliniar( allpoint[index_a].reshape(1, 12), allpoint[index_b].reshape(1, 12)) if (good_index + 1 == len(temp_points)): beast = Peza Peza += 5 index_a = mid + Peza index_b = mid - Peza if (not beast == 0): tempend = allpoint[(mid + beast - 1):, :] tempstart = allpoint[:(mid - beast + 1), :] if (not len(tempend)): tempend = allpoint[-1, :].reshape((1, 12)) if (not len(tempstart)): tempend = allpoint[0, :].reshape((1, 12)) tempmid = self.midpoint(tempstart[-1, :], tempend[0, :]) temp = np.append(tempstart, tempmid[1:-1], axis=0) allpoint = np.append(temp, tempend, axis=0) print("old : {} \t new : {}".format(a, len(allpoint))) print(-t + time.time()) return allpoint def improved_path2(self, allpoint): print("2") t = time.time() templen = 1000000 for i in range(1, 4): mid = int(len(allpoint) / 4 * i) Peza = 5 index_a = mid + Peza index_b = mid - Peza beast = 0 templen = len(allpoint) while (index_a <= len(allpoint) - 1 and index_b >= 0): good_index, temp_points = self.goliniar( allpoint[index_a].reshape(1, 12), allpoint[index_b].reshape(1, 12)) if (good_index + 1 == len(temp_points)): beast = Peza else: break Peza += 5 index_a = mid + Peza index_b = mid - Peza if (not beast == 0): tempend = allpoint[(mid + beast - 1):, :] tempstart = allpoint[:(mid - beast + 1), :] if (not len(tempend)): tempend = allpoint[-1, :].reshape((1, 12)) if (not len(tempstart)): tempend = allpoint[0, :].reshape((1, 12)) tempmid = self.midpoint(tempstart[-1, :], tempend[0, :]) temp = np.append(tempstart, tempmid[1:-1], axis=0) allpoint = np.append(temp, tempend, axis=0) # ============================================================================= # i=len(allpoint)-5 # while(i<0): # good_index, temp_points =self.goliniar(allpoint[i].reshape(1,12),allpoint[-1].reshape(1,12)) # if (good_index+1==len(temp_points)): # beast = i # else: # break # i -= 5 # if (not beast == 0): # tempmid = self.midpoint( allpoint[beast-1],allpoint[-1]) # allpoint = np.append(allpoint[(beast-1):],tempmid, axis=0) # ============================================================================= print(-t + time.time()) return allpoint def improved_path3(self, allpoint): print("4") t = time.time() templen = 1000000 a = len(allpoint) i = 5 a = len(allpoint) while (i < len(allpoint)): good_index, temp_points = self.goliniar(allpoint[0].reshape(1, 12), allpoint[i].reshape(1, 12)) if (good_index + 1 == len(temp_points)): beast = i else: break i += 5 if (not beast == 0): tempmid = self.midpoint(allpoint[0], allpoint[beast - 1]) allpoint = np.append(tempmid, allpoint[(beast - 1):], axis=0) i = len(allpoint) - 1 print("old : {} \t new : {}".format(a, len(allpoint))) a = len(allpoint) while (i > 0): good_index, temp_points = self.goliniar( allpoint[i].reshape(1, 12), allpoint[-1].reshape(1, 12)) if (good_index + 1 == len(temp_points)): beast = i else: break i -= 5 if (not beast == len(allpoint) - 1): tempmid = self.midpoint(allpoint[beast - 1], allpoint[-1]) allpoint = np.append(allpoint[:(beast - 1)], tempmid, axis=0) print("old : {} \t new : {}".format(a, len(allpoint))) a = len(allpoint) mid = int(len(allpoint) / 2) Peza = 5 index_a = mid + Peza index_b = mid - Peza beast = 0 templen = len(allpoint) while (index_a <= len(allpoint) - 1 and index_b >= 0): good_index, temp_points = self.goliniar( allpoint[index_a].reshape(1, 12), allpoint[index_b].reshape(1, 12)) if (good_index + 1 == len(temp_points)): beast = Peza Peza += 5 index_a = mid + Peza index_b = mid - Peza if (not beast == 0): tempend = allpoint[(mid + beast - 1):, :] tempstart = allpoint[:(mid - beast + 1), :] if (not len(tempend)): tempend = allpoint[-1, :].reshape((1, 12)) if (not len(tempstart)): tempend = allpoint[0, :].reshape((1, 12)) tempmid = self.midpoint(tempstart[-1, :], tempend[0, :]) temp = np.append(tempstart, tempmid[1:-1], axis=0) allpoint = np.append(temp, tempend, axis=0) print("old : {} \t new : {}".format(a, len(allpoint))) print(-t + time.time()) return allpoint def get_winindex(self): print(self.winindex) print("Extracting the route") dis = self.tree.depth(self.winindex[0]) * self.step + np.linalg.norm( self.tree[self.winindex[0]].data - self.desir) min_val = 99999 for i in self.winindex: dis = self.tree.depth(i) * self.step + np.linalg.norm( self.tree[i].data - self.desir) print(i, "\t", self.tree.depth(i), "\t", np.linalg.norm(self.tree[i].data - self.desir), '\t', dis) if (dis < min_val): min_val = dis win = i print(win) self.winindex = Manager().list() return win # The function that activates everything def let_the_magic_begin(self, ros_fun): # self.open_prosses(self.current[0],self.desir[0],self.NOP - 1) self.pool_point.append(self.current[0]) self.pool_index.append(0) self.open_prosses() NOL = 0 while (not self.statos.value): if (not NOL % 10): self.size += 10 self.size = min(self.size, 360) newpoint = (np.random.rand(1, 12) * self.kin.limit * self.size - self.kin.offset) self.add_point(newpoint) if (len(self.badindex)): for i in range(len(self.badindex)): self.add_to_tree(self.badindex[0], self.badpoint[0].reshape(1, 12)) del self.badindex[0] del self.badpoint[0] NOL += 1 print("Waiting for all processes to die") for job in self.jobs: job.join() best_point_index = self.get_winindex() allrote = self.get_direction1(self.tree, best_point_index) print(time.time() - self.t) input("preace any key to improved path\n") if (best_point_index == 0): print("no need to improve path") allrote1 = allrote else: for i in allrote: print( self.kin.configure_check1(i.reshape(1, 12), self.obstacle_gpu)) if not (self.kin.configure_check1(i.reshape(1, 12), self.obstacle_gpu)): print("faild") # ============================================================================= # # allrote2=self.improved_path1(allrote) # for i in allrote2: # if not (self.kin.configure_check1(i.reshape(1,12),self.obstacle_gpu)): # print("faild") # # allrote3=self.improved_path2(allrote) # for i in allrote3: # if not (self.kin.configure_check1(i.reshape(1,12),self.obstacle_gpu)): # print("faild") # # allrote4=self.improved_path3(allrote) # for i in allrote4: # if not (self.kin.configure_check1(i.reshape(1,12),self.obstacle_gpu)): # print(0) # print("faild") # ============================================================================= # ============================================================================= # # # allrote2=self.improved_path(allrote) # print("old -> : {}".format(len(allrote))) # print("1 -> : {} imp -> {}".format(len(allrote2),len(allrote)-len(allrote2))) # print("2 -> : {} imp -> {}".format(len(allrote3),len(allrote)-len(allrote3))) # print("3 -> : {} imp -> {}".format(len(allrote4),len(allrote)-len(allrote4))) # ============================================================================= input("preace any key to run the simulation\n") ros_fun.send_to_arm(allrote * np.pi / 180) input("preace any key to run the simulation\n") ros_fun.send_to_arm(np.flip(allrote, axis=0) * np.pi / 180) def run_serce(self, ros_fun, isinvers, invers_pos): self.obstacle_gpu = th.from_numpy(self.obstacle).float().to('cuda') if (isinvers): self.desir = self.kin.invers_arms_cfg(invers_pos) self.t = time.time() if self.kin.configure_check1(self.desir, self.obstacle_gpu): self.let_the_magic_begin(ros_fun) else: print("can't go to disre location")
class SentenceTree(object): def __init__(self): self.tree = Tree() self.tree_logger = Logger() self.sentence = [] # Builds a tree by backtracking the sentence matrix def build_tree(self, sentence_matrix): self.tree_logger.start_timer() self.matrix = sentence_matrix sentence_length = len(sentence_matrix)-1 # Saves the ST's sentence as a list of strings for i in range(1,sentence_length+1): self.sentence.append(self.matrix[0][i][0]) # Check if the sentence resolves to a tree if len(sentence_matrix[sentence_length][1])==0: return None # Find the most probable sentence option maximum = 0 index = None for option in sentence_matrix[sentence_length][1]: if option.probability > maximum: maximum = option.probability index = sentence_matrix[sentence_length][1].index(option) if index is None: return None # Build the tree self._nid = sentence_length+2 root = sentence_matrix[sentence_length][1][index] self.tree.create_node(root.constituent, self._nid) self._create_children(root, self._nid) # Call recursive function self.tree_logger.stop_timer() # Ensures unique node id in _create_children() def _nnid(self): self._nid +=1 return self._nid # Recursive function which builds the children nodes of a given parse_option # and then builds their children def _create_children(self, parse_option, pid): if parse_option is None: return None else: # If parse_option has children, extract those if parse_option.left_coord is not None: left_child = self.matrix[parse_option.left_coord[0]][parse_option.left_coord[1]][parse_option.left_coord[2]] right_child = self.matrix[parse_option.right_coord[0]][parse_option.right_coord[1]][parse_option.right_coord[2]] # Create left child as node (plus extra word node if leaf) cid = self._nnid() self.tree.create_node(left_child.constituent, cid, parent=pid) if left_child.left_coord is None: #If left_child is a leaf node, append a word node nid = parse_option.left_coord[1]-1 word = self.matrix[parse_option.left_coord[0]-1][parse_option.left_coord[1]][0] self.tree.create_node(word, nid, parent=cid) else: self._create_children(left_child, cid) # Create children of left_child # Create right child as node (plus extra word node if leaf) cid = self._nnid() self.tree.create_node(right_child.constituent, cid, parent=pid) if right_child.right_coord is None: #If left_child is a leaf node, append a word node nid = parse_option.right_coord[1]-1 word = self.matrix[parse_option.right_coord[0]-1][parse_option.right_coord[1]][0] self.tree.create_node(word, nid, parent=cid) else: self._create_children(right_child, cid) # Create children of right_child # Returns the sentence's sentiment score def get_sentiment_score(self, sentimentDict, term): total_score = 0 # placeholder dictionaries -TESTING PURPOSES negationList = ["ikke", "liden"] # Check the term against every sentiment word n1 = self.sentence.index(term) for key in sentimentDict: n2 = self._in_sentence(key) if n2 is not False: d = self._get_distance(n1, n2) score = float(sentimentDict[key]) / float(d) # If SentWord is negated, flip the score derived from it if self._is_negated(key, negationList): score = score * -1 print ">>SENTIMENTSCORE: Term: %s | SentWord: %s | Distance: %s | Score: %s" % (term, key, d,score) total_score += score return total_score # Checks whether a word is within a specified threshold distance of a negation word def _is_negated(self, w, negationList): negationThreshold = 3 n1 = self._in_sentence(w) if n1 is None: return False for nw in negationList: n2 = self._in_sentence(nw) if n2 is not None: if (self._get_distance(n1, n2)) < negationThreshold: print "negating word", w return True return False # Checks whether word w exists in the ST's sentence def _in_sentence(self, w): if w in self.sentence: return self.sentence.index(w) return False # Returns distance between two nodes n1 and n2 def _get_distance(self, n1, n2): LCA = self._get_LCA(self.tree.root, n1, n2) distance = self.tree.depth(self.tree.get_node(n1)) + self.tree.depth(self.tree.get_node(n2)) distance = distance - 2 * self.tree.depth(self.tree.get_node(LCA)) return distance-2 # Returns lowest common ancestor of two nodes n1 and n2 # Supporting method of _get_distance() def _get_LCA(self, root, n1, n2): if root is None: return None if root == n1 or root == n2: return root if len(self.tree.get_node(root).fpointer) == 0: return None #if leaf, return None if len(self.tree.get_node(root).fpointer) == 1: #if terminal node, check its single leaf node return self._get_LCA(self.tree.get_node(root).fpointer[0], n1, n2) if len(self.tree.get_node(root).fpointer) == 2: left = self._get_LCA(self.tree.get_node(root).fpointer[0],n1,n2) right = self._get_LCA(self.tree.get_node(root).fpointer[1],n1,n2) if left is not None and right is not None: return root if left is not None: return left if right is not None:return right return None
class AcquisitionChainIter(object): def __init__(self, acquisition_chain, parallel_prepare=True): self.__sequence_index = -1 self._parallel_prepare = parallel_prepare self.__acquisition_chain_ref = weakref.ref(acquisition_chain) # set all slaves into master for master in (x for x in acquisition_chain._tree.expand_tree() if isinstance(x, AcquisitionMaster)): del master.slaves[:] master.slaves.extend( acquisition_chain._tree.get_node(master).fpointer) # create iterators tree self._tree = Tree() self._root_node = self._tree.create_node("acquisition chain", "root") device2iter = dict() for dev in acquisition_chain._tree.expand_tree(): if not isinstance(dev, (AcquisitionDevice, AcquisitionMaster)): continue dev_node = acquisition_chain._tree.get_node(dev) parent = device2iter.get(dev_node.bpointer, "root") try: it = iter(dev) except TypeError: one_shot = self.acquisition_chain._device2one_shot_flag.get( dev, True) dev_iter = DeviceIterator(dev, one_shot) else: dev_iter = DeviceIteratorWrapper(it) device2iter[dev] = dev_iter self._tree.create_node( tag=dev.name, identifier=dev_iter, parent=parent) @property def acquisition_chain(self): return self.__acquisition_chain_ref() def prepare(self, scan, scan_info): preset_tasks = list() if self.__sequence_index == 0: preset_tasks.extend([gevent.spawn(preset.prepare) for preset in self.acquisition_chain._presets_list]) scan.prepare(scan_info, self.acquisition_chain._tree) self._execute( "_prepare", wait_between_levels=not self._parallel_prepare) if self.__sequence_index == 0: gevent.joinall(preset_tasks, raise_error=True) def start(self): if self.__sequence_index == 0: preset_tasks = [gevent.spawn( preset.start) for preset in self.acquisition_chain._presets_list] gevent.joinall(preset_tasks, raise_error=True) self._execute("_start") def stop(self): self._execute("stop", master_to_slave=True, wait_all_tasks=True) preset_tasks = [gevent.spawn(preset.stop) for preset in self.acquisition_chain._presets_list] gevent.joinall(preset_tasks) # wait to call all stop on preset gevent.joinall(preset_tasks, raise_error=True) def next(self): self.__sequence_index += 1 gevent.joinall([gevent.spawn(dev_iter.wait_ready) for dev_iter in self._tree.expand_tree() if dev_iter is not 'root'], raise_error=True) try: if self.__sequence_index: for dev_iter in self._tree.expand_tree(): if dev_iter is 'root': continue dev_iter.next() except StopIteration: # should we stop all devices? for acq_dev_iter in (x for x in self._tree.expand_tree() if x is not 'root' and isinstance(x.device, (AcquisitionDevice, AcquisitionMaster))): if hasattr(acq_dev_iter, 'wait_reading'): acq_dev_iter.wait_reading() dispatcher.send("end", acq_dev_iter.device) raise return self def _execute(self, func_name, master_to_slave=False, wait_between_levels=True, wait_all_tasks=False): tasks = list() prev_level = None if master_to_slave: devs = list(self._tree.expand_tree(mode=Tree.WIDTH))[1:] else: devs = reversed(list(self._tree.expand_tree(mode=Tree.WIDTH))[1:]) for dev in devs: node = self._tree.get_node(dev) level = self._tree.depth(node) if wait_between_levels and prev_level != level: gevent.joinall(tasks, raise_error=True) tasks = list() prev_level = level func = getattr(dev, func_name) tasks.append(gevent.spawn(func)) # ensure that all tasks are executed # (i.e: don't raise the first exception on stop) if wait_all_tasks: gevent.joinall(tasks) gevent.joinall(tasks, raise_error=True)
"Select extra knowledge to load mirror infomation") if filename is not None: with open(filename, 'r') as load_f: load_dict = json.load(load_f) mirror = get_mirror(root.startEA) procs = get_all_procs() tree.create_node(fname, hex(root.startEA), data=Xref_node(fname, hex(root.startEA), XType.code, mirror)) if mirror == fname: add_xrefs(root.startEA, XType.code) Message("Reference Tree:\n\n") tree.show(line_type="ascii-em", idhidden=False, data_property='mirror') Message("Unique references:\n") for node in tree.all_nodes_itr(): if type(node.data.mirror) is str: print node.identifier #hierarchical output for level in range(1, tree.depth()): Message("\nLevel %d: %d\n" % (level, tree.size(level))) for node in tree.all_nodes(): if tree.level(node.identifier) == level and type( node.data.mirror) is str: print node.identifier Message("\n%d subroutines in routine %s need transplanting.\n" % (Xref_node.xrefTrans - 1, fname)) conn.close() else: Warning("No function found at location %x" % here())
class PrePruneTree: def __init__(self, vehsInfo): self.tree = Tree() self.root = self.tree.create_node("Root", "root") # root node self.vehsInfo = vehsInfo self.vehList = list(vehsInfo.keys()) self.pruneList = None @tail_call_optimized def _build(self, currentNode, vehList): ''' :param vehList: A dict, keys is the set of vehicles, value is a tuple which represents (lane, position) :param currentNode: The current node in the tree ''' s = [currentNode.tag.find(vid) for vid in vehList] # the quit condition in recursion if (np.array(s) >= 0).all() or self._testIllegel(currentNode.tag): return for vehId in vehList: if vehId not in currentNode.tag: if currentNode.is_root: prefix = currentNode.tag.replace("Root", "") else: prefix = currentNode.tag self.tree.create_node(prefix + vehId + "-", prefix + vehId, parent=currentNode) # self.show() for node in self.tree.all_nodes(): if node.is_leaf() and not self._testPrePrune(node.tag): self._build(currentNode=node, vehList=vehList) def _testIllegel(self, tag): ''' test whether need to stop recursion :param tag: Node tag :return: boolean (if true, the recursion will stop) ''' # upToRight.1-leftToBelow.18-belowToRight.2-belowToRight.3- flag = False tmp = tag.split("-") try: tmp.remove('') except: pass if len(tmp) < len(list(self.vehsInfo.keys())) - 1: return flag surplusVeh = list(set(self.vehList) - set(tmp)) for veh1 in surplusVeh: for veh2 in list(set(self.vehsInfo.keys()) - set(tmp)): for subList in self.pruneList: if surplusVeh in subList and veh2 in subList: if subList.index(veh2) > subList.index(veh1): flag = True return flag def _testPrePrune(self, tag): ''' test whether need to preprune :param tag: :return: ''' flag = False vehs = tag.split("-") try: vehs.remove('') except: pass for veh1 in vehs: for veh2 in list(set(self.vehsInfo.keys()) - set(vehs)): for subList in self.pruneList: if veh1 in subList and veh2 in subList: if subList.index(veh2) < subList.index(veh1): flag = True return flag def obtainPruneList(self): laneId = [value[0] for value in self.vehsInfo.values()] sortedList = [] for i in list(set(laneId)): lane_info = {k: v[1] for k, v in self.vehsInfo.items() if v[0] == i} # Vehicles in front are at the front of the lane sortedList.append([vid[0] for vid in sorted(lane_info.items(), key=itemgetter(1), reverse=True)]) pruneList = [sublist for sublist in sortedList if len(sublist) > 1] return pruneList def build(self): threading.stack_size(20000000) self.pruneList = self.obtainPruneList() # start a new thread to generate tree thread = threading.Thread(target=self._build(self.root, self.vehList)) thread.start() def show(self): self.tree.show() def _leaves(self): ''' :return: All the plan for vehicle passing currently. ''' all_nodes = self.tree.all_nodes() return [node for node in all_nodes if node.is_leaf()] def legal_orders(self): leaves = self._leaves() orders = [] for pattern in leaves: # upToRight.1-leftToBelow.18-belowToRight.2-belowToRight.3- tmp = pattern.tag.split("-") try: tmp.remove('') except: pass if len(tmp) == self.tree.depth(): orders.append(tmp) return orders
if __name__ == '__main__': tree = Tree() node_id = 0 root_node = (3, 3, 1, node_id) all_nodes = [root_node] tree.create_node(tuple_to_string(root_node), root_node) depth = int(input("Enter the depth: ")) for node in all_nodes: if node[2] == 1: temp_node = (node[0] - 1, node[1] - 1, 0, node_id) place_in_tree(temp_node) if tree.depth() > depth: tree.remove_node(temp_node) break temp_node = (node[0] - 2, node[1], 0, node_id) place_in_tree(temp_node) temp_node = (node[0], node[1] - 2, 0, node_id) place_in_tree(temp_node) temp_node = (node[0] - 1, node[1], 0, node_id) place_in_tree(temp_node) temp_node = (node[0], node[1] - 1, 0, node_id) place_in_tree(temp_node) else: temp_node = (node[0] + 1, node[1] + 1, 1, node_id) place_in_tree(temp_node) if tree.depth() > depth: tree.remove_node(temp_node)
tree.create_node(tag=json_data[0].get("mid"), identifier=json_data[0].get("mid"), data=Info(json_data[0], param=param)) uu = 1000 if len(json_data) >= 1000 else len(json_data) # print('uu:',uu) for j in range(1, uu): try: tree.create_node(tag=json_data[j].get("mid"), identifier=json_data[j].get("mid"), parent=json_data[j].get("parent"), data=Info(json_data[j], param=param)) except: pass # 单颗传播树构建完成 # 对传播树进行简化 print('简化前:tree_depth:', tree.depth(), 'tree_nodes:', len(tree.all_nodes())) tree = simplify_tree(tree) print('***********简化后:tree_depth:', tree.depth(), 'tree_nodes:', len(tree.all_nodes())) trees.append(tree) # print(eid,"---trees simplified") # avg_tree_size += tree.size() # avg_trees_size.append(int(avg_tree_size/data_array.shape[0])) # plt.plot(params, avg_trees_size,'ro-') # plt.xlabel('alpha') # plt.ylabel('trees avg size') # plt.show() # plt.savefig('./img/img_1.jpg') # # 提取新闻原文的相关特征,并写入文件
file = open(sys.argv[1], 'r') line = file.readline() d = dict() while (line): m = line.split(')') key = m[0] value = m[1].strip('\n') if key not in d.keys(): d.setdefault(key, []) d[key].append(value) else: d[key].append(value) line = file.readline() tree = Tree() tree.create_node(identifier='COM') add_orbits(tree, 'COM', d) grandparents = [] for node in tree.all_nodes(): if tree.is_ancestor(node.identifier, 'YOU') and tree.is_ancestor( node.identifier, 'SAN'): grandparents.append(node.identifier) common = grandparents[-1:] m = tree.depth(tree.get_node(common[0])) you = tree.depth(tree.get_node('YOU')) san = tree.depth(tree.get_node('SAN')) print(you - m + san - m - 2)
class search_tree(object): def __init__(self, mcst, q_id, p_sen_list, max_depth, max_search_time, beta, m_value, l_passages, ref_answer, vocab): self.tree = Tree() self.q_id = q_id self.tree.create_node(identifier='question_' + str(q_id), data=node()) root_node = self.tree.get_node('question_' + str(q_id)) root_node.data.num = 1.0 self.node_map = {} self.l_passages = l_passages self.m_value = m_value self.ref_answer = ref_answer self.max_search_time = max_search_time self.count = 0.0 self.carpe_diem = mcst self.max_depth = max_depth self.beta = beta self.vocab = vocab self.p_sen_list = p_sen_list self.expand(self.tree.get_node(self.tree.root)) def expand(self, leaf_node): #print '---------------------- start expand: -----------------------' time_tree_start = time.time() sens_list = leaf_node.data.sen # print 'word_list:' # print words_list sens_id_list = map(eval, sens_list) p_sen_id_list, p_pred_list = self.carpe_diem.get_policy( sens_id_list, self.l_passages) # print 'candidate_id: ' # print np.shape(p_word_id) # print 'p_pred' # print np.shape(p_pred) for sen in p_sen_id_list: self.node_map[' '.join(sens_list + [str(sen)])] = len( self.node_map) new_node = node() new_node.sen = sens_list + [str(sen)] #print ('new_node.sen', new_node.sen) new_node.p = p_pred_list[p_sen_id_list.index(sen)] self.tree.create_node(identifier=self.node_map[' '.join( new_node.sen)], data=new_node, parent=leaf_node.identifier) #print ('&&&&&&&&&&&&&&& tree expand time = %3.2f s &&&&&&&&&&&&' % (time.time() - time_tree_start)) def update(self, node_list, value): #print '----update' for node_id in node_list: tmp_node = self.tree.get_node(node_id) tmp_node.data.Q = (tmp_node.data.Q * tmp_node.data.num + value) / (tmp_node.data.num + 1) tmp_node.data.num += 1 def search(self, start_node_id): #print '----tree search' tmp_node = self.tree.get_node(start_node_id) #print tmp_node.data.num has_visit_num = tmp_node.data.num - 1 self.count = has_visit_num if int(self.max_search_time - has_visit_num) > 0: start_node_search_time = int(self.max_search_time - has_visit_num) else: start_node_search_time = 0 for tm in range(start_node_search_time): if tm % 10 == 0: batch_start_time = time.time() #print ('search time',tm) search_list = [start_node_id] tmp_node = self.tree.get_node(start_node_id) #print 'search time :'+ str(time) while not tmp_node.is_leaf(): max_score = float("-inf") max_id = -1 for child_id in tmp_node.fpointer: child_node = self.tree.get_node(child_id) score = self.beta * child_node.data.p * ( (tmp_node.data.num)**0.5 / (1 + child_node.data.num)) #print 'child_node.data.Q: ' #print child_node.data.Q score += child_node.data.Q #print 'score: ' #print score #print '**************' if score > max_score: max_id = child_id max_score = score search_list.append(max_id) tmp_node = self.tree.get_node(max_id) if not tmp_node.data.value == None: v = tmp_node.data.value else: if tmp_node.data.sen[-1] == str(self.l_passages - 1): pred_answer = tmp_node.data.sen # print 'search to end pred_answer: ' # print pred_answer # print 'listSelectedSet' listSelectedSet_sens = [] listSelectedSet = map(eval, pred_answer) # print listSelectedSet for idx in listSelectedSet: listSelectedSet_sens.append(self.p_sen_list[idx]) # print 'pred_answer ' pred_answer_str = '' for sen in listSelectedSet_sens: str123_list = self.carpe_diem.vocab.recover_from_ids( sen, 0) for s in str123_list: pred_answer_str += s # print 'pred_answer_str: ' # print pred_answer_str # print 'ref_answer_str: ' # print list2string(self.ref_answer[0]['answers']) pred_answers = [] pred_answers.append({ 'question_id': [self.q_id], 'question_type': [], 'answers': [''.join(pred_answer_str)], 'entity_answers': [[]], 'yesno_answers': [] }) if len(self.ref_answer) > 0: pred_dict, ref_dict = {}, {} for pred, ref in zip(pred_answers, self.ref_answer): question_id = ref['question_id'] if len(ref['answers']) > 0: pred_dict[question_id] = normalize( pred['answers']) ref_dict[question_id] = normalize( ref['answers']) # print '========compare in tree=======' # print pred_dict[question_id] # print '----------------------' # print ref_dict[question_id] bleu_rouge = compute_bleu_rouge(pred_dict, ref_dict) else: bleu_rouge = None # print 'last words ++++++++++++++ ' # print bleu_rouge v = input_v = bleu_rouge['Rouge-L'] * self.m_value['Rouge-L'] \ + bleu_rouge['Bleu-4'] * self.m_value['Bleu-4'] \ + bleu_rouge['Bleu-1'] * self.m_value['Bleu-1'] \ + bleu_rouge['Bleu-3'] * self.m_value['Bleu-3'] \ + bleu_rouge['Bleu-2'] * self.m_value['Bleu-2'] else: v = self.carpe_diem.value_function(tmp_node.data.sen)[0][0] tmp_node.data.value = v # if tmp_node.data.sen[-1] == str(self.l_passages - 1): # pred_answer = tmp_node.data.sen # listSelectedSet_sens = [] # listSelectedSet = map(eval, pred_answer) # # print listSelectedSet # for idx in listSelectedSet: # listSelectedSet_sens.append(self.p_sen_list[idx]) # # print 'pred_answer ' # pred_answer_str = '' # for sen in listSelectedSet_sens: # str123_list = self.carpe_diem.vocab.recover_from_ids(sen, 0) # for s in str123_list: # pred_answer_str += s # # pred_answers = [] # # pred_answers.append({'question_id': [self.q_id], # 'question_type': [], # 'answers': [''.join(pred_answer_str)], # 'entity_answers': [[]], # 'yesno_answers': []}) # if len(self.ref_answer) > 0: # pred_dict, ref_dict = {}, {} # for pred, ref in zip(pred_answers, self.ref_answer): # question_id = ref['question_id'] # if len(ref['answers']) > 0: # pred_dict[question_id] = normalize(pred['answers']) # ref_dict[question_id] = normalize(ref['answers']) # bleu_rouge = compute_bleu_rouge(pred_dict, ref_dict) # else: # bleu_rouge = None # v = bleu_rouge['Rouge-L'] * self.m_value['Rouge-L'] \ # + bleu_rouge['Bleu-4'] * self.m_value['Bleu-4'] \ # + bleu_rouge['Bleu-1'] * self.m_value['Bleu-1'] \ # + bleu_rouge['Bleu-3'] * self.m_value['Bleu-3'] \ # + bleu_rouge['Bleu-2'] * self.m_value['Bleu-2'] # else: # v = self.carpe_diem.value_function(tmp_node.data.sen)[0][0] self.update(search_list, v) self.count += 1 if tmp_node.is_leaf() and ( self.tree.depth(tmp_node) < self.max_depth ) and tmp_node.data.sen[-1] != str(self.l_passages - 1): self.expand(tmp_node) # if tm %10 == 0: # print ('==================== search 10 time = %3.2f s ====================' % (time.time() - batch_start_time)) ########### ''' if time % 100 == 0: tmp_policy = self.get_ppolicy(start_node_id) print tmp_policy.values() print sum(tmp_policy.values()) print time ''' #print tmp_node.data.word #print '------finish search ' #print '===== finish all search ======' def search_eval(self, start_node_id): #print '----tree search' tmp_node = self.tree.get_node(start_node_id) #print tmp_node.data.num has_visit_num = tmp_node.data.num - 1 self.count = has_visit_num if int(self.max_search_time - has_visit_num) > 0: start_node_search_time = int(self.max_search_time - has_visit_num) else: start_node_search_time = 0 for time in range(start_node_search_time): #print ('search time',time) search_list = [start_node_id] tmp_node = self.tree.get_node(start_node_id) #print 'search time :'+ str(time) while not tmp_node.is_leaf(): max_score = float("-inf") max_id = -1 for child_id in tmp_node.fpointer: child_node = self.tree.get_node(child_id) score = self.beta * child_node.data.p * ( (tmp_node.data.num)**0.5 / (1 + child_node.data.num)) #print 'child_node.data.Q: ' #print child_node.data.Q score += child_node.data.Q #print 'score: ' #print score #print '**************' if score > max_score: max_id = child_id max_score = score search_list.append(max_id) tmp_node = self.tree.get_node(max_id) #if tmp_node.data.word[-1] == str(self.l_passages-1): v = self.carpe_diem.value_function(tmp_node.data.sen)[0][0] #print 'v: ' #print v self.update(search_list, v) self.count += 1 if tmp_node.is_leaf() and ( self.tree.depth(tmp_node) < self.max_depth ) and tmp_node.data.sen[-1] != str(self.l_passages - 1): self.expand(tmp_node) ########### ''' if time % 100 == 0: tmp_policy = self.get_ppolicy(start_node_id) print tmp_policy.values() print sum(tmp_policy.values()) print time ''' #print tmp_node.data.word #print '------finish search ' #print '===== finish all search ======' def take_action(self, start_node_id): #print '----take action: ' tmp_node = self.tree.get_node(start_node_id) max_time = -1 prob = {} for child_id in tmp_node.fpointer: child_node = self.tree.get_node(child_id) prob[child_node.data.sen[-1]] = child_node.data.num / self.count if child_node.data.num > max_time: #print child_node.data.num #print max_time #print 'child_node.data.num > max_time' max_time = child_node.data.num select_word = child_node.data.sen[-1] select_word_node_id = child_node.identifier #else: #print 'not > max time ' #print select_word #print select_word_node_id #print '-----take action end' return prob, select_word, select_word_node_id def get_ppolicy(self, start_node_id): tmp_node = self.tree.get_node(start_node_id) max_time = -1 prob = {} for child_id in tmp_node.fpointer: child_node = self.tree.get_node(child_id) if self.count == 0: prob[child_node.data.sen[-1]] = 0.0 else: prob[ child_node.data.sen[-1]] = child_node.data.num / self.count return prob
json_data = json.load(load_f) #构建新闻事件传播树 tree = Tree() tree.create_node(tag=json_data[0].get("mid"),identifier=json_data[0].get("mid"),data=Info(json_data[0],param = 10)) uu = 1000 if len(json_data) >=1000 else len(json_data) # print('uu:',uu) for j in range(1,uu): try: tree.create_node(tag=json_data[j].get("mid"),identifier=json_data[j].get("mid"),parent=json_data[j].get("parent"),data=Info(json_data[j])) except: pass #单颗传播树构建完成 #对传播树进行简化 print('简化前:tree_depth:',tree.depth(),'tree_nodes:',len(tree.all_nodes())) tree = simplify_tree(tree) print('简化后:tree_depth:', tree.depth(),'tree_nodes:',len(tree.all_nodes())) trees.append(tree) # print(eid,"---trees simplified") #所有新闻事件的传播树构建结束 #3折交叉法 pd_data = pd.read_csv('id_label.txt', sep='\t', header=None) # wb_data = pd_data.as_matrix()[0:20,]
if len(tree.get_node(root).fpointer) == 2: left = get_LCA(tree.get_node(root).fpointer[0],n1,n2) right = get_LCA(tree.get_node(root).fpointer[1],n1,n2) if left is not None and right is not None: return root if left is not None: return left if right is not None:return right return None LCA = get_LCA(tree.root, s.index(entity), s.index(opinionw)) # Calculate distance between two nodes distance = tree.depth(tree.get_node(s.index(entity))) + tree.depth(tree.get_node(s.index(opinionw))) distance = distance - 2 * tree.depth(tree.get_node(LCA)) print "Looking for words:", entity, opinionw print "LCA is", tree.get_node(LCA).tag print "Distance is", distance def get_distance(word1, word2, tree): pass
class RST_DT: def load(self, path2file): self.id_EDUs = [] self.EDU = {} self.treeNS = Tree() self.tree = Tree() # nombre max d'espace pour init id_parents with open(path2file, "r") as f: max_space = 0 nb_line = 0 for i, line in enumerate(f): nb_space = 0 for c in line: if c == " ": nb_space += 1 else: break if nb_space > max_space: max_space = nb_space nb_line += 1 with open(path2file, "r") as f: id_parents = [0] * max_space NS_parents = [0] * max_space for i, line in enumerate(f): # nombre d'espace détermine le parent nb_space = 0 for c in line: if c == " ": nb_space += 1 else: break space = nb_space / 2 id_parents[space] = i parent = id_parents[space - 1] reg = "\(([\w\-\[\]]+)|(_!.+!_)" # récupération du contenu match = re.findall(reg, line)[0] if match[0] == "": content = match[1] # feuille EDU self.id_EDUs.append(i) # print content self.EDU[i] = re.findall("_!(.*)!_", content) else: content = match[0] reg2 = "\[(N|S)\]" # récupération NS match2 = re.findall(reg2, content) NS_parents[space] = match2 # ['N','S'] # création du noeud if i == 0: self.tree.create_node(content, 0) self.treeNS.create_node("Root", 0) else: id_NS = len(self.tree.is_branch(parent)) # 0 ou 1 car arbre binaire self.tree.create_node(content, i, parent=parent) self.treeNS.create_node(NS_parents[space - 1][id_NS], i, parent=parent) def toDEP(self): ############################### # Etape 1 : construction du head_tree # parcours en largeur de tree afin de récupérer chaque id_node # pour chaque profondeur (init à 0) _! sans compter !_ les feuilles (EDUs) nodes_depth = [-1] * self.tree.size() for i in xrange(self.tree.size()): id_nodes = [0] depth = [999] * self.tree.size() while id_nodes: # False if empty id_node = id_nodes.pop(0) node = self.tree.get_node(id_node) if node.bpointer != None: node_parent = self.tree.get_node(node.bpointer) depth[node.identifier] = depth[node_parent.identifier] + 1 else: depth[node.identifier] = 0 if id_node == i: # print 'noeud ',i,' en profondeur', depth[node.identifier] if node.fpointer: nodes_depth[i] = depth[i] break if node.fpointer: id_nodes.append(node.fpointer[0]) id_nodes.append(node.fpointer[1]) # print nodes_depth id_nodes_depth = [] for d in xrange(self.tree.depth()): id_nodes_depth.append([]) for i in xrange(self.tree.size()): if nodes_depth[i] == d: id_nodes_depth[d].append(i) # print id_nodes_depth # # construction du head_tree head_tree = [-1] * self.treeNS.size() # pour chaque noeud (non EDU/feuille) en partant de la plus grande profondeur dans l'arbre for d in range(len(id_nodes_depth) - 1, -1, -1): for id_node in id_nodes_depth[d]: node = self.treeNS.get_node(id_node) node_left = self.treeNS.get_node(node.fpointer[0]) node_right = self.treeNS.get_node(node.fpointer[1]) if node_left.tag == "N": if head_tree[node_left.identifier] == -1: identifier = node_left.identifier else: identifier = head_tree[node_left.identifier] else: if head_tree[node_right.identifier] == -1: identifier = node_right.identifier else: identifier = head_tree[node_right.identifier] head_tree[id_node] = identifier # print head_tree ############################### # Etape 2 : construction du DEP # # construction du DEP # init # root est le premier noeud de head # pour chaque EDU son père est le root dans DEP dep_tree = Tree() id_root = head_tree[0] root = self.tree.get_node(id_root) # dep_tree.create_node(root.tag, root.identifier) dep_tree.create_node(root.tag, root.identifier) for id_EDU in xrange(len(head_tree)): if head_tree[id_EDU] == -1 and id_EDU != id_root: node = self.tree.get_node(id_EDU) # dep_tree.create_node(node.tag, node.identifier, parent=id_root) # dep_tree.create_node(str(id_EDU), node.identifier, parent=id_root) dep_tree.create_node(node.tag, node.identifier, parent=id_root) # print '//////////////////////' # print 'EDU', id_root # pour chaque EDU for id_EDU in xrange(len(head_tree)): if head_tree[id_EDU] == -1 and id_EDU != id_root: EDU_NS = self.treeNS.get_node(id_EDU) # print '.......................' # print 'EDU', id_EDU # print 'TAG', EDU_NS.tag if EDU_NS.tag == "N": # parcours en largeur jusqu'à trouver un S avec un head donc qui soit pas EDU id_nodes = [EDU_NS.identifier] visited = [False] * self.treeNS.size() while id_nodes: id_node = id_nodes.pop(0) EDU = self.tree.get_node(id_node) # print 'visited EDU', EDU.identifier visited[EDU.identifier] = True # cas d'arret head_EDU = head_tree[EDU.identifier] == -1 head_EDU = False node_tag = self.treeNS.get_node(EDU.identifier).tag # print ' head_EDU', head_EDU # print ' node_tag', node_tag if not head_EDU and node_tag == "S": break if EDU.bpointer: if not visited[EDU.bpointer]: id_nodes.append(EDU.bpointer) if EDU.fpointer: # sécurité if not visited[EDU.fpointer[0]]: id_nodes.append(EDU.fpointer[0]) if not visited[EDU.fpointer[1]]: id_nodes.append(EDU.fpointer[1]) # puis ajouter au DEP comme enfant du head du parent du noeud S id_head = head_tree[EDU.bpointer] # si parent S else: # parcours en largeur des ancêtre jusqu'à trouver un ancêtre avec un head parent = self.treeNS.get_node(EDU_NS.bpointer) id_head = head_tree[parent.identifier] # puis ajouter au DEP comme enfant de ce head if id_EDU != id_head: dep_tree.move_node(id_EDU, id_head) EDU = self.tree.get_node(id_EDU) # print '---- ajout de',EDU.identifier,' à',id_head # if id_EDU == id_head: # dep_tree.show() return dep_tree # showDepth(dep_tree, 4) # dep_tree.show() # node = dep_tree. def toString(self): """ affiche comme la sortie de Hilda """ showDepth(self.tree, 0)
def do_md_section(sysid, levelid, connection_str, output_format, output_file, compact, doc_handler, dotree): """ Given the MD ids, dump the content in the output file and produce the product tree diagrams :param sysid: MagicDraw subsystem id :param levelid: MagicDraw level id (package id containing the tree to extract) :param connection_str: MagicDraw encoded connection string :param output_format: OutputFormat :param output_file: File to dump :param compact: True if the portrait diagrams have to be in compact form :param doc_handler: the document id :param sub_name: name of the subsystem :return: none """ global template_path global productTree global tree_dict productTree = Tree() products = [] tree_dict = {} # get the information from MagicDraw mdr = build_md_tree(sysid, levelid, connection_str) print("\n Product tree depth:", productTree.depth()) nodes = productTree.expand_tree(key=lambda x: x.data.index, sorting=True) for n in nodes: products.append(productTree[n].data) tree_dict[productTree[n].data.id] = productTree[n].data print(f" Found {{np}} products (including container folders).".format( np=len(tree_dict))) envs = Environment(loader=ChoiceLoader([ FileSystemLoader(Config.TEMPLATE_DIRECTORY), PackageLoader('ptree', 'templates') ]), lstrip_blocks=True, trim_blocks=True, autoescape=None) # dump a csv file if dotree: do_csv(products, output_file) # create the diagrams tex files do_trees_diagrams(productTree, output_file, products[0].shortname, compact) # define the template template_path = f"section.{Config.TEMPLATE_LANGUAGE}.jinja2" else: # no diagrams are created # use a simplified template template_path = f"notree_section.{Config.TEMPLATE_LANGUAGE}.jinja2" # sort tree dictionary based mdp = productTree.to_dict(with_data=False) # get ordered dictionary template = envs.get_template(template_path) new_mdpt = dict() for k0 in mdp: new_mdpt[k0] = order_tree_level(mdp[k0]) # dump the tex section metadata = dict() metadata["template"] = template.filename text = template.render(metadata=metadata, mdrev=mdr, filename=output_file, doc_handler=doc_handler, mdt_dict=tree_dict, mdp=new_mdpt, mdps=products) tex_file_name = output_file + ".tex" file = open(tex_file_name, "w") print(_as_output_format(text, output_format), file=file) file.close() return tree_dict
class TreeT(object): def __init__(self, max_id=0): self.tree = Tree() def from_ptb_to_tree(self, line, max_id=0, leaf_id=1, parent_id=None): # starts by ['(', 'pos'] pos_tag = line[1] if parent_id is None: pos_id = 0 else: pos_id = max_id max_id += 1 self.tree.create_node(pos_tag, pos_id, parent_id, TreeData()) parent_id = pos_id total_offset = 2 if line[2] != '(': # sub-tree is leaf # line[0:3] = ['(', 'pos', 'word', ')'] word_tag = line[2] self.tree.create_node(word_tag, leaf_id, parent_id, TreeData()) return 4, max_id, leaf_id + 1 line = line[2:] while line[0] != ')': offset, max_id, leaf_id = self.from_ptb_to_tree( line, max_id, leaf_id, parent_id) total_offset += offset line = line[offset:] return total_offset + 1, max_id, leaf_id def add_height(self, tree_dep): for n in self.tree.all_nodes(): n.data.leaves = [] for leaf in self.tree.leaves(): lid = leaf.identifier hid = tree_dep[lid] if hid == self.tree.root: self.tree[lid].data.height = self.tree.depth(self.tree[lid]) for cid in [ p for p in self.tree.paths_to_leaves() if lid in p ][0]: self.tree[cid].data.leaves += [lid] else: height = -1 cid = lid cond = True while cond: self.tree[cid].data.leaves += [lid] height += 1 cid = self.tree.parent(cid).identifier cid_leaves = [l.identifier for l in self.tree.leaves(cid)] cid_l_dep = [tree_dep[l] for l in cid_leaves if l != lid] cond = set(cid_l_dep).issubset(set(cid_leaves)) self.tree[lid].data.height = height x_nodes = [ n.identifier for n in self.tree.all_nodes() if n.data.leaves == [] ] for x_node in x_nodes[::-1]: min_id = min(self.tree.children(x_node), key=lambda c: c.data.height) _lid = min_id.data.leaves[0] self.tree[_lid].data.height += 1 self.tree[x_node].data.leaves += [_lid] return True def _from_tree_to_ptb(self, nid): nid = self.tree.subtree(nid).root if self.tree[nid].is_leaf(): return ' (' + self.tree[nid].tag + ' ' + self.tree[ nid].data.word + ')' res = ' (' + self.tree[nid].tag for c_nid in sorted(self.tree.children(nid), key=lambda x: x.identifier): res += self._from_tree_to_ptb(c_nid.identifier) return res + ')' def from_tree_to_ptb(self): return self._from_tree_to_ptb(self.tree.root) def from_tag_to_tree(self, tag, word, pos_id=0): parent_id = None for tag_nodes in tag: if tag_nodes[0] in [CL, CR]: c_side = tag_nodes[0] _tag_nodes = tag_nodes[1:] if len(tag_nodes) > 1 else [''] else: c_side = '' _tag_nodes = tag_nodes self.tree.create_node(_tag_nodes[0], pos_id, parent=parent_id, data=TreeData(comb_side=c_side)) parent_id = pos_id pos_id += 1 for tag_node in _tag_nodes[1:]: self.tree.create_node(tag_node[1:], pos_id, parent=parent_id, data=TreeData(miss_side=tag_node[0])) pos_id += 1 for l in self.tree.leaves(): if l.data.miss_side == '': l.data.word = word break return pos_id @memoize def is_combine_to(self, side): return self.tree[self.tree.root].data.comb_side == side @memoize def is_combine_right(self): return self.is_combine_to(CR) @memoize def is_combine_left(self): return self.is_combine_to(CL) @memoize def is_complete_tree(self): return all([n.data.miss_side == '' for n in self.tree.all_nodes()]) @memoize def get_missing_leaves_to(self, miss_val, side): return [ l.identifier for l in self.tree.leaves(self.tree.root) if l.data.miss_side == side and l.tag == miss_val ] @memoize def get_missing_leaves_left(self, miss_val): return self.get_missing_leaves_to(miss_val, L) @memoize def get_missing_leaves_right(self, miss_val): return self.get_missing_leaves_to(miss_val, R) @memoize def root_tag(self): return self.tree[self.tree.root].tag @memoize def is_no_missing_leaves(self): return all( [l.data.miss_side == '' for l in self.tree.leaves(self.tree.root)]) @memoize def combine_tree(self, _tree, comb_leaf): self.tree.paste(comb_leaf, _tree.tree) self.tree.link_past_node(comb_leaf) return self def tree_to_path(self, nid, path): # Stop condition if self.tree[nid].is_leaf(): path[nid] = [] return nid, self.tree[nid].data.height # Recursion flag = CR for child in self.tree.children(nid): cid = child.identifier leaf_id, height = self.tree_to_path(cid, path) if (height == 0): # Reached end of path can add flag path[leaf_id].insert(0, flag) # path[leaf_id].append(flag) if height > 0: path[leaf_id].insert(0, nid) # only single child will have height>0 # and its value will be the one that is returned # to the parent ret_leaf_id, ret_height = leaf_id, height - 1 # once we reached a height>0, it means that # this path includes the parent, and thus flag # direction should flip flag = CL return ret_leaf_id, ret_height def path_to_tags(self, path): tags = [] for p in path: _res = [] _p = copy.copy(p) if _p[0] in [CL, CR]: _res.append(_p[0]) _p = _p[1:] while _p[:-1]: el_p = _p.pop(0) _res.append(self.tree[el_p].tag) for c in self.tree.children(el_p): if c.identifier != _p[0]: _res.append(R + c.tag if c.identifier > _p[0] else L + c.tag) _res.append(self.tree[_p[0]].tag) tags.append(_res) return tags def path_to_words(self, path): return [self.tree[k].tag for k in path] def from_tree_to_tag(self): path = {} self.tree_to_path(self.tree.root, path) return { 'tags': self.path_to_tags(path.values()), 'words': self.path_to_words(path.keys()) } def from_ptb_to_tag(self, line, max_id, depend): self.from_ptb_to_tree(line, max_id) self.add_height(depend) path = {} self.tree_to_path(self.tree.root, path) return self.path_to_tags(path.values())
class StepParse: def __init__(self): pass def load_step(self, step_filename): self.nauo_lines = [] self.prod_def_lines = [] self.prod_def_form_lines = [] self.prod_lines = [] self.filename = os.path.splitext(step_filename)[0] line_hold = '' line_type = '' # Find all search lines with open(step_filename) as f: for line in f: # TH: read pointer of lines as they are read, so if the file has text wrap it will notice and add it to the following lines index = re.search("#(.*)=", line) if index: # TH: if not none then it is the start of a line so read it # want to hold line until it has checked next line # if next line is a new indexed line then save previous line if line_hold: if line_type == 'nauo': self.nauo_lines.append(line_hold) elif line_type == 'prod_def': self.prod_def_lines.append(line_hold) elif line_type == 'prod_def_form': self.prod_def_form_lines.append(line_hold) elif line_type == 'prod': self.prod_lines.append(line_hold) line_hold = '' line_type = '' prev_index = True # TH remember previous line had an index if 'NEXT_ASSEMBLY_USAGE_OCCURRENCE' in line: line_hold = line.rstrip() line_type = 'nauo' elif ('PRODUCT_DEFINITION ' in line or 'PRODUCT_DEFINITION(' in line): line_hold = line.rstrip() line_type = 'prod_def' elif 'PRODUCT_DEFINITION_FORMATION' in line: line_hold = line.rstrip() line_type = 'prod_def_form' elif ('PRODUCT ' in line or 'PRODUCT(' in line): line_hold = line.rstrip() line_type = 'prod' else: prev_index = False #TH: if end of file and previous line was held if 'ENDSEC;' in line: if line_hold: if line_type == 'nauo': self.nauo_lines.append(line_hold) elif line_type == 'prod_def': self.prod_def_lines.append(line_hold) elif line_type == 'prod_def_form': self.prod_def_form_lines.append(line_hold) elif line_type == 'prod': self.prod_lines.append(line_hold) line_hold = '' line_type = '' else: #TH: if not end of file line_hold = line_hold + line.rstrip() self.nauo_refs = [] self.prod_def_refs = [] self.prod_def_form_refs = [] self.prod_refs = [] # TH: added 'replace(","," ").' to replace ',' with a space to make the spilt easier if there are not spaces inbetween the words' # Find all (# hashed) line references and product names # TH: it might be worth finding a different way of extracting data we do want rather than fixes to get rid of the data we don't for j, el_ in enumerate(self.nauo_lines): self.nauo_refs.append([ el.rstrip(',') for el in el_.replace(",", " ").replace("=", " ").split() if el.startswith('#') ]) for j, el_ in enumerate(self.prod_def_lines): self.prod_def_refs.append([ el.rstrip(',') for el in el_.replace(",", " ").replace("=", " ").split() if el.startswith('#') ]) for j, el_ in enumerate(self.prod_def_form_lines): self.prod_def_form_refs.append([ el.rstrip(',') for el in el_.replace(",", " ").replace("=", " ").split() if el.startswith('#') ]) for j, el_ in enumerate(self.prod_lines): self.prod_refs.append([ el.strip(',') for el in el_.replace(",", " ").replace( "(", " ").replace("=", " ").split() if el.startswith('#') ]) self.prod_refs[j].append(el_.split("'")[1]) # Get first two items in each sublist (as third is shape ref) # # First item is 'PRODUCT_DEFINITION' ref # Second item is 'PRODUCT_DEFINITION_FORMATION <etc>' ref self.prod_all_refs = [el[:2] for el in self.prod_def_refs] # Match up all references down to level of product name for j, el_ in enumerate(self.prod_all_refs): # Add 'PRODUCT_DEFINITION' ref for i, el in enumerate(self.prod_def_form_refs): if el[0] == el_[1]: el_.append(el[1]) break # Add names from 'PRODUCT_DEFINITION' lines for i, el in enumerate(self.prod_refs): if el[0] == el_[2]: el_.append(el[2]) break # Find all parent and child relationships (3rd and 2nd item in each sublist) self.parent_refs = [el[1] for el in self.nauo_refs] self.child_refs = [el[2] for el in self.nauo_refs] # Find distinct parts and assemblies via set operations; returns list, so no repetition of items self.all_type_refs = set(self.child_refs) | set(self.parent_refs) self.ass_type_refs = set(self.parent_refs) self.part_type_refs = set(self.child_refs) - set(self.parent_refs) #TH: find root node self.root_type_refs = set(self.parent_refs) - set(self.child_refs) # Create simple parts dictionary (ref + label) self.part_dict = {el[0]: el[3] for el in self.prod_all_refs} # self.part_dict_inv = {el[3]:el[0] for el in self.prod_all_refs} def show_values(self): # TH: basic testing, if needed these could be spilt up print(self.nauo_lines) print(self.prod_def_lines) print(self.prod_def_form_lines) print(self.prod_lines) print(self.nauo_refs) print(self.prod_def_refs) print(self.prod_def_form_refs) print(self.prod_refs) # HR: "create_dict" replaced by list comprehension elsewhere # # def create_dict(self): # # # TH: links nauo number with a name and creates dict # self.part_dict = {} # for part in self.all_type_refs: # for sublist in self.prod_def_refs: # if sublist[0] == part: # prod_loc = '#' + re.findall('\d+',sublist[1])[0] # pass # for sublist in self.prod_def_form_refs: # if sublist[0] == prod_loc: # prod_loc = '#' + str(re.findall('\d+',sublist[1])[0]) # pass # for sublist in self.prod_refs: # if sublist[0] == prod_loc: # part_name = sublist[2] # # self.part_dict[part] = part_name def create_tree(self): #TH: create tree diagram in newick format #TH: find root node self.tree = Tree() #TH: check if there are any parts to make a tree from, if not don't bother if self.part_dict == {}: return root_node_ref = list(self.root_type_refs)[0] # HR added part reference as data for later use self.tree.create_node(self.part_dict[root_node_ref], 0, data={'ref': root_node_ref}) #TH: created root node now fill in next layer #TH: create dict for tree, as each node needs a unique name i = [0] # Iterates through nodes self.tree_dict = {} self.tree_dict[i[0]] = root_node_ref def tree_next_layer(self, parent): root_node = self.tree_dict[i[0]] for line in self.nauo_refs: if line[1] == root_node: i[0] += 1 self.tree_dict[i[0]] = str(line[2]) # HR added part reference as data for later use self.tree.create_node(self.part_dict[line[2]], i[0], parent=parent, data={'ref': str(line[2])}) tree_next_layer(self, i[0]) tree_next_layer(self, 0) self.appended = False self.get_levels() def get_levels(self): # Initialise dict and get first level (leaves) self.levels = {} self.levels_set_p = set() self.levels_set_a = set() self.leaf_ids = [el.identifier for el in self.tree.leaves()] self.all_ids = [el for el in self.tree.nodes] self.non_leaf_ids = set(self.all_ids) - set(self.leaf_ids) self.part_level = 1 def do_level(self, tree_level): # Get all nodes within this level node_ids = [ el for el in self.tree.nodes if self.tree.level(el) == tree_level ] for el in node_ids: # If leaf, then n_p = 1 and n_a = 1 if el in self.leaf_ids: self.levels[el] = {} self.levels[el]['n_p'] = self.part_level self.levels[el]['n_a'] = self.part_level # If assembly, then get all children and sum all parts + assemblies else: # Get all children of node and sum levels child_ids = self.tree.is_branch(el) child_sum_p = 0 child_sum_a = 0 for el_ in child_ids: child_sum_p += self.levels[el_]['n_p'] child_sum_a += self.levels[el_]['n_a'] self.levels[el] = {} self.levels[el]['n_p'] = child_sum_p self.levels[el]['n_a'] = child_sum_a + 1 self.levels_set_p.add(child_sum_p) self.levels_set_a.add(child_sum_a + 1) # Go up through tree levels and populate lattice level dict for i in range(self.tree.depth(), -1, -1): do_level(self, i) self.create_lattice() self.levels_p_sorted = sorted(list(self.levels_set_p)) self.levels_a_sorted = sorted(list(self.levels_set_a)) # Function to return dictionary of item IDs for each lattice level def get_levels_inv(list_in, key): #Initialise levels_inv = {} levels_inv[self.part_level] = [] for el in list_in: levels_inv[el] = [] for k, v in self.levels.items(): levels_inv[v[key]].append(k) return levels_inv self.levels_p_inv = get_levels_inv(self.levels_p_sorted, 'n_p') self.levels_a_inv = get_levels_inv(self.levels_a_sorted, 'n_a') def get_all_children(self, id_): ancestors = [el.identifier for el in self.tree.children(id_)] parents = ancestors while parents: children = [] for parent in parents: children = [el.identifier for el in self.tree.children(parent)] ancestors.extend(children) parents = children return ancestors def create_lattice(self): # Create lattice self.g = nx.DiGraph() self.default_colour = 'r' # Get root node and set parent to -1 to maintain data type of "parent" # Set position to top/middle node_id = self.tree.root label_text = self.tree.get_node(node_id).tag self.g.add_node(node_id, parent=-1, label=label_text, colour=self.default_colour) # Do nodes from treelib "nodes" dictionary for key in self.tree.nodes: # Exclude root if key != self.tree.root: parent_id = self.tree.parent(key).identifier label_text = self.tree.get_node(key).tag # Node IDs same as for tree self.g.add_node(key, parent=parent_id, label=label_text, colour=self.default_colour) # Do edges from nodes for key in self.tree.nodes: # Exclude root if key != self.tree.root: parent_id = self.tree.parent(key).identifier self.g.add_edge(key, parent_id) # Escape if only one node # HR 6/3/20 QUICK BUG FIX: SINGLE-NODE TREE DOES NOT PLOT # IMPROVE LATER; SHOULD BE PART OF A GENERAL METHOD if self.tree.size() == 1: id_ = [el.identifier for el in self.tree.leaves()] self.g.nodes[id_[-1]]['pos'] = (0, 0) return # Get set of parents of leaf nodes leaf_parents = set( [self.tree.parent(el).identifier for el in self.leaf_ids]) # For each leaf_parent, set position of leaf nodes sequentially i = 0 no_leaves = len(self.tree.leaves()) for el in leaf_parents: for el_ in self.tree.is_branch(el): child_ids = [el.identifier for el in self.tree.leaves()] if el_ in child_ids: self.g.nodes[el_]['pos'] = ((i / (no_leaves)), 1) i += 1 # To set plot positions of nodes from lattice levels # --- # Traverse upwards from leaves for el in sorted(list(self.levels_set_a)): # Get all nodes at that level node_ids = [k for k, v in self.levels.items() if v['n_a'] == el] # Get all positions of children of that node # and set position as mean value of them for el_ in node_ids: child_ids = self.tree.is_branch(el_) pos_sum = 0 for el__ in child_ids: pos_ = self.g.nodes[el__]['pos'][0] pos_sum += pos_ pos_sum = pos_sum / len(child_ids) self.g.nodes[el_]['pos'] = (pos_sum, el) def print_tree(self): try: self.tree.show() except: self.create_tree() self.tree.show() def tree_to_json(self, save_to_file=False, filename='file', path=''): #TH: return json format tree, can also save to file if self.tree.size() != 0: data = self.tree.to_json() j = json.loads(data) if save_to_file == True: if path: file_path = os.path.join(path, filename) else: file_path = filename with open(file_path + '.json', 'w') as outfile: json.dump(j, outfile) return data else: print("no tree to print") return
def hierarchy(fna_mapping, dist): pending = list(fna_mapping.keys()) node_id = max(pending) + 1 mapping = bidict.bidict() cls_dist = [] cls_dist_temp = {} index = 0 pending.sort() for i in pending: mapping[i] = index index += 1 for i in range(0, len(pending)): temp1 = [] for j in range(0, i): temp1.append(cls_dist_temp[(mapping[pending[j]], mapping[pending[i]])]) for j in range(i, len(pending)): temp = cal_cls_dist(dist, fna_mapping[pending[i]], fna_mapping[pending[j]]) temp1.append(temp) cls_dist_temp[(mapping[pending[i]], mapping[pending[j]])] = temp cls_dist.append([np.array(temp1)]) cls_dist = np.concatenate(cls_dist) cls_dist_recls = cls_dist.copy() mapping_recls = mapping.copy() tree_relationship = {} pending = set(pending) while (len(pending) > 1): (child_a, child_b) = divmod(np.argmax(cls_dist), cls_dist.shape[1]) temp1 = [ np.concatenate([[cls_dist[child_a]], [cls_dist[child_b]]]).max(axis=0) ] cls_dist = np.concatenate([cls_dist, temp1], axis=0) temp1 = np.append(temp1, -1) temp1 = np.vstack(temp1) cls_dist = np.concatenate([cls_dist, temp1], axis=1) cls_dist = np.delete(cls_dist, [child_a, child_b], axis=0) cls_dist = np.delete(cls_dist, [child_a, child_b], axis=1) # change mapping cluster_a = mapping.inv[child_a] cluster_b = mapping.inv[child_b] # cluster id tree_relationship[node_id] = (cluster_a, cluster_b) del mapping[cluster_a], mapping[cluster_b] pending.remove(cluster_a) pending.remove(cluster_b) pending = sorted(list(pending)) for i in pending: if (mapping[i] > min([child_a, child_b]) and mapping[i] < max([child_a, child_b])): mapping[i] -= 1 elif (mapping[i] > max([child_a, child_b])): mapping[i] -= 2 mapping[node_id] = len(cls_dist) - 1 pending = set(pending) pending.add(node_id) node_id += 1 # build tree structure pending = list(pending) tree = Tree() T0 = Node(identifier=pending[0]) tree.add_node(T0) while (len(pending) > 0): parent = pending[0] for i in tree_relationship[parent]: tree.add_node(Node(identifier=i), parent=parent) if (i in tree_relationship): pending.append(i) pending.remove(parent) # load depth info depths = {} depths_mapping = defaultdict(set) leaves = set(tree.leaves()) for i in tree.all_nodes(): depths[i] = tree.depth(node=i) if (i in leaves): depths_mapping[depths[i]].add(i) return cls_dist_recls, mapping_recls, tree, depths, depths_mapping
class Blockchain(object): def __init__(self, genesis): # TODO: figure out if genesis should be passed in or created here # self.tinput = tinput self.blockCount = 0 self.blockchain = Tree() self.genesis = genesis self.addGenesisBlock(genesis) #Add the genesis block to chain def addGenesisBlock(self, genesis): self.blockchain.create_node("Genesis Block" + " ID: " + genesis.proofOfWork[:12], genesis.proofOfWork, data=genesis) def printBlockchain(self): self.blockchain.show() def addBlock(self, block): # TODO: run proof of work verification before adding block # Add block to chain & return true if POW valid # Else return false self.blockCount += 1 self.blockchain.create_node("Block " + str(self.blockCount) + " ID: " + block.proofOfWork[:12], block.proofOfWork, parent=block.prevBlockHash, data=block) def getGenesisID(self): return self.blockchain.root def getLongestChainBlocks(self): allNodes = self.blockchain.all_nodes() forkNum = 0 #number of leaves at longest branch treeDepth = self.blockchain.depth() longestPathLeaves = [ ] #WIll hold leaves with treeDepth depth ie longest branch(es) for node in allNodes: currentDepth = self.blockchain.depth(node) if (currentDepth == treeDepth): forkNum += 1 longestPathLeaves.append(node) return forkNum, longestPathLeaves def blockchainLength(self): # returns the depth of the tree ie the length of # the longest chain return self.blockchain.depth() def numBlocks(self): return self.blockchain.size() def printChain(self, chain): chain.show(data_property="humanID") def tailBlocks(self, chain): leaves = chain.leaves() print("Num leaves" + str(len(leaves))) print(leaves) def checkBlock(self): # Check the proof work work # return true if proof of work is valid # else rerturn false print("printing block") def createBlockchainGraph(self, outfilename): print("creating graph") self.blockchain.to_graphviz(filename=outfilename + '.gv', shape=u'box', graph=u'digraph') g = Source.from_file(outfilename + '.gv') g.render() def createBlockchainImg(self, outfilename): print("creating graph") self.blockchain.to_graphviz(filename=outfilename + '.gv', shape=u'box', graph=u'digraph') g = Source.from_file(outfilename + '.png') g.render()
class AcquisitionChainIter(object): def __init__(self, acquisition_chain, parallel_prepare=True): self.__sequence_index = -1 self._parallel_prepare = parallel_prepare self.__acquisition_chain_ref = weakref.ref(acquisition_chain) # set all slaves into master for master in (x for x in acquisition_chain._tree.expand_tree() if isinstance(x, AcquisitionMaster)): del master.slaves[:] master.slaves.extend( acquisition_chain._tree.get_node(master).fpointer) # create iterators tree self._tree = Tree() self._root_node = self._tree.create_node("acquisition chain", "root") device2iter = dict() for dev in acquisition_chain._tree.expand_tree(): if not isinstance(dev, (AcquisitionDevice, AcquisitionMaster)): continue dev_node = acquisition_chain._tree.get_node(dev) parent = device2iter.get(dev_node.bpointer, "root") try: it = iter(dev) except TypeError: one_shot = self.acquisition_chain._device2one_shot_flag.get( dev, True) dev_iter = DeviceIterator(dev, one_shot) else: dev_iter = DeviceIteratorWrapper(it) device2iter[dev] = dev_iter self._tree.create_node(tag=dev.name, identifier=dev_iter, parent=parent) @property def acquisition_chain(self): return self.__acquisition_chain_ref() def prepare(self, scan, scan_info): preset_tasks = list() if self.__sequence_index == 0: preset_tasks.extend([ gevent.spawn(preset.prepare) for preset in self.acquisition_chain._presets_list ]) scan.prepare(scan_info, self.acquisition_chain._tree) self._execute("_prepare", wait_between_levels=not self._parallel_prepare) if self.__sequence_index == 0: gevent.joinall(preset_tasks, raise_error=True) def start(self): if self.__sequence_index == 0: preset_tasks = [ gevent.spawn(preset.start) for preset in self.acquisition_chain._presets_list ] gevent.joinall(preset_tasks, raise_error=True) self._execute("_start") def stop(self): self._execute("stop", master_to_slave=True, wait_all_tasks=True) preset_tasks = [ gevent.spawn(preset.stop) for preset in self.acquisition_chain._presets_list ] gevent.joinall(preset_tasks) # wait to call all stop on preset gevent.joinall(preset_tasks, raise_error=True) def next(self): self.__sequence_index += 1 gevent.joinall([ gevent.spawn(dev_iter.wait_ready) for dev_iter in self._tree.expand_tree() if dev_iter is not 'root' ], raise_error=True) try: if self.__sequence_index: for dev_iter in self._tree.expand_tree(): if dev_iter is 'root': continue dev_iter.next() except StopIteration: # should we stop all devices? for acq_dev_iter in ( x for x in self._tree.expand_tree() if x is not 'root' and isinstance(x.device, ( AcquisitionDevice, AcquisitionMaster))): if hasattr(acq_dev_iter, 'wait_reading'): acq_dev_iter.wait_reading() dispatcher.send("end", acq_dev_iter.device) raise return self def _execute(self, func_name, master_to_slave=False, wait_between_levels=True, wait_all_tasks=False): tasks = list() prev_level = None if master_to_slave: devs = list(self._tree.expand_tree(mode=Tree.WIDTH))[1:] else: devs = reversed(list(self._tree.expand_tree(mode=Tree.WIDTH))[1:]) for dev in devs: node = self._tree.get_node(dev) level = self._tree.depth(node) if wait_between_levels and prev_level != level: gevent.joinall(tasks, raise_error=True) tasks = list() prev_level = level func = getattr(dev, func_name) tasks.append(gevent.spawn(func)) # ensure that all tasks are executed # (i.e: don't raise the first exception on stop) if wait_all_tasks: gevent.joinall(tasks) gevent.joinall(tasks, raise_error=True)
prev = None a.md5 = str(a.md5).replace("\\x00", "")[2:-1] a.path = str(a.path).replace("\\x00", "")[2:-5] a.path = str(a.path).replace("\\\\", "/") a.path = str(a.path).replace("\r", "") a.path = str(a.path).replace("\n", "") a.clone = str(a.clone).replace("\\x00", "")[2:-5] a.clone = str(a.clone).replace("\\\\", "/") a.clone = str(a.clone).replace("\r", "") a.clone = str(a.clone).replace("\n", "") master = "" for folder in a.path.split("/")[0:-1]: master += folder + "/" if tree.get_node(master) == None: tree.create_node(folder, master, parent=prev) prev = master folder = a.path.split("/")[-1:][0] master += folder + "/" if tree.get_node(master) == None: tree.create_node(str(a.isUnique), master, parent=prev, data=a) prev = master #print(tree.show()) print(tree.depth()) #time.sleep(1) DFS(tree) toPrune(tree)
class Route(object): def __init__(self, universe): self.route = Tree() self.universe = universe self.max_hops = 4 def show(self): self.route.show() def asString(self): return (','.join([self.route[node].tag for node in self.route.expand_tree(mode=Tree.DEPTH)])) def getRoute(self): return self.route def byScore_key(self, s): return s.score def findRoute(self, start): parent = self.universe.findSystem(start) self.route.create_node(start, start, data=parent) systems = self.findNextSystems(start, start) self.buildRoute(systems, start) return self.route def buildRoute(self, systems, parent): for s in systems: n = s.name h = 0 if (self.route.contains(n) == False): self.route.create_node(n, n, parent=parent, data=s) hop = h + self.route.depth(n) if (hop < self.max_hops): sub_systems = self.findNextSystems(parent, n) self.buildRoute(sub_systems, n) else: n = parent + ' --> ' + n self.route.create_node(n, n, parent=parent, data=s) def getSystemId(self, name, i=0): if (self.route.contains(name) == False): return name else: i += 1 n = name + '(' + str(i) + ')' return self.getSystemId(n) def findNextSystems(self, parent, start): systems = [] optimal = self.universe.distances.findOptimalSystems(start) for s in sorted(set(optimal)): if (s != parent): i = self.universe.findSystem(s) if (i.permit == False): systems.append(i) s = sorted(systems, key = self.byScore_key) return s[:self.max_hops] # http://xiaming.me/treelib/examples.html # # class SystemTree(object): # def __init__(self): # self.tree = Tree() # # def addNode(self, id, o): # self.tree.create_node(o, id) # # def addChildNode(self, p, id, o): # self.tree.create_node(o, id, parent=p) # # def getNode(self, id): # return self.tree.subtree(id) # # def __repr__(self): # return self.tree.to_json(with_data=True) # # # t = SystemTree() # t.addNode('Aerial', 'Aerial') # t.addChildNode('Aerial', 'Jotun', 'Jotun') # t.addChildNode('Jotun', 'Rusani', 'Rusani') # n = t.getNode('Jotun') # print(n) # n = t.tree.contains('Invalid') # print(n) # t.tree.show()