def parse_file(json_file):
with open(json_file, "r") as fp:
contents = json.load(fp)
root = importer.import_(contents['symbols'])
zr = find(root, lambda node: node.name == 'ZEPHYR_BASE')
ws = find(root, lambda node: node.name == 'WORKSPACE')
data = {}
if zr and ws:
trees = [zr, ws]
else:
trees = [root]
for node in PreOrderIter(root, maxlevel=2):
if node.name not in ['WORKSPACE', 'ZEPHYR_BASE']:
if node.name in ['Root', 'Symbols']:
data['all'] = node.size
else:
data[node.name] = node.size
for t in trees:
root = t.name
for node in PreOrderIter(t, maxlevel=2):
if node.name == root:
continue
comp = node.name
if comp in ['Root', 'Symbols']:
data['all'] = node.size
else:
data[comp] = node.size
return data
def main():
args = parse_args()
with open(args.file1, "r") as f:
data = json.load(f)
root1 = importer.import_(data['symbols'])
with open(args.file2, "r") as f:
data = json.load(f)
root2 = importer.import_(data['symbols'])
for node in PreOrderIter(root1):
# pylint: disable=undefined-loop-variable
n = find(root2, lambda node2: node2.identifier == node.identifier)
if n:
if n.size != node.size:
diff = n.size - node.size
if diff == 0:
continue
if not n.children or not n.parent:
if diff < 0:
print(f"{n.identifier} -> {Fore.GREEN}{diff}{Fore.RESET}")
else:
print(f"{n.identifier} -> {Fore.RED}+{diff}{Fore.RESET}")
else:
if not node.children:
print(f"{node.identifier} ({Fore.GREEN}-{node.size}{Fore.RESET}) disappeared.")
for node in PreOrderIter(root2):
n = find(root1, lambda node2: node2.identifier == node.identifier)
if not n:
if not node.children and node.size != 0:
print(f"{node.identifier} ({Fore.RED}+{node.size}{Fore.RESET}) is new.")
def prepare_sememe_sim(hownet, sememe_root, alpha, eps): # 任意两个sememe之间的相似度表格
sememe_sim_table = {}
sememe = list(hownet.sememe)
total_calc = int((1 + len(sememe)) * len(sememe) / 2)
for i in range(len(sememe)):
sememe_node1 = search.find(sememe_root,
lambda node: node.name == sememe[i])
flag = False
for ancestor in sememe_node1.ancestors:
if ancestor.name in ['Attribute|属性', 'Secondary Feature']:
flag = True
break
for j in range(i, len(sememe)):
sememe_node2 = search.find(sememe_root,
lambda node: node.name == sememe[j])
if sememe_node1 is None:
print(sememe[i])
if sememe_node2 is None:
print(sememe[j])
walk_result = walker.Walker().walk(sememe_node1, sememe_node2)
sememe_sim = alpha * (sememe_node1.depth + sememe_node2.depth) / \
(alpha * (sememe_node1.depth + sememe_node2.depth) + \
abs(sememe_node1.depth - sememe_node2.depth) + len(walk_result[0]) + len(walk_result[2]))
if flag:
sememe_sim = sememe_sim * eps
sememe_sim_table[(sememe[i], sememe[j])] = sememe_sim
if len(sememe_sim_table) % 1000 == 0:
print(len(sememe_sim_table), '/', total_calc)
return sememe_sim_table
def add_item(self, parent_object_id, text):
parent_node = find(self.tree, lambda n: n.id == parent_object_id)
existing_node = find(parent_node, lambda n: n.name == text, maxlevel=2)
new_item = None
if existing_node:
new_item = existing_node
elif isinstance(parent_node, ResponseStoryNode) or isinstance(
parent_node, BaseNode):
add_node = find(self.nlu.tree, lambda n: n.name == text)
new_item = IntentStoryNode(item=add_node.item, parent=parent_node)
elif isinstance(parent_node, IntentStoryNode):
add_node = find(self.resp.tree, lambda n: n.name == text)
new_item = ResponseStoryNode(item=add_node.item,
parent=parent_node)
return new_item
def add(self,
index=-1,
timemade=-1,
proof_of_work_input=-1,
effort=-1,
transactions=-1,
previous_hash=-1,
update_db=True):
func = inspect.currentframe().f_back.f_code
block = Block(index, timemade, proof_of_work_input, effort,
transactions, previous_hash)
execute_sql = False
if self.__root is None:
execute_sql = True
self.__root = block
else:
found = search.find(self.__root,
lambda node: node.hash == previous_hash)
if found != None:
block.parent = found
execute_sql = True
self.__analyze()
self.__set_last_added(block)
if execute_sql and update_db:
dict_to_use = block.getdict()
self.cursor.execute(
"INSERT INTO unverified_blocks VALUES (:index,:timemade,:proof_of_work,:effort,:transactions,:hash,:previous_hash)",
dict_to_use)
self.connection.commit()
self.__added_to_db += 1
self.__to_store()
def import_data(self):
current_intent = None
try:
with open(self.filename, "r", encoding="utf-8") as df:
nlu = json.loads(df.read())
for example in nlu["rasa_nlu_data"]["common_examples"]:
if example["intent"] not in self.items:
current_intent = IntentNode(
Intent(name=example["intent"]), parent=self.tree
)
self.add_to_items(example["intent"])
else:
current_intent = find(
self.tree,
lambda node: node.name == example["intent"],
maxlevel=2,
)
IntentExample(name=example["example"], parent=current_intent)
self.add_to_items(example["example"])
except json.JSONDecodeError: # suggest it's markdown
with open(self.filename, "r", encoding="utf-8") as df:
nlu = df.readlines()
for line in nlu:
if line.startswith("## intent:"):
heading = line.split("## intent:")[1].strip()
current_intent = IntentNode(
Intent(name=heading), parent=self.tree
)
self.add_to_items(heading)
if line.startswith("- "):
text_example = line.split("- ")[1].strip()
IntentExample(name=text_example, parent=current_intent)
self.add_to_items(text_example)
def add(self, block, update_db=True):
func = inspect.currentframe().f_back.f_code
execute_sql = False
if self.__root is None:
execute_sql = True
self.__root = block
else:
# print("trying to add {}".format(block))
found = search.find(self.__root,
lambda node: node.hash == block.previous_hash)
if found != None:
block.parent = found
execute_sql = True
self.__analyze()
else:
print("MAJOR PROBLEM couldn't find parent")
self.__set_last_added(block)
if execute_sql and update_db:
print("adding", block, "to db")
dict_to_use = block.get_block_as_dictionary()
self.cursor.execute(
"INSERT INTO unverified_blocks VALUES (:index,:timemade,:proof_of_work,:effort,:transactions,:hash,:previous_hash)",
dict_to_use)
self.connection.commit()
self.__added_to_db += 1
self.__to_store()
def run(self, dispatcher, tracker, domain):
func = tracker.get_slot('function')
if (func != None):
result_list = search.findall(self.ITSM_tree, lambda node:
(func) in node.id)
length = len(result_list)
temp = ""
if (length > 1):
for x in range(0, length):
result = result_list[x]
temp += str(x + 1) + ". " + result.id + ": "
temp += result.url
if (x != length - 1):
temp += "\n"
response = """Are you looking for...\n{}""".format(temp)
elif (length == 1):
url = result_list[0].url
title = result_list[0].id
response = """I think you are looking for this!\n{}\n{}""".format(
title, url)
else:
temp = search.find(self.ITSM_tree, lambda node:
("homepage") == node.id)
url = temp.url
response = "Sorry! Content related to '{}' is not found! Please checkout our homepage {}".format(
func, url)
else:
response = "Sorry! I don't understand what you are asking.. Please try asking in other ways!"
# print(response)
dispatcher.utter_message(response)
return [SlotSet('function', None)]
def update_node_value(self, node, new_value):
node = find(self.tree, lambda n: n.name == node, maxlevel=3)
self.items.remove(node.name)
self.add_to_items(new_value)
node.name = new_value
if isinstance(node, self.parent_nodes_class):
node.item.name = new_value
for story_item in node.item.story_tree:
story_item.name = new_value
def get_object_type_and_handler_by_id(self, object_id):
node = find(self.tree, lambda n: n.id == object_id)
if isinstance(node, ResponseStoryNode):
result = {"type": TYPE_RESPONSE, "handler": self.nlu}
elif isinstance(node, IntentStoryNode):
result = {"type": TYPE_INTENT, "handler": self.resp}
else:
result = {"type": TYPE_ROOT, "handler": None}
return result
def __init__(self, userId):
self.userId = userId
self.node = search.find(ROOT, lambda node: node.name == userId)
self.group = [node.name
for node in PostOrderIter(self.node)]
self.child = [node.name
for node in self.node.children]
self.alliance = getAlliance(self.node.children)
self.status = [2]
def remove_node(self, node):
node = find(self.tree, lambda n: n.name == node, maxlevel=3)
if not hasattr(node, "item") or not node.item.story_tree:
node.parent = None
self.items.remove(node.name)
for kid in node.children:
self.items.remove(kid.name)
else:
raise ValueError("Item is used in stories and can't be removed.")
def performance_at_get_key():
global dt_root, max_items
dt = dt_root
key_list = []
for i in range(max_items):
key_list.append(None)
for ii in range(max_items):
key_list[-1] = '%i_%i' % (i, ii)
key = '/'.join(key_list)
b = search.find(dt, lambda node: node.name == key)
def getUserTree(userFind):
root = Node('root')
for user in userFind:
Node(user['_id'], shareCodeMemList=user.get(
'shareCodeMemList'), parent=root)
for node in root.children:
if isinstance(node.shareCodeMemList, (list)):
for userId in node.shareCodeMemList:
search.find(root, lambda node: node.name ==
userId).parent = node
return root
def get_available_children_by_parent_id(self, parent_object_id):
parent_node = find(self.tree, lambda n: n.id == parent_object_id)
available_values = None
if isinstance(parent_node, ResponseStoryNode) or isinstance(
parent_node, BaseNode):
available_values = sorted(
[child.name for child in self.nlu.tree.children])
elif isinstance(parent_node, IntentStoryNode):
available_values = sorted(
[child.name for child in self.resp.tree.children])
return available_values
def add_node(p,cp,age,gender,loc,travel):
#Add localtion if does not exist.
loc_node = search.find(root, lambda node: node.name == loc)
if (loc_node == None):
# print("Adding location %s node to" %loc,root)
loc_node = Node(loc,parent=root,age='',gender='',loc='',travel='')
#Under loc, add local if no travel, else create under travel.
if (cp == ''):
travel_his = search.find(loc_node,lambda node: node.name == travel)
if (travel_his == None):
#Adding travel
# print("Adding new node with travel ", p,loc_node, travel)
travel_his = Node(travel,parent=loc_node,age='',gender='',loc='',travel='')
par = travel_his
else:
par = search.find(root, lambda node: node.name == cp)
# print("Adding patient %s to " % p,par,cp,loc)
Node(p,parent=par,age=age,gender=gender,loc=loc,travel=travel)
def get_child_type_and_handler_by_id(self, object_id):
node = find(self.tree, lambda n: n.id == object_id)
child_type = None
child_handler = None
if isinstance(node, ResponseStoryNode) or isinstance(node, BaseNode):
child_type = TYPE_INTENT
child_handler = self.nlu
elif isinstance(node, IntentStoryNode):
child_type = TYPE_RESPONSE
child_handler = self.resp
return child_type, child_handler
def get_tree_list_at_hash(self, hash):
"""Return all blocks after the given hash.
:param hash: Hash of the block whose decedents should be returned.
:return: [] if hash is not part of the chain.
"""
selected_node = find(self.chain_tree,
lambda node: node.block.hash == hash)
if selected_node:
return [node.block for node in selected_node.descendants]
else:
return []
def add_example_to_node(self, parent, text):
selected_node = find(self.tree,
lambda node: node.name == parent,
maxlevel=3)
if isinstance(selected_node, self.parent_nodes_class):
parent = selected_node
elif isinstance(selected_node, self.child_nodes_class):
parent = selected_node.parent
else:
raise ValueError("Can't find item to add example.")
self.child_nodes_class(name=text, parent=parent)
self.add_to_items(text)
def findRoute(self, nodeName):
anytreeNodePath = search.find(self.organizations,
lambda node: node.name == nodeName)
#print ("anytree node path:",anytreeNodePath)
try:
if anytreeNodePath:
rt = str(anytreeNodePath).split('(')[1].replace(
"'.", "", 1).replace("')", '', 1)
#print ("designated route:",rt)
return rt
except Exception as anytreeError:
#print ("anytree error occured",anytreeError)
return None
def print_node(loc='',travel=''):
rnode = root
rdepth = 3
sdepth = 2
(uknown_total, uknown_primary, uknown_spread) = (0,0,0)
(travel_total, travel_primar, travel_spread, primary_carriers, primary_travel) = (0,0,0,0,0)
total_cases = 0
total_spread = 0
if loc != '':
rnode = search.find(root, lambda node: node.name == loc)
if (rnode != root):
rdepth = 3
sdepth = 2
for pre, fill, node in RenderTree(rnode):
if (str.isdigit(node.name)):
total_cases += 1
if (node.depth == rdepth):
primary_carriers += 1
if (node.name == 'Local'):
(t, p, s) = get_count_from_travel(node, 'Local')
uknown_total += t
uknown_primary += p
uknown_spread += s
if( node.name == travel ):
(t,p,s) = get_count_from_travel(node, travel, True)
travel_total += t
travel_primar += p
travel_spread += s
if (node.depth == sdepth):
primary_travel += 1
#print("%s%s %s-%s-%s %s" % (pre, node.name,node.age,node.gender,node.loc,node.travel))
if node.depth > sdepth+1:
total_spread += 1
if ( travel == ''):
print("%s%s %s-%s-%s %s" % (pre, node.name,node.age,node.gender,node.loc,node.travel))
print("Total Cases in %s: \t\t\t\t %s" %(loc,total_cases))
print("Total Primar carriers in %s: \t\t\t %s" %(loc,primary_carriers))
print("Total Impact due to travel in %s: \t\t %s" %(loc,total_cases - uknown_total))
print("Total Spread in %s from Primary carriers: \t %s" %(loc,total_spread))
print("Total Unknown Local cases in %s: \t\t %s" %(loc,uknown_total))
print("Total Spread from uknown Local cases in %s: \t %s" %(loc,uknown_spread))
print("Impact in %s due to Travel from %s...."%(loc,travel))
print("\tTotal Impact : \t %s" %(travel_total))
print("\tPrimary Carrier : \t %s" % travel_primar)
print("\tTotal Spread : \t %s" %(travel_spread))
def create_new_tree(cls, blobs: Iterable[Blob]) -> TreeNode:
root = TreeNode(".")
for blob in blobs:
file_content = blob.compressed_data
curr_node = root
for file in blob.filename.split(os.sep):
curr_node.update_hash(file_content)
child = search.find(curr_node, lambda node: node.name == file)
if child:
curr_node = child
else:
curr_node = TreeNode(file, parent=curr_node)
curr_node.update_hash(file_content)
return root
def find_by_name(self, name) -> Node:
"""
Retrieve a node by its name
Parameters
----------
name: str holding the name to be looked for
Returns
-------
node: an anytree Node instance with the seeked node
"""
root = self.nodes[self.root_key]
node = find(root, lambda node: node.name == name)
return node
def import_data(self):
with open(self.filename, "r", encoding="utf-8") as stories_file:
stories = stories_file.readlines()
for line in stories:
if line.startswith("## "):
# new story should start from root
current_response = self.tree
current_intent = None
elif line.startswith("* "): # intent
intent_heading = line.split("* ")[-1].strip()
if find(
current_response,
lambda n: n.name == intent_heading and isinstance(
n, IntentStoryNode),
maxlevel=2,
):
current_intent = find(
current_response,
lambda n: n.name == intent_heading and isinstance(
n, IntentStoryNode),
maxlevel=2,
)
else:
nlu_node = find(self.nlu.tree,
lambda n: n.name == intent_heading)
current_intent = IntentStoryNode(
item=nlu_node.item, parent=current_response)
elif line.strip().startswith("- "): # response
response_heading = line.split("- ")[-1].split(
"utter_")[-1].strip()
if find(
current_intent,
lambda n: n.name == response_heading and
isinstance(n, ResponseStoryNode),
maxlevel=2,
):
current_response = find(
current_intent,
lambda n: n.name == response_heading and
isinstance(n, ResponseStoryNode),
maxlevel=2,
)
else:
resp_node = find(self.resp.tree,
lambda n: n.name == response_heading)
current_response = ResponseStoryNode(
item=resp_node.item, parent=current_intent)
def geoparse_findings(self):
for _, region, finding, city in self.findings_df.itertuples():
print('\n* finding description>{}\nregion>{}'.format(
finding, region))
node = search.find(self, lambda node: node.name == region)
if node:
for child in node.children:
bounds_str = self.bound_string(child)
queries = [(q, bounds_str)
for q in self.queries_generator(child, finding)]
# print('queries>{}\n'.format(queries))
for (query, bounds) in queries:
print('\nq>{}\nbound name>{}\nbound coords>{}'.format(
query, child.name, bounds))
response = self.call_opencage(query, bounds)
str_response = json.dumps(response,
ensure_ascii=False,
encoding='utf-8',
indent=2)
print('{}\n{}'.format(query, str_response))
if len(response) > 0:
for r in response:
try:
x1 = r['bounds']['southwest']['lng']
y1 = r['bounds']['southwest']['lat']
x2 = r['bounds']['northeast']['lng']
y2 = r['bounds']['northeast']['lat']
geometry_r = box(x1, y1, x2, y2)
except Exception as e:
x1 = r['geometry']['lng']
y1 = r['geometry']['lat']
geometry_r = Point()
if child.geometry.contains(geometry_r):
print('---- FOUND {} ----'.format(query))
print('insert>{} as child of {}'.format(
r['formatted'], child.name))
GeospatialTree(name=r['formatted'],
geometry=geometry_r,
parent=child)
def service(self, name):
return find(self,
lambda node: node.name == name and node.resource_type ==
ResourceType.ServiceModel,
maxlevel=2)
def entity(self, ent_name):
return find(self,
lambda node: node.name == ent_name and node.resource_type
== ResourceType.EntityModel,
maxlevel=2)
def find_(self, name, parent):
return find(self.tree,
lambda node: node.name == name and node.parent == parent)
def create_fm_tree(fm_programme_df, fm_policytc_df, fm_profile_df, fm_summary_df):
missing_node_link = False
def get_policy_tc(agg_id, level_id):
policytc = fm_policytc_df.loc[
(fm_policytc_df['agg_id'] == agg_id) & (fm_policytc_df['level_id'] == level_id)
]
policy_list = []
for _, policy in policytc.iterrows():
# Find calc_rule
profile = fm_profile_df.loc[fm_profile_df.policytc_id == policy.policytc_id]
calc_rules = []
for _, step in profile.iterrows():
trig_start = step.trigger_start if hasattr(step, 'trigger_start') else 0
trig_end = step.trigger_end if hasattr(step, 'trigger_end') else 0
is_step_rule = (trig_end > 0 or trig_start > 0)
calc_rules.append(CalcRuleTuple(
policytc_id=int(policy.policytc_id),
calcrule_id=int(step.calcrule_id),
is_step=is_step_rule,
trig_start=trig_start,
trig_end=trig_end,
))
policy_list.append(
PolicyTuple(
layer_id=int(policy.layer_id),
agg_id=int(policy.agg_id),
calc_rules=calc_rules,
)
)
return len(policytc), policy_list
level_ids = sorted(list(fm_programme_df.level_id.unique()), reverse=True)
root = anytree.Node('Insured Loss', agg_id=1, level_id=max(level_ids)+1, policy_tc=None)
for level in level_ids:
agg_id_idxs = list(fm_programme_df[fm_programme_df.level_id == level].drop_duplicates(subset=['level_id','to_agg_id'], keep="first").index)
for node_idx in agg_id_idxs:
node_info = fm_programme_df.iloc[node_idx]
layer_max, policy_list = get_policy_tc(node_info.to_agg_id, node_info.level_id)
# Set parent node as root or find based on level/agg ids
if level == max(level_ids):
parent_node = root
else:
try:
matched_id = fm_programme_df.loc[(fm_programme_df.level_id == level+1) & (fm_programme_df.from_agg_id == node_info.to_agg_id)].to_agg_id.item()
parent_node = find(root, filter_=lambda node: node.level_id == level+1 and node.agg_id == matched_id)
except ValueError:
missing_node_link = True
print('Missing node link: agg_id={}, level_id={}'.format(node_info.to_agg_id,level+1))
# Set node names based on attrs in FM files
if level >= 3:
node_name = "policy term {} \nlevel: {}".format(
node_info.to_agg_id,
node_info.level_id
)
elif level == 2:
node_name = "loc term {} ".format(node_info.to_agg_id)
else:
node_name = "cov term {}".format(node_info.to_agg_id)
for policy in policy_list:
node_name += "\n\nlayer_id: {}".format(policy.layer_id)
for rule in policy.calc_rules:
if rule.is_step:
node_name += "\n policytc_id {}: step_rule:{}, start:{} end:{}".format(
rule.policytc_id,
rule.calcrule_id,
rule.trig_start,
rule.trig_end
)
else:
node_name += "\npolicytc_id: {} \ncalc_rule: {}".format(
rule.policytc_id,
rule.calcrule_id,
)
# Create Node in FM tree
node = anytree.Node(
node_name,
agg_id=node_info.to_agg_id,
level_id=level,
parent=parent_node,
layer_max=layer_max,
policy_tc=policy_list,
)
# Add item level data
item_agg_idx = list(fm_summary_df[['agg_id']].drop_duplicates().index)
for item in item_agg_idx:
item_info = fm_summary_df.iloc[item]
matched_id = fm_programme_df.loc[(fm_programme_df.level_id == 1) & (fm_programme_df.from_agg_id == item_info.agg_id)].to_agg_id.item()
parent_node = find(root, filter_=lambda node: node.level_id == 1 and node.agg_id == matched_id)
node_name = "\n".join([
"item {}\n".format(int(item_info.agg_id)),
"locnumber: {}".format(item_info.locnumber),
"accnumber: {}".format(item_info.accnumber),
"polnumber: {}".format(item_info.polnumber),
"portnumber: {}".format(item_info.portnumber),
"cov_type: {}".format(item_info.coverage_type_id),
"peril_id: {}".format(item_info.peril_id),
"tiv: {}".format(item_info.tiv),
])
node = anytree.Node(
node_name,
agg_id=item_info.agg_id,
level_id=0,
parent=parent_node,
locnumber=item_info.locnumber,
accnumber=item_info.accnumber,
polnumber=item_info.polnumber,
portnumber=item_info.polnumber,
tiv=item_info.tiv,
coverage_id=item_info.coverage_id,
coverage_type=item_info.coverage_type_id,
peril_id=item_info.peril_id,
)
return root, missing_node_link
def update_node_data(self, new_data):
node = find(self.root_, lambda node: node.name == new_data.name)
assert node
node.data = new_data
self.build_tree_(node)
