def test_dict_importer():
"""Dict Importer."""
importer = DictImporter()
exporter = DictExporter()
refdata = {
'id':
'root',
'children': [{
'id': 'sub0',
'children': [{
'id': 'sub0B'
}, {
'id': 'sub0A'
}]
}, {
'id':
'sub1',
'children': [{
'id': 'sub1A'
}, {
'id': 'sub1B'
}, {
'id': 'sub1C',
'children': [{
'id': 'sub1Ca'
}]
}]
}]
}
data = deepcopy(refdata)
root = importer.import_(data)
eq_(data, refdata)
eq_(exporter.export(root), data)
def generate_json(self):
nested_dict = self.build()
importer = DictImporter()
tree = importer.import_(nested_dict)
hr_entries = []
for node in LevelOrderIter(tree):
item = {
'PreferredFirstName': getattr(node, 'PreferredFirstName'),
'Preferred_Name_-_Last_Name': getattr(node, 'Preferred_Name_-_Last_Name'),
'businessTitle': getattr(node, 'businessTitle'),
'EmployeeID': getattr(node, 'EmployeeID'),
'PrimaryWorkEmail': getattr(node, 'PrimaryWorkEmail'),
'IsManager': len(node.children) > 0,
}
if node.parent:
manager_first_name = getattr(node.parent, 'PreferredFirstName')
manager_last_name = getattr(node.parent, 'Preferred_Name_-_Last_Name')
manager = {
'WorkersManager': '{} {}'.format(manager_first_name, manager_last_name),
'WorkersManagersEmployeeID': getattr(node.parent, 'EmployeeID')
}
item.update(manager)
hr_entries.append(item)
hr_data = {
'Report_Entry': hr_entries
}
return json.dumps(hr_data)
def _prune_leaves(self, root_node) -> dict:
"""Prune all the leaves that does not have 'value' attribute in them.
In other words, remove the leaves that user did not set in file.
Parameters
----------
root_node : dict
Branch to be pruned.
Returns
-------
dict
Pruned branch.
"""
importer = DictImporter()
exporter = DictExporter()
inptree = importer.import_(root_node)
# keep checking for leaves without value attribute
while True:
all_has_value = all(
hasattr(leaf, 'value') for leaf in inptree.leaves)
if all_has_value:
break
for leaf in inptree.leaves:
if not hasattr(leaf, 'value'):
# prune leaves without value attribute
leaf.parent = None
return exporter.export(inptree)
def intercept():
"""判断用户是否top_root"""
# redis-cli
rc = current_app.redis_cli
root_data = json.loads(rc.hget(f"drp_relation_member_{g.user_id}", 0).decode())
importer = DictImporter()
root = importer.import_(root_data)
return root.height
def sync_to_restore(self, opt_model):
self.opt_model = opt_model
if hasattr(self, 'root_node'):
root_node_compressed = self.root_node
importer = DictImporter()
self.root_node = importer.import_(root_node_compressed)
sync_part_tree_on_restore(self.opt_model.ele_model,
self.opt_model.seq_model, self.root_node)
def get_path_descendants(path):
""" Get the set of descendants for a tree like path dict.
"""
importer = DictImporter()
root = importer.import_(path)
descendants = set(
[descendant_node.name for descendant_node in root.descendants])
return descendants
def load_tree(self, directory=None):
data_path = (
directory or '.'
) + '/nn_data/' + self.config.experiment_name + '/spectral_tree.dict'
if os.path.exists(data_path):
importer = DictImporter()
with open(data_path, 'rb') as file:
treedict = pickle.load(file)
self.root = importer.import_(treedict)
print(RenderTree(self.root).by_attr('id'))
def path_differences(model, paths_labels, type_analysis='production'):
"""
Parameters
----------
model: PySB model
Model used to do dominant path analysis
paths_labels: dict
Dictionary of pathways generated by dominant path analysis
type_analysis: str
Type of analysis used in the dominant path analysis.
It can either be `production` or `consumption`
Returns
-------
A pandas dataframe where the column names and row indices are the labels of the
pathways and the cells contain the edges that are present in the
row pathway index but not in the column pathway.
"""
generate_equations(model)
importer = DictImporter()
path_edges = {}
def find_numbers(dom_r_str):
n = map(int, re.findall('\d+', dom_r_str))
return n
def nodenamefunc(node):
node_idx = list(find_numbers(node.name))[0]
node_sp = model.species[node_idx]
node_name = parse_name(node_sp)
return node_name
def edgeattrfunc(node, child):
return 'dir="back"'
for keys, values in paths_labels.items():
root = importer.import_(values)
dot = DotExporter(root, graph='strict digraph', options=["rankdir=RL;"], nodenamefunc=nodenamefunc,
edgeattrfunc=edgeattrfunc)
data = ''
for line in dot:
data += line
pydot_graph = graph_from_dot_data(data)
graph = from_pydot(pydot_graph[0])
if type_analysis == 'production':
graph = graph.reverse()
edges = set(graph.edges())
path_edges[keys] = edges
path_diff = pd.DataFrame(index=paths_labels.keys(), columns=paths_labels.keys())
for row in path_diff.columns:
for col in path_diff.columns:
path_diff.loc[row, col] = path_edges[row].difference(path_edges[col])
return path_diff
def load_from_file(cls, file_path, game_def):
"""
Creates a Tree from a file with the tree in json format
Args:
file_path: Path to the json file
"""
with open(file_path) as feedsjson:
tree_dic = json.load(feedsjson)
importer = DictImporter()
root = importer.import_(tree_dic['tree'])
for n in PreOrderIter(root):
n = cls.node_class.from_dic(n.name,game_def,tree_dic['main_player'],parent=n.parent,children=n.children)
t = cls(root)
return t
def render(self, reform):
from anytree.importer import DictImporter
from anytree import RenderTree
if isinstance(reform, dict):
reform = DeppReform.from_data(reform)
importer = DictImporter()
# tree_root = importer.import_(r['hierplane_tree']['root'])
print([(w, h) for w, h in zip(reform.words, reform.heads)])
tree_root = importer.import_(reform.tree['root'])
tree = RenderTree(tree_root)
for pre, fill, node in tree:
print("%s%s: %s (%s)" % (pre, node.nodeType, node.word, ','.join(
node.attributes).lower()))
def __init__(self, vectorifier=None):
dict_importer = DictImporter(nodecls=Node)
importer = JsonImporter(dictimporter=dict_importer)
self.root = importer.import_(symptom_json)
if vectorifier is not None:
# normal mode, if it is None is eval mode
self.vectorifier = vectorifier
self.vector_dimension = vectorifier.d
"""
if concept name vectors file not exists, create it
"""
if vectorifier.internal_representation == "glove":
if not os.path.isfile(COM.CSV_ST_CONCEPT_NAMES_GLOVE_PATH + \
str(self.vector_dimension) + "d.csv"):
print(
"Concept names glove vectors file not found. \nComputing file..."
)
self.__save_csv_concept_name_glove_vectors()
self.concept_name_vectors_df = pd.read_csv(COM.CSV_ST_CONCEPT_NAMES_GLOVE_PATH + \
str(self.vector_dimension) + "d.csv",
header=None)
elif vectorifier.internal_representation == "bert":
if not os.path.isfile(COM.CSV_ST_CONCEPT_NAMES_BERT_PATH + \
COM.FILENAME_CSV_ST_CONCEPT_NAMES_BERT):
print(
"Concept names bert vectors file not found. \nComputing file..."
)
self.__save_csv_concept_name_bert_vectors()
self.concept_name_vectors_df = pd.read_csv(COM.CSV_ST_CONCEPT_NAMES_BERT_PATH + \
COM.FILENAME_CSV_ST_CONCEPT_NAMES_BERT,
header=None)
def get_childs(query_term="radiological finding"):
tree = None
s = {}
path = root + '/Rx-thorax-automatic-captioning/' + filetree
stream = open(path, "r")
docs = yaml.load_all(stream, Loader=yaml.FullLoader)
print("Starting to search " + query_term + " in the following trees:")
for doc in docs:
tree_root = DictImporter().import_(doc)
print("\t" + tree_root.label)
r = search.findall(tree_root, lambda node: node.label == query_term)
if r:
tree = RenderTree(r[0])
for pre, _, node in tree:
CUI = None
try:
CUI = node.concept
m = re.search('\((\w+)\)', CUI)
if m:
CUI = m.group(0).strip('(').strip(')')
cui.add(CUI)
except:
pass
s[node.label] = CUI
if tree:
t = str(tree.by_attr(lambda n: n.label + '\n'))
st.sidebar.markdown("## Tree of child terms")
st.sidebar.markdown(t)
return s, t
def load(self, dirname):
importer = DictImporter()
(self.tree, self.node_to_class, self.node_to_classes,
self.class_maps) = load_tree(join(dirname, 'tree'))
self.models = {}
models_dirname = join(dirname, 'models')
with open(join(models_dirname, 'models_fnames.yaml'),
'r',
encoding='utf-8') as file:
models_dct = yaml_load(file)
for node_id, fname in models_dct.items():
model_path = join(models_dirname, fname)
self.models[node_id] = load_model(model_path)
self.encoders = {}
encoders_dirname = join(dirname, 'encoders')
with open(join(encoders_dirname, 'encoders_fnames.yaml'),
'r',
encoding='utf-8') as file:
encoders_dct = yaml_load(file)
for node_id, fname in encoders_dct.items():
encoder_path = join(encoders_dirname, fname)
self.encoders[node_id] = joblib_load(encoder_path)
self._fitted = True
def load_tree(dirname):
importer = DictImporter()
with open(join(dirname, 'tree.yaml'), 'r', encoding='utf-8') as file:
tree = importer.import_(yaml_load(file))
with open(join(dirname, 'node_to_classes.yaml'), 'r',
encoding='utf-8') as file:
node_to_classes = yaml_load(file)
with open(join(dirname, 'node_to_class.yaml'), 'r',
encoding='utf-8') as file:
node_to_class = yaml_load(file)
with open(join(dirname, 'class_maps.yaml'), 'r', encoding='utf-8') as file:
class_maps = yaml_load(file)
return tree, node_to_class, node_to_classes, class_maps
def undo(self):
importer = DictImporter(nodecls=QAnyTreeItem)
parent = getIndexFromLocations(self.indexLocations, self.model)
parentItem = self.model.getItem(parent)
self.result = self.model.beginInsertRows(parent, self.position,
self.position + self.rows - 1)
for row, data in enumerate(self.data):
print(data)
item = importer.import_(data)
# Reconstruct branch
item.parent = parentItem
parentItem.moveChild(parentItem.childCount() - 1,
self.position + row)
self.model.endInsertRows()
def buildTreeCounts( ):
stream = open("manual_review/trees.txt", "r")
docs = yaml.load_all(stream)
study_labels_file = 'manual_review/all_info_studies_labels.csv'
path = root + '/Rx-thorax-automatic-captioning/' + study_labels_file
all_studies_labels_DF = pd.read_csv(path, sep = ',' , header = 0)
row_labels = all_studies_labels_DF.groupby('StudyID').head(1)
row_labels['labels'] = row_labels['labels'].astype('str')
pattern = re.compile('[^a-zA-Z,\s]+')
row_labels['label_list'] = row_labels['labels'].apply(lambda r: set([item.strip() for item in pattern.sub('', r).split(",")]))
study_localizations_count = 'manual_review/unique_localizations_28K.csv'
study_labels_count = 'manual_review/unique_labels_28K.csv'
path = root + '/Rx-thorax-automatic-captioning/' + study_labels_count
study_labels_count_DF = pd.read_csv(path, sep = ',' ,names = ['label','counts'], dtype={'label': str, 'counts': np.int32})
path = root + '/Rx-thorax-automatic-captioning/' + study_localizations_count
study_localizations_count_DF = pd.read_csv(path, sep = ',' ,names = ['label','counts'], dtype={'label': str, 'counts': np.int32})
study_labels_count_DF = pd.concat([study_labels_count_DF,study_localizations_count_DF], axis = 0)
for doc in docs:
tree_root = DictImporter().import_(doc)
for pre, _, node in RenderTree(tree_root):
try:
if node.label == 'normal': #Normal if single label (studies with reports without pathological findings: [Normal + exclude], [Normal], [Normal + Suboptimal]
s = {'normal','exclude','suboptimal study'}
node.study_counts = row_labels[row_labels['label_list'] <= s].shape[0]
elif node.label == 'exclude': #Exclude if single label (reports not informative, neither normal not pathological finding reported): [Exclude], [Exclude, suboptimal]
s = {'exclude','suboptimal study'}
node.study_counts = row_labels[row_labels['label_list'] <= s].shape[0]
else:
node.study_counts = study_labels_count_DF.loc[study_labels_count_DF.label == node.label,'counts'].values[0]
except:
node.study_counts = 0
with open("manual_review/tree_term_counts.txt", "a") as text_file:
for pre, _, node in RenderTree(tree_root):
node.study_node_counts = node.study_counts + sum(i.study_counts for i in node.descendants)
print("%s%s %s" % (pre, node.label, node.study_node_counts), file=text_file)
#with open("manual_review/tree_term_counts.txt", "a") as text_file:
#print(RenderTree(tree_root), file=text_file)
#labels (pathological findings) associated to differential diagnoses
#load differential diagnosis subtree
#for each diagnosis collect all other pathological findings in all_info_study_labels
if tree_root.label == "differential diagnosis":
for n in tree_root.descendants:
n.ordered_near_labels = pd.Series([l for sublist in row_labels[row_labels['label_list'] >= {n.label}]['label_list'] for l in sublist ]).value_counts()
#with open("manual_review/tree_term_counts.txt", "a") as text_file:
#print(RenderTree(tree_root), file=text_file)
return
def read(self) -> Entry:
content = None
if os.path.isfile(self._filepath):
fp = open(Path(self._filepath))
d_imp = DictImporter(nodecls=Entry)
importer = JsonImporter(dictimporter=d_imp)
content = importer.import_(fp.read())
fp.close()
return content
def from_file(cls, filename):
"""
Load parameter space as a ternary tree from pickle file.
"""
# load dict from pickle
if not filename.endswith(PKL_EXT):
filename += PKL_EXT
with open(filename, "rb") as f:
loaded_dict = pickle.load(f)
def _sanitize_dict(raw_dict, keys_to_delete):
"""
Remove keys from dict - possibly nested.
"""
sanit_dict = {}
for key, value in raw_dict.items():
if key not in keys_to_delete:
if isinstance(value, (list, tuple)):
sanit_dict[key] = [
_sanitize_dict(list_val, keys_to_delete)
if isinstance(list_val, dict)
else list_val
for list_val in value
]
elif isinstance(value, dict):
sanit_dict[key] = _sanitize_dict(value, keys_to_delete)
else:
sanit_dict[key] = value
return sanit_dict
# sanitise dict for correct init
keys_to_delete = set(loaded_dict.keys()) - set(cls.INIT_ATTRS)
sanitised_dict = _sanitize_dict(loaded_dict, keys_to_delete)
# import as `LeafNode` class
importer = DictImporter(nodecls=LeafNode)
root_leaf_node = importer.import_(sanitised_dict)
# force `ParameterSpace` class which extends `LeafNode` with some
# useful methods
root_leaf_node.__class__ = ParameterSpace
return root_leaf_node
def load_from_file(path):
path = osp.expanduser(path)
if osp.exists(osp.dirname(path)):
dict_imp = DictImporter(nodecls=DPNode)
imp = JsonImporter(dictimporter=dict_imp,
object_hook=tools.object_hook)
with open(path, "r") as f:
res = imp.read(f)
return RemoteTree(res)
else:
print("Error saving to disk. Dir {} not existing.".format(
osp.dirname(path)))
def generate_json(self):
nested_dict = self.build()
importer = DictImporter()
tree = importer.import_(nested_dict)
hr_entries = []
for node in LevelOrderIter(tree):
item = {
'PreferredFirstName':
getattr(node, 'PreferredFirstName'),
'Preferred_Name_-_Last_Name':
getattr(node, 'Preferred_Name_-_Last_Name'),
'businessTitle':
getattr(node, 'businessTitle'),
'EmployeeID':
getattr(node, 'EmployeeID'),
'PrimaryWorkEmail':
getattr(node, 'PrimaryWorkEmail'),
'IsManager':
len(node.children) > 0,
}
if node.parent:
manager_first_name = getattr(node.parent, 'PreferredFirstName')
manager_last_name = getattr(node.parent,
'Preferred_Name_-_Last_Name')
manager = {
'WorkersManager':
'{} {}'.format(manager_first_name, manager_last_name),
'WorkersManagersEmployeeID':
getattr(node.parent, 'EmployeeID')
}
item.update(manager)
hr_entries.append(item)
hr_data = {'Report_Entry': hr_entries}
return json.dumps(hr_data)
def vis_trees(trees: List[Dict[Text, Any]], word_info=True):
from anytree.importer import DictImporter
from anytree import RenderTree
importer = DictImporter()
for index, data in enumerate(trees):
word = data['word']
if word_info:
pprint(word)
tree_root = importer.import_(data['tree'])
tree = RenderTree(tree_root)
tc.emp(
'green',
f"Display #{index} sememe tree: {word['en_word']}|{word['ch_word']}.{word['No']}"
)
for pre, fill, node in tree:
if node.role and node.role != 'None':
cl = 'magenta'
role = node.role
else:
cl = 'yellow'
role = '✔'
tc.emp(cl, "%s%s: %s" % (pre, role, node.name))
def analyze_paths(df: pd.DataFrame) -> Tuple[Node, int]:
use_prepruning = st.checkbox("Use Pre-Pruning?")
tree, max_count = build_page_hierarchy(
df, aggregation_nodes if use_prepruning else None)
root = DictImporter().import_(tree)
# if st.checkbox("Show pagetree in ASCII?", value=False):
# st.text(RenderTree(root, style=ContRoundStyle()))
# if st.checkbox("Show pagetree as graph?", value=True):
min_counts = st.slider("Minimum number of counts:",
value=2,
min_value=0,
max_value=50)
root = prune_tree(root, min_counts)
return root, max_count
def get_all_terms():
labels = {}
stream = open("trees_code.txt", "r")
docs = yaml.load_all(stream, Loader=yaml.FullLoader)
for doc in docs:
tree_root = DictImporter().import_(doc)
labels[tree_root.label] = ''
for pre, _, node in RenderTree(tree_root):
CUI = ''
try:
CUI = node.concept
m = re.search('\((\w+)\)', CUI)
if m:
CUI = m.group(0).strip('(').strip(')')
except:
pass
labels[node.label] = CUI
return labels
def get_all_terms():
labels = {}
path = root + '/Rx-thorax-automatic-captioning/' + filetree
stream = open(path, "r")
docs = yaml.load_all(stream, Loader=yaml.FullLoader)
for doc in docs:
tree_root = DictImporter().import_(doc)
labels[tree_root.label] = ''
for pre, _, node in RenderTree(tree_root):
CUI = ''
try:
CUI = node.concept
m = re.search('\((\w+)\)', CUI)
if m:
CUI = m.group(0).strip('(').strip(')')
except:
pass
labels[node.label] = CUI
return labels