def test_dict_importer():
"""Dict Importer."""
importer = DictImporter()
exporter = DictExporter()
refdata = {
'id':
'root',
'children': [{
'id': 'sub0',
'children': [{
'id': 'sub0B'
}, {
'id': 'sub0A'
}]
}, {
'id':
'sub1',
'children': [{
'id': 'sub1A'
}, {
'id': 'sub1B'
}, {
'id': 'sub1C',
'children': [{
'id': 'sub1Ca'
}]
}]
}]
}
data = deepcopy(refdata)
root = importer.import_(data)
eq_(data, refdata)
eq_(exporter.export(root), data)
def _prune_leaves(self, root_node) -> dict:
"""Prune all the leaves that does not have 'value' attribute in them.
In other words, remove the leaves that user did not set in file.
Parameters
----------
root_node : dict
Branch to be pruned.
Returns
-------
dict
Pruned branch.
"""
importer = DictImporter()
exporter = DictExporter()
inptree = importer.import_(root_node)
# keep checking for leaves without value attribute
while True:
all_has_value = all(
hasattr(leaf, 'value') for leaf in inptree.leaves)
if all_has_value:
break
for leaf in inptree.leaves:
if not hasattr(leaf, 'value'):
# prune leaves without value attribute
leaf.parent = None
return exporter.export(inptree)
def generate_json(self):
nested_dict = self.build()
importer = DictImporter()
tree = importer.import_(nested_dict)
hr_entries = []
for node in LevelOrderIter(tree):
item = {
'PreferredFirstName': getattr(node, 'PreferredFirstName'),
'Preferred_Name_-_Last_Name': getattr(node, 'Preferred_Name_-_Last_Name'),
'businessTitle': getattr(node, 'businessTitle'),
'EmployeeID': getattr(node, 'EmployeeID'),
'PrimaryWorkEmail': getattr(node, 'PrimaryWorkEmail'),
'IsManager': len(node.children) > 0,
}
if node.parent:
manager_first_name = getattr(node.parent, 'PreferredFirstName')
manager_last_name = getattr(node.parent, 'Preferred_Name_-_Last_Name')
manager = {
'WorkersManager': '{} {}'.format(manager_first_name, manager_last_name),
'WorkersManagersEmployeeID': getattr(node.parent, 'EmployeeID')
}
item.update(manager)
hr_entries.append(item)
hr_data = {
'Report_Entry': hr_entries
}
return json.dumps(hr_data)
def sync_to_restore(self, opt_model):
self.opt_model = opt_model
if hasattr(self, 'root_node'):
root_node_compressed = self.root_node
importer = DictImporter()
self.root_node = importer.import_(root_node_compressed)
sync_part_tree_on_restore(self.opt_model.ele_model,
self.opt_model.seq_model, self.root_node)
def load_tree(path):
importer = DictImporter()
with gzip.open(path, 'rb') as f:
dataRead = pickle.load(f)
logging.debug("dataRead=" + str(dataRead))
rootNode = importer.import_(dataRead)
return rootNode
def intercept():
"""判断用户是否top_root"""
# redis-cli
rc = current_app.redis_cli
root_data = json.loads(rc.hget(f"drp_relation_member_{g.user_id}", 0).decode())
importer = DictImporter()
root = importer.import_(root_data)
return root.height
def get_path_descendants(path):
""" Get the set of descendants for a tree like path dict.
"""
importer = DictImporter()
root = importer.import_(path)
descendants = set(
[descendant_node.name for descendant_node in root.descendants])
return descendants
def load_tree(self, directory=None):
data_path = (
directory or '.'
) + '/nn_data/' + self.config.experiment_name + '/spectral_tree.dict'
if os.path.exists(data_path):
importer = DictImporter()
with open(data_path, 'rb') as file:
treedict = pickle.load(file)
self.root = importer.import_(treedict)
print(RenderTree(self.root).by_attr('id'))
def path_differences(model, paths_labels, type_analysis='production'):
"""
Parameters
----------
model: PySB model
Model used to do dominant path analysis
paths_labels: dict
Dictionary of pathways generated by dominant path analysis
type_analysis: str
Type of analysis used in the dominant path analysis.
It can either be `production` or `consumption`
Returns
-------
A pandas dataframe where the column names and row indices are the labels of the
pathways and the cells contain the edges that are present in the
row pathway index but not in the column pathway.
"""
generate_equations(model)
importer = DictImporter()
path_edges = {}
def find_numbers(dom_r_str):
n = map(int, re.findall('\d+', dom_r_str))
return n
def nodenamefunc(node):
node_idx = list(find_numbers(node.name))[0]
node_sp = model.species[node_idx]
node_name = parse_name(node_sp)
return node_name
def edgeattrfunc(node, child):
return 'dir="back"'
for keys, values in paths_labels.items():
root = importer.import_(values)
dot = DotExporter(root, graph='strict digraph', options=["rankdir=RL;"], nodenamefunc=nodenamefunc,
edgeattrfunc=edgeattrfunc)
data = ''
for line in dot:
data += line
pydot_graph = graph_from_dot_data(data)
graph = from_pydot(pydot_graph[0])
if type_analysis == 'production':
graph = graph.reverse()
edges = set(graph.edges())
path_edges[keys] = edges
path_diff = pd.DataFrame(index=paths_labels.keys(), columns=paths_labels.keys())
for row in path_diff.columns:
for col in path_diff.columns:
path_diff.loc[row, col] = path_edges[row].difference(path_edges[col])
return path_diff
def load_from_file(cls, file_path, game_def):
"""
Creates a Tree from a file with the tree in json format
Args:
file_path: Path to the json file
"""
with open(file_path) as feedsjson:
tree_dic = json.load(feedsjson)
importer = DictImporter()
root = importer.import_(tree_dic['tree'])
for n in PreOrderIter(root):
n = cls.node_class.from_dic(n.name,game_def,tree_dic['main_player'],parent=n.parent,children=n.children)
t = cls(root)
return t
def render(self, reform):
from anytree.importer import DictImporter
from anytree import RenderTree
if isinstance(reform, dict):
reform = DeppReform.from_data(reform)
importer = DictImporter()
# tree_root = importer.import_(r['hierplane_tree']['root'])
print([(w, h) for w, h in zip(reform.words, reform.heads)])
tree_root = importer.import_(reform.tree['root'])
tree = RenderTree(tree_root)
for pre, fill, node in tree:
print("%s%s: %s (%s)" % (pre, node.nodeType, node.word, ','.join(
node.attributes).lower()))
def undo(self):
importer = DictImporter(nodecls=QAnyTreeItem)
parent = getIndexFromLocations(self.indexLocations, self.model)
parentItem = self.model.getItem(parent)
self.result = self.model.beginInsertRows(parent, self.position,
self.position + self.rows - 1)
for row, data in enumerate(self.data):
print(data)
item = importer.import_(data)
# Reconstruct branch
item.parent = parentItem
parentItem.moveChild(parentItem.childCount() - 1,
self.position + row)
self.model.endInsertRows()
def load_tree(dirname):
importer = DictImporter()
with open(join(dirname, 'tree.yaml'), 'r', encoding='utf-8') as file:
tree = importer.import_(yaml_load(file))
with open(join(dirname, 'node_to_classes.yaml'), 'r',
encoding='utf-8') as file:
node_to_classes = yaml_load(file)
with open(join(dirname, 'node_to_class.yaml'), 'r',
encoding='utf-8') as file:
node_to_class = yaml_load(file)
with open(join(dirname, 'class_maps.yaml'), 'r', encoding='utf-8') as file:
class_maps = yaml_load(file)
return tree, node_to_class, node_to_classes, class_maps
def test_dict_importer_node():
"""Dict Importer."""
importer = DictImporter(Node)
exporter = DictExporter()
refdata = {
'name':
'root',
'children': [{
'name': 'sub0',
'children': [{
'name': 'sub0B'
}, {
'name': 'sub0A'
}]
}, {
'name':
'sub1',
'children': [{
'name': 'sub1A'
}, {
'name': 'sub1B'
}, {
'name': 'sub1C',
'children': [{
'name': 'sub1Ca'
}]
}]
}]
}
data = deepcopy(refdata)
root = importer.import_(data)
eq_(data, refdata)
eq_(exporter.export(root), data)
r = RenderTree(root)
expected = u"\n".join([
u"Node('/root')",
u"├── Node('/root/sub0')",
u"│ ├── Node('/root/sub0/sub0B')",
u"│ └── Node('/root/sub0/sub0A')",
u"└── Node('/root/sub1')",
u" ├── Node('/root/sub1/sub1A')",
u" ├── Node('/root/sub1/sub1B')",
u" └── Node('/root/sub1/sub1C')",
u" └── Node('/root/sub1/sub1C/sub1Ca')",
])
eq_str(str(r), expected)
def vis_tree(ds:Dict[Text, Any], lang, trans=False):
""" 可视化domains """
from anytree.importer import DictImporter
from anytree import RenderTree
from sagas.nlu.translator import get_contrast
data = treeing(ds)
importer = DictImporter()
tree_root = importer.import_(data)
tree = RenderTree(tree_root)
for pre, fill, node in tree:
if node.dependency_relation=='punct':
addons='_'
else:
addons=f"{node.lemma}; {get_contrast(node.text, lang)}" \
if trans else f"{node.lemma}"
print(f"{pre}{node.dependency_relation}: "
f"{node.text}({addons}, {node.upos.lower()}, {node.index})")
def from_file(cls, filename):
"""
Load parameter space as a ternary tree from pickle file.
"""
# load dict from pickle
if not filename.endswith(PKL_EXT):
filename += PKL_EXT
with open(filename, "rb") as f:
loaded_dict = pickle.load(f)
def _sanitize_dict(raw_dict, keys_to_delete):
"""
Remove keys from dict - possibly nested.
"""
sanit_dict = {}
for key, value in raw_dict.items():
if key not in keys_to_delete:
if isinstance(value, (list, tuple)):
sanit_dict[key] = [
_sanitize_dict(list_val, keys_to_delete)
if isinstance(list_val, dict)
else list_val
for list_val in value
]
elif isinstance(value, dict):
sanit_dict[key] = _sanitize_dict(value, keys_to_delete)
else:
sanit_dict[key] = value
return sanit_dict
# sanitise dict for correct init
keys_to_delete = set(loaded_dict.keys()) - set(cls.INIT_ATTRS)
sanitised_dict = _sanitize_dict(loaded_dict, keys_to_delete)
# import as `LeafNode` class
importer = DictImporter(nodecls=LeafNode)
root_leaf_node = importer.import_(sanitised_dict)
# force `ParameterSpace` class which extends `LeafNode` with some
# useful methods
root_leaf_node.__class__ = ParameterSpace
return root_leaf_node
def get_hierarchy(self, attribute_name):
"""
Returns the hierarchy for a given attribute, None if there is no hierarchy
Parameters
----------
attribute_name: str
The attribute name.
Returns
-------
AnyNode
The generalization hierarchy.
"""
attribute = self.attributes[attribute_name]
if "hierarchy" in attribute:
importer = DictImporter()
root = importer.import_(attribute['hierarchy'])
for node in LevelOrderIter(root):
node_range = name_to_range(node.name)
node.range = node_range
return root
return None
def generate_json(self):
nested_dict = self.build()
importer = DictImporter()
tree = importer.import_(nested_dict)
hr_entries = []
for node in LevelOrderIter(tree):
item = {
'PreferredFirstName':
getattr(node, 'PreferredFirstName'),
'Preferred_Name_-_Last_Name':
getattr(node, 'Preferred_Name_-_Last_Name'),
'businessTitle':
getattr(node, 'businessTitle'),
'EmployeeID':
getattr(node, 'EmployeeID'),
'PrimaryWorkEmail':
getattr(node, 'PrimaryWorkEmail'),
'IsManager':
len(node.children) > 0,
}
if node.parent:
manager_first_name = getattr(node.parent, 'PreferredFirstName')
manager_last_name = getattr(node.parent,
'Preferred_Name_-_Last_Name')
manager = {
'WorkersManager':
'{} {}'.format(manager_first_name, manager_last_name),
'WorkersManagersEmployeeID':
getattr(node.parent, 'EmployeeID')
}
item.update(manager)
hr_entries.append(item)
hr_data = {'Report_Entry': hr_entries}
return json.dumps(hr_data)
def vis_trees(trees: List[Dict[Text, Any]], word_info=True):
from anytree.importer import DictImporter
from anytree import RenderTree
importer = DictImporter()
for index, data in enumerate(trees):
word = data['word']
if word_info:
pprint(word)
tree_root = importer.import_(data['tree'])
tree = RenderTree(tree_root)
tc.emp(
'green',
f"Display #{index} sememe tree: {word['en_word']}|{word['ch_word']}.{word['No']}"
)
for pre, fill, node in tree:
if node.role and node.role != 'None':
cl = 'magenta'
role = node.role
else:
cl = 'yellow'
role = '✔'
tc.emp(cl, "%s%s: %s" % (pre, role, node.name))
def main():
parser = argparse.ArgumentParser()
parser.add_argument('-v',
'--verbose',
action='store_true',
help='one-time computations')
parser.add_argument('-p',
'--path',
default='embs.npy',
type=str,
help='path to language embeddings')
args = parser.parse_args()
embs = np.load(args.path)
if not os.path.exists('../outputs'):
os.makedirs('../outputs')
# only considering languages relatively close to current language (e.g. not cross-continent)
radius = 500
# for each k, we return fraction of languages that have a top k embedding distance
# neighbor that's in the family/tree
ks = {2, 4, 8, 16, 32}
metrics = {} # where all the metrics are added
lang_list_pth = '../metadata/LangList.txt'
with open(lang_list_pth, 'r') as inf:
lines = inf.readlines()
langlist_lines = [l.strip() for l in lines]
lang_to_link = {l.split()[0]: l.split()[2] for l in langlist_lines}
# lang_to_link['ABIWBT'] = 'https://en.wikipedia.org/wiki/Abidji_language'
lang_to_3 = {l.split()[0]: l.split()[1] for l in langlist_lines}
# lang_to_3['ABIWBT'] = 'abi'
lang_to_coord = {
l.split()[0]: (l.split()[4], l.split()[5])
for l in langlist_lines
} # latitude, longitude
# lang_to_coord['ABIWBT'] = ('5.65656', '-4.58421')
with open('../metadata/lang195.txt', 'r') as inf:
lines = inf.readlines()
lang195_langs = [l.strip() for l in lines] # ['ABIWBT', ...]
lang195_langs_set = set(lang195_langs)
lons = [float(lang_to_coord[l][1]) for l in lang195_langs]
lats = [float(lang_to_coord[l][0]) for l in lang195_langs]
if not os.path.exists('../metadata/dists_km.npy'):
coords = [(lat, lon) for lat, lon in zip(lats, lons)]
dists = [[geopy.distance.distance(c1, c2) for c1 in coords]
for c2 in coords]
dists_km = [[d.km for d in ds] for ds in dists]
# dists_km[i][j] = dist btw lang_i and lang_j
np.save('../metadata/dists_km.npy', dists_km)
dists_km = np.load('../metadata/dists_km.npy')
emb_dists = [[np.linalg.norm(a - b) for a in embs] for b in embs]
correlation_mean = np.mean(
[pearsonr(dists_km[i], emb_dists[i])[0] for i in range(len(dists_km))])
correlation_std = np.std(
[pearsonr(dists_km[i], emb_dists[i])[0] for i in range(len(dists_km))])
metrics['correlation_mean'] = correlation_mean
metrics['correlation_std'] = correlation_std
idxs_under_threshold = [[i for i, d in enumerate(ds) if d < radius]
for ds in dists_km]
# row i has the languages < 500 km of lang i
dists_km_subsets = [[ds[i] for i in idxs]
for idxs, ds in zip(idxs_under_threshold, dists_km)]
emb_dists_subsets = [[ds[i] for i in idxs]
for idxs, ds in zip(idxs_under_threshold, emb_dists)]
rad_correlation_mean = np.mean([
pearsonr(dists_km_subsets[i], emb_dists_subsets[i])[0]
for i in range(len(dists_km)) if len(dists_km_subsets[i]) > 1
])
rad_correlation_std = np.std([
pearsonr(dists_km_subsets[i], emb_dists_subsets[i])[0]
for i in range(len(dists_km)) if len(dists_km_subsets[i]) > 1
])
metrics['rad_correlation_mean'] = rad_correlation_mean
metrics['rad_correlation_std'] = rad_correlation_std
ethnologue_tree_path = '../outputs/ethnologue_tree.txt'
ethnologue_ancestor_mat_path = '../metadata/ethnologue_lang195_ancestor_mat.npy'
ethnologue_avg_ancestor_mat_path = '../metadata/ethnologue_lang195_avg_ancestor_mat.npy'
ethnologue_tree_idxs_path = '../metadata/ethnologue_lang195_tree_idxs.npy'
if not (os.path.exists(ethnologue_tree_path)
and os.path.exists(ethnologue_ancestor_mat_path)
and os.path.exists(ethnologue_avg_ancestor_mat_path)
and os.path.exists(ethnologue_tree_idxs_path)):
def contains_pattern(p, node):
return p in node.name
filename = "../metadata/ethnologue_forest.json"
with open(filename, 'r') as f:
ethnologue_forest = json.load(f)
importer = DictImporter()
ethnologue_trees = []
for cur in ethnologue_forest:
root = importer.import_(cur)
ethnologue_trees.append(root)
output_str = ''
for root in ethnologue_trees:
for pre, _, node in RenderTree(root):
output_str += "%s%s\n" % (pre, node.name)
with open(ethnologue_tree_path, 'w+') as ouf:
ouf.write(output_str)
def get_node_ethnologue(lang):
l1 = lang_to_3[lang]
f = partial(contains_pattern, l1)
for ti, t in enumerate(ethnologue_trees):
nodes = anytree.search.findall(t, filter_=f)
if len(nodes) > 0:
return nodes[0], ti
return None, -1
def tree_dist(node0, node1, mode='min'):
'''assumes nodes are in the same tree
e.g. if node0 is depth 7 and node1 is depth 5 and their common ancestor is depth 3,
then we return 5-3 = 2
'''
ancestor_depth = len(anytree.util.commonancestors(node0, node1))
if mode == 'avg':
return (node0.depth + node1.depth) / 2 - ancestor_depth
else:
return min(node0.depth, node1.depth) - ancestor_depth
def mk_ancestor_mat(languages, mode='min'):
'''
Args:
languages: list of strings (each string is 6-letter language id)
Returns:
ancestor_mat: entry [i][j] is the tree_dist between lang_i and lang_j
'''
nodes = []
tree_idxs = []
for lang in languages:
n, ti = get_node_ethnologue(lang)
nodes.append(n)
tree_idxs.append(ti)
ancestor_mat = np.zeros((len(languages), len(languages)),
dtype=int)
for i, lang1 in enumerate(languages):
for j, lang2 in enumerate(languages):
if lang1 == lang2:
ancestor_mat[i][j] = 0
elif i < j:
node0 = nodes[i]
node1 = nodes[j]
if tree_idxs[i] != tree_idxs[j]:
ancestor_mat[i][j] = -1
else:
ancestor_mat[i][j] = tree_dist(node0,
node1,
mode=mode)
else:
ancestor_mat[i][j] = ancestor_mat[j][i]
return ancestor_mat
if not os.path.exists(ethnologue_ancestor_mat_path):
lang195_ancestor_mat = mk_ancestor_mat(lang195_langs)
np.save(ethnologue_ancestor_mat_path, lang195_ancestor_mat)
if not os.path.exists(ethnologue_avg_ancestor_mat_path):
lang195_ancestor_mat = mk_ancestor_mat(lang195_langs, mode='avg')
np.save(ethnologue_avg_ancestor_mat_path, lang195_ancestor_mat)
if not os.path.exists(ethnologue_tree_idxs_path):
lang195_tree_idxs = []
for lang in lang195_langs:
n, ti = get_node_ethnologue(lang)
lang195_tree_idxs.append(ti)
np.save(ethnologue_tree_idxs_path, lang195_tree_idxs)
ethnologue_lang195_ancestor_mat = np.load(ethnologue_ancestor_mat_path)
ethnologue_lang195_avg_ancestor_mat = np.load(
ethnologue_avg_ancestor_mat_path)
lang195_tree_idxs = np.load(ethnologue_tree_idxs_path)
emb_dists_np = np.array(emb_dists)
# for each k, we return fraction of languages that have a top k embedding distance
# neighbor that's in the family/tree
langs_with_fam = [] # list of (index in lang195, lang)
for i, lang in enumerate(lang195_langs):
if np.sum(ethnologue_lang195_ancestor_mat[i] >= 0) > 1:
langs_with_fam.append((i, lang))
tot = len(langs_with_fam)
all_closest_idxs = {
i: np.argsort(emb_dists_np[i])[1:]
for i, lang in langs_with_fam
}
max_k = max(ks)
num_outliers = 10
num_non_outliers = tot - num_outliers
for ancestor_mat in [ethnologue_lang195_ancestor_mat]:
k_outlier = -1
outliers = [] # list of 6-letter language ids
for k in range(1, tot):
count = 0
curr_outliers = []
for i, lang in langs_with_fam:
closest_idxs = all_closest_idxs[i][:k]
ti = lang195_tree_idxs[i]
ctis = lang195_tree_idxs[closest_idxs]
has_fam = (ctis == ti).sum() > 0
if not has_fam:
curr_outliers.append(lang)
count += has_fam
if count >= num_non_outliers and k > max_k:
k_outlier = k
outliers = curr_outliers
break
if k in ks:
metrics['k-%d' % k] = count / tot
metrics['k_outlier'] = k_outlier
metrics['outliers'] = outliers
ti_to_195_idxs = {}
ti_to_195_idxs_set = {}
for i, ti in enumerate(lang195_tree_idxs):
if ti in ti_to_195_idxs:
ti_to_195_idxs[ti].append(i)
else:
ti_to_195_idxs[ti] = [i]
for ti in ti_to_195_idxs:
ti_to_195_idxs_set[ti] = set(ti_to_195_idxs[ti])
# languages on map colored by language family
if args.verbose:
import matplotlib
k = 10
tis = lang195_tree_idxs
ti_list = []
new_tis = []
for ti in tis:
if ti in ti_list:
new_tis.append(ti_list.index(ti))
else:
ti_list.append(ti)
new_tis.append(len(ti_list) - 1)
tis = new_tis
plt.figure(figsize=(20, 20))
ax = plt.axes(projection=ccrs.PlateCarree())
ax.stock_img()
plt.scatter(lons, lats, marker='o', c=tis, cmap=plt.cm.jet)
plt.title('Language Families')
plt.colorbar(fraction=0.02, pad=0.01)
plt.savefig('../outputs/lang_fams.png')
kmeans = KMeans(n_clusters=5, random_state=0).fit(embs)
cd = ['b', 'c', 'y', 'm', 'r'] # color scheme, but can also use colormap
plt.figure(figsize=(20, 20))
ax = plt.axes(projection=ccrs.PlateCarree())
ax.stock_img()
plt.scatter(lons, lats, c=[cd[i] for i in kmeans.labels_], marker='o')
plt.savefig('../outputs/cluster_map.png')
# tsne plot: same language family has same color
tis = lang195_tree_idxs
min_fam_size = 5
min_fam_ti = -1
for ti in tis:
if len(ti_to_195_idxs_set[ti]) < min_fam_size:
min_fam_ti = ti
break
idxs_kept = []
for i, ti in enumerate(tis):
if len(ti_to_195_idxs_set[ti]) < min_fam_size:
tis[i] = min_fam_ti
else:
idxs_kept.append(i)
ti_to_color_id = {min_fam_ti: 0}
count = 1
for ti in tis:
if ti not in ti_to_color_id:
ti_to_color_id[ti] = count
count += 1
tis = np.array([ti_to_color_id[ti] for ti in tis])[idxs_kept]
perplexity = 15.0
learning_rate = 3e2
n_iter = 600
n_iter_without_progress = 300
random_state = 0
X_embedded = TSNE(n_components=2,
perplexity=perplexity,
learning_rate=learning_rate,
n_iter=n_iter,
n_iter_without_progress=n_iter_without_progress,
random_state=random_state).fit_transform(embs)
plt.figure(figsize=(10, 10))
plt.scatter(X_embedded[idxs_kept, 0],
X_embedded[idxs_kept, 1],
c=tis,
cmap=plt.cm.jet)
plt.savefig('../outputs/tnse.png')
with open('../outputs/metrics.json', 'w+') as f:
json.dump(metrics, f, indent=2)
start = timer()
build_oct(data, "root", 0, 0, 0, 0, 0, 0, 0, 0, 0, 0)
end = timer()
elif tree_choice == 2:
start = timer()
build_kd(data, 6, 0, "root", "root")
end = timer()
print("The tree has been built successfully.")
print(end - start)
elif choise_2 == 2:
importer = DictImporter()
if tree_choice == 1:
dict = ast.literal_eval(open("tree_save/oct_export.txt", encoding="utf-8").read())
nodes_oct = []
nodes2 = []
nodes2 = importer.import_(dict)
nodes_oct.append(nodes2)
elif tree_choice == 2:
dict = ast.literal_eval(open("tree_save/kd_export.txt", encoding="utf-8").read())
nodes = []
nodes2 = []
nodes2 = importer.import_(dict)
nodes.append(nodes2)
print(" The tree has been imported successfully")
while True:
print("==================================================")
print("Type the number of the action you want to execute")
print("1. Search")
print("2. Delete")
def load_labels_tree(data_path):
"""Loads a tree from json format."""
data = load_json_data(data_path)
importer = DictImporter(nodecls=AnyNode)
return importer.import_(data)
def word_print(word_str, word, verbosity):
# pretty print
title_styles = {
"noun": Fore.LIGHTRED_EX + "Noun" + Style.RESET_ALL,
"verb": Fore.LIGHTGREEN_EX + "Verb" + Style.RESET_ALL,
"adjective": Fore.LIGHTYELLOW_EX + "Adjective" + Style.RESET_ALL,
"adverb": Fore.LIGHTMAGENTA_EX + "Adverb" + Style.RESET_ALL
}
index = 0
if verbosity["audio"]:
audio_path = (Path("Vocabot") / "audio" / word_str).with_suffix(".mp3")
if audio_path.exists():
playsound(str(audio_path))
if "ordinal" in word: # a word that is scraped
# print main text
if word["short"] is not None and verbosity["short"]:
print(Fore.LIGHTGREEN_EX + word["short"] + Style.RESET_ALL + "\n")
if word["long"] is not None and verbosity["long"]:
print(Fore.GREEN + word["long"] + Style.RESET_ALL + "\n")
# print definitions
print(Fore.LIGHTWHITE_EX + "Definitions:" + Style.RESET_ALL)
for ordinal_index in range(len(word["ordinal"])):
print(Fore.LIGHTWHITE_EX + ("\n" if ordinal_index != 0 else "") +
str(ordinal_index + 1) + "-----" + Style.RESET_ALL)
for i in range(index, index + word["ordinal"][ordinal_index]):
print(Fore.LIGHTWHITE_EX + str(i - index + 1) +
Style.RESET_ALL)
meaning = word["meaning"][i]
if "title" in meaning and verbosity["title"]:
print(title_styles[meaning["title"]])
if "definition" in verbosity and verbosity["definition"]:
print(Fore.LIGHTGREEN_EX + meaning["definition"] +
Style.RESET_ALL)
if "example" in meaning:
for example in meaning["example"][:verbosity["example"]]:
print(Fore.GREEN + '"' + example + '"' +
Style.RESET_ALL)
instances = {
instance: dict(
list(meaning[instance].items())[-verbosity[instance]:])
for instance in
["antonyms", "synonyms", "types", "type of", "examples"]
if instance in meaning and meaning[instance] != {}
}
for instance in instances:
str_list = []
for (word_str, definition) in instances[instance].items():
if definition is None:
str_list.append("|" + word_str)
else:
str_list.append("|" + word_str + " > " +
definition)
print(Fore.LIGHTCYAN_EX + instance + Style.RESET_ALL +
"\n" + Fore.CYAN + "\n".join(str_list) +
Style.RESET_ALL)
index += word["ordinal"][ordinal_index]
# print family
print(Fore.LIGHTWHITE_EX + "\n\nWord Family:" + Style.RESET_ALL)
importer = DictImporter()
family = importer.import_(word["family"])
family = RenderTree(family, style=ContRoundStyle()).by_attr("freq")
family = "\n".join(family.split("\n")[1:verbosity["family"] + 1])
family += "\n╰──────"
print(Fore.LIGHTGREEN_EX + family + Style.RESET_ALL)
# print usage
if word["usage"] is not None:
print(Fore.LIGHTWHITE_EX + "\n\nUsage:" + Style.RESET_ALL)
for usage in word["usage"][:verbosity["usage"]]:
print(Fore.LIGHTCYAN_EX + usage["sentence"] + Style.RESET_ALL +
Fore.CYAN + "\n--" + usage["title"] + " " +
(usage["date"] if usage["date"] is not None else "") +
Style.RESET_ALL)
else: # a word from a list
word = word["meaning"][0]
if "definition" in word:
print(Fore.LIGHTGREEN_EX + word["definition"] + Style.RESET_ALL)
if "example" in word:
print(Fore.GREEN + word["example"] + Style.RESET_ALL)
if "examples" in word:
if "Example" in word["examples"]:
print(word["examples"]["Example"])
def get_tbs(self):
tb_url = "{}/api/content/v1/search".format(self.content_search)
payload = """{
"request": {
"filters": {
"contentType": ["Textbook"],
"status": ["Live"]
},
"sort_by": {"createdOn":"desc"},
"limit": 10000
}
}"""
headers = {
'content-type': "application/json; charset=utf-8",
'cache-control': "no-cache"
}
retry_count = 0
while 1:
try:
response = requests.request("POST",
tb_url,
data=payload,
headers=headers)
break
except requests.exceptions.ConnectionError:
print("Retry {} for textbook list".format(retry_count + 1))
retry_count += 1
sleep(10)
if retry_count == 5:
print("Max retries reached...")
break
list_of_textbooks = pd.DataFrame(response.json()['result']['content'])
list_of_textbooks = list_of_textbooks[[
'identifier', 'channel', 'board', 'gradeLevel', 'medium', 'name',
'subject'
]]
tb_list = list(list_of_textbooks.identifier.unique())
list_of_textbooks.drop_duplicates(subset=['identifier'],
keep='first',
inplace=True)
dialcode_df = pd.DataFrame()
tb_count = 0
for tb_id in tb_list:
tb_count = tb_count + 1
print("currently running for textbook number %d(%s)/%d" %
(tb_count, tb_id, len(tb_list)))
retry_count = 0
url = "{}/api/course/v1/hierarchy/{}".format(
self.content_hierarchy, tb_id)
while 1:
try:
response = requests.request("GET", url, headers=headers)
break
except requests.exceptions.ConnectionError:
print("Retry {} for TOC {}".format(retry_count + 1, tb_id))
retry_count += 1
sleep(10)
if retry_count == 5:
print("Max retries reached...")
print("Skipping the run for TB ID %s" % (tb_id))
failed_tbs.append(tb_id)
break
if response.json()['result'] != {}:
tb = response.json()['result']['content']
if 'children' in tb:
pass
else:
continue
if tb['children'] == None:
continue
if 'index' not in tb['children'][0]:
continue
tree_obj = self.traverse(tb)
importer = DictImporter()
root = importer.import_(tree_obj)
resources = findall(root,
filter_=lambda node: node.contentType in
("Resource"))
dialcodes = findall(root,
filter_=lambda node: node.dialcode not in
(""))
dialcodes_with_content = []
for resource in resources:
for ancestor in resource.ancestors:
dialcodes_with_content.append(
(ancestor.dialcode, ancestor.index))
dialcodes_with_content = set(
[x for x in dialcodes_with_content if (x[0] != '')])
dialcodes_all = []
for dialcode in dialcodes:
dialcodes_all.append((dialcode.dialcode, dialcode.index))
dialcodes_all = set([x for x in dialcodes_all if (x != '')])
no_content = pd.DataFrame(list(dialcodes_all -
dialcodes_with_content),
columns=['QR', 'Index'])
no_content['TB_ID'] = tb_id
no_content['status'] = 'no content'
with_content = pd.DataFrame(list(dialcodes_with_content),
columns=['QR', 'Index'])
with_content['TB_ID'] = tb_id
with_content['status'] = 'content linked'
final_df = with_content.copy()
final_df = final_df.append(no_content)
final_df['Index'].fillna(int(0), inplace=True)
final_df['Index'].loc[final_df['Index'] == ''] = 0
final_df.Index = final_df.Index.astype('category')
final_df.Index.cat.reorder_categories(natsorted(
set(final_df.Index)),
inplace=True,
ordered=True)
final_df_sorted_by_index = final_df.sort_values('Index')
ranks_to_be_assigned_for_positions_of_QR = list(
range(len(final_df_sorted_by_index.QR) + 1))[1:]
final_df_ranked_for_QR = final_df_sorted_by_index
final_df_ranked_for_QR[
'Position of QR in a TB'] = ranks_to_be_assigned_for_positions_of_QR
final_df_ranked_for_QR[
'Position of QR in a TB'] = final_df_ranked_for_QR[
'Position of QR in a TB'].astype(int)
dialcode_df = dialcode_df.append(final_df_sorted_by_index,
ignore_index=True)
dialcode_state = dialcode_df.merge(list_of_textbooks,
how='left',
left_on='TB_ID',
right_on='identifier')
dialcode_state_final = dialcode_state[[
'board', 'gradeLevel', 'QR', 'medium', 'subject', 'TB_ID', 'name',
'status', 'Index', 'Position of QR in a TB', 'channel'
]]
execution_date_str = datetime.strptime(self.execution_date,
"%d/%m/%Y").strftime('%Y-%m-%d')
os.makedirs(self.data_store_location.joinpath('tb_metadata',
execution_date_str),
exist_ok=True)
dialcode_state_final.to_csv(self.data_store_location.joinpath(
'tb_metadata', execution_date_str, 'qr_code_state.csv'),
index=False,
encoding='UTF-8')
post_data_to_blob(self.data_store_location.joinpath(
'tb_metadata', execution_date_str, 'qr_code_state.csv'),
backup=True)
a = open('7.txt').read().split('\n')[:-1]
a = [x.split(' -> ') for x in a]
a = [x[0].split() + [x[1].split(', ') if len(x) > 1 else []] for x in a]
a = {x[0]: (int(x[1][1:-1]), x[2]) for x in a}
root = (set(x for x in a) - set(chain(*[a[x][1] for x in a]))).pop()
def get_total_weight(node):
children = a[node][1]
if len(children) == 0:
return a[node][0]
else:
return a[node][0] + sum([get_total_weight(x) for x in children])
def create_dict(node):
children = a[node][1]
if len(children) == 0:
return {"a": a[node][0]}
else:
return {
"a": (a[node][0], get_total_weight(node)),
"children": [create_dict(x) for x in children]
}
importer = DictImporter()
tree = importer.import_(create_dict(root))
print RenderTree(tree)
fNaive = io.open("./runs/TreeTest2.csv", 'a')
fTrees = io.open("./DictStore.pkl", 'r').read().splitlines()
singleRunResults = RandomDict()
key = "DEFAULT, SHOULD NEVER BE SEEN"
for line in fTrees:
if line[0] == '.':
key = line
singleRunResults[key] = []
else:
tDict = eval(line)
root = importer.import_(tDict)
singleRunResults[key].append(root)
done = 0
for x in singleRunResults.keys:
for y in range(0, 10):
t = time.time()
z = singleRunResults.random_key()
while (z == x or z == x[:-5] + '2.txt'):
z = singleRunResults.random_key()
comp = treeTester.runDouble(singleRunResults[x], singleRunResults[z])
t = time.time() - t
output = x + ", " + z + ", " + str(comp) + ", " + str(t) + "\n"
print output
done += 1
node_ids = [i for i in node_ids]
return node_ids
if __name__ == '__main__':
input_directory = constants.input_data
output_directory = constants.output_data
input_files_term_hierarchy = constants.input_files_term_hierarchy
node_filter = output_directory + 'filtered_nodes/'+'node_filter_1_input.txt'
node_filter_output = output_directory + 'filtered_nodes/'+'node_filter_1_output.json'
with codecs.open(input_directory+input_files_term_hierarchy,encoding = "utf-8-sig") as json_file:
treedict = json.load(json_file)
# preprocess data with this pattern ,\n^\s*"children": null
# create a tree object
root = importer.import_(treedict)
#open the nodes file
main_nodes = open(node_filter).readlines()
complete_results = []
for node_name in main_nodes:
# Check if a combination of nodes
if not '+' in node_name:
if '[' not in node_name:
result = find_container_subnodes([node_name.strip()])
complete_results.append({node_name.strip():result})
else:
result = find_nodes_by_name_pattern(node_name.strip().split('[')[0])
complete_results.append({node_name.strip().split('[')[0]:result})
else:
partial_result = []
def run_cidm_chain(circuit_folder, conf_template_file, conf_output_file,
conf_cell_file, sub_folder, fitting_type):
conf_template_file = os.path.join(circuit_folder, conf_template_file)
conf_output_file = os.path.join(circuit_folder, conf_output_file)
conf_cell_file = os.path.join(circuit_folder, conf_cell_file)
fitting_type = FittingType(fitting_type)
with open(conf_cell_file) as json_file:
data = json.load(json_file)
# instead of generating a deepcopy for the results dictionary
with open(conf_cell_file) as json_file:
data_results = json.load(json_file)
simulations = data['simulations']
simulation_results = data_results['simulations']
replacements = data['replacements']
dependency_tree_dict = data['dependency_tree']
importer = DictImporter()
dependency_tree = importer.import_(dependency_tree_dict)
# IDM_STAR_FWD and IDM_STAR_BWD must have a dependency tree
assert (not (fitting_type == FittingType.IDM_STAR_FWD
or fitting_type == FittingType.IDM_STAR_BWD)
or dependency_tree)
if not dependency_tree:
dependency_tree = Node("Top")
for idx, simulation in enumerate(simulations):
Node(idx, parent=dependency_tree)
traverse_order = PreOrderIter
if fitting_type == FittingType.IDM_STAR_BWD:
traverse_order = PostOrderIter
counter = 0
simulate_all = True
if simulate_all:
for node in traverse_order(dependency_tree):
if node.name == 'Top':
continue
counter = counter + 1
print("===========================")
print("Progress: {counter}/{total}".format(counter=counter,
total=len(simulations)))
print("===========================")
# counter = counter + 1
# if counter == 4:
# break
simulation = simulations[node.name]
print(node.name, simulation)
perform_simulation(simulation, conf_template_file,
conf_output_file, replacements, circuit_folder,
sub_folder, fitting_type, simulations,
simulation_results, node, conf_cell_file, data)
else:
node_ids = [76, 126]
for node_id in node_ids:
simulation = simulations[node_id]
node = None
for n in traverse_order(dependency_tree):
if n.name == node_id:
node = n
break
print(node.name, simulation)
perform_simulation(simulation, conf_template_file,
conf_output_file, replacements, circuit_folder,
sub_folder, fitting_type, simulations,
simulation_results, node, conf_cell_file, data)
else:
obj[new_key] = "Node_type: " + obj[
key] + "\n" + "Value: " + obj[
'value'] + "\n" + "Line: " + str(obj['line'])
del obj[key]
return obj
if __name__ == '__main__':
arg_parser = argparse.ArgumentParser(description='Tree Gnerator!')
arg_parser.add_argument("--i",
default="AST.json",
type=str,
help="Input JSON file with Nodes")
arg_parser.add_argument("--o",
default="tree.png",
type=str,
help="Output image for tree ")
args: argparse.Namespace = arg_parser.parse_args()
input_filename: str = args.i
output_filename: str = args.o
with open(input_filename) as json_file:
data = json.load(json_file)
new_json = json.loads(json.dumps(data), object_hook=remove_dots)
root = importer.import_(new_json)
UniqueDotExporter(root).to_picture(output_filename)
with open('courses.csv', newline='') as file:
coursesInFile = csv.DictReader(file, delimiter=',')
for course in coursesInFile:
courses[course['course_code']] = course
selectedCourse = ''
print('--------------Welcome to the UCalgary Prerequisites Wizard!-----------')
selectedCourse = input(
'Select a UCalgary course you want to generate a prerequisites-tree from:')
prerequisitesTree = getPrerequisites(selectedCourse, courses)
importer = DictImporter()
root = importer.import_(prerequisitesTree)
RenderTreeGraph(root).to_picture("tree.png")
print('`tree.png` has been generated.')
if sys.platform == 'win32':
subprocess.Popen(['start', 'tree.png'], shell=True)
elif sys.platform == 'darwin':
subprocess.Popen(['open', 'tree.png'])
else:
try:
subprocess.Popen(['xdg-open', 'tree.png'])
except OSError:
print('Can\'t open file in system.')
# er, think of something else to try
output = file1 + ", " + file2 + ", " + str(len(f1)) + ", " + str(
len(f2)) + ", " + str(mDist1) + ", " + str(mDist2) + ", " + str(
mDist3) + ", " + str(runTime)
print output
return output
if __name__ == "__main__":
rrp = RerankingParser.fetch_and_load('WSJ-PTB3', verbose=False)
newTweet = "The quick brown fox jumps over the lazy dog."
newTree = rrp.simple_parse(newTweet)
sTree = parse_sexp(newTree)
pprint(sTree)
dTree = tMake(sTree[0])
root = importer.import_(dTree)
print RenderTree(root)
newTweet = "The quick brown fox, it jumped over the lazy dog slowly."
newTree = rrp.simple_parse(newTweet)
sTree = parse_sexp(newTree)
pprint(sTree)
dTree = tMake(sTree[0])
root = importer.import_(dTree)
print RenderTree(root)
#print runSingle(sys.argv[1], rrp)