def main(input_data, basement, num_layers, multiplier, num_patches, features):
if os.path.isfile(input_data):
data = load_file(input_data)
elif os.path.isdir(input_data):
data = load_folder(input_data)
else:
raise Exception('{} is neither file nor directory'.format(input_data))
if features:
try:
features = [int(f) for f in features.split(',')]
except:
raise Exception(
'Bad features: {}. Expecting a bunch of integers separated by comma'
.format(features))
if len(features) != num_layers:
raise Exception(
'{} features specified, but total number of layers is {}'.
format(len(features), num_layers))
H = Hierarchy(data)
for i in xrange(num_layers):
mult = 0 if i == 0 else multiplier
f = features[i] if features else 100
H.add_layer(basement * (multiplier**i), f, num_patches, mult)
H.learn()
H.visualize_layer(i)
def __init__(self, dir_name, org_name, peering_name):
self.dir = dir_name
self.org_name = org_name
self.peering_name = peering_name
self.prob = dict()
self.init_prob = dict()
self.siblings = set()
self.edge_infer = set()
self.edge_finish = set()
self.provider = defaultdict(set)
self.peer = defaultdict(set)
self.customer = defaultdict(set)
self.tripletRel = dict()
self.nonpath = dict()
self.distance2Clique = dict()
self.vp = dict()
self.ixp = defaultdict(int)
self.facility = defaultdict(int)
self.adjanceTypeRatio = dict()
self.degreeRatio = dict()
self.vppos = dict()
self.tier = Hierarchy(self.dir + 'asrel_prime_prob.txt')
self.ingestProb()
self.extractSiblings()
def initUI(self):
self.setWindowTitle(self.title)
self.setGeometry(self.left, self.top, self.width, self.height)
#tabbed view that displays the design hierarchy in tree form
self.folderView = QTabWidget()
self.constraintView = FileTree(self, None, None)
self.hierarchy = Hierarchy(self)
self.fileTree = FileTree(self, self.hierarchy, self.constraintView)
self.folderView.addTab(self.fileTree, "Files")
self.folderView.addTab(self.hierarchy, "Hierarchy")
self.folderView.addTab(self.constraintView, "Constraints")
#parts of the search bar to search for certain files in the tabbed view
searchBarLabel = QLabel()
searchBarLabel.setText("Search modules:")
self.searchBar = QLineEdit()
self.searchBar.setStatusTip('Search modules')
self.searchBar.textChanged.connect(self.fileTree.searchModule)
#tabbed view to view and edit files
self.fileView = QTabWidget()
self.fileView.tabBarClicked.connect(self.openTab)
self.fileView.tabCloseRequested.connect(self.closeTab)
self.fileView.setTabsClosable(True)
self.fileView.setMovable(True)
#helpers to keep track of which editor is being viewed by the user
self.textEdit = CodeEditor("verilog")
self.editors = {}
self.filePaths = {}
#creates the basic layout for the user interface
wid = QWidget(self)
self.setCentralWidget(wid)
rightSide = QWidget(self)
leftSide = QWidget(self)
hbox = QHBoxLayout(wid)
vrbox = QVBoxLayout(rightSide)
vrbox.addWidget(self.fileView)
vlbox = QVBoxLayout(leftSide)
vlbox.addWidget(searchBarLabel)
vlbox.addWidget(self.searchBar)
vlbox.addWidget(self.folderView)
rightSide.setLayout(vrbox)
leftSide.setLayout(vlbox)
main_splitter = QSplitter(Qt.Horizontal)
main_splitter.addWidget(leftSide)
main_splitter.addWidget(rightSide)
main_splitter.setStretchFactor(0, 1)
main_splitter.setStretchFactor(1, 6)
hbox.addWidget(main_splitter)
wid.setLayout(hbox)
# set up the status bar and file menus
self.statusBar()
self.setUpMenus()
self.center()
self.show()
def clusterSelected(self, event=None):
popup = Toplevel(self)
hierarchy = None
if (self.displayRepresentatives):
hierarchy = Hierarchy(representatives=map(
lambda s: self.findHierarchy(s).hierarchy, self.selected))
else:
reps = []
for tag in self.selected:
reps += self.findHierarchy(tag).hierarchy.representatives
hierarchy = Hierarchy(representatives=reps)
frame = GraphFrame(popup, hierarchy)
frame.pack(fill=BOTH, expand=1)
def newWindow(self, event=None):
#TODO: Pass off a subset of hierarchy to other graphFrame.
popup = Toplevel(self)
hierarchy = Hierarchy(representatives=map(
lambda s: self.findHierarchy(s).hierarchy, self.selected))
frame = GraphFrame(popup, hierarchy, showReps=True)
frame.pack(fill=BOTH, expand=1)
def get_questions_all(self):
h = Hierarchy()
questions_all = []
category_list = h.get_hierarchy_categories()
for x in category_list:
questions_all.extend(Category.get_category_questions(self, x))
return questions_all
def to_tree(self, root_id, priority_list=[]):
"""
Use a BFS from the root to decide the level of each node in the graph.
Associate the children to their parents according to a priority_list.
Input:
dag - a directer acyclic graph, format should be a dictionary
whose keys are all the node IDs and values are lists
of theirs children
root - the ID of the node that should be the root of the tree
"""
if root_id not in self:
raise KeyError('The chosen root is not in the GODag %s:' % root_id)
tree = Hierarchy()
tree.create_node(self[root_id].name, root_id, parent=None)
parents = [root_id]
level = 0
while parents:
logging.info("Tree level %02d, %d nodes" % (level, len(parents)))
children = set()
child_to_parent_list = {}
for parent in parents:
for child in [c.id for c in self[parent].children]:
if tree.get_node(child) is not None:
continue
children.add(child)
child_to_parent_list.setdefault(child, []).append(parent)
# associate each child with the parent with the highest priority
# in the priority list
for child, parent_list in child_to_parent_list.iteritems():
parent = parent_list[
0] # by default, choose the first parent in the list
# if one of the IDs in the priority list is a parent, use that
# one instead
for go_id in priority_list:
if go_id in parent_list:
parent = go_id
break
# we have to get rid of all colons in the names of the
# tree because Paver treats them as delimiters between
# display name and systematic name
child_name = self[child].name.replace(':', ';')
tree.create_node(child_name, child, parent)
parents = list(children)
level += 1
return tree
def __init__(self, groupSize, dirs):
self.groupSize = groupSize
self.dir = dirs
if not os.path.exists(self.dir):
os.mkdir(self.dir)
self.VP2AS = defaultdict(set)
self.VP2path = defaultdict(set)
self.fullVP = set()
self.partialVP = set()
self.VPGroup = list()
self.fileNum = -1
self.tier = Hierarchy('asrel.txt')
def to_shallow_tree(self, root_id):
"""
Use a BFS from the root to decide the level of each node in the graph.
For each child, keep only one of the edges leading to it so it will
have a single parent, from the previous level. Choose these edges
in a way which will keep the tree balanced (probably not optimally
though). This will assure that only one of the shortest paths from
the root to every node is kept.
Input:
dag - a directer acyclic graph, format should be a dictionary
whose keys are all the node IDs and values are lists
of theirs children
root - the ID of the node that should be the root of the tree
"""
if root_id not in self:
raise KeyError('The chosen root is not in the GODag %s:' % root_id)
tree = Hierarchy()
tree.create_node(self[root_id].name, root_id, parent=None)
parents = [root_id]
while parents:
children = set()
child_to_parent_list = {}
for parent in parents:
for child in [c.id for c in self[parent].children]:
if tree.get_node(child) is not None:
continue
children.add(child)
child_to_parent_list.setdefault(child, []).append(parent)
# the following list will contain pairs, counting the number of
# children that each parent has.
child_counts = [[0, p] for p in parents]
for child, parent_list in child_to_parent_list.iteritems():
# give this child to the parent with the least children
for i, (counter, parent) in enumerate(child_counts):
if parent in parent_list:
break
# we have to get rid of all colons in the names of the
# tree because Paver treats them as delimiters between
# display name and systematic name
child_name = self[child].name.replace(':', ';')
tree.create_node(child_name, child, parent)
child_counts[i][0] += 1
child_counts.sort()
return tree
def run(S: List, save_file_name: str, c: float = 1, d: int = 1):
logging.basicConfig(filename='run_logs/' + save_file_name + '.log',
format='%(asctime)s - %(message)s',
datefmt='%H:%M:%S',
level=logging.INFO)
logging.info('Entered debug mode.')
distances = utils.calc_euclidean_distances(np.array(S))
delta = utils.get_delta(distances)
t = int(np.ceil(np.log2(1 / delta)))
logging.info('Points:\n{}'.format(str(S)))
# logging.info('Distances:\n{}'.format(str(distances)))
# logging.info('Minimal distance (delta): {}'.format(str(delta)))
logging.info('t: {}'.format(str(t)))
# print(str(arr_scaler(distances)))
logging.info('Building hierarchy.')
h = Hierarchy(S, distances, c, t)
logging.info('Done.')
logging.info('Building LP.')
lp = LinearProgram(h, d, delta, save_file_name)
logging.info('Done.')
logging.info('Solving.')
lp.solve()
logging.info('Done.')
w_hierarchy = []
for var in [var for var in lp.model.variables() if var.name.startswith('z') and var.value() == 1]:
split = var.name.split('_')
i = int(split[1][1:])
j = int(split[2][1:])
while len(w_hierarchy) <= i:
w_hierarchy.append([])
w_hierarchy[i].append(j)
w_hierarchy = [sorted(w) for w in w_hierarchy]
W = [S[w] for w in w_hierarchy[-1]]
logging.info(f"W hierarchy: {w_hierarchy}")
logging.info(f"Recall S: {S}")
S_np = np.array(S)
logging.info(f"W: {W}")
W_np = np.array(W)
if W == S:
print("(W == S)")
return W
def __init__(self,
config,
environment,
choice="prop",
llb=False,
explo="babbling",
n_explo_points=0,
choose_children_mode='competence',
choose_children_local=True,
allow_split_mod1=False):
Observable.__init__(self)
self.config = config
self.log = None
self.environment = environment
self.choice = choice
self.llb = llb
self.explo = explo
self.n_explo_points = n_explo_points
self.choose_children_mode = choose_children_mode
self.ccm_local = choose_children_local
self.allow_split_mod1 = allow_split_mod1
self.conf = self.config.agent
self.expl_dims = self.config.agent.m_dims
self.inf_dims = self.config.agent.s_dims
self.t = 1
self.modules = {}
self.chosen_modules = {}
self.mid_control = ''
self.last_space_children_choices = {}
self.split_mod1_maxlen = 10
self.previous_ms = deque([None] * self.split_mod1_maxlen,
maxlen=self.split_mod1_maxlen)
self.hierarchy = Hierarchy() # Build Hierarchy
for motor_space in self.config.m_spaces.values():
self.hierarchy.add_motor_space(motor_space)
for mid in self.config.modules.keys(): # Build modules
self.init_module(mid)
#[set.add(self.modules[mid].controled_vars, cvar) for cmid in self.config.modules[mid]['children'] if self.hierarchy.is_mod(cmid) for cvar in self.modules[cmid].controled_vars]
for mid in self.modules.keys():
self.last_space_children_choices[mid] = Queue.Queue()
def __pb_classificate_click(self):
self.__remove_chart_from_layout()
self.__init_chart()
try:
self.__amount_of_objects = int(self.ui.le_amount_of_objects.text())
except ValueError:
pass
# self.__distances = [[0.0, 6.0, 5.0], [6.0, 0.0, 2.0], [5.0, 2.0, 0.0]]
self.__distances = self.__generate_random_distances(
self.__amount_of_objects)
print(self.__distances)
hierarchy = Hierarchy(self.__distances, self.__amount_of_objects)
hierarchy.find_groups()
hierarchy.draw(self.__chart)
self.__chart.createDefaultAxes()
self.__update_form()
def main():
d = Diagnosis()
c = Category()
l = Level()
q = Question()
e = Emergency_followup()
hist = Historik()
value_list = [3, 4, 5, 4, 5, 6, 6, 5, 4]
#answer = q.get_question("sex.txt", 1)
#print(answer)
#cat_list = c.get_category_questions("sex.txt")
#print(cat_list)
#answer = l.get_level_category(3)
#print(answer)
h = Hierarchy()
#answer = h.get_hierarchy_categories()
#print(answer)
I = Interface()
#answer = I.get_questions_all()
#print(answer)
#answer = I.get_question_akut()
#print(answer)
#answer = d.setup_diagnosis()
#answer = d.read_diagnosis()
#answer = d.write_diagnosis("this is question nr 3", "A diagnos")
#answer = d.get_diagnosis("this is question nr 3")
#answer = e.setup_followup()
#answer = e.change_followup("How have your general health been today?", "Go for a run")
#answer = e.read_followup("How have your general health been today?")
#answer = e.new_followup()
#answer = I.interface_questions_akut()
#answer = I.interface_daily_questions()
#answer = hist.set_historik_total([1, 2 , 3, 6])
#answer = hist.get_historik_all_total()
answer = l.level_count()
print(answer)
def loadData(dataname):
dirpath = "data/" + dataname
dat_claims, dat_gt, dat_h = loadDataUnicode(dirpath)
#Reform claims
claims, names, srcnames = reform_claims(dat_claims)
claimsByObj = recordsByObject(claims, len(names))
#Build Hierarchy
h = Hierarchy()
h.build(dat_h)
#h.printH()
h.setMaxDepth()
if "heritages" in dataname:
h.preprocessed = True
else:
h.preprocessed = False
print("#claims: %d, #entities: %d, claims/entities: %f" %
(len(claims), len(names), float(len(claims)) / len(names)))
return claims, claimsByObj, names, dat_gt, srcnames, h
def print_s3_contents_boto3(connection, bucket):
#for bucket in connection.buckets.all():
for key in bucket.objects.all():
print(key.key)
if __name__ == '__main__':
boto3_connection = boto3.resource('s3')
s3_client = boto3.client('s3')
bucket = boto3_connection.Bucket('econ-demog')
bucket_name = 'econ-demog'
quote_page = 'https://www2.census.gov/acs/downloads/Core_Tables/rural_statistics_area/2007/Wyoming/'
quote_list = Hierarchy(quote_page)
#key = boto.s3.key.Key(bucket,file_object)
#filepath = '/home/david/PublicData/acs_files/'
#
for i in quote_list.files:
file_name = i[i.rfind('/') + 1:]
folder = file_name[:-4]
filepath = '/home/david/PublicData/acs_files/' + folder + '/'
if not os.path.exists(filepath):
os.makedirs(filepath)
quote_list.load_zip(i, filepath)
acs_files = [f for f in listdir(filepath) if isfile(join(filepath, f))]
#k = bucket.new_key(folder)
except:
print('ERROR')
af.write('ERROR\n')
qf = open('LCQ_SQ')
qlines = qf.readlines()
ef = open('LCQ_SE')
elines = ef.readlines()
pf = open('LCQ_SP')
plines = pf.readlines()
rf = open('LCQ_SR')
rlines = rf.readlines()
sf = open('LCQ_SS')
slines = sf.readlines()
h = Hierarchy('./hierarchy.txt')
# qline = qlines[38].strip()
# eline = elines[38].strip()
# pline, answerp = tuple(plines[38].strip().split('|'))
# pline = ast.literal_eval(pline)
# rline = rlines[38].strip()
# answerp = int(answerp)
# singleEMultiRQuick(qline, eline, pline, rline, answerp)
af = open('LCQ_SA_BERT', 'w')
for i in range(len(qlines)):
print(i)
qline = qlines[i].strip()
eline = elines[i].strip()
# eline = eline.split(';')
#Read in dataset into annotations and vocabulary
for filename in listdir(getcwd() + '/datasets/' + dataset):
with open(Path(getcwd() + '/datasets/' + dataset + '/' + filename),
encoding='utf-8') as input_file:
lines = input_file.readlines()
assert len(lines) == 1
annotations[filename] = lines[0].lower().strip().split(" ")
for tag in annotations[filename]:
if tag not in vocabulary:
vocabulary.append(tag)
#Run smict, the first part of the algorithm
subsumption_axioms, root_tag = smict(annotations, vocabulary, alpha)
#Initialize hierarchy from subsumption axioms
hierarchy = Hierarchy(subsumption_axioms, root_tag)
#Perform subject clustering, the second part of our algorithm
hierarchy = subject_clustering(hierarchy, annotations)
#Prune the tree of empty clusters, the final part of our algorithm
hierarchy.prune(hierarchy.get_root())
#Write pretty printed hierarchy and json summary of hierarchy
with Path('smich_cluster_hierarchy_' + dataset).open(
'w', encoding="utf-8") as output_file:
write_hierarchy(output_file, hierarchy.get_root())
with open('smich_cluster_hierarchy_json_' + dataset, 'w') as json_file:
dump(generate_json_hierarchy(hierarchy), json_file)
