def parse(edges_path, source_header='source', target_header='target', weight_header='weight',
edge_limit=None) -> Graph:
"""
Parse graph from input file
:param edges_path: input file
:param source_header: header for source node
:param target_header: header for target node
:param weight_header: header for weight
:param edge_limit: limit the number of edges to parse from file
:return: Graph object
"""
# ToDo: parse by streaming to avoid loading a large graph to memmory
# ToDo: parse nodes and edges properties
# ToDo: validate inputs
name = get_name_from_path(edges_path)
edges = list()
with open(edges_path, mode='r') as edges_file:
edges_reader = csv.DictReader(edges_file, delimiter=',')
for i, row in enumerate(edges_reader):
if edge_limit is not None and i >= edge_limit:
break
edges.append((
int(row[source_header]),
int(row[target_header]),
float(row[weight_header])
))
graph = Graph(name)
graph.add_edges(edges)
return graph
def test_3_generalisation_sort_order(self):
# START AGAIN - more tests, ensure children nodes with children themselves, are prioritised
# C2,C --|> B
# B,B2 --|> A
g = Graph()
c = GraphNode('C', 0, 0, 200, 200)
c2 = GraphNode('C2', 0, 0, 200, 200)
b = GraphNode('B', 0, 0, 200, 200)
b2 = GraphNode('B2', 0, 0, 200, 200)
a = GraphNode('A', 0, 0, 200, 200)
# add out of order
g.AddNode(b2)
g.AddNode(b)
g.AddNode(c)
g.AddNode(c2)
g.AddNode(a)
g.AddEdge(c, b)['uml_edge_type'] = 'generalisation'
g.AddEdge(c2, b)['uml_edge_type'] = 'generalisation'
g.AddEdge(b2, a)['uml_edge_type'] = 'generalisation'
g.AddEdge(b, a)['uml_edge_type'] = 'generalisation'
nodelist_normal = [node.id for node in g.nodes]
nodelist_sorted = [node.id for node,annotation in g.nodes_sorted_by_generalisation]
nodelist_sorted_expected = ['A', 'B', 'B2', 'C', 'C2']
nodelist_sorted_expected2 = ['A', 'B', 'B2', 'C2', 'C']
#print "nodelist_normal", nodelist_normal
#print "nodelist_sorted_expected", nodelist_sorted_expected
#print "nodelist_sorted", nodelist_sorted
assert nodelist_sorted_expected == nodelist_sorted or \
nodelist_sorted_expected2 == nodelist_sorted
def train(model_config):
print(list_config(model_config))
data = TrainData(model_config)
graph = Graph(True, model_config, data)
graph.create_model_multigpu()
with tf.train.MonitoredTrainingSession(
checkpoint_dir=model_config.logdir,
save_checkpoint_secs=model_config.save_model_secs,
config=get_session_config(),
) as sess:
# with tf.Session(config=get_session_config()) as sess:
# sess.run(tf.initialize_all_variables())
ckpt = tf.train.get_checkpoint_state(model_config.logdir)
if ckpt:
graph.saver.restore(sess, ckpt.model_checkpoint_path)
perplexitys = []
start_time = datetime.now()
previous_step = 0
while True:
input_feed, _, _ = get_feed(graph.objs, data, model_config, True)
fetches = [graph.train_op, graph.increment_global_step, graph.global_step,
graph.perplexity]
_, _, step, perplexity = sess.run(fetches, input_feed)
perplexitys.append(perplexity)
if (step - previous_step) > model_config.model_print_freq:
end_time = datetime.now()
time_span = end_time - start_time
start_time = end_time
print('Perplexity:\t%f at step %d using %s.' % (perplexity, step, time_span))
perplexitys.clear()
previous_step = step
def setUp(self):
# Hyperparameters
num_nodes = 100
num_edges = 500
weight_max = 100
self.edges = []
self.nodes = [Node()]
# Build connected nodes
for _ in range(num_nodes - 1):
new = Node()
self.nodes.append(new)
old = choice(self.nodes)
edge = Edge(randint(0, weight_max))
self.edges.append(edge)
old.connect_to(new, edge)
# Add random connections
missing_connections = [
(n1, n2) for n1 in self.nodes
for n2 in list(set(self.nodes) - set(n1.get_neighbors()))
]
for i in range(num_edges - (num_nodes - 1)):
a, b = missing_connections.pop(
randint(0,
len(missing_connections) - 1))
edge = Edge(randint(0, weight_max))
self.edges.append(edge)
a.connect_to(b, edge)
self.edges = list(set(self.edges))
self.graph = Graph(self.nodes)
def getGraph(self):
if self.graph == None:
self.graph = Graph()
self.graph.initFromLists(self.pageNumber, self.node_rk_generator(),
self.edge_generator())
return self.graph
def test():
it = data.get_data_iter()
graph = Graph(is_train=False)
tf.reset_default_graph()
graph.create_model()
sv = tf.train.Supervisor(logdir=get_path(args.logdir),
global_step=graph.global_step)
sess = sv.PrepareSession()
losses = []
scores = 0
total = 0
while True:
input_feed, gt_last_events, effective_batch_size, is_end = get_data(
graph.inputs_ph, it)
fetches = [graph.last_event, graph.loss, graph.global_step]
last_events, loss, step = sess.run(fetches, input_feed)
for i in range(effective_batch_size):
losses.append(loss)
total += 1
if last_events[i] == gt_last_events[i]:
scores += 1
if is_end:
break
file = 'Accurary%sLoss%s.txt' % (scores / total, np.mean(losses))
output = 'Accurary:\t%s Loss:\t%s' % (scores / total, np.mean(losses))
if not os.path.exists(get_path(args.resultdir)):
os.mkdir(get_path(args.resultdir))
f = open(get_path(args.resultdir) + file, mode='w', encoding='utf-8')
f.write(output)
f.close()
def test_7_generalisation_multiple_inhertitance(self):
# START AGAIN - more tests, check multiple inheritance trees
# See 'python-in/testmodule08_multiple_inheritance.py'
# for another related unit test
# F --|> M
# F --|> S
g = Graph()
f = GraphNode('F', 0, 0, 200, 200)
m = GraphNode('M', 0, 0, 200, 200)
s = GraphNode('S', 0, 0, 200, 200)
g.AddEdge(f, m)['uml_edge_type'] = 'generalisation'
g.AddEdge(f, s)['uml_edge_type'] = 'generalisation'
nodelist_normal = [node.id for node in g.nodes]
#print "nodelist_normal", nodelist_normal
nodelist_sorted = [node.id for node,annotation in g.nodes_sorted_by_generalisation]
nodelist_sorted_expected = ['M', 'F', 'S']
assert nodelist_sorted_expected == nodelist_sorted, nodelist_sorted
#print "nodelist_sorted_expected", nodelist_sorted_expected
#print "nodelist_sorted", nodelist_sorted
nodelist_sorted_annotated = [(node.id, annotation) for node,annotation in g.nodes_sorted_by_generalisation]
nodelist_sorted_expected_annotated = [('M', 'root'), ('F', 'root'), ('S', 'root')]
assert nodelist_sorted_expected_annotated == nodelist_sorted_annotated, nodelist_sorted_annotated
def test_7_generalisation_multiple_inhertitance(self):
# START AGAIN - more tests, check multiple inheritance trees
# See 'python-in/testmodule08_multiple_inheritance.py'
# for another related unit test
# F --|> M
# F --|> S
g = Graph()
f = GraphNode('F', 0, 0, 200, 200)
m = GraphNode('M', 0, 0, 200, 200)
s = GraphNode('S', 0, 0, 200, 200)
g.AddEdge(f, m)['uml_edge_type'] = 'generalisation'
g.AddEdge(f, s)['uml_edge_type'] = 'generalisation'
nodelist_normal = [node.id for node in g.nodes]
#print "nodelist_normal", nodelist_normal
nodelist_sorted = [
node.id for node, annotation in g.nodes_sorted_by_generalisation
]
nodelist_sorted_expected = ['M', 'F', 'S']
assert nodelist_sorted_expected == nodelist_sorted, nodelist_sorted
#print "nodelist_sorted_expected", nodelist_sorted_expected
#print "nodelist_sorted", nodelist_sorted
nodelist_sorted_annotated = [
(node.id, annotation)
for node, annotation in g.nodes_sorted_by_generalisation
]
nodelist_sorted_expected_annotated = [('M', 'root'), ('F', 'root'),
('S', 'root')]
assert nodelist_sorted_expected_annotated == nodelist_sorted_annotated, nodelist_sorted_annotated
def setUp(self):
class FakeGui:
def stateofthenation(self, recalibrate=False, auto_resize_canvas=True):
pass
self.g = Graph()
self.overlap_remover = OverlapRemoval(self.g, margin=5, gui=FakeGui())
def __init__(self,
geo_hiddens,
rnn_type,
rnn_hiddens,
graph_type,
graph,
input_dim,
output_dim,
use_sampling,
cl_decay_steps,
prefix=None):
super(Seq2Seq, self).__init__(prefix=prefix)
# initialize encoder
with self.name_scope():
encoder_cells = []
encoder_graphs = []
for i, hidden_size in enumerate(rnn_hiddens):
pre_hidden_size = input_dim if i == 0 else rnn_hiddens[i - 1]
c = RNNCell.create(rnn_type[i],
pre_hidden_size,
hidden_size,
prefix='encoder_c%d_' % i)
g = Graph.create_graphs('None' if i == len(rnn_hiddens) -
1 else graph_type[i],
graph,
hidden_size,
prefix='encoder_g%d_' % i)
encoder_cells.append(c)
encoder_graphs.append(g)
self.encoder = Encoder(encoder_cells, encoder_graphs)
# initialize decoder
with self.name_scope():
decoder_cells = []
decoder_graphs = []
for i, hidden_size in enumerate(rnn_hiddens):
pre_hidden_size = input_dim if i == 0 else rnn_hiddens[i - 1]
c = RNNCell.create(rnn_type[i],
pre_hidden_size,
hidden_size,
prefix='decoder_c%d_' % i)
g = Graph.create_graphs(graph_type[i],
graph,
hidden_size,
prefix='decoder_g%d_' % i)
decoder_cells.append(c)
decoder_graphs.append(g)
self.decoder = Decoder(decoder_cells, decoder_graphs, input_dim,
output_dim, use_sampling, cl_decay_steps)
# initalize geo encoder network
self.geo_encoder = MLP(geo_hiddens,
act_type='relu',
out_act=True,
prefix='geo_encoder_')
def graph_list(cls, user_name, type, is_published=True):
if is_published:
graphs = Graph.select().filter(Graph.type == type,
Graph.is_published).order_by(
Graph.created_at.desc()).all()
else:
graphs = Graph.select().filter(
Graph.type == type, Graph.user_name == user_name).order_by(
Graph.created_at.desc()).all()
return [graph.get_json() for graph in graphs]
def predict_from_checkpoint(model_config, ckpt):
"""
Make prediction using transformer, return list of InstancePred.
:param model_config:
:param ckpt:
:return:
"""
# Eval only uses single GPU
assert model_config.num_gpus == 1
train_data = TrainData(model_config)
graph = Graph(False, model_config, train_data)
tf.reset_default_graph()
graph.create_model_multigpu()
sess = tf.train.MonitoredTrainingSession(config=get_session_config())
try:
graph.saver.restore(sess, ckpt)
except tf.errors.NotFoundError:
# Partial restore
import tensorflow.contrib.slim as slim
var_list = slim.get_variables_to_restore()
available_vars = {}
reader = tf.train.NewCheckpointReader(ckpt)
var_dict = {var.op.name: var for var in var_list}
for var in var_dict:
if reader.has_tensor(var):
var_ckpt = reader.get_tensor(var)
var_cur = var_dict[var]
if any([
var_cur.shape[i] != var_ckpt.shape[i]
for i in range(len(var_ckpt.shape))
]):
print('Variable %s missing due to shape.', var)
else:
available_vars[var] = var_dict[var]
else:
print('Variable %s missing.', var)
partial_restore_ckpt = slim.assign_from_checkpoint_fn(
ckpt_path,
available_vars,
ignore_missing_vars=True,
reshape_variables=False)
partial_restore_ckpt(sess)
# instance_collections = evaluate_on_testsets(sess, graph, train_data)
evaluate_and_write_to_disk(sess,
graph,
model_config,
train_data,
output_file_path=model_config.logdir)
def setUp(self):
self.graph = Graph()
v1 = self.graph.add_actor_vertex("actor1", {'age':38})
v2 = self.graph.add_actor_vertex("actor2", {'age':26})
v3 = self.graph.add_actor_vertex("actor3", {'age':10})
v4 = self.graph.add_movie_vertex("movie1", {'year':2017, 'gross':17000, 'lang': ["English"]})
v5 = self.graph.add_movie_vertex("movie2", {'year':2017, 'gross':170000, 'country': ["China"]})
self.graph.add_edge("actor1", "movie1", 3)
self.graph.add_edge("actor2", "movie1", 4)
self.graph.add_edge("actor3", "movie2", 5)
self.graph.add_edge("actor3", "movie3", 6)
class OverlapTests(unittest.TestCase):
def setUp(self):
class FakeGui:
def stateofthenation(self,
recalibrate=False,
auto_resize_canvas=True):
pass
self.g = Graph()
self.overlap_remover = OverlapRemoval(self.g, margin=5, gui=FakeGui())
def tearDown(self):
pass
def testStress1(self):
for i in range(10):
self.g.LoadGraphFromStrings(TEST_GRAPH5_STRESS)
print i,
were_all_overlaps_removed = self.overlap_remover.RemoveOverlaps()
self.assertTrue(were_all_overlaps_removed)
self.g.Clear()
def testStress2_InitialBoot(self):
"""
This is the slowest stress test because it runs the spring layout several times.
"""
from layout.layout_spring import GraphLayoutSpring
from layout.coordinate_mapper import CoordinateMapper
self.g.LoadGraphFromStrings(
GRAPH_INITIALBOOT) # load the scenario ourselves
layouter = GraphLayoutSpring(self.g)
coordmapper = CoordinateMapper(self.g, (800, 800))
def AllToLayoutCoords():
coordmapper.AllToLayoutCoords()
def AllToWorldCoords():
coordmapper.AllToWorldCoords()
for i in range(8):
print i,
AllToLayoutCoords()
layouter.layout(keep_current_positions=False)
AllToWorldCoords()
were_all_overlaps_removed = self.overlap_remover.RemoveOverlaps()
self.assertTrue(were_all_overlaps_removed)
class TestGraph(TestCase):
def setUp(self):
init_db()
self.graph = Graph(db_session)
def tearDown(self):
Base.metadata.drop_all(bind=engine)
@classmethod
def tearDownClass(cls):
db_session.remove()
def test_add(self):
self.graph.add(actor_item)
self.assertEqual(self.graph.get_actor("a").wiki_page, actor_item.get("wiki_page"))
def get_share_img(cls, user_name, id):
graph = Graph.select().get(id)
if graph is None:
raise ServerException(msg='图片不存在')
if not graph.is_published and user_name != graph.user_name:
raise ServerException(msg='该图未发布')
return graph
def test_4(self):
"""
Upgrade 1.1 to 1.1
"""
g = Graph()
filedata = """
# PynSource Version 1.1
{'type':'meta', 'info1':'Lorem ipsum dolor sit amet, consectetur adipiscing elit.'}
{'type':'umlshape', 'id':'UmlShapeCanvas', 'x':237, 'y':65, 'width':226, 'height':781, 'attrs':'scrollStepX|scrollStepY|classnametoshape|log|frame|save_gdi|working|font1|font2|umlworkspace|layout|coordmapper|layouter|overlap_remover', 'meths':'__init__|AllToLayoutCoords|AllToWorldCoords|onKeyPress|CmdInsertNewumlshape|CmdZapShape|Clear|ConvertParseModelToUmlModel|BuildEdgeModel|Go|CreateUmlShape|newRegion|CreateUmlEdge|OnWheelZoom|ChangeScale|stage1|stage2|stateofthenation|stateofthespring|RedrawEverything|ReLayout|LayoutAndPositionShapes|setSize|DecorateShape|createumlshapeShape|AdjustShapePosition|Redraw222|get_umlboxshapes|OnDestroy|OnLeftClick|DeselectAllShapes'}
{'type':'umlshape', 'id':'Log', 'x':1089, 'y':222, 'width':82, 'height':67, 'attrs':'', 'meths':'WriteText'}
{'type':'edge', 'id':'UmlShapeCanvas_to_MainApp', 'source':'UmlShapeCanvas', 'target':'MainApp', 'uml_edge_type':'composition'}
"""
model.graph_persistence.PERSISTENCE_CURRENT_VERSION = 1.1
self.assertTrue(
g.persistence.UpgradeToLatestFileFormatVersion(filedata))
#print g.persistence.filedata_list
assert g.persistence.filedata_list[
0] == "# PynSource Version 1.1", g.persistence.filedata_list[0]
assert "meta" in g.persistence.filedata_list[
1], g.persistence.filedata_list[1]
assert g.persistence.ori_file_version == 1.1
# now check type node has been converted to type umlshape
data = eval(g.persistence.filedata_list[2])
self.assertEquals('umlshape', data.get('type'))
def query(cls, id):
graph = Graph.select().get(id)
data = QiniuService.get_doc(graph.data_url)
graph_data = json.loads(data)
result = graph.get_json()
result['graph_data'] = graph_data
return result
def test_3(self):
"""
Upgrade 1.0 to 1.1
"""
g = Graph()
filedata = """
# PynSource Version 1.0
{'type':'node', 'id':'UmlShapeCanvas', 'x':237, 'y':65, 'width':226, 'height':781, 'attrs':'scrollStepX|scrollStepY|classnametoshape|log|frame|save_gdi|working|font1|font2|umlworkspace|layout|coordmapper|layouter|overlap_remover', 'meths':'__init__|AllToLayoutCoords|AllToWorldCoords|onKeyPress|CmdInsertNewNode|CmdZapShape|Clear|ConvertParseModelToUmlModel|BuildEdgeModel|Go|CreateUmlShape|newRegion|CreateUmlEdge|OnWheelZoom|ChangeScale|stage1|stage2|stateofthenation|stateofthespring|RedrawEverything|ReLayout|LayoutAndPositionShapes|setSize|DecorateShape|createNodeShape|AdjustShapePosition|Redraw222|get_umlboxshapes|OnDestroy|OnLeftClick|DeselectAllShapes'}
{'type':'node', 'id':'Log', 'x':1089, 'y':222, 'width':82, 'height':67, 'attrs':'', 'meths':'WriteText'}
{'type':'node', 'id':'MainApp', 'x':788, 'y':217, 'width':234, 'height':717, 'attrs':'log|andyapptitle|frame|notebook|umlwin|yuml|asciiart|multiText|popupmenu|next_menu_id|printData|box|canvas|preview', 'meths':'OnInit|OnResizeFrame|OnRightButtonMenu|OnBuildGraphFromUmlWorkspace|OnSaveGraphToConsole|OnSaveGraph|OnLoadGraphFromText|OnLoadGraph|LoadGraph|OnTabPageChanged|InitMenus|Add|FileImport|FileImport2|FileImport3|FileNew|FilePrint|OnAbout|OnVisitWebsite|OnCheckForUpdates|OnHelp|OnDeleteNode|OnLayout|OnRefreshUmlWindow|MessageBox|OnButton|OnCloseFrame'}
{'type':'node', 'id':'MyEvtHandler', 'x':10, 'y':173, 'width':170, 'height':285, 'attrs':'log|frame|shapecanvas|popupmenu', 'meths':'__init__|UpdateStatusBar|OnLeftClick|_SelectNodeNow|OnEndDragLeft|OnSizingEndDragLeft|OnMovePost|OnPopupItemSelected|OnRightClick|RightClickDeleteNode'}
{'type':'edge', 'id':'UmlShapeCanvas_to_MainApp', 'source':'UmlShapeCanvas', 'target':'MainApp', 'uml_edge_type':'composition'}
{'type':'edge', 'id':'ImageViewer_to_MainApp', 'source':'ImageViewer', 'target':'MainApp', 'uml_edge_type':'composition'}
{'type':'edge', 'id':'UmlShapeCanvas_to_MainApp', 'source':'UmlShapeCanvas', 'target':'MainApp', 'uml_edge_type':'composition'}
"""
model.graph_persistence.PERSISTENCE_CURRENT_VERSION = 1.1
self.assertTrue(g.persistence.UpgradeToLatestFileFormatVersion(filedata))
#print g.persistence.filedata_list
assert g.persistence.filedata_list[0] == "# PynSource Version 1.1", g.persistence.filedata_list[0]
assert "meta" in g.persistence.filedata_list[1], g.persistence.filedata_list[1]
assert g.persistence.ori_file_version == 1.0
# now check type node has been converted to type umlshape
data = eval(g.persistence.filedata_list[2])
self.assertEquals('umlshape', data.get('type'))
def test_connect(self):
graph = Graph()
self.assertEqual([], graph.get_nodes())
self.assertEqual([], graph.get_edges())
n1, n2, e = Node(), Node(), Edge()
graph.connect(n1, n2, e)
self.assertEqual(set((n1, n2)), set(graph.get_nodes()))
self.assertEqual([e], graph.get_edges())
def __init__(self, agent, kb, env):
super(GraphNeuralSession, self).__init__(agent, kb, env)
self.graph = GraphBatch([Graph(kb)])
self.utterances = None
self.context = None
self.graph_data = None
self.init_checklists = None
def test_2_generalisation_sort_order(self):
# C --|> B --|> A
g = Graph()
c = GraphNode('C', 0, 0, 200, 200)
b = GraphNode('B', 0, 0, 200, 200) # parent of C
a = GraphNode('A', 0, 0, 200, 200) # parent of B
# add out of order
g.AddNode(b)
g.AddNode(c)
g.AddNode(a)
g.AddEdge(c, b)['uml_edge_type'] = 'generalisation'
g.AddEdge(b, a)['uml_edge_type'] = 'generalisation'
nodelist_normal = [node.id for node in g.nodes]
nodelist_sorted = [node.id for node,annotation in g.nodes_sorted_by_generalisation]
nodelist_sorted_expected = ['A', 'B', 'C']
#print "nodelist_normal", nodelist_normal
#print "nodelist_sorted_expected", nodelist_sorted_expected
#print "nodelist_sorted", nodelist_sorted
assert nodelist_sorted_expected == nodelist_sorted
# D --|> C --|> B --|> A
d = GraphNode('D', 0, 0, 200, 200)
g.AddNode(d)
g.AddEdge(d, c)['uml_edge_type'] = 'generalisation'
nodelist_sorted = [node.id for node,annotation in g.nodes_sorted_by_generalisation]
nodelist_sorted_expected = ['A', 'B', 'C', 'D']
assert nodelist_sorted_expected == nodelist_sorted
# E node not connected to anything
e = GraphNode('E', 0, 0, 200, 200)
g.AddNode(e)
nodelist_sorted = [node.id for node,annotation in g.nodes_sorted_by_generalisation]
nodelist_sorted_expected = ['A', 'B', 'C', 'D', 'E']
assert nodelist_sorted_expected == nodelist_sorted
# D --|> C --|> B --|> A
# E
# C2 --|> B
c2 = GraphNode('C2', 0, 0, 200, 200)
g.AddNode(c2)
g.AddEdge(c2, b)['uml_edge_type'] = 'generalisation'
nodelist_sorted = [node.id for node,annotation in g.nodes_sorted_by_generalisation]
nodelist_sorted_expected = ['A', 'B', 'C', 'C2', 'D', 'E']
assert nodelist_sorted_expected == nodelist_sorted
def test_6(self):
"""
Empty file
"""
g = Graph()
filedata = """
"""
model.graph_persistence.PERSISTENCE_CURRENT_VERSION = 1.1
self.assertFalse(g.persistence.can_I_read(filedata)[0])
self.assertFalse(g.persistence.UpgradeToLatestFileFormatVersion(filedata))
def delete(cls, user_name, id):
graph = Graph.select().get(id)
assert graph is not None
assert graph.user_name == user_name
assert QiniuService.delete_file(
bucket_name=config.QI_NIU.get('doc_bucket_name'),
file_name=graph.data_key)
assert QiniuService.delete_file(
bucket_name=config.QI_NIU.get('img_bucket_name'),
file_name=graph.img_key)
graph.delete()
return True
def test_4_generalisation_sort_order(self):
# START AGAIN - more tests, ensure children nodes with children themselves, are prioritised
# and furthermore, children with the most descendants are prioritised even more.
# B,B1 --|> A
# C --|> B
g = Graph()
c = GraphNode('C', 0, 0, 200, 200)
b = GraphNode('B', 0, 0, 200, 200)
b1 = GraphNode('B1', 0, 0, 200, 200)
a = GraphNode('A', 0, 0, 200, 200)
c2 = GraphNode('C2', 0, 0, 200, 200)
d = GraphNode('D', 0, 0, 200, 200)
# add out of order
g.AddNode(b1)
g.AddNode(b)
g.AddNode(a)
g.AddNode(c)
g.AddNode(c2)
g.AddNode(d)
g.AddEdge(c2, b1)['uml_edge_type'] = 'generalisation'
g.AddEdge(d, c)['uml_edge_type'] = 'generalisation'
g.AddEdge(c, b)['uml_edge_type'] = 'generalisation'
g.AddEdge(b1, a)['uml_edge_type'] = 'generalisation'
g.AddEdge(b, a)['uml_edge_type'] = 'generalisation'
nodelist_normal = [node.id for node in g.nodes]
nodelist_sorted = [node.id for node,annotation in g.nodes_sorted_by_generalisation]
nodelist_sorted_expected = ['A', 'B', 'B1', 'C', 'D', 'C2']
#print "nodelist_normal", nodelist_normal
#print "nodelist_sorted_expected", nodelist_sorted_expected
#print "nodelist_sorted", nodelist_sorted
assert nodelist_sorted_expected == nodelist_sorted
def train():
graph = Graph(is_train=True)
graph.create_model()
sv = tf.train.Supervisor(logdir=get_path(args.logdir),
global_step=graph.global_step,
saver=graph.saver,
save_model_secs=600)
sess = sv.PrepareSession()
losses = []
while True:
input_feed = get_data(graph.inputs_ph)
fetches = [graph.train_op, graph.loss, graph.global_step]
_, loss, step = sess.run(fetches, input_feed)
losses.append(loss)
if step % 100 == 0:
print('Loss\t%s.' % np.mean(losses))
losses = []
from model.test import test
test()
def main():
"""Draw a sample graph"""
nodes = [1, 2, 3, 4, 5]
edges = [(1, 2), (1, 3), (1, 5)]
graph = Graph('Visualizer Sample')
graph.add_nodes(nodes)
graph.add_edges(edges)
draw(graph)
def save_graph(cls, user_name, id, type, title, data, img):
if data:
data = json.dumps(data)
if title is None:
title = 'Untitled'
if id is not None:
graph: Graph = Graph.select().get(id)
graph.type = type
graph.title = title
data_key = md5(title + data)
if graph.data_key != data_key:
data_url = QiniuService.upload_doc(data, graph.data_key,
data_key)
graph.data_key = data_key
graph.data_url = data_url
img_key = md5(img)
if graph.img_key != img_key:
img_url = QiniuService.upload_img(img, graph.img_key, img_key)
graph.img_key = img_key
graph.img_url = img_url
img_info = QiniuService.get_img_info(img_url)
graph.width = img_info.get('width')
graph.height = img_info.get('height')
graph.size = img_info.get('size')
graph.format = img_info.get('format')
graph.color_model = img_info.get('colorModel')
Graph.update(graph)
return id
else:
data_key = md5(title + data + datetime.now().timestamp().__str__())
data_url = QiniuService.upload_doc(data, file_name=data_key)
img_key = md5(img + datetime.now().timestamp().__str__())
img_url = QiniuService.upload_img(img, file_name=img_key)
img_info = QiniuService.get_img_info(img_url)
graph = Graph(user_name=user_name,
title=title,
data_key=data_key,
data_url=data_url,
type=type,
img_key=img_key,
img_url=img_url,
width=img_info.get('width'),
height=img_info.get('height'),
size=img_info.get('size'),
format=img_info.get('format'),
color_model=img_info.get('colorModel'))
graph.insert()
return Graph.select().filter(Graph.data_key == data_key).one().id
def test_5(self):
"""
Upgrade 1.1 to 1.0 - simulate loading a newer file format into an older version of the app
Cannot read. (a bit strict, I know - but gui allows for forcing the read - see filemgmt.py)
"""
g = Graph()
filedata = """
# PynSource Version 1.1
{'type':'meta', 'info1':'Lorem ipsum dolor sit amet, consectetur adipiscing elit.'}
{'type':'umlshape', 'id':'UmlShapeCanvas', 'x':237, 'y':65, 'width':226, 'height':781, 'attrs':'scrollStepX|scrollStepY|classnametoshape|log|frame|save_gdi|working|font1|font2|umlworkspace|layout|coordmapper|layouter|overlap_remover', 'meths':'__init__|AllToLayoutCoords|AllToWorldCoords|onKeyPress|CmdInsertNewumlshape|CmdZapShape|Clear|ConvertParseModelToUmlModel|BuildEdgeModel|Go|CreateUmlShape|newRegion|CreateUmlEdge|OnWheelZoom|ChangeScale|stage1|stage2|stateofthenation|stateofthespring|RedrawEverything|ReLayout|LayoutAndPositionShapes|setSize|DecorateShape|createumlshapeShape|AdjustShapePosition|Redraw222|get_umlboxshapes|OnDestroy|OnLeftClick|DeselectAllShapes'}
{'type':'umlshape', 'id':'Log', 'x':1089, 'y':222, 'width':82, 'height':67, 'attrs':'', 'meths':'WriteText'}
{'type':'edge', 'id':'UmlShapeCanvas_to_MainApp', 'source':'UmlShapeCanvas', 'target':'MainApp', 'uml_edge_type':'composition'}
"""
model.graph_persistence.PERSISTENCE_CURRENT_VERSION = 1.0
self.assertFalse(g.persistence.UpgradeToLatestFileFormatVersion(filedata))
self.assertFalse(g.persistence.can_I_read(filedata)[0])
def layout(self, keep_current_positions=False, optimise=True):
if len(self.graph.nodes) == 0:
print "Layout aborted - nothing to lay out."
return
if not keep_current_positions:
self.layoutPrepare()
if self.gui:
self.gui.kill_layout = False # initialise
memento1 = self.graph.GetMementoOfLayoutPoints()
break_pending = 0
for i in range(0, self.iterations):
self.layoutIteration()
if i % 100 == 0: # i%50==0:
if self.gui:
self.layoutCalcBounds(
) # this is the only time you need to call this explicitly since are in the MIDDLE of a layout and about to visualise
self.gui.stateofthenation(
recalibrate=True,
auto_resize_canvas=False) # refresh gui
if self.gui.kill_layout:
print "Layout aborted early, due to user interrupt"
break
if i % 20 == 0:
if optimise:
memento2 = self.graph.GetMementoOfLayoutPoints()
if Graph.MementosEqual(memento1, memento2, 0.01):
break_pending += 1
#print "!",
if break_pending > 5:
#print "break"
break
else:
break_pending = 0
#print ".",
memento1 = memento2
#print
self.layoutCalcBounds()
def initialize_graph(self):
# Get neural network
self.network = self._get_neural_network()
# Build computational graph
self.graph = Graph(network=self.network,
save_path=self.save_path,
max_to_keep=self.max_to_keep)
# Start session
self.sess = tf.Session()
# Initialize global variables
self.sess.run(self.graph.init_global)
# Save network object
self._pickle_network()
def test_2(self):
"""
Upgrade 0.9 (no version number) to 1.1
"""
g = Graph()
filedata = """
{'type':'node', 'id':'c', 'x':230, 'y':174, 'width':60, 'height':120}
{'type':'node', 'id':'c1', 'x':130, 'y':174, 'width':60, 'height':120}
{'type':'edge', 'id':'c_to_c1', 'source':'c', 'target':'c1'}
"""
model.graph_persistence.PERSISTENCE_CURRENT_VERSION = 1.1
self.assertTrue(g.persistence.UpgradeToLatestFileFormatVersion(filedata))
#print g.persistence.filedata_list
assert g.persistence.filedata_list[0] == "# PynSource Version 1.1", g.persistence.filedata_list[0]
assert "meta" in g.persistence.filedata_list[1], g.persistence.filedata_list[1]
assert g.persistence.ori_file_version == 0.9
# now check type node has been converted to type umlshape
data = eval(g.persistence.filedata_list[2])
self.assertEquals('umlshape', data.get('type'))
def test_1(self):
"""
Upgrade 0.9 (no version number) to 1.0
"""
g = Graph()
filedata = """
{'type':'node', 'id':'c', 'x':230, 'y':174, 'width':60, 'height':120}
{'type':'node', 'id':'c1', 'x':130, 'y':174, 'width':60, 'height':120}
{'type':'edge', 'id':'c_to_c1', 'source':'c', 'target':'c1'}
"""
#g.Clear()
assert len(g.nodes) == 0
#assert g.GraphToString().strip() == ""
model.graph_persistence.PERSISTENCE_CURRENT_VERSION = 1.0
#g.LoadGraphFromStrings(filedata)
self.assertTrue(g.persistence.UpgradeToLatestFileFormatVersion(filedata))
#print g.persistence.filedata_list
assert g.persistence.filedata_list[0] == "# PynSource Version 1.0", g.persistence.filedata_list[0]
assert g.persistence.ori_file_version == 0.9
def __init__(self,
in_channels=2,
in_planes=128,
edge_importance_weighting=True):
super().__init__()
graph = Graph()
A = torch.tensor(graph.A, dtype=torch.float32, requires_grad=False)
self.register_buffer('A', A)
# Networks.
spatial_kernel_size = A.size(0)
temporal_kernel_size = 9
kernel_size = (temporal_kernel_size, spatial_kernel_size)
self.data_bn = nn.BatchNorm1d(in_channels * A.size(1))
self.st_gcn_networks = nn.ModuleList(
(st_gcn(in_channels, 64, kernel_size, 1,
residual=False), st_gcn(64, 64, kernel_size,
1), st_gcn(64, 64, kernel_size, 1),
st_gcn(64, 64, kernel_size,
1), st_gcn(64, 128, kernel_size,
2), st_gcn(128, 128, kernel_size,
1), st_gcn(128, 128, kernel_size, 1),
st_gcn(128, 256, kernel_size,
2), st_gcn(256, 256, kernel_size,
1), st_gcn(256, 256, kernel_size, 1)))
if edge_importance_weighting:
self.edge_importance = nn.ParameterList([
nn.Parameter(torch.ones(A.size()))
for i in self.st_gcn_networks
])
else:
self.edge_importance = [1] * len(self.st_gcn_networks)
self.cls = nn.Conv2d(256, in_planes, kernel_size=1)
def test_6_generalisation_sort_order(self):
# START AGAIN - more tests, check stranger trees
# B,D,F --|> A
# G --|> C --|> B
# E --|> D
g = Graph()
a = GraphNode('A', 0, 0, 200, 200)
b = GraphNode('B', 0, 0, 200, 200)
c = GraphNode('C', 0, 0, 200, 200)
d = GraphNode('D', 0, 0, 200, 200)
e = GraphNode('E', 0, 0, 200, 200)
f = GraphNode('F', 0, 0, 200, 200)
h = GraphNode('H', 0, 0, 200, 200)
# add out of order
g.AddNode(f)
g.AddNode(b)
g.AddNode(a)
g.AddNode(h)
g.AddNode(c)
g.AddNode(e)
g.AddNode(d)
g.AddEdge(b, a)['uml_edge_type'] = 'generalisation'
g.AddEdge(d, a)['uml_edge_type'] = 'generalisation'
g.AddEdge(f, a)['uml_edge_type'] = 'generalisation'
g.AddEdge(h, c)['uml_edge_type'] = 'generalisation'
g.AddEdge(c, b)['uml_edge_type'] = 'generalisation'
g.AddEdge(e, d)['uml_edge_type'] = 'generalisation'
nodelist_normal = [node.id for node in g.nodes]
nodelist_sorted = [node.id for node,annotation in g.nodes_sorted_by_generalisation]
nodelist_sorted_expected = ['A', 'B', 'D', 'F', 'C', 'H', 'E']
assert nodelist_sorted_expected == nodelist_sorted
nodelist_sorted_annotated = [(node.id, annotation) for node,annotation in g.nodes_sorted_by_generalisation]
nodelist_sorted_expected_annotated = [('A', 'root'), ('B', 'fc'), ('D', 'tab'), ('F', 'tab'), ('C', 'fc'), ('H', 'fc'), ('E', 'root')]
assert nodelist_sorted_expected_annotated == nodelist_sorted_annotated
def test_5_generalisation_sort_order(self):
# START AGAIN - more tests, check stranger trees, though the algorithm
# is proving pretty smart, prioritising children who have children to the left
# B,B1,C,K --|> A
# D --|> C
g = Graph()
a = GraphNode('A', 0, 0, 200, 200)
b = GraphNode('B', 0, 0, 200, 200)
b1 = GraphNode('B1', 0, 0, 200, 200)
c = GraphNode('C', 0, 0, 200, 200)
k = GraphNode('K', 0, 0, 200, 200)
d = GraphNode('D', 0, 0, 200, 200)
# add out of order
g.AddNode(b1)
g.AddNode(b)
g.AddNode(a)
g.AddNode(c)
g.AddNode(k)
g.AddNode(d)
g.AddEdge(k, a)['uml_edge_type'] = 'generalisation'
g.AddEdge(d, c)['uml_edge_type'] = 'generalisation'
g.AddEdge(c, a)['uml_edge_type'] = 'generalisation'
g.AddEdge(b1, a)['uml_edge_type'] = 'generalisation'
g.AddEdge(b, a)['uml_edge_type'] = 'generalisation'
nodelist_normal = [node.id for node in g.nodes]
nodelist_sorted = [node.id for node,annotation in g.nodes_sorted_by_generalisation]
#print "nodelist_normal", nodelist_normal
#print "nodelist_sorted_expected", nodelist_sorted_expected
#print "nodelist_sorted", nodelist_sorted
assert nodelist_sorted[0] == 'A'
assert nodelist_sorted[1] == 'C'
assert nodelist_sorted[-1] == 'D'
nodelist_sorted_annotated = [(node.id, annotation) for node,annotation in g.nodes_sorted_by_generalisation]
assert nodelist_sorted_annotated[0] == ('A', 'root')
assert nodelist_sorted_annotated[1] == ('C', 'fc')
assert nodelist_sorted_annotated[-1] == ('D', 'fc')
assert ('K', 'tab') in nodelist_sorted_annotated
assert ('B', 'tab') in nodelist_sorted_annotated
assert ('B1', 'tab') in nodelist_sorted_annotated
def __setstate__(self, state):
(self.hyperparameters, self.parameters, self.trainer) = state
self.graph = Graph(self.hyperparameters, self.parameters)
def __init__(self, hyperparameters):
self.hyperparameters = hyperparameters
self.parameters = Parameters(self.hyperparameters)
self.trainer = Trainer()
self.graph = Graph(self.hyperparameters, self.parameters)
def test_8_multiple_inhertitance_render(self):
# F --|> M
# F --|> S
g = Graph()
f = GraphNode('F', 0, 0, 200, 200)
m = GraphNode('M', 0, 0, 200, 200)
s = GraphNode('S', 0, 0, 200, 200)
g.AddEdge(f, m)['uml_edge_type'] = 'generalisation'
g.AddEdge(f, s)['uml_edge_type'] = 'generalisation'
nodelist_normal = [node.id for node in g.nodes]
"""
Custom ordering allows us to bypass the graph 'nodes_sorted_by_generalisation'
algorithm which might either be crashing or have unwanted ordering results.
Thus we can experiment with how different experimental orderings will render.
"""
mycustom_ordering = [(m, 'root'), (s, 'root'), (f, 'root')]
from ascii_uml.layout_ascii import model_to_ascii_builder
m = model_to_ascii_builder()
s = m.main(g, nodes_annotated_and_sorted=mycustom_ordering)
expected_s = """
+---+
| M |
+---+
+---+ [ S ][ M ]
| S | .
+---+ /_\
|
|
+---+
| F |
+---+
"""
def remove_blank_lines(str):
return os.linesep.join([s for s in str.splitlines() if s.strip()])
# remove blank lines, since different margins and paddings in ascii uml layout
# could cause difference
expected_s = remove_blank_lines(expected_s)
s = remove_blank_lines(s)
#print
#print "*"*88
#print expected_s
#print "*"*88
#print s
#print "*"*88
if s.strip() <> expected_s.strip():
# Write to file
with open('logs/test_8_out_actual_.txt','w') as f: f.write(s)
with open('logs/test_8_out_expected.txt','w') as f: f.write(expected_s)
import difflib
# delta = difflib.ndiff(s.strip(), expected_s.strip()) # this will always emit something, a visual of the original with changes.
delta = difflib.unified_diff(s.strip(), expected_s.strip(), n=0,
fromfile='actual', tofile='expected')
diff_s = ''.join(delta)
print diff_s
assert s.strip() == expected_s.strip()
class Model:
def __init__(self, hyperparameters):
self.hyperparameters = hyperparameters
self.parameters = Parameters(self.hyperparameters)
self.trainer = Trainer()
self.graph = Graph(self.hyperparameters, self.parameters)
def __getstate__(self):
return (self.hyperparameters, self.parameters, self.trainer)
def __setstate__(self, state):
(self.hyperparameters, self.parameters, self.trainer) = state
self.graph = Graph(self.hyperparameters, self.parameters)
def embed(self, window):
seq = [self.parameters.embeddings[word] for word in window]
return numpy.dstack([numpy.resize(s, (1, s.size, 1)) for s in seq])
def embeds(self, sequences):
return numpy.vstack([self.embed(seq) for seq in sequences])
def corrupt_example(self, e):
import copy, random
e = copy.deepcopy(e)
pos = - self.hyperparameters.window_size // 2
mid = e[pos]
while e[pos] == mid: e[pos] = random.randint(0, self.hyperparameters.curriculum_size - 1)
pr = 1. / self.hyperparameters.curriculum_size
weight = 1. / pr
return e, numpy.float32(weight)
def corrupt_examples(self, correct_sequences):
return zip(*[self.corrupt_example(e) for e in correct_sequences])
def train(self, correct_sequences):
noise_sequences, weights = self.corrupt_examples(correct_sequences)
for w in weights: assert w == weights[0]
learning_rate = self.hyperparameters.learning_rate
r = self.graph.train(self.embeds(correct_sequences), self.embeds(noise_sequences), numpy.float32(learning_rate * weights[0]))
correct_inputs_gradient, noise_inputs_gradient, losses, correct_scores, noise_scores = r
to_normalize = set()
for example in range(len(correct_sequences)):
correct_sequence = correct_sequences[example]
noise_sequence = noise_sequences[example]
loss, correct_score, noise_score = losses[example], correct_scores[example], noise_scores[example]
import pdb
pdb.set_trace()
correct_input_gradient = correct_inputs_gradient[example]
noise_input_gradient = noise_inputs_gradient[example]
# self.trainer.update(numpy.sum(loss), correct_score, noise_score)
for w in weights: assert w == weights[0]
embedding_learning_rate = self.hyperparameters.embedding_learning_rate * weights[0]
if numpy.sum(loss) == 0:
for di in correct_input_gradient + noise_input_gradient:
assert (di == 0).all()
else:
for (i, di) in zip(correct_sequence, correct_input_gradient.T):
self.parameters.embeddings[i] -= 1.0 * embedding_learning_rate * di
to_normalize.add(i)
for (i, di) in zip(noise_sequence, noise_input_gradient.T):
self.parameters.embeddings[i] -= 1.0 * embedding_learning_rate * di
to_normalize.add(i)
self.parameters.normalize(list(to_normalize))
def predict(self, sequence):
(score) = self.graph.predict(self.embed(sequence))
return score
def verbose_predict(self, sequence):
(score, prehidden) = self.graph.verbose_predict(self.embed(sequence))
return score, prehidden
def validate(self, sequence):
import copy
corrupt_sequence = copy.copy(sequence)
rank = 1
correct_score = self.predict(sequence)
mid = self.hyperparameters.window_size // 2
for i in range(self.hyperparameters.curriculum_size - 1):
if i == sequence[mid]: continue
corrupt_sequence[mid] = i
corrupt_score = self.predict(corrupt_sequence)
rank += (correct_score <= corrupt_score)
return rank