class NetTests(unittest.TestCase):
""" The tests of the core module net feature """
def setUp(self):
""" Setup the test case. """
self.net = Net("001")
def tearDown(self):
""" Teardown the test case. """
pass
def test_create_new_net(self):
""" Test the creation of a new empty net. """
assert self.net.net_id == "001"
def test_bounds_simple(self):
"""Make sure bounds() uses all the included NetPoints"""
for (x, y) in ((1, 3), (3, 2), (4, 3), (3, 5)):
net_pt = NetPoint(str((x, y)), x, y)
self.net.add_point(net_pt)
# NetPoints don't actually need to be connected to affect bounds()
top_left, btm_right = self.net.bounds()
self.assertEqual(top_left.x, 1)
self.assertEqual(top_left.y, 2)
self.assertEqual(btm_right.x, 4)
self.assertEqual(btm_right.y, 5)
def test_activation(activation_layer, expect_range):
"""Test expected output range of activation layers"""
input_ = np.random.normal(size=(100, 5))
net = Net([Dense(1), activation_layer])
output = net.forward(input_)
lower_bound, upper_bound = expect_range
assert np.all((output >= lower_bound) & (output <= upper_bound))
def main(args):
if args.seed >= 0:
random_seed(args.seed)
train_set, valid_set, test_set = prepare_dataset(args.data_dir)
train_x, train_y = train_set
test_x, test_y = test_set
# train_y = get_one_hot(train_y, 2)
net = Net([Dense(100), ReLU(), Dense(30), ReLU(), Dense(1)])
model = Model(net=net,
loss=SigmoidCrossEntropy(),
optimizer=Adam(lr=args.lr))
iterator = BatchIterator(batch_size=args.batch_size)
loss_list = list()
for epoch in range(args.num_ep):
t_start = time.time()
for batch in iterator(train_x, train_y):
pred = model.forward(batch.inputs)
loss, grads = model.backward(pred, batch.targets)
model.apply_grad(grads)
loss_list.append(loss)
print("Epoch %d time cost: %.4f" % (epoch, time.time() - t_start))
# evaluate
model.set_phase("TEST")
test_y_idx = np.asarray(test_y).reshape(-1)
test_pred = model.forward(test_x)
test_pred[test_pred > 0] = 1
test_pred[test_pred <= 0] = 0
test_pred_idx = test_pred.reshape(-1)
res = accuracy(test_pred_idx, test_y_idx)
print(res)
model.set_phase("TRAIN")
def calc_nets(self, segments):
""" Return a set of Nets from segments """
points = {} # (x,y) -> NetPoint
def get_point(p):
if p not in points:
points[p] = NetPoint('%da%d' % p, *p)
return points[p]
# turn the (x, y) points into unique NetPoint objects
segments = set((get_point(p1), get_point(p2)) for p1, p2 in segments)
nets = []
# Iterate over the segments, removing segments when added to a net
while segments:
seg = segments.pop() # pick a point
newnet = Net('')
newnet.connect(seg)
found = True
while found:
found = set()
for seg in segments: # iterate over segments
if newnet.connected(seg): # segment touching the net
newnet.connect(seg) # add the segment
found.add(seg)
for seg in found:
segments.remove(seg)
nets.append(newnet)
return nets
def main(args):
if args.seed >= 0:
random_seed(args.seed)
# data preparing
train_x, train_y, img_shape = prepare_dataset(args.img)
net = Net([
Dense(30),
ReLU(),
Dense(100),
ReLU(),
Dense(100),
ReLU(),
Dense(30),
ReLU(),
Dense(3),
Sigmoid()
])
model = Model(net=net, loss=MSE(), optimizer=Adam())
iterator = BatchIterator(batch_size=args.batch_size)
for epoch in range(args.num_ep):
for batch in iterator(train_x, train_y):
preds = model.forward(batch.inputs)
loss, grads = model.backward(preds, batch.targets)
model.apply_grad(grads)
# evaluate
preds = net.forward(train_x)
mse = mean_square_error(preds, train_y)
print("Epoch %d %s" % (epoch, mse))
# generate painting
if epoch % 5 == 0:
preds = preds.reshape(img_shape[0], img_shape[1], -1)
preds = (preds * 255.0).astype("uint8")
name, ext = os.path.splitext(args.img)
filename = os.path.basename(name)
out_filename = filename + "-paint-epoch" + str(epoch) + ext
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
out_path = os.path.join(args.output_dir, out_filename)
Image.fromarray(preds).save(out_path)
print("save painting to %s" % out_path)
def mlp_D():
w_init = Normal(0.0, 0.02)
return Net([
Dense(300, w_init=w_init),
LeakyReLU(),
Dense(100, w_init=w_init),
LeakyReLU(),
Dense(1, w_init=w_init)
])
def cnn_model():
net = Net([
Conv2D(kernel=[3, 3, 1, 2]),
MaxPool2D(pool_size=[2, 2], stride=[2, 2]),
Conv2D(kernel=[3, 3, 2, 4]),
MaxPool2D(pool_size=[2, 2], stride=[2, 2]),
Flatten(),
Dense(1)
])
return Model(net, loss=MSE(), optimizer=SGD())
def mlp_G():
w_init = Normal(0.0, 0.02)
return Net([
Dense(100, w_init=w_init),
LeakyReLU(),
Dense(300, w_init=w_init),
LeakyReLU(),
Dense(784, w_init=w_init),
Sigmoid()
])
def parse_net(self, args):
""" Assembles a net from a list of junctions, segments, and labels. """
thisnet = Net(args)
subdata = defaultdict(list)
for phrase in self.stream:
print phrase
cmd, _sep, args = phrase.partition(' ')
if cmd not in ('J', 'S', 'A', 'L', 'Q', 'B'):
self.stream.push(phrase)
break
print args
k, v = self.parsenode(cmd)(args)
subdata[k].append(v)
# finish building thisnet
for netpt in subdata['netpoint'][:]:
# using a copy so that we can modify subdata['netpoint'] inside loop
if netpt.point_id not in thisnet.points:
thisnet.add_point(netpt)
else:
# oh yeah, a net can have a point more than once, because that
# makes *great* sense.
for point in netpt.connected_points:
thisnet.points[netpt.point_id].add_connected_point(point)
for comp in netpt.connected_components:
thisnet.points[netpt.point_id].add_connected_component(comp)
# update subdata['netpoint'] so that ref to netpt points to the
# new combined point
i = subdata['netpoint'].index(netpt)
subdata['netpoint'][i] = thisnet.points[netpt.point_id]
# yuck, passing in-band
thisnet.ibpts = subdata['netpoint']
for pt_a, pt_b in subdata['segment']:
thisnet.connect((subdata['netpoint'][pt_a - 1],
subdata['netpoint'][pt_b - 1]))
for annot in subdata['annot']:
thisnet.add_annotation(annot)
if '=' in annot.value:
thisnet.add_attribute(*(annot.value.split('=', 1)))
return ('net', thisnet)
def __init__(self):
self.vocab_file = os.path.join(DATA_DIR, FLAGS.version, 'word_vocab')
self.neter = Net(self.vocab_file, FLAGS.embed_dim, FLAGS.filter_num,
map(int, FLAGS.filter_sizes.split(',')),
FLAGS.drop_rate, FLAGS.sen_len)
self.data_loader_train = DataLoader(FLAGS.batch_size)
self.data_loader_test = DataLoader(FLAGS.batch_size)
self.train_path = os.path.join(DATA_DIR, FLAGS.version, "train_id")
self.test_path = os.path.join(DATA_DIR, FLAGS.version, "validate_id")
self.load_res()
self.init_env()
pass
def test_struct_param():
net = Net([Dense(10, 1)])
params = net.params
assert isinstance(params, StructuredParam)
assert isinstance(params.values, np.ndarray)
assert len(params.values) == 2 # w and b
# test operations on StructParam
bigger = params * 10
assert isinstance(bigger, StructuredParam)
assert params.shape == bigger.shape
assert (params.values[-1] * 10 == bigger.values[-1]).all()
def calc_nets(self, segments):
""" Return a set of Nets from segments """
points = {} # (x,y) -> NetPoint
def get_point(p):
if p not in points:
points[p] = NetPoint("%da%d" % p, *p)
return points[p]
# turn the (x, y) points into unique NetPoint objects
segments = set((get_point(p1), get_point(p2)) for p1, p2 in segments)
nets = []
# Iterate over the segments, removing segments when added to a net
while segments:
seg = segments.pop() # pick a point
newnet = Net("")
newnet.connect(seg)
found = True
while found:
found = set()
for seg in segments: # iterate over segments
if newnet.connected(seg): # segment touching the net
newnet.connect(seg) # add the segment
found.add(seg)
for seg in found:
segments.remove(seg)
nets.append(newnet)
return nets
def parse_nets(self, nets):
""" Extract nets. """
for net in nets:
net_id = net.get('net_id')
ret_net = Net(net_id)
# Add Annotations
for annotation in net.get('annotations'):
anno = self.parse_annotation(annotation)
ret_net.add_annotation(anno)
# Get the Attributes
for key, value in net.get('attributes').items():
ret_net.add_attribute(key, value)
# Get the Points
for net_point in net.get('points'):
npnt = self.parse_net_point(net_point)
ret_net.add_point(npnt)
self.design.add_net(ret_net)
def test_parameters_change(fake_dataset):
# make sure the parameters does change after apply gradients
# fake dataset
X, y = fake_dataset
# simple model
net = Net([Dense(10), Dense(1)])
loss = MSE()
opt = SGD(lr=1.0)
model = Model(net, loss, opt)
# forward and backward
pred = model.forward(X)
loss, grads = model.backward(pred, y)
# parameters change test
params_before = model.net.params.values
model.apply_grad(grads)
params_after = model.net.params.values
for p1, p2 in zip(params_before, params_after):
assert np.all(p1 != p2)
def test_write_net(self):
"""
The write_net method creates the correct kicad wires from an
openjson net.
"""
net = Net('')
p1 = NetPoint('p1', 0, 0)
p2 = NetPoint('p2', 1, 0)
p3 = NetPoint('p3', 0, 1)
net.add_point(p1)
net.add_point(p2)
net.add_point(p3)
net.conn_point(p1, p2)
net.conn_point(p1, p3)
writer = KiCAD()
buf = StringIO()
writer.write_net(buf, net)
self.assertEqual(
buf.getvalue(),
'Wire Wire Line\n\t0 0 0 -11\nWire Wire Line\n\t0 0 11 0\n')
def calc_nets(self, segments):
""" Return a set of Nets from segments """
points = {} # (x, y) -> NetPoint
def get_point(point):
""" Return a new or existing NetPoint for an (x,y) point """
point = (make_length(point[0]), make_length(point[1]))
if point not in points:
points[point] = NetPoint('%da%d' % point, point[0], point[1])
return points[point]
# turn the (x, y) points into unique NetPoint objects
segments = set((get_point(p1), get_point(p2)) for p1, p2 in segments)
nets = []
# Iterate over the segments, removing segments when added to a net
while segments:
seg = segments.pop() # pick a point
newnet = Net('')
newnet.connect(seg)
found = True
while found:
found = set()
for seg in segments: # iterate over segments
if newnet.connected(seg): # segment touching the net
newnet.connect(seg) # add the segment
found.add(seg)
for seg in found:
segments.remove(seg)
nets.append(newnet)
return nets
def generate_substrate_network():
substrate_network = Net()
substrate_network.init_bandwidth_capacity(1, 6, 100)
substrate_network.init_bandwidth_capacity(1, 2, 100)
substrate_network.init_bandwidth_capacity(2, 3, 100)
substrate_network.init_bandwidth_capacity(3, 4, 100)
substrate_network.init_bandwidth_capacity(4, 5, 100)
substrate_network.init_bandwidth_capacity(5, 6, 100)
substrate_network.init_bandwidth_capacity(2, 6, 100)
substrate_network.init_bandwidth_capacity(2, 5, 100)
substrate_network.init_bandwidth_capacity(3, 5, 100)
substrate_network.init_link_latency(1, 6, 2)
substrate_network.init_link_latency(1, 2, 2)
substrate_network.init_link_latency(2, 3, 2)
substrate_network.init_link_latency(3, 4, 2)
substrate_network.init_link_latency(4, 5, 2)
substrate_network.init_link_latency(5, 6, 2)
substrate_network.init_link_latency(2, 6, 2)
substrate_network.init_link_latency(2, 5, 2)
substrate_network.init_link_latency(3, 5, 2)
substrate_network.init_node_cpu_capacity(1, 100)
substrate_network.init_node_cpu_capacity(2, 100)
substrate_network.init_node_cpu_capacity(3, 100)
substrate_network.init_node_cpu_capacity(4, 100)
substrate_network.init_node_cpu_capacity(5, 100)
substrate_network.init_node_cpu_capacity(6, 100)
return substrate_network
def generate_substrate_network():
substrate_network = Net()
substrate_network.init_bandwidth_capacity(0, 1, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(0, 2, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(1, 2, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(1, 3, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(2, 3, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(2, 4, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(3, 5, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(3, 16, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(4, 5, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(4, 6, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(4, 8, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(5, 16, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(6, 16, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(6, 7, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(7, 8, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(7, 9, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(7, 10, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(8, 11, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(8, 9, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(9, 11, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(9, 12, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(10, 9, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(10, 13, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(10, 18, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(10, 16, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(11, 12, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(12, 14, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(12, 13, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(13, 14, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(13, 15, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(14, 19, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(15, 19, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(15, 18, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(15, 23, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(16, 17, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(16, 18, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(17, 20, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(17, 18, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(18, 23, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(19, 23, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(20, 21, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(21, 26, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(22, 23, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(22, 25, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(23, 24, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(24, 25, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(24, 31, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(24, 46, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(25, 26, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(26, 46, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(26, 27, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(26, 33, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(26, 28, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(27, 46, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(27, 28, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(27, 31, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(28, 30, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(29, 32, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(29, 47, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(30, 32, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(30, 36, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(31, 29, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(32, 47, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(32, 34, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(33, 30, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(33, 35, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(34, 47, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(34, 40, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(35, 37, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(36, 37, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(36, 34, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(37, 38, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(38, 40, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(38, 41, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(38, 39, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(39, 40, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(39, 44, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(40, 42, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(41, 43, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(42, 44, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(43, 45, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(44, 45, bandwidth_capacity)
##############
substrate_network.init_link_latency(0, 1, 1.5884322213336)
substrate_network.init_link_latency(0, 2, 1.36694566212203)
substrate_network.init_link_latency(1, 2, 0.595078345833503)
substrate_network.init_link_latency(1, 3, 0.606753089165439)
substrate_network.init_link_latency(2, 3, 0.424627093187248)
substrate_network.init_link_latency(2, 4, 0.612090114688609)
substrate_network.init_link_latency(3, 5, 0.70582162543929)
substrate_network.init_link_latency(3, 16, 0.910629979890955)
substrate_network.init_link_latency(4, 5, 0.203807662166071)
substrate_network.init_link_latency(4, 6, 0.26351563520654)
substrate_network.init_link_latency(4, 8, 0.818566289616265)
substrate_network.init_link_latency(5, 16, 0.623764858020544)
substrate_network.init_link_latency(6, 16, 0.600748935451872)
substrate_network.init_link_latency(6, 7, 0.544710167458582)
substrate_network.init_link_latency(7, 8, 0.318887275009433)
substrate_network.init_link_latency(7, 9, 0.264182763396936)
substrate_network.init_link_latency(7, 10, 0.355245761386032)
substrate_network.init_link_latency(8, 11, 0.425294221377644)
substrate_network.init_link_latency(8, 9, 0.390269991381838)
substrate_network.init_link_latency(9, 11, 0.220485866925979)
substrate_network.init_link_latency(9, 12, 0.10106992084504)
substrate_network.init_link_latency(10, 9, 0.24917237911302)
substrate_network.init_link_latency(10, 13, 0.321555787771019)
substrate_network.init_link_latency(10, 18, 0.39160424776263)
substrate_network.init_link_latency(10, 16, 0.76352821390857)
substrate_network.init_link_latency(11, 12, 0.130757125317676)
substrate_network.init_link_latency(12, 14, 0.0960664594170678)
substrate_network.init_link_latency(12, 13, 0.158109381123924)
substrate_network.init_link_latency(13, 14, 0.121750894747325)
substrate_network.init_link_latency(13, 15, 0.289200070536798)
substrate_network.init_link_latency(14, 19, 0.505016040130002)
substrate_network.init_link_latency(15, 19, 0.408282452522538)
substrate_network.init_link_latency(15, 18, 0.547045116124969)
substrate_network.init_link_latency(15, 23, 0.876606442180744)
substrate_network.init_link_latency(16, 17, 0.847586365898504)
substrate_network.init_link_latency(16, 18, 0.704820933153695)
substrate_network.init_link_latency(17, 20, 0.198137072547702)
substrate_network.init_link_latency(17, 18, 0.643111575542037)
substrate_network.init_link_latency(18, 23, 0.836578750756965)
substrate_network.init_link_latency(19, 23, 0.619762088878167)
substrate_network.init_link_latency(20, 21, 0.255843661016983)
substrate_network.init_link_latency(21, 26, 0.494008424988463)
substrate_network.init_link_latency(22, 23, 0.10106992084504)
substrate_network.init_link_latency(22, 25, 0.357914274147617)
substrate_network.init_link_latency(23, 24, 0.221820123306771)
substrate_network.init_link_latency(24, 25, 0.28152809634724)
substrate_network.init_link_latency(24, 31, 1.21884320385405)
substrate_network.init_link_latency(24, 46, 0.298539865202346)
substrate_network.init_link_latency(25, 26, 0.0333564095198152)
substrate_network.init_link_latency(26, 46, 0.139096227697629)
substrate_network.init_link_latency(26, 27, 0.130089997127279)
substrate_network.init_link_latency(26, 33, 0.845584981327315)
substrate_network.init_link_latency(26, 28, 0.25817860968337)
substrate_network.init_link_latency(27, 46, 0.173453329503039)
substrate_network.init_link_latency(27, 28, 0.123085151128118)
substrate_network.init_link_latency(27, 31, 0.253842276445794)
substrate_network.init_link_latency(28, 30, 0.47833091251415)
substrate_network.init_link_latency(29, 32, 0.248505250922623)
substrate_network.init_link_latency(29, 47, 0.522694937175504)
substrate_network.init_link_latency(30, 32, 0.239499020352273)
substrate_network.init_link_latency(30, 36, 0.538038885554619)
substrate_network.init_link_latency(31, 29, 0.219485174640384)
substrate_network.init_link_latency(32, 47, 0.531367603650656)
substrate_network.init_link_latency(32, 34, 0.648448601065208)
substrate_network.init_link_latency(33, 30, 0.47299388699098)
substrate_network.init_link_latency(33, 35, 0.405613939760953)
substrate_network.init_link_latency(34, 47, 0.837913007137758)
substrate_network.init_link_latency(34, 40, 0.555384218504923)
substrate_network.init_link_latency(35, 37, 0.85559190418326)
substrate_network.init_link_latency(36, 37, 0.442973118423146)
substrate_network.init_link_latency(36, 34, 0.220819431021177)
substrate_network.init_link_latency(37, 38, 0.493341296798067)
substrate_network.init_link_latency(38, 40, 0.662124728968332)
substrate_network.init_link_latency(38, 41, 0.17879035502621)
substrate_network.init_link_latency(38, 39, 0.394939888714612)
substrate_network.init_link_latency(39, 40, 0.493674860893265)
substrate_network.init_link_latency(39, 44, 0.568726782312849)
substrate_network.init_link_latency(40, 42, 0.690477677060175)
substrate_network.init_link_latency(41, 43, 0.334564787483746)
substrate_network.init_link_latency(42, 44, 0.419957195854473)
substrate_network.init_link_latency(43, 45, 2.52841584160199)
substrate_network.init_link_latency(44, 45, 2.24722130934995)
for i in range(0, 48):
substrate_network.init_node_cpu_capacity(i, cpu_capacity)
substrate_network.pre_get_single_source_minimum_latency_path()
substrate_network.update()
return substrate_network
def generate_substrate_network():
substrate_network = Net()
substrate_network.init_bandwidth_capacity(0, 1, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(0, 2, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(0, 3, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(1, 2, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(1, 7, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(2, 5, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(3, 4, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(3, 10, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(4, 5, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(4, 6, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(5, 9, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(5, 12, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(6, 7, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(7, 8, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(8, 9, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(8, 11, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(8, 13, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(10, 11, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(10, 13, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(11, 12, bandwidth_capacity)
substrate_network.init_bandwidth_capacity(12, 13, bandwidth_capacity)
substrate_network.init_link_latency(0, 1, 1)
substrate_network.init_link_latency(0, 2, 2)
substrate_network.init_link_latency(0, 3, 3)
substrate_network.init_link_latency(1, 2, 3)
substrate_network.init_link_latency(1, 7, 7)
substrate_network.init_link_latency(2, 5, 4)
substrate_network.init_link_latency(3, 4, 3)
substrate_network.init_link_latency(3, 10, 8)
substrate_network.init_link_latency(4, 5, 2)
substrate_network.init_link_latency(4, 6, 2)
substrate_network.init_link_latency(5, 9, 6)
substrate_network.init_link_latency(5, 12, 8)
substrate_network.init_link_latency(6, 7, 2)
substrate_network.init_link_latency(7, 8, 2)
substrate_network.init_link_latency(8, 9, 6)
substrate_network.init_link_latency(8, 11, 5)
substrate_network.init_link_latency(8, 13, 5)
substrate_network.init_link_latency(10, 11, 4)
substrate_network.init_link_latency(10, 13, 6)
substrate_network.init_link_latency(11, 12, 6)
substrate_network.init_link_latency(12, 13, 3)
substrate_network.init_node_cpu_capacity(0, cpu_capacity)
substrate_network.init_node_cpu_capacity(1, cpu_capacity)
substrate_network.init_node_cpu_capacity(2, cpu_capacity)
substrate_network.init_node_cpu_capacity(3, cpu_capacity)
substrate_network.init_node_cpu_capacity(4, cpu_capacity)
substrate_network.init_node_cpu_capacity(5, cpu_capacity)
substrate_network.init_node_cpu_capacity(6, cpu_capacity)
substrate_network.init_node_cpu_capacity(7, cpu_capacity)
substrate_network.init_node_cpu_capacity(8, cpu_capacity)
substrate_network.init_node_cpu_capacity(9, cpu_capacity)
substrate_network.init_node_cpu_capacity(10, cpu_capacity)
substrate_network.init_node_cpu_capacity(11, cpu_capacity)
substrate_network.init_node_cpu_capacity(12, cpu_capacity)
substrate_network.init_node_cpu_capacity(13, cpu_capacity)
substrate_network.pre_get_single_source_minimum_latency_path()
substrate_network.update()
return substrate_network
def fc_model():
net = Net([Dense(10), Dense(1)])
loss = MSE()
opt = SGD()
return Model(net, loss, opt)
def main(args):
if args.seed >= 0:
random_seed(args.seed)
# create output directory for saving result images
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
# prepare and read dataset
train_set, _, test_set = mnist(args.data_dir)
train_x, train_y = train_set
test_x, test_y = test_set
# specify the encoder and decoder net structure
encoder_net = Net([Dense(256), ReLU(), Dense(64)])
decoder_net = Net([ReLU(), Dense(256), Tanh(), Dense(784), Tanh()])
nets = (encoder_net, decoder_net)
optimizers = (Adam(args.lr), Adam(args.lr))
model = AutoEncoder(nets, loss=MSE(), optimizer=optimizers)
# for pre-trained model, test generated images from latent space
if args.load_model is not None:
# load pre-trained model
model.load(os.path.join(args.output_dir, args.load_model))
print("Loaded model fom %s" % args.load_model)
# transition from test[from_idx] to test[to_idx] in n steps
idx_arr, n = [2, 4, 32, 12, 82], 160
print("Transition in numbers", [test_y[i] for i in idx_arr],
"in %d steps ..." % n)
stops = [model.en_net.forward(test_x[i]) for i in idx_arr]
k = int(n / (len(idx_arr) - 1)) # number of code per transition
# generate all transition codes
code_arr = []
for i in range(len(stops) - 1):
t = [c.copy() for c in transition(stops[i], stops[i + 1], k)]
code_arr += t
# apply decoding all n "code" from latent space...
batch = None
for code in code_arr:
# translate latent space to image
genn = model.de_net.forward(code)
# save decoded results in a batch
if batch is None:
batch = np.array(genn)
else:
batch = np.concatenate((batch, genn))
output_path = os.path.join(args.output_dir, "genn-latent.png")
save_batch_as_images(output_path, batch)
quit()
# train the auto-encoder
iterator = BatchIterator(batch_size=args.batch_size)
for epoch in range(args.num_ep):
for batch in iterator(train_x, train_y):
origin_in = batch.inputs
# make noisy inputs
m = origin_in.shape[0] # batch size
mu = args.gaussian_mean # mean
sigma = args.gaussian_std # standard deviation
noises = np.random.normal(mu, sigma, (m, 784))
noises_in = origin_in + noises # noisy inputs
# forward
genn = model.forward(noises_in)
# back-propagate
loss, grads = model.backward(genn, origin_in)
# apply gradients
model.apply_grad(grads)
print("Epoch: %d Loss: %.3f" % (epoch, loss))
# save all the generated images and original inputs for this batch
noises_in_path = os.path.join(args.output_dir,
"ep%d-input.png" % epoch)
genn_path = os.path.join(args.output_dir, "ep%d-genn.png" % epoch)
save_batch_as_images(noises_in_path, noises_in, titles=batch.targets)
save_batch_as_images(genn_path, genn, titles=batch.targets)
# save the model after training
model.save(os.path.join(args.output_dir, args.save_model))
def setUp(self):
self.node = Node()
self.substrate_node = SubstrateNode(200)
self.net = Net()
self.vnf = VNF('test')
class MyTestCase(unittest.TestCase):
def setUp(self):
self.node = Node()
self.substrate_node = SubstrateNode(200)
self.net = Net()
self.vnf = VNF('test')
def test_Node(self):
self.assertEqual(self.node.test_function(), "test function: Node")
def test_substrate_node(self):
self.assertEqual(self.substrate_node.get_id(), 200)
self.assertEqual(self.substrate_node.set_cpu_capacity(100), 100)
self.assertEqual(self.substrate_node.get_cpu_free(), 100)
self.assertEqual(self.substrate_node.get_cpu_capacity(), 100)
self.assertEqual(self.substrate_node.allocate_cpu_resource(20), 20)
self.assertEqual(self.substrate_node.get_cpu_used(), 20)
self.assertEqual(self.substrate_node.get_cpu_free(), 80)
self.substrate_node.reset()
self.assertEqual(self.substrate_node.get_cpu_capacity(), 100)
self.assertEqual(self.substrate_node.get_cpu_free(), 100)
self.assertEqual(self.substrate_node.get_cpu_used(), 0)
self.substrate_node.reset_with_cpu_capacity(200)
self.assertEqual(self.substrate_node.get_cpu_capacity(), 200)
self.assertEqual(self.substrate_node.get_cpu_used(), 0)
self.assertEqual(self.substrate_node.get_cpu_free(), 200)
self.assertEqual(self.substrate_node.allocate_cpu_resource(150), 150)
self.assertEqual(self.substrate_node.get_cpu_capacity(), 200)
self.assertEqual(self.substrate_node.get_cpu_free(), 50)
self.assertEqual(self.substrate_node.get_cpu_used(), 150)
def test_net_node(self):
self.net.init_node_cpu_capacity(1, 100)
self.assertEqual(self.net.get_node_cpu_capacity(1), 100)
self.assertEqual(self.net.get_node_cpu_free(1), 100)
self.assertEqual(self.net.get_node_cpu_used(1), 0)
self.assertTrue(self.net.allocate_cpu_resource(1, 10))
self.assertEqual(self.net.get_node_cpu_capacity(1), 100)
self.assertEqual(self.net.get_node_cpu_free(1), 90)
self.assertEqual(self.net.get_node_cpu_used(1), 10)
self.assertFalse(self.net.allocate_cpu_resource(1, 95))
self.net.reset_node_cpu_capacity(1, 100)
self.assertEqual(self.net.get_node_cpu_capacity(1), 100)
self.assertEqual(self.net.get_node_cpu_free(1), 100)
self.assertEqual(self.net.get_node_cpu_used(1), 0)
self.assertEqual(1, 1)
def test_net_link(self):
self.net.init_bandwidth_capacity(1, 2, 100)
self.assertEqual(self.net.get_link_bandwidth_capacity(1, 2), 100)
self.assertEqual(self.net.get_link_bandwidth_used(1, 2), 0)
self.assertEqual(self.net.get_link_bandwidth_free(1, 2), 100)
self.assertTrue(self.net.allocate_bandwidth_resource(1, 2, 80))
self.assertFalse(self.net.allocate_bandwidth_resource(1, 2, 90))
self.assertEqual(self.net.get_link_bandwidth_used(1, 2), 80)
self.assertEqual(self.net.get_link_bandwidth_free(1, 2), 20)
self.net.init_link_latency(1, 2, 2)
self.assertEqual(self.net.get_link_latency(
1,
2,
), 2)
pass
def test_vnf(self):
self.vnf.set_cpu_request(100)
self.assertEqual(self.vnf.get_cpu_request(), 100)
self.vnf.set_income_interface_bandwidth(105)
self.assertEqual(self.vnf.get_income_interface_bandwidth(), 105)
self.vnf.set_outcome_interface_banwdith(110)
self.assertEqual(self.vnf.get_outcome_interface_bandwidth(), 110)
vnf2 = VNF(2)
vnf3 = VNF(3)
self.vnf.set_next_vnf(vnf2)
self.vnf.set_previous_vnf(vnf3)
self.assertEqual(self.vnf.get_previous_vnf().id, 3)
self.assertEqual(self.vnf.get_next_vnf().id, 2)
pass
def generate_substrate_network():
substrate_network = Net()
substrate_network.init_bandwidth_capacity(1, 6, 100)
substrate_network.init_bandwidth_capacity(1, 2, 100)
substrate_network.init_bandwidth_capacity(2, 3, 100)
substrate_network.init_bandwidth_capacity(3, 4, 100)
substrate_network.init_bandwidth_capacity(4, 5, 100)
substrate_network.init_bandwidth_capacity(5, 6, 100)
substrate_network.init_bandwidth_capacity(2, 6, 100)
# substrate_network.init_bandwidth_capacity(2, 5, 100)
substrate_network.init_bandwidth_capacity(3, 5, 100)
substrate_network.init_link_latency(1, 6, 2)
substrate_network.init_link_latency(1, 2, 1)
substrate_network.init_link_latency(2, 3, 4)
substrate_network.init_link_latency(3, 4, 30)
substrate_network.init_link_latency(4, 5, 2)
substrate_network.init_link_latency(5, 6, 2)
substrate_network.init_link_latency(2, 6, 5)
# substrate_network.init_link_latency(2, 5, 2)
substrate_network.init_link_latency(3, 5, 30)
substrate_network.init_node_cpu_capacity(1, 100)
substrate_network.init_node_cpu_capacity(2, 100)
substrate_network.init_node_cpu_capacity(3, 100)
substrate_network.init_node_cpu_capacity(4, 100)
substrate_network.init_node_cpu_capacity(5, 100)
substrate_network.init_node_cpu_capacity(6, 100)
substrate_network.pre_get_single_source_minimum_latency_path()
substrate_network.update()
return substrate_network
def build_net(self):
q_net = Net([Dense(100), ReLU(), Dense(self.action_dim)])
return q_net
def generate_random_network(number_of_node, probability):
# create a new substrate network
substrate_random_network = Net()
number_of_nodes = number_of_node
probability = probability
topology = nx.erdos_renyi_graph(number_of_nodes,
probability,
seed=None,
directed=False)
network_create_counter = 0
while not nx.is_connected(topology):
if network_create_counter >= 10000:
break
network_create_counter += 1
topology = nx.erdos_renyi_graph(number_of_nodes,
probability,
seed=None,
directed=False)
for edge in topology.edges():
# bw = random.randint(50, 100)
bw = 1000
substrate_random_network.init_bandwidth_capacity(edge[0], edge[1], bw)
lt = random.uniform(1, 5)
substrate_random_network.init_link_latency(edge[0], edge[1], lt)
for node in topology.nodes():
# cpu_capacity = random.randint(50, 100)
cpu_capacity = 100
substrate_random_network.init_node_cpu_capacity(node, cpu_capacity)
substrate_random_network.pre_get_single_source_minimum_latency_path()
substrate_random_network.update()
return substrate_random_network
def setUp(self):
""" Setup the test case. """
self.net = Net("001")
def main(args):
if args.seed >= 0:
random_seed(args.seed)
train_set, valid_set, test_set = mnist(args.data_dir)
train_x, train_y = train_set
test_x, test_y = test_set
train_y = get_one_hot(train_y, 10)
if args.model_type == "cnn":
train_x = train_x.reshape((-1, 28, 28, 1))
test_x = test_x.reshape((-1, 28, 28, 1))
if args.model_type == "cnn":
# a LeNet-5 model with activation function changed to ReLU
net = Net([
Conv2D(kernel=[5, 5, 1, 6], stride=[1, 1], padding="SAME"),
ReLU(),
MaxPool2D(pool_size=[2, 2], stride=[2, 2]),
Conv2D(kernel=[5, 5, 6, 16], stride=[1, 1], padding="SAME"),
ReLU(),
MaxPool2D(pool_size=[2, 2], stride=[2, 2]),
Flatten(),
Dense(120),
ReLU(),
Dense(84),
ReLU(),
Dense(10)
])
elif args.model_type == "dense":
net = Net([
Dense(200),
ReLU(),
Dense(100),
ReLU(),
Dense(70),
ReLU(),
Dense(30),
ReLU(),
Dense(10)
])
else:
raise ValueError("Invalid argument: model_type")
model = Model(net=net,
loss=SoftmaxCrossEntropy(),
optimizer=Adam(lr=args.lr))
iterator = BatchIterator(batch_size=args.batch_size)
loss_list = list()
for epoch in range(args.num_ep):
t_start = time.time()
for batch in iterator(train_x, train_y):
pred = model.forward(batch.inputs)
loss, grads = model.backward(pred, batch.targets)
model.apply_grad(grads)
loss_list.append(loss)
print("Epoch %d time cost: %.4f" % (epoch, time.time() - t_start))
# evaluate
model.set_phase("TEST")
test_pred = model.forward(test_x)
test_pred_idx = np.argmax(test_pred, axis=1)
test_y_idx = np.asarray(test_y)
res = accuracy(test_pred_idx, test_y_idx)
print(res)
model.set_phase("TRAIN")
import networkx as nx
# import matplotlib.pyplot as plt
from core.net import Net
from core.sfc import SFC
from core.vnf import VNF
import random
# create a new substrate network
substrate_network = Net()
number_of_nodes = 50
probability = 0.2
topology = nx.erdos_renyi_graph(number_of_nodes,
probability,
seed=None,
directed=False)
network_create_counter = 0
while not nx.is_connected(topology):
if network_create_counter >= 10000:
break
network_create_counter += 1
topology = nx.erdos_renyi_graph(number_of_nodes,
probability,
seed=None,
directed=False)
for edge in topology.edges():
bw = random.randint(50, 100)
substrate_network.init_bandwidth_capacity(edge[0], edge[1], bw)
lt = random.uniform(1, 5)
substrate_network.init_link_latency(edge[0], edge[1], lt)