def test_avgpool_module():
shape = (10, 3, 32, 32)
input = smo.Input(nn.Variable(shape))
inp_module = smo.Input(value=input)
pool = smo.AvgPool(parents=[inp_module], kernel=(3, 3), stride=(1, 1))
assert pool.shape == (10, 3, 30, 30)
def __init__(self, parents):
smo.Graph.__init__(self, parents=parents)
join_param = Parameter(shape=(len(parents) + 3, ))
join_prob = F.softmax(join_param)
self.append(
smo.Input(name='input_1',
value=nn.Variable((10, 20, 32, 32)),
eval_prob=join_prob[0] + join_prob[1]))
self.append(
smo.Conv(name='conv',
parents=[self[-1]],
in_channels=20,
out_channels=20,
kernel=(3, 3),
pad=(1, 1),
eval_prob=join_prob[0]))
self.append(
smo.Input(name='input_2',
value=nn.Variable((10, 20, 32, 32)),
eval_prob=join_prob[2]))
self.append(
smo.Join(name='join',
parents=parents + [mi for mi in self],
mode='linear',
join_parameters=join_param))
def test_collapse_module():
shape = (10, 3, 1, 1)
input = smo.Input(nn.Variable(shape))
inp_module = smo.Input(value=input)
collapse = smo.Collapse(parents=[inp_module])
assert collapse.shape == (10, 3)
def test_dwconv_module():
shape = (10, 3, 32, 32)
input = smo.Input(nn.Variable(shape))
inp_module = smo.Input(value=input)
conv = smo.DwConv(parents=[inp_module], in_channels=3, kernel=(3, 3))
assert conv.shape == (10, 3, 30, 30)
def test_merging_module():
shape = (10, 3, 32, 32)
input_1 = smo.Input(nn.Variable(shape))
input_2 = smo.Input(nn.Variable(shape))
merge_add = smo.Merging([input_1, input_2], mode='add')
merge_con = smo.Merging([input_1, input_2], mode='concat')
assert merge_add().shape == (10, 3, 32, 32)
assert merge_con().shape == (10, 6, 32, 32)
def test_zophcell_module():
shape = (10, 3, 32, 32)
input_1 = smo.Input(nn.Variable(shape))
input_2 = smo.Input(nn.Variable(shape))
zc = zoph.ZophCell(parents=[input_1, input_2],
candidates=ZOPH_CANDIDATES,
channels=64)
assert zc().shape == (10, 320, 32, 32)
assert zc.shape == (10, 320, 32, 32)
def _init_modules_from_graph(self, graph):
adj_matrix = nx.adjacency_matrix(graph).todense()
sorted_nodes = np.argsort(graph.nodes)
for i, ii in enumerate(sorted_nodes):
p_idxs = np.where(np.ravel(adj_matrix[sorted_nodes, ii]) > 0)[0]
if len(p_idxs) == 0:
self.append(
smo.Input(name='{}/input'.format(self.name),
value=nn.Variable(self._input_shape)))
else:
rnd_class = self._candidates[np.random.randint(
0, len(self._candidates), 1)[0]]
rnd_channels = np.random.randint(self._min_channels,
self._max_channels, 1)[0]
parents = [self[pi] for pi in p_idxs]
self.append(
rnd_class(name='{}/{}'.format(self.name, i),
parents=parents,
channels=rnd_channels))
self.append(
smo.GlobalAvgPool(name='{}/global_average_pool'.format(self.name),
parents=[self[-1]]))
self.append(
smo.Collapse(name='{}/output_reshape'.format(self.name),
parents=[self[-1]]))
def test_static_module():
module_1 = smo.Module(name='module_1')
module_2 = smo.Module(parents=[module_1], name='module_2')
assert module_1.children[0] == module_2
assert module_2.parents[0] == module_1
class MyModule(smo.Module):
def __init__(self, parents):
smo.Module.__init__(self, parents=parents)
self.linear = mo.Linear(in_features=5, out_features=3)
def call(self, *input):
return self.linear(*input)
input = smo.Input(value=nn.Variable((8, 5)))
my_mod = MyModule(parents=[input])
output = my_mod()
assert 'linear' in my_mod.modules
assert len(my_mod.modules) == 1
assert output.shape == (8, 3)
assert my_mod.shape == (8, 3)
my_mod.reset_value()
assert my_mod._value is None
assert my_mod._shape == -1
input.value = nn.Variable((10, 5))
assert my_mod.shape == (10, 3)
params = my_mod.get_parameters()
assert len(params) == 2
def test_sepconv5x5_module():
shape = (10, 3, 32, 32)
input = smo.Input(nn.Variable(shape))
conv = zoph.SepConv5x5(parents=[input], channels=64)
assert conv.shape == (10, 64, 32, 32)
def test_identity_module():
shape = (10, 3, 32, 32)
input = nn.Variable(shape)
inp_module = smo.Input(value=input)
identity = smo.Identity(parents=[inp_module])
assert identity() == input
def test_avgpool3x3_module():
shape = (10, 3, 32, 32)
n_channels = 20
input = smo.Input(nn.Variable(shape))
pool = zoph.AveragePool3x3(parents=[input], channels=n_channels)
assert pool.input_shapes[0] == (10, 3, 32, 32)
def test_zophblock_module():
shape = (10, 3, 32, 32)
input_1 = smo.Input(nn.Variable(shape))
input_2 = smo.Input(nn.Variable(shape))
zb = zoph.ZophBlock(parents=[input_1,
input_2],
candidates=ZOPH_CANDIDATES,
channels=64)
out = zb()
assert out.shape == (10, 64, 32, 32)
assert zb.shape == (10, 64, 32, 32)
assert len(zb) == 11
cand = zb[2:-1]
for ci, cri in zip(cand, ZOPH_CANDIDATES):
assert type(ci) == cri
def test_maxpool3x3_module():
shape = (10, 3, 32, 32)
input = smo.Input(nn.Variable(shape))
n_channels = 15
pool = zoph.MaxPool3x3(parents=[input], channels=n_channels)
assert pool.input_shapes[0] == (10, 3, 32, 32)
def test_input_module():
shape = (10, 3, 32, 32)
input = nn.Variable(shape)
inp_module = smo.Input(value=input)
assert inp_module() == input
assert inp_module.shape == shape
inp_module.reset_value()
assert inp_module._value == input
def test_zero_module():
shape = (10, 3, 32, 32)
input = nn.Variable(shape)
inp_module = smo.Input(value=input)
zero = smo.Zero(parents=[inp_module])
assert zero.shape == input.shape
inp_module.value = nn.Variable((20, 3, 32, 32))
zero.reset_value()
assert zero.shape == (20, 3, 32, 32)
def test_join_module():
shape = (10, 3, 32, 32)
input_1 = smo.Input(nn.Variable(shape))
input_2 = smo.Input(nn.Variable(shape))
alpha = Parameter(shape=(2, ))
alpha.d = np.array([1, 2])
join_linear = smo.Join([input_1, input_2],
join_parameters=alpha,
mode='linear')
join_sample = smo.Join([input_1, input_2],
join_parameters=alpha,
mode='sample')
join_max = smo.Join([input_1, input_2], join_parameters=alpha, mode='max')
assert join_linear().shape == (10, 3, 32, 32)
assert join_sample().shape == (10, 3, 32, 32)
assert join_max().shape == (10, 3, 32, 32)
assert join_max._idx == 1
assert join_max() == input_2._value
def test_sepconv_module():
shape = (10, 3, 32, 32)
input = smo.Input(nn.Variable(shape))
conv = zoph.SepConv(parents=[input],
in_channels=3,
out_channels=64,
kernel=(3, 3),
pad=(0, 0),
dilation=(1, 1),
with_bias=False)
assert conv.shape == (10, 64, 30, 30)
def test_sepconvbn_module():
shape = (10, 3, 32, 32)
input = smo.Input(nn.Variable(shape))
conv = zoph.SepConvBN(
parents=[input],
out_channels=64,
kernel=(3, 3),
dilation=(1, 1),
)
assert conv.shape == (10, 64, 32, 32)
mod_names = [
mi.name for _, mi in conv.get_modules() if isinstance(mi, smo.Module)
]
ex_mod_names = [
'', '/SepConv_1', '/SepConv_1/dwconv', '/SepConv_1/pwconv',
'/SepConv_2', '/SepConv_2/dwconv', '/SepConv_2/pwconv', '/bn', '/relu'
]
for mni, emni in zip(mod_names, ex_mod_names):
assert mni == emni
def test_graph_module():
class MyGraph(smo.Graph):
def __init__(self, parents):
smo.Graph.__init__(self, parents=parents)
join_param = Parameter(shape=(len(parents) + 3, ))
join_prob = F.softmax(join_param)
self.append(
smo.Input(name='input_1',
value=nn.Variable((10, 20, 32, 32)),
eval_prob=join_prob[0] + join_prob[1]))
self.append(
smo.Conv(name='conv',
parents=[self[-1]],
in_channels=20,
out_channels=20,
kernel=(3, 3),
pad=(1, 1),
eval_prob=join_prob[0]))
self.append(
smo.Input(name='input_2',
value=nn.Variable((10, 20, 32, 32)),
eval_prob=join_prob[2]))
self.append(
smo.Join(name='join',
parents=parents + [mi for mi in self],
mode='linear',
join_parameters=join_param))
input = smo.Input(value=nn.Variable((10, 20, 32, 32)))
my_graph = MyGraph(parents=[input])
output = my_graph()
assert output.shape == (10, 20, 32, 32)
mods = [mi for _, mi in my_graph.get_modules()]
assert len(mods) == 5
def test_relu_module():
input = smo.Input(value=nn.Variable((8, 5)))
relu = smo.ReLU(parents=[input])
output = relu()
assert output.shape == (8, 5)
def test_global_avgpool_module():
shape = (10, 3, 32, 32)
input = smo.Input(nn.Variable(shape))
inp_module = smo.Input(value=input)
pool = smo.GlobalAvgPool(parents=[inp_module])
assert pool.shape == (10, 3, 1, 1)
def test_linear_module():
input = smo.Input(value=nn.Variable((8, 5)))
linear = smo.Linear(parents=[input], in_features=5, out_features=3)
output = linear()
assert output.shape == (8, 3)
str_summary += mi.name + " chosen with probability "
str_summary += str(mi._eval_prob.d) + "\n"
return str_summary
def save_graph(self, path):
"""
save whole network/graph (in a PDF file)
Args:
path
"""
gvg = self.get_gv_graph()
gvg.render(path + '/graph')
if __name__ == '__main__':
input_1 = smo.Input(name='input_1', value=nn.Variable((10, 16, 32, 32)))
input_2 = smo.Input(name='input_2', value=nn.Variable((10, 32, 16, 16)))
conv = Conv(name='test_conv',
parents=[input_1, input_2],
channels=64,
kernel=(3, 3),
pad=(1, 1))
c3x3 = Conv3x3(name='test_c3x3', parents=[input_1, input_2], channels=64)
c5x5 = Conv5x5(name='test_c5x5', parents=[input_1, input_2], channels=64)
mp3x3 = MaxPool2x2(name='test_mp3x3',
parents=[input_1, input_2],
channels=64)
ap3x3 = AvgPool2x2(name='test_ap3x3',
parents=[input_1, input_2],
channels=64)
def test_batchnorm_module():
input = smo.Input(value=nn.Variable((10, 3, 32, 32)))
bn = smo.BatchNormalization(parents=[input], n_features=3, n_dims=4)
output = bn()
assert output.shape == (10, 3, 32, 32)
def calc_latency_and_onnx(exp_nr, calc_latency=False, ext_name='cpu',
device_id=0, onnx=False):
nn.clear_parameters()
N = 10 # number of random networks to sample
estim_net = None
estim_accum_by_graph = None
estim_accum_by_mod = None
# 10 **************************
if exp_nr == 10:
from nnabla_nas.contrib import zoph
ctx = get_extension_context(ext_name=ext_name, device_id=device_id)
nn.set_default_context(ctx)
OUTPUT_DIR = './logs/zoph/one_net/'
# Sample one ZOPH network from the search space
shape = (1, 3, 32, 32)
input = nn.Variable(shape)
zn = zoph.SearchNet()
zn.apply(training=False)
output = zn(input)
estim_net, estim_accum_by_graph, estim_accum_by_mod = \
init_calc_latency(output, ext_name=ext_name, device_id=device_id)
# zn_unique_active_modules = get_active_and_profiled_modules(zn)
# SAVE GRAPH in PDF
zn.save_graph(OUTPUT_DIR + 'zn')
# SAVE WHOLE NET. Calc whole net (real) latency using [Profiler]
# Calculate also layer-based latency
# The modules are discovered using the nnabla graph of the whole net
# The latency is then calculated based on each individual module's
# nnabla graph [LatencyGraphEstimator]
net_lat, acc_lat = zn.save_net_nnp(
OUTPUT_DIR + 'zn', input, output,
calc_latency=calc_latency,
func_real_latency=estim_net,
func_accum_latency=estim_accum_by_graph
)
# reset estim_accum_by_graph
estim_net, estim_accum_by_graph, estim_accum_by_mod = \
init_calc_latency(output, ext_name=ext_name, device_id=device_id)
# SAVE ALL MODULES. Calc layer-based latency.
# The modules are discovered by going over the module list.
# The latency is then calculated based on each individual module's
# nnabla graph [LatencyGraphEstimator]
acc_lat_g = zn.save_modules_nnp(
OUTPUT_DIR + 'zn_by_graph',
active_only=True,
calc_latency=calc_latency,
func_latency=estim_accum_by_graph
)
# SAVE ALL MODULES. Calc layer-based latency.
# The modules are discovered by going over the module list.
# The latency is then calculated using the module [LatencyEstimator]
# **** This function is deprecated ****
acc_lat_m = zn.save_modules_nnp_by_mod(
OUTPUT_DIR + 'zn_by_module',
active_only=True,
calc_latency=calc_latency,
func_latency=estim_accum_by_mod
)
# reset estim_accum_by_graph, estim_accum_by_mod
estim_net, estim_accum_by_graph, estim_accum_by_mod = \
init_calc_latency(output, ext_name=ext_name, device_id=device_id)
# Just obtain the latencies of all modules without saving files
# By graph
latencies_1, acc_lat_1 = zn.get_latency(estim_accum_by_graph)
# By module (deprecated)
latencies_2, acc_lat_2 = zn.get_latency_by_mod(estim_accum_by_mod)
print(net_lat, acc_lat, acc_lat_g, acc_lat_m, acc_lat_1, acc_lat_2)
# CONVERT ALL TO ONNX
if onnx:
zn.convert_npp_to_onnx(OUTPUT_DIR)
# VERBOSITY - INFO OF NETWORK CONTENT
# with open(OUTPUT_DIR + 'zn.txt', 'w') as f:
# print_me(zn, f)
# 11 **************************
if exp_nr == 11:
from nnabla_nas.contrib import zoph
ctx = get_extension_context(ext_name=ext_name, device_id=device_id)
nn.set_default_context(ctx)
OUTPUT_DIR = './logs/zoph/many_different_nets/'
shape = (1, 3, 32, 32)
input = nn.Variable(shape)
# Sample N zoph networks from the search space
for i in range(0, N):
nn.clear_parameters()
zn = zoph.SearchNet()
zn.apply(training=False)
output = zn(input)
if calc_latency:
estim_net, estim_accum_by_graph, estim_accum_by_mod = \
init_calc_latency(output,
ext_name=ext_name,
device_id=device_id
)
zn.save_graph(OUTPUT_DIR + 'zn' + str(i))
zn.save_net_nnp(OUTPUT_DIR + 'zn' + str(i), input, output,
calc_latency=calc_latency,
func_real_latency=estim_net,
func_accum_latency=estim_accum_by_graph
)
zn.save_modules_nnp(OUTPUT_DIR + 'zn' + str(i), active_only=True,
calc_latency=calc_latency,
func_latency=estim_accum_by_graph
)
if onnx:
zn = zoph.SearchNet()
zn.convert_npp_to_onnx(OUTPUT_DIR, opset='opset_snpe')
# 12 **************************
if exp_nr == 12:
from nnabla_nas.contrib import zoph
import time
ctx = get_extension_context(ext_name=ext_name, device_id=device_id)
nn.set_default_context(ctx)
OUTPUT_DIR = './logs/zoph/same_net_many_times/'
shape = (1, 3, 32, 32)
input = nn.Variable(shape)
zn = zoph.SearchNet()
zn.apply(training=False)
output = zn(input)
# Measure add-hoc latency of zoph network
for i in range(0, N):
n_run = 100
# warm up
output.forward()
result = 0.0
for i in range(n_run):
start = time.time()
output.forward()
stop = time.time()
result += stop - start
mean_time = result / n_run
print(mean_time*1000)
# Measure latency on same zoph network N times
for i in range(0, N):
if calc_latency:
estim_net, estim_accum_by_graph, estim_accum_by_mod = \
init_calc_latency(output,
ext_name=ext_name,
device_id=device_id
)
zn.save_net_nnp(OUTPUT_DIR + 'zn' + str(i), input, output,
calc_latency=calc_latency,
func_real_latency=estim_net,
func_accum_latency=estim_accum_by_graph
)
zn.save_modules_nnp(OUTPUT_DIR + 'zn' + str(i), active_only=True,
calc_latency=calc_latency,
func_latency=estim_accum_by_graph
)
zn.save_graph(OUTPUT_DIR)
if onnx:
zn.convert_npp_to_onnx(OUTPUT_DIR)
# 20 **************************
if exp_nr == 20:
from nnabla_nas.contrib import random_wired
ctx = get_extension_context(ext_name=ext_name, device_id=device_id)
nn.set_default_context(ctx)
OUTPUT_DIR = './logs/rdn/one_net/'
# Sample one random wired network from the search space
shape = (1, 3, 32, 32)
input = nn.Variable(shape)
rw = random_wired.TrainNet()
rw.apply(training=False)
output = rw(input)
if calc_latency:
estim_net, estim_accum_by_graph, estim_accum_by_mod = \
init_calc_latency(output,
ext_name=ext_name,
device_id=device_id
)
rw.save_graph(OUTPUT_DIR + 'rw')
rw.save_net_nnp(OUTPUT_DIR + 'rw', input, output,
calc_latency=calc_latency,
func_real_latency=estim_net,
func_accum_latency=estim_accum_by_graph
)
rw.save_modules_nnp(OUTPUT_DIR + 'rw', active_only=True,
calc_latency=calc_latency,
func_latency=estim_accum_by_graph
)
if onnx:
rw.convert_npp_to_onnx(OUTPUT_DIR)
# 21 **************************
if exp_nr == 21:
from nnabla_nas.contrib import random_wired
ctx = get_extension_context(ext_name=ext_name, device_id=device_id)
nn.set_default_context(ctx)
OUTPUT_DIR = './logs/rdn/many_different_nets/'
shape = (1, 3, 32, 32)
input = nn.Variable(shape)
# Measure latency on same rdn network N times
for i in range(0, N):
nn.clear_parameters()
rw = random_wired.TrainNet()
rw.apply(training=False)
output = rw(input)
if calc_latency:
estim_net, estim_accum_by_graph, estim_accum_by_mod = \
init_calc_latency(output,
ext_name=ext_name,
device_id=device_id
)
rw.save_graph(OUTPUT_DIR + 'rw' + str(i))
rw.save_net_nnp(OUTPUT_DIR + 'rw' + str(i), input, output,
calc_latency=calc_latency,
func_real_latency=estim_net,
func_accum_latency=estim_accum_by_graph
)
rw.save_modules_nnp(OUTPUT_DIR + 'rw' + str(i), active_only=True,
calc_latency=calc_latency,
func_latency=estim_accum_by_graph
)
if onnx:
rw = random_wired.TrainNet()
rw.convert_npp_to_onnx(OUTPUT_DIR, opset='opset_snpe')
# 22 **************************
if exp_nr == 22:
from nnabla_nas.contrib import random_wired
import time
OUTPUT_DIR = './logs/rdn/same_net_many_times/'
ctx = get_extension_context(ext_name=ext_name, device_id=device_id)
nn.set_default_context(ctx)
shape = (1, 3, 32, 32)
input = nn.Variable(shape)
rw = random_wired.TrainNet()
rw.apply(training=False)
output = rw(input)
for i in range(0, N):
n_run = 10
# warm up
output.forward()
result = 0.0
for i in range(n_run):
start = time.time()
output.forward()
stop = time.time()
result += stop - start
mean_time = result / n_run
print(mean_time*1000)
# Measure latency on same rdn network N times
for i in range(0, N):
if calc_latency:
estim_net, estim_accum_by_graph, estim_accum_by_mod = \
init_calc_latency(output,
ext_name=ext_name,
device_id=device_id
)
rw.save_net_nnp(OUTPUT_DIR + 'rw' + str(i), input, output,
calc_latency=calc_latency,
func_real_latency=estim_net,
func_accum_latency=estim_accum_by_graph
)
rw.save_modules_nnp(OUTPUT_DIR + 'rw' + str(i), active_only=True,
calc_latency=calc_latency,
func_latency=estim_accum_by_graph
)
rw.save_graph(OUTPUT_DIR + 'rw' + str(i))
if onnx:
rw.convert_npp_to_onnx(OUTPUT_DIR)
# 31 **************************
if exp_nr == 31:
from nnabla_nas.contrib.classification.mobilenet import SearchNet
ctx = get_extension_context(ext_name=ext_name, device_id=device_id)
nn.set_default_context(ctx)
OUTPUT_DIR = './logs/mobilenet/many_different_nets/'
input = nn.Variable((1, 3, 224, 224))
# number of random networks to sample
# Sample N networks from the search space
for i in range(0, N):
nn.clear_parameters()
mobile_net = SearchNet(num_classes=1000)
mobile_net.apply(training=False)
output = mobile_net(input)
if calc_latency:
estim_net, estim_accum_by_graph, estim_accum_by_mod = \
init_calc_latency(output,
ext_name=ext_name,
device_id=device_id
)
# This calculates the actual latency of the network
# and the accum. latency by adding the latencies of
# each module, following the nnabla graph
mobile_net.save_net_nnp(OUTPUT_DIR + 'mn' + str(i), input, output,
calc_latency=calc_latency,
func_real_latency=estim_net,
func_accum_latency=estim_accum_by_graph
)
"""
# This calculates the latency of each module going
# over the nnabla graph (deprecated)
mobile_net.calc_latency_all_modules(
OUTPUT_DIR + 'graph_mn' + str(i), output,
func_latency=estim_accum_by_graph)
"""
# This calculates the latency of each module going
# the tree of the modules
mobile_net.save_modules_nnp(
OUTPUT_DIR + 'mn' + str(i), active_only=True,
calc_latency=calc_latency,
func_latency=estim_accum_by_graph
)
if onnx:
mobile_net = SearchNet()
mobile_net.convert_npp_to_onnx(OUTPUT_DIR, opset='opset_snpe')
# 32 **************************
if exp_nr == 32:
from nnabla_nas.contrib.classification.mobilenet import SearchNet
import time
OUTPUT_DIR = './logs/mobilenet/same_net_many_times/'
ctx = get_extension_context(ext_name=ext_name, device_id=device_id)
nn.set_default_context(ctx)
input = nn.Variable((1, 3, 224, 224))
mobile_net = SearchNet(num_classes=1000)
mobile_net.apply(training=False)
output = mobile_net(input)
for i in range(0, N):
n_run = 100
# warm up
output.forward()
result = 0.0
for i in range(n_run):
start = time.time()
output.forward()
stop = time.time()
result += stop - start
mean_time = result / n_run
print(mean_time*1000)
# Measure latency on same network N times
for i in range(0, N):
if calc_latency:
estim_net, estim_accum_by_graph, estim_accum_by_mod = \
init_calc_latency(output,
ext_name=ext_name,
device_id=device_id
)
mobile_net.save_net_nnp(OUTPUT_DIR + 'mn' + str(i), input, output,
calc_latency=calc_latency,
func_real_latency=estim_net,
func_accum_latency=estim_accum_by_graph
)
mobile_net.save_modules_nnp(
OUTPUT_DIR + 'mn' + str(i), active_only=True,
calc_latency=calc_latency,
func_latency=estim_accum_by_graph
)
if onnx:
mobile_net.convert_npp_to_onnx(OUTPUT_DIR)
# 4 **************************
if exp_nr == 4:
from nnabla_nas.module import static as smo
input1 = nn.Variable((1, 256, 32, 32))
input2 = nn.Variable((1, 384, 32, 32))
input3 = nn.Variable((1, 128, 32, 32))
input4 = nn.Variable((1, 768, 32, 32))
input5 = nn.Variable((1, 1280, 32, 32))
input6 = nn.Variable((1, 2048, 32, 32))
input7 = nn.Variable((1, 512, 32, 32))
input8 = nn.Variable((1, 192, 32, 32))
input9 = nn.Variable((1, 224, 32, 32))
static_input1 = smo.Input(value=input1)
static_input2 = smo.Input(value=input2)
static_input3 = smo.Input(value=input3)
static_input4 = smo.Input(value=input4)
static_input5 = smo.Input(value=input5)
static_input6 = smo.Input(value=input6)
static_input7 = smo.Input(value=input7)
static_input8 = smo.Input(value=input8)
static_input9 = smo.Input(value=input9)
myconv1 = smo.Conv(parents=[static_input1], in_channels=256,
out_channels=128, kernel=(1, 1), pad=None, group=1)
myconv2 = smo.Conv(parents=[static_input2], in_channels=384,
out_channels=128, kernel=(1, 1), pad=None, group=1)
myconv3 = smo.Conv(parents=[static_input3], in_channels=128,
out_channels=256, kernel=(1, 1), pad=None, group=1)
myconv4 = smo.Conv(parents=[static_input4], in_channels=768,
out_channels=256, kernel=(1, 1))
myconv5 = smo.Conv(parents=[static_input5], in_channels=1280,
out_channels=256, kernel=(1, 1), pad=None, group=1)
myconv6 = smo.Conv(parents=[static_input6], in_channels=2048,
out_channels=256, kernel=(1, 1), pad=None, group=1)
myconv7 = smo.Conv(parents=[static_input7], in_channels=512,
out_channels=512, kernel=(3, 3), pad=(1, 1), group=1
)
myconv8 = smo.Conv(parents=[static_input8], in_channels=192,
out_channels=512, kernel=(7, 7), pad=(3, 3), group=1
)
myconv9 = smo.Conv(parents=[static_input9], in_channels=224,
out_channels=128, kernel=(5, 5), pad=(2, 2), group=1
)
output1 = myconv1()
output2 = myconv2()
output3 = myconv3()
output4 = myconv4()
output5 = myconv5()
output6 = myconv6()
output7 = myconv7()
output8 = myconv8()
output9 = myconv9()
N = 10
for i in range(0, N):
mean_time = estim_fwd(output1)
print("1, ", mean_time)
mean_time = estim_fwd(output2)
print("2, ", mean_time)
mean_time = estim_fwd(output3)
print("3, ", mean_time)
mean_time = estim_fwd(output4)
print("4, ", mean_time)
mean_time = estim_fwd(output5)
print("5, ", mean_time)
mean_time = estim_fwd(output6)
print("6, ", mean_time)
mean_time = estim_fwd(output7)
print("7, ", mean_time)
mean_time = estim_fwd(output8)
print("8, ", mean_time)
mean_time = estim_fwd(output9)
print("9, ", mean_time)
N = 100
from nnabla_nas.utils.estimator.latency import LatencyGraphEstimator
for i in range(0, N):
estimation = LatencyGraphEstimator(n_run=100, ext_name='cpu')
latency = estimation.get_estimation(myconv1)
latency = estimation.get_estimation(myconv2)
latency = estimation.get_estimation(myconv3)
latency = estimation.get_estimation(myconv4)
latency = estimation.get_estimation(myconv5)
latency = estimation.get_estimation(myconv6)
latency = estimation.get_estimation(myconv7)
latency = estimation.get_estimation(myconv8)
latency = estimation.get_estimation(myconv9)
estimation = LatencyGraphEstimator(n_run=100, ext_name='cpu')
latency = estimation.get_estimation(myconv9)
latency = estimation.get_estimation(myconv8)
latency = estimation.get_estimation(myconv7)
latency = estimation.get_estimation(myconv6)
latency = estimation.get_estimation(myconv5)
latency = estimation.get_estimation(myconv4)
latency = estimation.get_estimation(myconv3)
latency = estimation.get_estimation(myconv2)
latency = estimation.get_estimation(myconv1)
estimation = LatencyGraphEstimator(n_run=100, ext_name='cpu')
latency = estimation.get_estimation(myconv6)
latency = estimation.get_estimation(myconv9)
latency = estimation.get_estimation(myconv1)
latency = estimation.get_estimation(myconv4)
latency = estimation.get_estimation(myconv8)
latency = estimation.get_estimation(myconv3)
latency = estimation.get_estimation(myconv5)
latency = estimation.get_estimation(myconv7)
latency = estimation.get_estimation(myconv2)
latency += 0 # to avoid lint/flake8 error
# 5 **************************
if exp_nr == 5:
from nnabla_nas.module import static as smo
from nnabla_nas.utils.estimator.latency import LatencyGraphEstimator
from numpy.random import permutation
import numpy as np
run_also_ours_at_the_end = True
N_conv = 50 # number of different convolutions tried
in_sizes = np.random.randint(low=1, high=1000, size=N_conv)
out_sizes = np.random.randint(low=1, high=600, size=N_conv)
kernel_sizes = np.random.randint(low=1, high=7, size=N_conv)
feat_sizes = np.random.randint(low=16, high=48, size=N_conv)
N = 100
for j in range(N):
estimation = LatencyGraphEstimator(n_run=100, ext_name='cpu')
print('****************** RUN ********************')
for i in permutation(N_conv):
input = nn.Variable((1, in_sizes[i],
feat_sizes[i], feat_sizes[i]))
static_input = smo.Input(value=input)
myconv = smo.Conv(parents=[static_input],
in_channels=in_sizes[i],
out_channels=out_sizes[i],
kernel=(kernel_sizes[i], kernel_sizes[i]),
pad=None, group=1
)
output = myconv()
latency = estimation.get_estimation(myconv)
latency += 0 # to avoid lint/flake8 error
if run_also_ours_at_the_end is True:
print('*********** NOW IT IS OUR TURN ***********')
for i in range(N_conv):
input = nn.Variable((1, in_sizes[i],
feat_sizes[i], feat_sizes[i]))
static_input = smo.Input(value=input)
myconv = smo.Conv(parents=[static_input],
in_channels=in_sizes[i],
out_channels=out_sizes[i],
kernel=(kernel_sizes[i], kernel_sizes[i]),
pad=None, group=1
)
output = myconv()
mean_time = estim_fwd(output, n_run=100) * 1000 # in ms
print('Our_Conv : 100 :', mean_time, ':',
'[(1, ' + str(in_sizes[i]) + ', ' + str(feat_sizes[i]) +
', ' + str(feat_sizes[i]) + ')]',
':', out_sizes[i], ':', kernel_sizes[i]
)
# 6 **************************
if exp_nr == 6:
import onnx
load_onnx = False
if len(sys.argv) > 2:
INPUT_DIR = sys.argv[2]
else:
INPUT_DIR = './logs/zoph/one_net/'
if len(sys.argv) > 3:
load_onnx = True
existing_networks = glob.glob(INPUT_DIR + '/*' + os.path.sep)
all_nets_latencies = dict.fromkeys([])
all_nets = dict.fromkeys([])
net_idx = 0
for network in existing_networks:
all_blocks = glob.glob(network + '**/*.acclat', recursive=True)
blocks = dict.fromkeys([])
block_idx = 0
this_net_accumulated_latency = 0.0
this_net_accumulated_latency_of_convs = 0.0
this_net_accumulated_latency_of_relus = 0.0
this_net_accumulated_latency_of_bns = 0.0
this_net_accumulated_latency_of_merges = 0.0
this_net_accumulated_latency_of_pools = 0.0
this_net_accumulated_latency_of_reshapes = 0.0
this_net_accumulated_latency_of_affines = 0.0
this_net_accumulated_latency_of_add2s = 0.0
for block_lat in all_blocks:
block = block_lat[:-7] + '.onnx'
print('.... READING .... --> ' + block)
# Reading latency for each of the blocks of layers
with open(block_lat, 'r') as f:
block_latency = float(f.read())
this_net_accumulated_latency += block_latency
# Layer-type-wise latencies tested
# for Zoph
# for Random Wired networks
# for mobilenet
layer_name = block.split('/')[-1].split('.')[-2]
if layer_name.find('bn') != -1:
this_net_accumulated_latency_of_bns += block_latency
elif layer_name.find('batchnorm') != -1:
this_net_accumulated_latency_of_bns += block_latency
elif layer_name.find('relu') != -1:
this_net_accumulated_latency_of_relus += block_latency
elif layer_name.find('conv') != -1:
this_net_accumulated_latency_of_convs += block_latency
elif layer_name.find('merg') != -1:
this_net_accumulated_latency_of_merges += block_latency
elif layer_name.find('pool') != -1:
this_net_accumulated_latency_of_pools += block_latency
elif layer_name.find('con') != -1: # from concat
this_net_accumulated_latency_of_merges += block_latency
elif layer_name.find('reshape') != -1:
this_net_accumulated_latency_of_reshapes += block_latency
elif layer_name.find('linear') != -1:
this_net_accumulated_latency_of_affines += block_latency
elif layer_name.find('add2') != -1:
this_net_accumulated_latency_of_add2s += block_latency
this_block = dict.fromkeys([])
this_block['latency'] = block_latency
if load_onnx:
# Interesting FIELDS in params.graph:
# 'input', 'name', 'node', 'output'
params = onnx.load(block)
this_block['name'] = params.graph.name
this_block['input'] = params.graph.input
this_block['output'] = params.graph.output
this_block['nodes'] = params.graph.node
blocks[block_idx] = this_block
block_idx += 1
net_realat_file = network[:-1] + '.realat'
with open(net_realat_file, 'r') as f:
this_net_real_latency = float(f.read())
net_acclat_file = network[:-1] + '.acclat'
with open(net_acclat_file, 'r') as f:
this_net_acc_latency = float(f.read())
this_net = dict.fromkeys([])
this_net['real_latency'] = this_net_real_latency
this_net['accum_latency_graph'] = this_net_acc_latency
this_net['accum_latency_module'] = this_net_accumulated_latency
if load_onnx:
net_file = network[:-1] + '.onnx'
print('xxxx READING xxxx --> ' + net_file)
params = onnx.load(net_file)
this_net['name'] = params.graph.name
this_net['input'] = params.graph.input
this_net['output'] = params.graph.output
this_net['nodes'] = params.graph.node
all_nets_latencies[net_idx] = [
this_net_real_latency,
this_net_acc_latency,
this_net_accumulated_latency,
this_net_accumulated_latency_of_convs,
this_net_accumulated_latency_of_bns,
this_net_accumulated_latency_of_relus,
this_net_accumulated_latency_of_pools,
this_net_accumulated_latency_of_merges,
this_net_accumulated_latency_of_reshapes,
this_net_accumulated_latency_of_affines,
this_net_accumulated_latency_of_add2s,
]
all_nets[net_idx] = this_net
net_idx += 1
# Compare accumulated latency to net latencies, do a plot:
print('LATENCY Results from ' + INPUT_DIR)
print('NETWORK, LAYER-WISE (by graph), ',
'LAYER-WISE (by module), of CONVs, of BNs, of RELUs, ',
'of POOLs, of MERGEs/CONCATs, of RESHAPEs, of AFFINEs, ',
'of ADD2 layers'
)
for i in range(len(all_nets_latencies)):
# print(all_nets_latencies[i])
print(['%7.3f' % val for val in all_nets_latencies[i]])
def test_dropout_module():
input = smo.Input(value=nn.Variable((8, 5)))
dropout = smo.Dropout(parents=[input])
output = dropout()
assert output.shape == (8, 5)
