def test_avgpool_module(): shape = (10, 3, 32, 32) input = smo.Input(nn.Variable(shape)) inp_module = smo.Input(value=input) pool = smo.AvgPool(parents=[inp_module], kernel=(3, 3), stride=(1, 1)) assert pool.shape == (10, 3, 30, 30)
def __init__(self, parents): smo.Graph.__init__(self, parents=parents) join_param = Parameter(shape=(len(parents) + 3, )) join_prob = F.softmax(join_param) self.append( smo.Input(name='input_1', value=nn.Variable((10, 20, 32, 32)), eval_prob=join_prob[0] + join_prob[1])) self.append( smo.Conv(name='conv', parents=[self[-1]], in_channels=20, out_channels=20, kernel=(3, 3), pad=(1, 1), eval_prob=join_prob[0])) self.append( smo.Input(name='input_2', value=nn.Variable((10, 20, 32, 32)), eval_prob=join_prob[2])) self.append( smo.Join(name='join', parents=parents + [mi for mi in self], mode='linear', join_parameters=join_param))
def test_collapse_module(): shape = (10, 3, 1, 1) input = smo.Input(nn.Variable(shape)) inp_module = smo.Input(value=input) collapse = smo.Collapse(parents=[inp_module]) assert collapse.shape == (10, 3)
def test_dwconv_module(): shape = (10, 3, 32, 32) input = smo.Input(nn.Variable(shape)) inp_module = smo.Input(value=input) conv = smo.DwConv(parents=[inp_module], in_channels=3, kernel=(3, 3)) assert conv.shape == (10, 3, 30, 30)
def test_merging_module(): shape = (10, 3, 32, 32) input_1 = smo.Input(nn.Variable(shape)) input_2 = smo.Input(nn.Variable(shape)) merge_add = smo.Merging([input_1, input_2], mode='add') merge_con = smo.Merging([input_1, input_2], mode='concat') assert merge_add().shape == (10, 3, 32, 32) assert merge_con().shape == (10, 6, 32, 32)
def test_zophcell_module(): shape = (10, 3, 32, 32) input_1 = smo.Input(nn.Variable(shape)) input_2 = smo.Input(nn.Variable(shape)) zc = zoph.ZophCell(parents=[input_1, input_2], candidates=ZOPH_CANDIDATES, channels=64) assert zc().shape == (10, 320, 32, 32) assert zc.shape == (10, 320, 32, 32)
def _init_modules_from_graph(self, graph): adj_matrix = nx.adjacency_matrix(graph).todense() sorted_nodes = np.argsort(graph.nodes) for i, ii in enumerate(sorted_nodes): p_idxs = np.where(np.ravel(adj_matrix[sorted_nodes, ii]) > 0)[0] if len(p_idxs) == 0: self.append( smo.Input(name='{}/input'.format(self.name), value=nn.Variable(self._input_shape))) else: rnd_class = self._candidates[np.random.randint( 0, len(self._candidates), 1)[0]] rnd_channels = np.random.randint(self._min_channels, self._max_channels, 1)[0] parents = [self[pi] for pi in p_idxs] self.append( rnd_class(name='{}/{}'.format(self.name, i), parents=parents, channels=rnd_channels)) self.append( smo.GlobalAvgPool(name='{}/global_average_pool'.format(self.name), parents=[self[-1]])) self.append( smo.Collapse(name='{}/output_reshape'.format(self.name), parents=[self[-1]]))
def test_static_module(): module_1 = smo.Module(name='module_1') module_2 = smo.Module(parents=[module_1], name='module_2') assert module_1.children[0] == module_2 assert module_2.parents[0] == module_1 class MyModule(smo.Module): def __init__(self, parents): smo.Module.__init__(self, parents=parents) self.linear = mo.Linear(in_features=5, out_features=3) def call(self, *input): return self.linear(*input) input = smo.Input(value=nn.Variable((8, 5))) my_mod = MyModule(parents=[input]) output = my_mod() assert 'linear' in my_mod.modules assert len(my_mod.modules) == 1 assert output.shape == (8, 3) assert my_mod.shape == (8, 3) my_mod.reset_value() assert my_mod._value is None assert my_mod._shape == -1 input.value = nn.Variable((10, 5)) assert my_mod.shape == (10, 3) params = my_mod.get_parameters() assert len(params) == 2
def test_sepconv5x5_module(): shape = (10, 3, 32, 32) input = smo.Input(nn.Variable(shape)) conv = zoph.SepConv5x5(parents=[input], channels=64) assert conv.shape == (10, 64, 32, 32)
def test_identity_module(): shape = (10, 3, 32, 32) input = nn.Variable(shape) inp_module = smo.Input(value=input) identity = smo.Identity(parents=[inp_module]) assert identity() == input
def test_avgpool3x3_module(): shape = (10, 3, 32, 32) n_channels = 20 input = smo.Input(nn.Variable(shape)) pool = zoph.AveragePool3x3(parents=[input], channels=n_channels) assert pool.input_shapes[0] == (10, 3, 32, 32)
def test_zophblock_module(): shape = (10, 3, 32, 32) input_1 = smo.Input(nn.Variable(shape)) input_2 = smo.Input(nn.Variable(shape)) zb = zoph.ZophBlock(parents=[input_1, input_2], candidates=ZOPH_CANDIDATES, channels=64) out = zb() assert out.shape == (10, 64, 32, 32) assert zb.shape == (10, 64, 32, 32) assert len(zb) == 11 cand = zb[2:-1] for ci, cri in zip(cand, ZOPH_CANDIDATES): assert type(ci) == cri
def test_maxpool3x3_module(): shape = (10, 3, 32, 32) input = smo.Input(nn.Variable(shape)) n_channels = 15 pool = zoph.MaxPool3x3(parents=[input], channels=n_channels) assert pool.input_shapes[0] == (10, 3, 32, 32)
def test_input_module(): shape = (10, 3, 32, 32) input = nn.Variable(shape) inp_module = smo.Input(value=input) assert inp_module() == input assert inp_module.shape == shape inp_module.reset_value() assert inp_module._value == input
def test_zero_module(): shape = (10, 3, 32, 32) input = nn.Variable(shape) inp_module = smo.Input(value=input) zero = smo.Zero(parents=[inp_module]) assert zero.shape == input.shape inp_module.value = nn.Variable((20, 3, 32, 32)) zero.reset_value() assert zero.shape == (20, 3, 32, 32)
def test_join_module(): shape = (10, 3, 32, 32) input_1 = smo.Input(nn.Variable(shape)) input_2 = smo.Input(nn.Variable(shape)) alpha = Parameter(shape=(2, )) alpha.d = np.array([1, 2]) join_linear = smo.Join([input_1, input_2], join_parameters=alpha, mode='linear') join_sample = smo.Join([input_1, input_2], join_parameters=alpha, mode='sample') join_max = smo.Join([input_1, input_2], join_parameters=alpha, mode='max') assert join_linear().shape == (10, 3, 32, 32) assert join_sample().shape == (10, 3, 32, 32) assert join_max().shape == (10, 3, 32, 32) assert join_max._idx == 1 assert join_max() == input_2._value
def test_sepconv_module(): shape = (10, 3, 32, 32) input = smo.Input(nn.Variable(shape)) conv = zoph.SepConv(parents=[input], in_channels=3, out_channels=64, kernel=(3, 3), pad=(0, 0), dilation=(1, 1), with_bias=False) assert conv.shape == (10, 64, 30, 30)
def test_sepconvbn_module(): shape = (10, 3, 32, 32) input = smo.Input(nn.Variable(shape)) conv = zoph.SepConvBN( parents=[input], out_channels=64, kernel=(3, 3), dilation=(1, 1), ) assert conv.shape == (10, 64, 32, 32) mod_names = [ mi.name for _, mi in conv.get_modules() if isinstance(mi, smo.Module) ] ex_mod_names = [ '', '/SepConv_1', '/SepConv_1/dwconv', '/SepConv_1/pwconv', '/SepConv_2', '/SepConv_2/dwconv', '/SepConv_2/pwconv', '/bn', '/relu' ] for mni, emni in zip(mod_names, ex_mod_names): assert mni == emni
def test_graph_module(): class MyGraph(smo.Graph): def __init__(self, parents): smo.Graph.__init__(self, parents=parents) join_param = Parameter(shape=(len(parents) + 3, )) join_prob = F.softmax(join_param) self.append( smo.Input(name='input_1', value=nn.Variable((10, 20, 32, 32)), eval_prob=join_prob[0] + join_prob[1])) self.append( smo.Conv(name='conv', parents=[self[-1]], in_channels=20, out_channels=20, kernel=(3, 3), pad=(1, 1), eval_prob=join_prob[0])) self.append( smo.Input(name='input_2', value=nn.Variable((10, 20, 32, 32)), eval_prob=join_prob[2])) self.append( smo.Join(name='join', parents=parents + [mi for mi in self], mode='linear', join_parameters=join_param)) input = smo.Input(value=nn.Variable((10, 20, 32, 32))) my_graph = MyGraph(parents=[input]) output = my_graph() assert output.shape == (10, 20, 32, 32) mods = [mi for _, mi in my_graph.get_modules()] assert len(mods) == 5
def test_relu_module(): input = smo.Input(value=nn.Variable((8, 5))) relu = smo.ReLU(parents=[input]) output = relu() assert output.shape == (8, 5)
def test_global_avgpool_module(): shape = (10, 3, 32, 32) input = smo.Input(nn.Variable(shape)) inp_module = smo.Input(value=input) pool = smo.GlobalAvgPool(parents=[inp_module]) assert pool.shape == (10, 3, 1, 1)
def test_linear_module(): input = smo.Input(value=nn.Variable((8, 5))) linear = smo.Linear(parents=[input], in_features=5, out_features=3) output = linear() assert output.shape == (8, 3)
str_summary += mi.name + " chosen with probability " str_summary += str(mi._eval_prob.d) + "\n" return str_summary def save_graph(self, path): """ save whole network/graph (in a PDF file) Args: path """ gvg = self.get_gv_graph() gvg.render(path + '/graph') if __name__ == '__main__': input_1 = smo.Input(name='input_1', value=nn.Variable((10, 16, 32, 32))) input_2 = smo.Input(name='input_2', value=nn.Variable((10, 32, 16, 16))) conv = Conv(name='test_conv', parents=[input_1, input_2], channels=64, kernel=(3, 3), pad=(1, 1)) c3x3 = Conv3x3(name='test_c3x3', parents=[input_1, input_2], channels=64) c5x5 = Conv5x5(name='test_c5x5', parents=[input_1, input_2], channels=64) mp3x3 = MaxPool2x2(name='test_mp3x3', parents=[input_1, input_2], channels=64) ap3x3 = AvgPool2x2(name='test_ap3x3', parents=[input_1, input_2], channels=64)
def test_batchnorm_module(): input = smo.Input(value=nn.Variable((10, 3, 32, 32))) bn = smo.BatchNormalization(parents=[input], n_features=3, n_dims=4) output = bn() assert output.shape == (10, 3, 32, 32)
def calc_latency_and_onnx(exp_nr, calc_latency=False, ext_name='cpu', device_id=0, onnx=False): nn.clear_parameters() N = 10 # number of random networks to sample estim_net = None estim_accum_by_graph = None estim_accum_by_mod = None # 10 ************************** if exp_nr == 10: from nnabla_nas.contrib import zoph ctx = get_extension_context(ext_name=ext_name, device_id=device_id) nn.set_default_context(ctx) OUTPUT_DIR = './logs/zoph/one_net/' # Sample one ZOPH network from the search space shape = (1, 3, 32, 32) input = nn.Variable(shape) zn = zoph.SearchNet() zn.apply(training=False) output = zn(input) estim_net, estim_accum_by_graph, estim_accum_by_mod = \ init_calc_latency(output, ext_name=ext_name, device_id=device_id) # zn_unique_active_modules = get_active_and_profiled_modules(zn) # SAVE GRAPH in PDF zn.save_graph(OUTPUT_DIR + 'zn') # SAVE WHOLE NET. Calc whole net (real) latency using [Profiler] # Calculate also layer-based latency # The modules are discovered using the nnabla graph of the whole net # The latency is then calculated based on each individual module's # nnabla graph [LatencyGraphEstimator] net_lat, acc_lat = zn.save_net_nnp( OUTPUT_DIR + 'zn', input, output, calc_latency=calc_latency, func_real_latency=estim_net, func_accum_latency=estim_accum_by_graph ) # reset estim_accum_by_graph estim_net, estim_accum_by_graph, estim_accum_by_mod = \ init_calc_latency(output, ext_name=ext_name, device_id=device_id) # SAVE ALL MODULES. Calc layer-based latency. # The modules are discovered by going over the module list. # The latency is then calculated based on each individual module's # nnabla graph [LatencyGraphEstimator] acc_lat_g = zn.save_modules_nnp( OUTPUT_DIR + 'zn_by_graph', active_only=True, calc_latency=calc_latency, func_latency=estim_accum_by_graph ) # SAVE ALL MODULES. Calc layer-based latency. # The modules are discovered by going over the module list. # The latency is then calculated using the module [LatencyEstimator] # **** This function is deprecated **** acc_lat_m = zn.save_modules_nnp_by_mod( OUTPUT_DIR + 'zn_by_module', active_only=True, calc_latency=calc_latency, func_latency=estim_accum_by_mod ) # reset estim_accum_by_graph, estim_accum_by_mod estim_net, estim_accum_by_graph, estim_accum_by_mod = \ init_calc_latency(output, ext_name=ext_name, device_id=device_id) # Just obtain the latencies of all modules without saving files # By graph latencies_1, acc_lat_1 = zn.get_latency(estim_accum_by_graph) # By module (deprecated) latencies_2, acc_lat_2 = zn.get_latency_by_mod(estim_accum_by_mod) print(net_lat, acc_lat, acc_lat_g, acc_lat_m, acc_lat_1, acc_lat_2) # CONVERT ALL TO ONNX if onnx: zn.convert_npp_to_onnx(OUTPUT_DIR) # VERBOSITY - INFO OF NETWORK CONTENT # with open(OUTPUT_DIR + 'zn.txt', 'w') as f: # print_me(zn, f) # 11 ************************** if exp_nr == 11: from nnabla_nas.contrib import zoph ctx = get_extension_context(ext_name=ext_name, device_id=device_id) nn.set_default_context(ctx) OUTPUT_DIR = './logs/zoph/many_different_nets/' shape = (1, 3, 32, 32) input = nn.Variable(shape) # Sample N zoph networks from the search space for i in range(0, N): nn.clear_parameters() zn = zoph.SearchNet() zn.apply(training=False) output = zn(input) if calc_latency: estim_net, estim_accum_by_graph, estim_accum_by_mod = \ init_calc_latency(output, ext_name=ext_name, device_id=device_id ) zn.save_graph(OUTPUT_DIR + 'zn' + str(i)) zn.save_net_nnp(OUTPUT_DIR + 'zn' + str(i), input, output, calc_latency=calc_latency, func_real_latency=estim_net, func_accum_latency=estim_accum_by_graph ) zn.save_modules_nnp(OUTPUT_DIR + 'zn' + str(i), active_only=True, calc_latency=calc_latency, func_latency=estim_accum_by_graph ) if onnx: zn = zoph.SearchNet() zn.convert_npp_to_onnx(OUTPUT_DIR, opset='opset_snpe') # 12 ************************** if exp_nr == 12: from nnabla_nas.contrib import zoph import time ctx = get_extension_context(ext_name=ext_name, device_id=device_id) nn.set_default_context(ctx) OUTPUT_DIR = './logs/zoph/same_net_many_times/' shape = (1, 3, 32, 32) input = nn.Variable(shape) zn = zoph.SearchNet() zn.apply(training=False) output = zn(input) # Measure add-hoc latency of zoph network for i in range(0, N): n_run = 100 # warm up output.forward() result = 0.0 for i in range(n_run): start = time.time() output.forward() stop = time.time() result += stop - start mean_time = result / n_run print(mean_time*1000) # Measure latency on same zoph network N times for i in range(0, N): if calc_latency: estim_net, estim_accum_by_graph, estim_accum_by_mod = \ init_calc_latency(output, ext_name=ext_name, device_id=device_id ) zn.save_net_nnp(OUTPUT_DIR + 'zn' + str(i), input, output, calc_latency=calc_latency, func_real_latency=estim_net, func_accum_latency=estim_accum_by_graph ) zn.save_modules_nnp(OUTPUT_DIR + 'zn' + str(i), active_only=True, calc_latency=calc_latency, func_latency=estim_accum_by_graph ) zn.save_graph(OUTPUT_DIR) if onnx: zn.convert_npp_to_onnx(OUTPUT_DIR) # 20 ************************** if exp_nr == 20: from nnabla_nas.contrib import random_wired ctx = get_extension_context(ext_name=ext_name, device_id=device_id) nn.set_default_context(ctx) OUTPUT_DIR = './logs/rdn/one_net/' # Sample one random wired network from the search space shape = (1, 3, 32, 32) input = nn.Variable(shape) rw = random_wired.TrainNet() rw.apply(training=False) output = rw(input) if calc_latency: estim_net, estim_accum_by_graph, estim_accum_by_mod = \ init_calc_latency(output, ext_name=ext_name, device_id=device_id ) rw.save_graph(OUTPUT_DIR + 'rw') rw.save_net_nnp(OUTPUT_DIR + 'rw', input, output, calc_latency=calc_latency, func_real_latency=estim_net, func_accum_latency=estim_accum_by_graph ) rw.save_modules_nnp(OUTPUT_DIR + 'rw', active_only=True, calc_latency=calc_latency, func_latency=estim_accum_by_graph ) if onnx: rw.convert_npp_to_onnx(OUTPUT_DIR) # 21 ************************** if exp_nr == 21: from nnabla_nas.contrib import random_wired ctx = get_extension_context(ext_name=ext_name, device_id=device_id) nn.set_default_context(ctx) OUTPUT_DIR = './logs/rdn/many_different_nets/' shape = (1, 3, 32, 32) input = nn.Variable(shape) # Measure latency on same rdn network N times for i in range(0, N): nn.clear_parameters() rw = random_wired.TrainNet() rw.apply(training=False) output = rw(input) if calc_latency: estim_net, estim_accum_by_graph, estim_accum_by_mod = \ init_calc_latency(output, ext_name=ext_name, device_id=device_id ) rw.save_graph(OUTPUT_DIR + 'rw' + str(i)) rw.save_net_nnp(OUTPUT_DIR + 'rw' + str(i), input, output, calc_latency=calc_latency, func_real_latency=estim_net, func_accum_latency=estim_accum_by_graph ) rw.save_modules_nnp(OUTPUT_DIR + 'rw' + str(i), active_only=True, calc_latency=calc_latency, func_latency=estim_accum_by_graph ) if onnx: rw = random_wired.TrainNet() rw.convert_npp_to_onnx(OUTPUT_DIR, opset='opset_snpe') # 22 ************************** if exp_nr == 22: from nnabla_nas.contrib import random_wired import time OUTPUT_DIR = './logs/rdn/same_net_many_times/' ctx = get_extension_context(ext_name=ext_name, device_id=device_id) nn.set_default_context(ctx) shape = (1, 3, 32, 32) input = nn.Variable(shape) rw = random_wired.TrainNet() rw.apply(training=False) output = rw(input) for i in range(0, N): n_run = 10 # warm up output.forward() result = 0.0 for i in range(n_run): start = time.time() output.forward() stop = time.time() result += stop - start mean_time = result / n_run print(mean_time*1000) # Measure latency on same rdn network N times for i in range(0, N): if calc_latency: estim_net, estim_accum_by_graph, estim_accum_by_mod = \ init_calc_latency(output, ext_name=ext_name, device_id=device_id ) rw.save_net_nnp(OUTPUT_DIR + 'rw' + str(i), input, output, calc_latency=calc_latency, func_real_latency=estim_net, func_accum_latency=estim_accum_by_graph ) rw.save_modules_nnp(OUTPUT_DIR + 'rw' + str(i), active_only=True, calc_latency=calc_latency, func_latency=estim_accum_by_graph ) rw.save_graph(OUTPUT_DIR + 'rw' + str(i)) if onnx: rw.convert_npp_to_onnx(OUTPUT_DIR) # 31 ************************** if exp_nr == 31: from nnabla_nas.contrib.classification.mobilenet import SearchNet ctx = get_extension_context(ext_name=ext_name, device_id=device_id) nn.set_default_context(ctx) OUTPUT_DIR = './logs/mobilenet/many_different_nets/' input = nn.Variable((1, 3, 224, 224)) # number of random networks to sample # Sample N networks from the search space for i in range(0, N): nn.clear_parameters() mobile_net = SearchNet(num_classes=1000) mobile_net.apply(training=False) output = mobile_net(input) if calc_latency: estim_net, estim_accum_by_graph, estim_accum_by_mod = \ init_calc_latency(output, ext_name=ext_name, device_id=device_id ) # This calculates the actual latency of the network # and the accum. latency by adding the latencies of # each module, following the nnabla graph mobile_net.save_net_nnp(OUTPUT_DIR + 'mn' + str(i), input, output, calc_latency=calc_latency, func_real_latency=estim_net, func_accum_latency=estim_accum_by_graph ) """ # This calculates the latency of each module going # over the nnabla graph (deprecated) mobile_net.calc_latency_all_modules( OUTPUT_DIR + 'graph_mn' + str(i), output, func_latency=estim_accum_by_graph) """ # This calculates the latency of each module going # the tree of the modules mobile_net.save_modules_nnp( OUTPUT_DIR + 'mn' + str(i), active_only=True, calc_latency=calc_latency, func_latency=estim_accum_by_graph ) if onnx: mobile_net = SearchNet() mobile_net.convert_npp_to_onnx(OUTPUT_DIR, opset='opset_snpe') # 32 ************************** if exp_nr == 32: from nnabla_nas.contrib.classification.mobilenet import SearchNet import time OUTPUT_DIR = './logs/mobilenet/same_net_many_times/' ctx = get_extension_context(ext_name=ext_name, device_id=device_id) nn.set_default_context(ctx) input = nn.Variable((1, 3, 224, 224)) mobile_net = SearchNet(num_classes=1000) mobile_net.apply(training=False) output = mobile_net(input) for i in range(0, N): n_run = 100 # warm up output.forward() result = 0.0 for i in range(n_run): start = time.time() output.forward() stop = time.time() result += stop - start mean_time = result / n_run print(mean_time*1000) # Measure latency on same network N times for i in range(0, N): if calc_latency: estim_net, estim_accum_by_graph, estim_accum_by_mod = \ init_calc_latency(output, ext_name=ext_name, device_id=device_id ) mobile_net.save_net_nnp(OUTPUT_DIR + 'mn' + str(i), input, output, calc_latency=calc_latency, func_real_latency=estim_net, func_accum_latency=estim_accum_by_graph ) mobile_net.save_modules_nnp( OUTPUT_DIR + 'mn' + str(i), active_only=True, calc_latency=calc_latency, func_latency=estim_accum_by_graph ) if onnx: mobile_net.convert_npp_to_onnx(OUTPUT_DIR) # 4 ************************** if exp_nr == 4: from nnabla_nas.module import static as smo input1 = nn.Variable((1, 256, 32, 32)) input2 = nn.Variable((1, 384, 32, 32)) input3 = nn.Variable((1, 128, 32, 32)) input4 = nn.Variable((1, 768, 32, 32)) input5 = nn.Variable((1, 1280, 32, 32)) input6 = nn.Variable((1, 2048, 32, 32)) input7 = nn.Variable((1, 512, 32, 32)) input8 = nn.Variable((1, 192, 32, 32)) input9 = nn.Variable((1, 224, 32, 32)) static_input1 = smo.Input(value=input1) static_input2 = smo.Input(value=input2) static_input3 = smo.Input(value=input3) static_input4 = smo.Input(value=input4) static_input5 = smo.Input(value=input5) static_input6 = smo.Input(value=input6) static_input7 = smo.Input(value=input7) static_input8 = smo.Input(value=input8) static_input9 = smo.Input(value=input9) myconv1 = smo.Conv(parents=[static_input1], in_channels=256, out_channels=128, kernel=(1, 1), pad=None, group=1) myconv2 = smo.Conv(parents=[static_input2], in_channels=384, out_channels=128, kernel=(1, 1), pad=None, group=1) myconv3 = smo.Conv(parents=[static_input3], in_channels=128, out_channels=256, kernel=(1, 1), pad=None, group=1) myconv4 = smo.Conv(parents=[static_input4], in_channels=768, out_channels=256, kernel=(1, 1)) myconv5 = smo.Conv(parents=[static_input5], in_channels=1280, out_channels=256, kernel=(1, 1), pad=None, group=1) myconv6 = smo.Conv(parents=[static_input6], in_channels=2048, out_channels=256, kernel=(1, 1), pad=None, group=1) myconv7 = smo.Conv(parents=[static_input7], in_channels=512, out_channels=512, kernel=(3, 3), pad=(1, 1), group=1 ) myconv8 = smo.Conv(parents=[static_input8], in_channels=192, out_channels=512, kernel=(7, 7), pad=(3, 3), group=1 ) myconv9 = smo.Conv(parents=[static_input9], in_channels=224, out_channels=128, kernel=(5, 5), pad=(2, 2), group=1 ) output1 = myconv1() output2 = myconv2() output3 = myconv3() output4 = myconv4() output5 = myconv5() output6 = myconv6() output7 = myconv7() output8 = myconv8() output9 = myconv9() N = 10 for i in range(0, N): mean_time = estim_fwd(output1) print("1, ", mean_time) mean_time = estim_fwd(output2) print("2, ", mean_time) mean_time = estim_fwd(output3) print("3, ", mean_time) mean_time = estim_fwd(output4) print("4, ", mean_time) mean_time = estim_fwd(output5) print("5, ", mean_time) mean_time = estim_fwd(output6) print("6, ", mean_time) mean_time = estim_fwd(output7) print("7, ", mean_time) mean_time = estim_fwd(output8) print("8, ", mean_time) mean_time = estim_fwd(output9) print("9, ", mean_time) N = 100 from nnabla_nas.utils.estimator.latency import LatencyGraphEstimator for i in range(0, N): estimation = LatencyGraphEstimator(n_run=100, ext_name='cpu') latency = estimation.get_estimation(myconv1) latency = estimation.get_estimation(myconv2) latency = estimation.get_estimation(myconv3) latency = estimation.get_estimation(myconv4) latency = estimation.get_estimation(myconv5) latency = estimation.get_estimation(myconv6) latency = estimation.get_estimation(myconv7) latency = estimation.get_estimation(myconv8) latency = estimation.get_estimation(myconv9) estimation = LatencyGraphEstimator(n_run=100, ext_name='cpu') latency = estimation.get_estimation(myconv9) latency = estimation.get_estimation(myconv8) latency = estimation.get_estimation(myconv7) latency = estimation.get_estimation(myconv6) latency = estimation.get_estimation(myconv5) latency = estimation.get_estimation(myconv4) latency = estimation.get_estimation(myconv3) latency = estimation.get_estimation(myconv2) latency = estimation.get_estimation(myconv1) estimation = LatencyGraphEstimator(n_run=100, ext_name='cpu') latency = estimation.get_estimation(myconv6) latency = estimation.get_estimation(myconv9) latency = estimation.get_estimation(myconv1) latency = estimation.get_estimation(myconv4) latency = estimation.get_estimation(myconv8) latency = estimation.get_estimation(myconv3) latency = estimation.get_estimation(myconv5) latency = estimation.get_estimation(myconv7) latency = estimation.get_estimation(myconv2) latency += 0 # to avoid lint/flake8 error # 5 ************************** if exp_nr == 5: from nnabla_nas.module import static as smo from nnabla_nas.utils.estimator.latency import LatencyGraphEstimator from numpy.random import permutation import numpy as np run_also_ours_at_the_end = True N_conv = 50 # number of different convolutions tried in_sizes = np.random.randint(low=1, high=1000, size=N_conv) out_sizes = np.random.randint(low=1, high=600, size=N_conv) kernel_sizes = np.random.randint(low=1, high=7, size=N_conv) feat_sizes = np.random.randint(low=16, high=48, size=N_conv) N = 100 for j in range(N): estimation = LatencyGraphEstimator(n_run=100, ext_name='cpu') print('****************** RUN ********************') for i in permutation(N_conv): input = nn.Variable((1, in_sizes[i], feat_sizes[i], feat_sizes[i])) static_input = smo.Input(value=input) myconv = smo.Conv(parents=[static_input], in_channels=in_sizes[i], out_channels=out_sizes[i], kernel=(kernel_sizes[i], kernel_sizes[i]), pad=None, group=1 ) output = myconv() latency = estimation.get_estimation(myconv) latency += 0 # to avoid lint/flake8 error if run_also_ours_at_the_end is True: print('*********** NOW IT IS OUR TURN ***********') for i in range(N_conv): input = nn.Variable((1, in_sizes[i], feat_sizes[i], feat_sizes[i])) static_input = smo.Input(value=input) myconv = smo.Conv(parents=[static_input], in_channels=in_sizes[i], out_channels=out_sizes[i], kernel=(kernel_sizes[i], kernel_sizes[i]), pad=None, group=1 ) output = myconv() mean_time = estim_fwd(output, n_run=100) * 1000 # in ms print('Our_Conv : 100 :', mean_time, ':', '[(1, ' + str(in_sizes[i]) + ', ' + str(feat_sizes[i]) + ', ' + str(feat_sizes[i]) + ')]', ':', out_sizes[i], ':', kernel_sizes[i] ) # 6 ************************** if exp_nr == 6: import onnx load_onnx = False if len(sys.argv) > 2: INPUT_DIR = sys.argv[2] else: INPUT_DIR = './logs/zoph/one_net/' if len(sys.argv) > 3: load_onnx = True existing_networks = glob.glob(INPUT_DIR + '/*' + os.path.sep) all_nets_latencies = dict.fromkeys([]) all_nets = dict.fromkeys([]) net_idx = 0 for network in existing_networks: all_blocks = glob.glob(network + '**/*.acclat', recursive=True) blocks = dict.fromkeys([]) block_idx = 0 this_net_accumulated_latency = 0.0 this_net_accumulated_latency_of_convs = 0.0 this_net_accumulated_latency_of_relus = 0.0 this_net_accumulated_latency_of_bns = 0.0 this_net_accumulated_latency_of_merges = 0.0 this_net_accumulated_latency_of_pools = 0.0 this_net_accumulated_latency_of_reshapes = 0.0 this_net_accumulated_latency_of_affines = 0.0 this_net_accumulated_latency_of_add2s = 0.0 for block_lat in all_blocks: block = block_lat[:-7] + '.onnx' print('.... READING .... --> ' + block) # Reading latency for each of the blocks of layers with open(block_lat, 'r') as f: block_latency = float(f.read()) this_net_accumulated_latency += block_latency # Layer-type-wise latencies tested # for Zoph # for Random Wired networks # for mobilenet layer_name = block.split('/')[-1].split('.')[-2] if layer_name.find('bn') != -1: this_net_accumulated_latency_of_bns += block_latency elif layer_name.find('batchnorm') != -1: this_net_accumulated_latency_of_bns += block_latency elif layer_name.find('relu') != -1: this_net_accumulated_latency_of_relus += block_latency elif layer_name.find('conv') != -1: this_net_accumulated_latency_of_convs += block_latency elif layer_name.find('merg') != -1: this_net_accumulated_latency_of_merges += block_latency elif layer_name.find('pool') != -1: this_net_accumulated_latency_of_pools += block_latency elif layer_name.find('con') != -1: # from concat this_net_accumulated_latency_of_merges += block_latency elif layer_name.find('reshape') != -1: this_net_accumulated_latency_of_reshapes += block_latency elif layer_name.find('linear') != -1: this_net_accumulated_latency_of_affines += block_latency elif layer_name.find('add2') != -1: this_net_accumulated_latency_of_add2s += block_latency this_block = dict.fromkeys([]) this_block['latency'] = block_latency if load_onnx: # Interesting FIELDS in params.graph: # 'input', 'name', 'node', 'output' params = onnx.load(block) this_block['name'] = params.graph.name this_block['input'] = params.graph.input this_block['output'] = params.graph.output this_block['nodes'] = params.graph.node blocks[block_idx] = this_block block_idx += 1 net_realat_file = network[:-1] + '.realat' with open(net_realat_file, 'r') as f: this_net_real_latency = float(f.read()) net_acclat_file = network[:-1] + '.acclat' with open(net_acclat_file, 'r') as f: this_net_acc_latency = float(f.read()) this_net = dict.fromkeys([]) this_net['real_latency'] = this_net_real_latency this_net['accum_latency_graph'] = this_net_acc_latency this_net['accum_latency_module'] = this_net_accumulated_latency if load_onnx: net_file = network[:-1] + '.onnx' print('xxxx READING xxxx --> ' + net_file) params = onnx.load(net_file) this_net['name'] = params.graph.name this_net['input'] = params.graph.input this_net['output'] = params.graph.output this_net['nodes'] = params.graph.node all_nets_latencies[net_idx] = [ this_net_real_latency, this_net_acc_latency, this_net_accumulated_latency, this_net_accumulated_latency_of_convs, this_net_accumulated_latency_of_bns, this_net_accumulated_latency_of_relus, this_net_accumulated_latency_of_pools, this_net_accumulated_latency_of_merges, this_net_accumulated_latency_of_reshapes, this_net_accumulated_latency_of_affines, this_net_accumulated_latency_of_add2s, ] all_nets[net_idx] = this_net net_idx += 1 # Compare accumulated latency to net latencies, do a plot: print('LATENCY Results from ' + INPUT_DIR) print('NETWORK, LAYER-WISE (by graph), ', 'LAYER-WISE (by module), of CONVs, of BNs, of RELUs, ', 'of POOLs, of MERGEs/CONCATs, of RESHAPEs, of AFFINEs, ', 'of ADD2 layers' ) for i in range(len(all_nets_latencies)): # print(all_nets_latencies[i]) print(['%7.3f' % val for val in all_nets_latencies[i]])
def test_dropout_module(): input = smo.Input(value=nn.Variable((8, 5))) dropout = smo.Dropout(parents=[input]) output = dropout() assert output.shape == (8, 5)