def info_tutorial():
nn = NeuralNet()
x_shape = dat.shape()
test_gen, _ = dat.testset(batch_size=cfg.BATCH_SIZE,
max_samples=cfg.TEST_SET_SIZE)
nn.test(test_gen, print_it=True)
nn.net.initialize_spatial_layers(x_shape, cfg.BATCH_SIZE, PATCH_SIZE)
nn.summary(x_shape, print_it=True)
nn.print_weights()
print(nn.output_size(x_shape))
# Spatial Operations, defined one the net itself. Remember that after enabling a layer, ops are affected
assert nn.net.num_spatial_layers() != 0
nn.net.print_spatial_status()
# nn.train(epochs=1, set_size=5000, lr=0.1, batch_size=cfg.BATCH_SIZE) # Train to see fully disabled performance
nn.net.print_ops_summary()
nn.net.print_ops_summary(
use_conv=True) # Count convlution operations instead of MAC
print(nn.net.num_ops()) # (ops_saved, total_ops)
# Given x, we generate all spatial layer requirement sizes:
spat_sizes = nn.net.generate_spatial_sizes(x_shape)
print(spat_sizes)
p_spat_sizes = nn.net.generate_padded_spatial_sizes(x_shape, PATCH_SIZE)
print(p_spat_sizes)
# Generate a constant 1 value mask over all spatial nets
print(nn.net.enabled_layers())
nn.net.fill_masks_to_val(1)
print(nn.net.enabled_layers())
print(nn.net.disabled_layers())
nn.net.print_spatial_status(
) # Now all are enabled, seeing the mask was set
nn.train(epochs=1, set_size=5000, lr=0.1, batch_size=cfg.BATCH_SIZE
) # Train to see all layers enabled performance
nn.net.print_ops_summary()
nn.net.print_ops_summary(
use_conv=True) # Count convlution operations instead of MAC
nn.net.reset_spatial() # Disables layers as well
nn.net.print_ops_summary()
nn.net.print_ops_summary(use_conv=True)
# Turns on 3 ids and turns off all others
chosen_victims = random.sample(range(nn.net.num_spatial_layers()), 4)
nn.net.strict_mask_update(update_ids=chosen_victims[0:3],
masks=[
torch.zeros(p_spat_sizes[chosen_victims[0]]),
torch.zeros(p_spat_sizes[chosen_victims[1]]),
torch.zeros(p_spat_sizes[chosen_victims[2]])
])
# Turns on one additional id and *does not* turn off all others
nn.net.lazy_mask_update(
update_ids=[chosen_victims[3]],
masks=[torch.zeros(p_spat_sizes[chosen_victims[3]])])
nn.net.print_spatial_status() #
print(nn.net.enabled_layers())
nn.train(epochs=1, set_size=5000, lr=0.1,
batch_size=cfg.BATCH_SIZE) # Run with 4 layers on
nn.net.print_ops_summary()
nn.net.print_ops_summary(use_conv=True)
def create_FR_after_retrain(self,
mode,
acc_loss,
retrain=True,
epochs=50,
lr=0.01):
final_rec = self.create_FR_with_different_acc_loss(mode, acc_loss)
if retrain:
self.retrain_with_mask(final_rec, epochs=epochs, lr=lr)
retrain_nn = NeuralNet(ckp_name_prefix=final_rec.get_retrain_prefix())
retrain_nn.net.initialize_spatial_layers(dat.shape(), cfg.BATCH_SIZE,
self.ps)
retrain_nn.net.reset_spatial()
retrain_nn.net.strict_mask_update(update_ids=list(
range(len(final_rec.mask))),
masks=final_rec.mask)
if INNAS_COMP:
test_acc = 100
ops_saved = 100
ops_total = 100
else:
_, test_acc, _ = retrain_nn.test(self.test_gen)
ops_saved, ops_total = retrain_nn.net.num_ops()
final_rec.retrain_update(test_acc, ops_saved, ops_total, epochs, lr)
print(final_rec)
save_to_file(final_rec, path=cfg.RESULTS_DIR)
def __init__(self, resume=True, ckp_name_prefix=None):
# Decide on device:
if torch.cuda.is_available():
# print('CUDA FOUND!')
self.device = torch.device('cuda')
cudnn.benchmark = True
if torch.cuda.device_count() > 1:
raise NotImplementedError
# This line enables multiple GPUs, but changes the layer names a bit
# self.net = torch.nn.DataParallel(self.net)
# Useful if you have multiple GPUs - does not hurt otherwise
else:
self.device = torch.device('cpu')
# torch.set_num_threads(4) # Presuming 4 cores
print('WARNING: Found no valid GPU device - Running on CPU')
# Build Model:
print(f'==> Building model {net.__name__} on the dataset {dat.name()}')
self.net = net(self.device, dat.num_classes(), dat.input_channels(),
dat.shape())
print(f'==> Detected family model of {self.net.family_name()}')
if resume:
print(
f'==> Resuming from checkpoint via sorting method: {cfg.RESUME_METHOD}'
)
assert os.path.isdir(
cfg.CHECKPOINT_DIR), 'Error: no checkpoint directory found!'
ck_file = self.__class__.resume_methods[cfg.RESUME_METHOD](
self.net.family_name(), ckp_name_prefix=ckp_name_prefix)
if ck_file is None:
print(
f'-E- Found no valid checkpoints for {net.__name__} on {dat.name()}'
)
self.best_val_acc = 0
self.start_epoch = 0
else:
checkpoint = torch.load(ck_file, map_location=self.device)
self._load_checkpoint(checkpoint['net'])
self.best_val_acc = checkpoint['acc']
self.start_epoch = checkpoint['epoch']
assert (dat.name() == checkpoint['dataset'])
print(
f'==> Loaded model with val-acc of {self.best_val_acc:.3f}'
)
else:
self.best_val_acc = 0
self.start_epoch = 0
self.net = self.net.to(self.device)
# Build SGD Algorithm:
self.criterion = torch.nn.CrossEntropyLoss()
self.train_gen, self.val_gen, self.classes = (None, None, None)
self.optimizer = None
def gen_first_lvl_results(self, mode):
rec_filename = self.record_finder.find_rec_filename(
mode, RecordType.FIRST_LVL_REC)
if rec_filename is not None:
rcs = load_from_file(rec_filename, path='')
st_point = rcs.find_resume_point()
if st_point is None:
return rcs
layers_layout = self.nn.net.generate_spatial_sizes(dat.shape())
self._init_nn()
if rec_filename is None:
if self.input_patterns is None:
rcs = Record(layers_layout, self.gran_thresh, True, mode,
self.init_acc, self.ps, self.ones_range)
else:
rcs = Record(layers_layout, self.gran_thresh, False, mode,
self.init_acc, self.input_patterns,
self.ones_range)
st_point = [0] * 4
if INNAS_COMP:
rcs.filename = 'DEBUG_' + rcs.filename
print('==> Result will be saved to ' +
os.path.join(cfg.RESULTS_DIR, rcs.filename))
save_counter = 0
for layer, channel, patch, pattern_idx, mask in tqdm(
mf.gen_masks_with_resume(self.ps,
rcs.all_patterns,
rcs.mode,
rcs.gran_thresh,
layers_layout,
resume_params=st_point)):
self.nn.net.strict_mask_update(update_ids=[layer],
masks=[torch.from_numpy(mask)])
if INNAS_COMP:
test_acc = 100
ops_saved = 100
ops_total = 100
else:
_, test_acc, _ = self.nn.test(self.test_gen)
ops_saved, ops_total = self.nn.net.num_ops()
self.nn.net.reset_spatial()
rcs.addRecord(ops_saved, ops_total, test_acc, layer, channel,
patch, pattern_idx)
save_counter += 1
if save_counter > cfg.SAVE_INTERVAL:
save_to_file(rcs, True, cfg.RESULTS_DIR)
save_counter = 0
save_to_file(rcs, True, cfg.RESULTS_DIR)
print('==> Result saved to ' +
os.path.join(cfg.RESULTS_DIR, rcs.filename))
return rcs
def training():
# dat.data_summary(show_sample=False)
nn = NeuralNet(resume=True) # Spatial layers are by default, disabled
nn.summary(dat.shape())
nn.train(epochs=50, lr=0.01)
test_gen, _ = dat.testset(batch_size=cfg.BATCH_SIZE,
max_samples=cfg.TEST_SET_SIZE)
test_loss, test_acc, count = nn.test(test_gen)
print(
f'==> Final testing results: test acc: {test_acc:.3f} with {count}, test loss: {test_loss:.3f}'
)
def plot_ops_saved_accuracy_uniform_network(self):
layers_layout = self.nn.net.generate_spatial_sizes(dat.shape())
rcs = Record(layers_layout, self.gran_thresh, True, Mode.UNIFORM_LAYER,
self.init_acc, self.ps, self.ones_range)
no_of_patterns = rcs.all_patterns.shape[2]
ops_saved_array = [None] * no_of_patterns
acc_array = [None] * no_of_patterns
self._init_nn()
for p_idx in range(no_of_patterns):
sp_list = [None] * len(layers_layout)
for layer, layer_mask in enumerate(
mf.base_line_mask(layers_layout,
self.ps,
pattern=rcs.all_patterns[:, :, p_idx])):
sp_list[layer] = torch.from_numpy(layer_mask)
self.nn.net.strict_mask_update(update_ids=list(
range(len(layers_layout))),
masks=sp_list)
_, test_acc, _ = self.nn.test(self.test_gen)
ops_saved, ops_total = self.nn.net.num_ops()
self.nn.net.reset_ops()
ops_saved_array[p_idx] = ops_saved / ops_total
acc_array[p_idx] = test_acc
plt.figure()
plt.subplot(211)
plt.plot(list(range(no_of_patterns)), ops_saved_array, 'o')
plt.xlabel('pattern index')
plt.ylabel('ops_saved [%]')
plt.title(f'ops saved for uniform network, patch_size:{self.ps}')
plt.subplot(212)
plt.plot(list(range(no_of_patterns)), acc_array, 'o')
plt.xlabel('pattern index')
plt.ylabel('accuracy [%]')
plt.title(f'accuracy for uniform network, patch_size:{self.ps}')
data = [rcs.all_patterns, ops_saved_array, acc_array]
save_to_file(
data, False, cfg.RESULTS_DIR,
'baseline_all_patterns_{cfg.NET.__name__}_{dat.name()}' +
f'acc{self.init_acc}_ps{self.ps}_ones{self.ones_range[0]}x{self.ones_range[1]}_mg{self.gran_thresh}.pkl'
)
plt.savefig(
f'{cfg.RESULTS_DIR}/baseline_all_patterns_{cfg.NET.__name__}_{dat.name()}'
+
f'acc{self.init_acc}_ps{self.ps}_ones{self.ones_range[0]}x{self.ones_range[1]}_mg{self.gran_thresh}.pdf'
)
return data
def base_line_result(self):
layers_layout = self.nn.net.generate_spatial_sizes(dat.shape())
self._init_nn()
sp_list = [None] * len(layers_layout)
for layer, layer_mask in enumerate(
mf.base_line_mask(layers_layout, self.ps)):
sp_list[layer] = torch.from_numpy(layer_mask)
self.nn.net.strict_mask_update(update_ids=list(
range(len(layers_layout))),
masks=sp_list)
_, test_acc, _ = self.nn.test(self.test_gen)
ops_saved, ops_total = self.nn.net.num_ops()
bl_rec = BaselineResultRc(self.init_acc, test_acc, ops_saved,
ops_total, self.ps, cfg.NET.__name__,
dat.name())
print(bl_rec)
save_to_file(bl_rec, True, cfg.RESULTS_DIR)
def __init__(self,
patch_size,
ones_range,
gran_thresh,
max_acc_loss,
init_acc=None,
test_size=cfg.TEST_SET_SIZE,
patterns_idx=None):
self.ps = patch_size
self.max_acc_loss = max_acc_loss
self.gran_thresh = gran_thresh
if patterns_idx is None:
self.ones_range = ones_range
self.input_patterns = None
else:
patterns_rec = load_from_file(
f'all_patterns_ps{self.ps}_cluster{patterns_idx}.pkl',
path=cfg.RESULTS_DIR)
self.ones_range = (patterns_rec[1], patterns_rec[1] + 1)
self.input_patterns = patterns_rec[2]
self.full_net_run_time = None
self.total_ops = None
self.nn = NeuralNet()
self.nn.net.initialize_spatial_layers(dat.shape(), cfg.BATCH_SIZE,
self.ps)
self.test_gen, _ = dat.testset(batch_size=cfg.BATCH_SIZE,
max_samples=cfg.TEST_SET_SIZE)
self.test_set_size = cfg.TEST_SET_SIZE
if INNAS_COMP:
init_acc = DEBUG_INIT_ACC
if init_acc is None:
_, test_acc, correct = self.nn.test(self.test_gen)
print(f'==> Asserted test-acc of: {test_acc} [{correct}]\n ')
self.init_acc = test_acc # TODO - Fix initialize bug
else:
self.init_acc = init_acc
self.record_finder = RecordFinder(cfg.NET.__name__, dat.name(),
patch_size, ones_range, gran_thresh,
max_acc_loss, self.init_acc)
def eval_run_time(self, mode, no_of_tries=5):
layers_layout = self.nn.net.generate_spatial_sizes(dat.shape())
if self.input_patterns is None:
recs_first_lvl = Record(layers_layout, self.gran_thresh, True,
mode, self.init_acc, self.ps,
self.ones_range)
else:
recs_first_lvl = Record(layers_layout, self.gran_thresh, False,
mode, self.init_acc, self.input_patterns,
self.ones_range)
first_lvl_runs = recs_first_lvl.size
self.nn.net.reset_spatial()
run_time_for_iter = 0
for idx in range(no_of_tries):
layer = random.randint(0, recs_first_lvl.no_of_layers - 1)
channel = random.randint(0,
recs_first_lvl.no_of_channels[layer] - 1)
patch = random.randint(0, recs_first_lvl.no_of_patches[layer] - 1)
pattern_idx = random.randint(
0, recs_first_lvl.no_of_patterns[layer] - 1)
pattern = recs_first_lvl.all_patterns[:, :, pattern_idx]
mask = mf.get_specific_mask(layers_layout[layer], channel, patch,
pattern,
recs_first_lvl.patch_sizes[layer],
mode)
st_time = time.time()
self.nn.net.reset_spatial()
self.nn.net.strict_mask_update(update_ids=[layer],
masks=[torch.from_numpy(mask)])
_, test_acc, _ = self.nn.test(self.test_gen)
end_time = time.time()
run_time_for_iter += (end_time - st_time)
run_time_for_iter = run_time_for_iter / no_of_tries
recs_first_lvl.fill_empty()
if mode == Mode.UNIFORM_LAYER:
second_lvl_runs = 0
lQ = LayerQuantizier(recs_first_lvl, self.init_acc, 0, self.ps,
self.ones_range, self.get_total_ops())
lQ_runs = lQ.number_of_iters()
elif mode == Mode.MAX_GRANULARITY:
pQ = PatchQuantizier(recs_first_lvl, self.init_acc, 0, self.ps)
pQ.output_rec.fill_empty()
cQ = ChannelQuantizier(pQ.output_rec, self.init_acc, 0, self.ps)
cQ.output_rec.fill_empty()
second_lvl_runs = pQ.number_of_iters() + cQ.number_of_iters()
lQ = LayerQuantizier(cQ.output_rec, self.init_acc, 0, self.ps,
self.ones_range, self.get_total_ops())
lQ_runs = lQ.number_of_iters()
elif mode == Mode.UNIFORM_FILTERS:
pQ = PatchQuantizier(recs_first_lvl, self.init_acc, 0, self.ps)
second_lvl_runs = pQ.number_of_iters()
pQ.output_rec.fill_empty()
lQ = LayerQuantizier(pQ.output_rec, self.init_acc, 0, self.ps,
self.ones_range, self.get_total_ops())
lQ_runs = lQ.number_of_iters()
elif mode == Mode.UNIFORM_PATCH:
cQ = ChannelQuantizier(recs_first_lvl, self.init_acc, 0, self.ps)
cQ.output_rec.fill_empty()
second_lvl_runs = cQ.number_of_iters()
lQ = LayerQuantizier(cQ.output_rec, self.init_acc, 0, self.ps,
self.ones_range, self.get_total_ops())
lQ_runs = lQ.number_of_iters()
no_of_runs = (first_lvl_runs, second_lvl_runs, lQ_runs)
run_times = (round(run_time_for_iter,
3), self.get_full_net_run_time(no_of_tries))
return no_of_runs, run_times