def showcase_cifar10_0001(config_data, verbose=250): print('showcase_cifar10_0001()') trainloader = load_cifar_0001(config_data, train=True, batch_size=None, shuffle=True, verbose=verbose) testloader = load_cifar_0001(config_data, train=False, batch_size=1, shuffle=False, verbose=verbose) print('\ntrain') X = [] for i, data in enumerate(trainloader, 0): if i >= 10: break x, y = data pm.printvm('%s, %s' % (str(x.shape), str(y))) x1 = x.detach().numpy() for j in range(len(x1)): X.append(x1[j]) print('\ntest') for i, data in enumerate(testloader, 0): if i >= 10: break x, y = data pm.printvm('%s, %s' % (str(x.shape), str(y))) X = np.concatenate(X[:10], axis=2) plt.figure() plt.imshow(np.transpose(X, (1, 2, 0))) plt.show() """
def showcase_cifar10_0003(config_data, verbose=250): print('showcase_cifar10_0003()') config_data['data_from_torch']['cifar']['resize'] = ( 128, 128) # set to None for original size trainloader = load_cifar_0001(config_data, train=True, batch_size=None, shuffle=True, verbose=verbose) testloader = load_cifar_0001(config_data, train=False, batch_size=1, shuffle=False, verbose=verbose) print('\ntrain') X = [] for i, data in enumerate(trainloader, 0): if i >= 3: break x, y = data pm.printvm('%s, %s' % (str(x.shape), str(y))) x1 = x.detach().numpy() for j in range(len(x1)): X.append(x1[j]) # print('\ntest') # for i, data in enumerate(testloader,0): # if i>=10: break # x,y = data # pm.printvm('%s, %s'%(str(x.shape),str(y))) X = np.concatenate(X[:3], axis=2) plt.figure() plt.imshow(np.transpose(X, (1, 2, 0))) plt.show()
def train_vgg_cifar_0001(config_data, tab_level=0, verbose=250): print('train_vgg_cifar_0001()') from pipeline.data.load_data_cifar import load_cifar_0001 from utils.loss import compute_loss trainloader = load_cifar_0001(config_data, batch_size=None, shuffle=True, verbose=verbose) state_tracker = setup_state_tracker(config_data, verbose=verbose, tab_level=tab_level) net = new_or_load_model(state_tracker, verbose=verbose, tab_level=tab_level) criterion, optimizer = setup_training_tools_0001(net, config_data, verbose=verbose, tab_level=tab_level+1) pm.printv('Start training...'%(), tab_level=tab_level) total_iter_in_this_run = 0 l_epoch = 1 + state_tracker.get_latest_saved_epoch() for n_epoch in range(l_epoch, l_epoch + config_data['general']['epoch']): state_tracker.setup_for_this_epoch(n_epoch) for i, data in enumerate(trainloader,0): optimizer.zero_grad() x, y0 = data if DEBUG_train_loop_0001(DEBUG_train_vgg_cifar_LOOP_SIGNAL, net, x, y0, tab_level=tab_level, verbose=verbose): return y = net(x.to(this_device)) loss = compute_loss(criterion, y.squeeze(3).squeeze(2).cpu(), y0) loss.backward() optimizer.step() # FOR LOGGING total_iter_in_this_run += 1 state_tracker.store_loss_by_epoch(loss.item(), n_epoch) stop_iter, stop_epoch = DEBUG_train_loop_0002(DEBUG_train_vgg_cifar_LOOP2_SIGNAL, i, n_epoch - l_epoch , tab_level=tab_level+1, verbose=verbose) if stop_iter: break state_tracker.update_epoch() if stop_epoch: break state_tracker.update_state(total_iter_in_this_run, config_data) save_model_by_n_th_run(net, state_tracker,tab_level=tab_level, verbose=verbose) state_tracker.display_end_state(tab_level=tab_level+1, verbose=verbose)
def eval_cifar_0001(config_data, state_tracker, net, debug_signal, train=True, tab_level=0, verbose=250): from utils.metrics import ClassAccuracy data_loader = load_cifar_0001(config_data, train=train, shuffle=False, batch_size=1, verbose=0) net.eval() Acc = ClassAccuracy() pm.printvm('Start evaluation...', tab_level=tab_level, verbose=verbose, verbose_threshold=250) for i, data in enumerate(data_loader, 0): x, y0 = data y = net(x.to(this_device)).squeeze(3).squeeze(2) y1 = torch.argmax(y.squeeze(1)).clone().detach().cpu().numpy() Acc.update_acc(int(y1), int(y0)) if DEBUG_eval_smallnet_mnist_LOOP_0001(i, x, y0, y, net, debug_signal, tab_level=tab_level + 1): break Acc.compute_acc() Acc.display_stats(tab_level=0, verbose=250) """
def drivethru_0001_alexnet_cifar(config_data, tab_level=0, verbose=250): print('drivethru_0001_alexnet_mnist()') PROGRESS_TRACK_EVERY_N_PERCENT = 5. tracker_name = 'drivethru_0001_alexnet_cifar' config_data = set_debug_config(IS_DEBUG, config_data) pm.print_recursive_dict(config_data, tab_level=tab_level + 2, verbose=verbose, verbose_threshold=None) data_loader = load_cifar_0001(config_data, train=True, shuffle=True, verbose=verbose) n_batch_train = len(data_loader) # print('n_batch_train:%s'%(str(n_batch_train))) #12500 data for batch size= 4 autoreloader = AutoReloaderTestCIFAR(config_data, shuffle=True) eval_every_n_iter = get_eval_every_n_iter( config_data, n_batch_train, config_data['general']['epoch'], NO_OF_EVALUATION_DESIRED=config_data['drivethru'] ['no_of_evals_per_run'], manual_specification=False, DEBUG_N_ITER_MAX_PER_EPOCH=DEBUG_N_ITER_MAX_PER_EPOCH) state_tracker = setup_state_tracker(config_data, tracker_name, for_training=True, verbose=250, tab_level=0) net = new_or_load_model(state_tracker, config_data, verbose=verbose, tab_level=tab_level) criterion, optimizer = setup_training_tools_0001(net, config_data, verbose=verbose, tab_level=tab_level + 1) save_data_by_iter_details = {} # split save data by runs total_iter_in_this_run = 0 total_global_iter = state_tracker.save_data_by_nth_run[ state_tracker.current_run]['total_iteration'] last_saved_epoch = 1 + state_tracker.get_latest_saved_epoch() progress_tracker = int(n_batch_train / (100 / PROGRESS_TRACK_EVERY_N_PERCENT)) pm.printv('Start drive through...' % (), tab_level=tab_level) for n_epoch in range(last_saved_epoch, last_saved_epoch + config_data['general']['epoch']): state_tracker.setup_for_this_epoch(n_epoch, tab_level=tab_level + 1, verbose=verbose) for i, data in enumerate(data_loader, 0): print_progress_percentage(i, progress_tracker, n_batch_train, verbose=250, tab_level=tab_level + 1) # if emergency_drivethru_loop(EMERGENCY_DRIVETHRU_LOOP_SIGNAL, total_iter_in_this_run, eval_every_n_iter): total_iter_in_this_run += 1; continue optimizer.zero_grad() x, y0 = data x = x.to(this_device) net.train() y = net(x) loss = compute_loss(criterion, y.squeeze(3).squeeze(2).cpu(), y0) loss.backward() optimizer.step() # Drive through LRP and evaluation if (total_iter_in_this_run + 1) % eval_every_n_iter == 0: save_data_by_iter_details, autoreloader = drive_thru_evaluation( net, autoreloader, x, y, y0, save_data_by_iter_details, i, total_global_iter + total_iter_in_this_run, n_epoch, n_of_test_data_per_eval=config_data['drivethru'] ['n_of_test_data_per_eval'], n_of_test_data_per_LRP_eval=config_data['drivethru'] ['n_of_test_data_per_LRP_eval'], DEBUG_DRIVE_TRHU_LOOP=DEBUG_DRIVE_TRHU_LOOP, tab_level=tab_level + 1, verbose=verbose) # just_in_time_display(save_data_by_iter_details, verbose=verbose, tab_level=tab_level+1) if DEBUG_DRIVE_TRHU_LOOP: return # FOR LOGGING total_iter_in_this_run += 1 state_tracker.store_loss_by_epoch(loss.item(), n_epoch) stop_iter, stop_epoch = DEBUG_train_loop_0002( DEBUG_N_ITER_MAX_PER_EPOCH, i, n_epoch - last_saved_epoch, tab_level=tab_level + 1, verbose=verbose) if stop_iter: break if DEBUG_DRIVE_TRHU_LOOP2: return state_tracker.update_epoch() if stop_epoch: break state_tracker.save_data_by_iter_details = save_data_by_iter_details state_tracker.update_state(total_iter_in_this_run, config_data) save_model_by_n_th_run(net, state_tracker, tab_level=tab_level, verbose=verbose) state_tracker.display_end_state(tab_level=tab_level + 1, verbose=verbose)