def ga_main(self): for i in range(self.nb_inv): model = self.make_init_model() model_l = self.get_model_list(model) graph = MyGraph(model_l) config = self.get_curr_config() self.population[config.name] = MyModel(config=config, graph=graph) self.max_ind += 1 # for test: save these weights according to iter and record the choice self.choice_weight = [] self.choice_idx = [] f_handle1 = file('../output/choice_prob.npy', 'a') f_handle2 = file('../output/choice_idx.npy', 'a') for i in range(self.gl_config.evoluation_time): logger.info("Now {} evolution ".format(i + 1)) if dbg: self.mutation_process() self.select_process() self.train_process() else: self.mutation_process() self.train_process() self.select_process() np.save(f_handle1, self.choice_weight) np.save(f_handle2, self.choice_idx) f_handle1.close() f_handle2.close()
def run(name, epochs=100, verbose=1, limit_data=False, dataset_type='cifar10'): try: import keras import tensorflow as tf os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' from Config import MyConfig from Model import MyModel from keras.utils.generic_utils import get_custom_objects from Model import IdentityConv, GroupIdentityConv get_custom_objects()['IdentityConv'] = IdentityConv get_custom_objects()['GroupIdentityConv'] = GroupIdentityConv config = tf.ConfigProto() config.gpu_options.allow_growth = True sess = tf.Session(config=config) with sess.as_default(): config = MyConfig(name=name, epochs=epochs, verbose=verbose, clean=False, limit_data=limit_data, dataset_type=dataset_type) # device = 0 # with tf.device(device): model = MyModel(config=config, model=keras.models.load_model(config.model_path)) logger.debug('model {} start fit epochs {}'.format(name, epochs)) score = model.comp_fit_eval() keras.models.save_model(model.model, model.config.model_path) return name, score except Exception as inst: print 'INST is' print str(inst) errors = [ 'ResourceExhaustedError', 'Resource exhausted: OOM', 'OOM', 'Failed to create session', 'CUDNN_STATUS_INTERNAL_ERROR', 'Chunk', 'CUDA_ERROR_OUT_OF_MEMORY' ] for error in errors: if error in str(inst): return NOMEM, NOMEM return None, None
def copy_model(self, model, config): from keras.utils.generic_utils import get_custom_objects from Model import IdentityConv, GroupIdentityConv get_custom_objects()['IdentityConv'] = IdentityConv get_custom_objects()['GroupIdentityConv'] = GroupIdentityConv new_model = MyModel(config, model.graph.copy(), keras.models.load_model(model.config.model_path)) keras.models.save_model(new_model.model, new_model.config.model_path) return new_model
def wider_conv2d(self, model, layer_name, new_width, config): # modify graph new_graph = model.graph.copy() new_node = new_graph.get_nodes(layer_name)[0] assert new_node.type == 'Conv2D' or new_node.type == 'Conv2D_Pooling', 'must wide a conv' new_node.config['filters'] = new_width # logger.debug(new_graph.to_json()) # construct model new_model = MyModel(config=config, graph=new_graph) self.copy_weight(model, new_model) # inherit weight w_conv1, b_conv1 = model.get_layers(layer_name)[0].get_weights() w_conv2, b_conv2 = model.get_layers(layer_name, next_layer=True, type=['Conv2D', 'Conv2D_Pooling' ])[0].get_weights() new_w_conv1, new_b_conv1, new_w_conv2 = Net2Net._wider_conv2d_weight( w_conv1, b_conv1, w_conv2, new_width, "net2wider") new_model.get_layers(layer_name)[0].set_weights( [new_w_conv1, new_b_conv1]) new_model.get_layers(layer_name, next_layer=True, type=['Conv2D', 'Conv2D_Pooling' ])[0].set_weights([new_w_conv2, b_conv2]) return new_model
def deeper_conv2d(self, model, layer_name, config, with_pooling, kernel_size=3, filters='same'): # construct graph new_graph = model.graph.copy() if with_pooling == False: type = 'Conv2D' else: type = 'Conv2D_Pooling' new_node = Node(type=type, name='new', config={ 'kernel_size': kernel_size, 'filters': filters }) new_graph.deeper(layer_name, new_node) # logger.debug(new_graph.to_json()) # construct model new_model = MyModel(config=config, graph=new_graph) # inherit weight # in fact there is no need to get w_conv1 and b_conv1 # what we actually need is only w_conv1's shape # more specifically, only kh, kw and filters are needed, num_channel is not necessary node_type = model.graph.get_nodes(layer_name)[0].type if node_type == 'Conv2D' or node_type == 'Conv2D_Pooling': w_conv1, b_conv1 = new_model.get_layers( layer_name)[0].get_weights() # tensorflow kernel format: [filter_height, filter_width, in_channels, out_channels] channels_last # theano kernel format: [output_channels, input_channels, filter_rows, filter_columns] channels_first if K.image_data_format() == "channels_first": # theano format # convert_kernel function Converts a Numpy kernel matrix from Theano format to TensorFlow format, vise versa w_conv1 = convert_kernel(w_conv1) kh, kw, num_channel, filters = w_conv1.shape elif node_type == 'Group': kh, kw, filters = 3, 3, model.graph.get_nodes( layer_name)[0].config['filters'] w_conv2, b_conv2 = new_model.get_layers( layer_name, next_layer=True)[0].get_weights() new_w_conv2, new_b_conv2 = Net2Net._deeper_conv2d_weight( kh=kh, kw=kw, filters=filters) new_model.get_layers(layer_name, next_layer=True)[0].set_weights( [new_w_conv2, new_b_conv2]) self.copy_weight(model, new_model) return new_model
def avepool_by_name(self, model, name, config): new_graph = model.graph.copy() node = new_graph.get_nodes(name)[0] node1 = new_graph.get_nodes(name, next_layer=True)[0] new_graph.update() new_name = 'AveragePooling2D' + \ str( 1 + max(new_graph.type2ind.get('AveragePooling2D', [0])) ) new_node = Node(type='AveragePooling2D', name=new_name, config={}) new_graph.add_node(new_node) new_graph.remove_edge(node, node1) new_graph.add_edge(node, new_node) new_graph.add_edge(new_node, node1) # logger.debug(new_graph.to_json()) new_model = MyModel(config=config, graph=new_graph) self.copy_weight(model, new_model) return new_model
def add_group(self, model, config): topo_nodes = nx.topological_sort(model.graph) names = [ node.name for node in topo_nodes if node.type == 'Conv2D' or node.type == 'Group' or node.type == 'Conv2D_Pooling' ] # random choose a layer to concat a group operation choice = names[np.random.randint(0, len(names) - 1)] group_num = np.random.randint(1, 5) # group number: [1, 2, 3, 4] logger.info('choose {} to add group, group_num is {}'.format( choice, group_num)) # add node and edge to the graph new_graph = model.graph.copy() node = new_graph.get_nodes(choice)[0] node_suc = new_graph.get_nodes(choice, next_layer=True)[0] new_graph.update() new_name = 'Group' + \ str( 1 + max(new_graph.type2ind.get('Group', [0])) ) filters = node.config['filters'] new_node = Node(type='Group', name=new_name, config={ 'group_num': group_num, 'filters': filters }) new_graph.add_node(new_node) new_graph.remove_edge(node, node_suc) new_graph.add_edge(node, new_node) new_graph.add_edge(new_node, node_suc) new_model = MyModel(config=config, graph=new_graph) self.copy_weight(model, new_model) return new_model, True
def skip(self, model, from_name, to_name, config): # original: node1-> node2 -> node3 # now: node1 -> node2 -> new_node -> node3 # --------------------> new_graph = model.graph.copy() node1 = new_graph.get_nodes(from_name)[0] node2 = new_graph.get_nodes(to_name)[0] node3 = new_graph.get_nodes(to_name, next_layer=True)[0] new_graph.update() new_name = 'Add' + \ str( 1 + max(new_graph.type2ind.get('Add', [0])) ) new_node = Node(type='Add', name=new_name, config={}) new_graph.add_node(new_node) new_graph.remove_edge(node2, node3) new_graph.add_edge(node1, new_node) new_graph.add_edge(node2, new_node) new_graph.add_edge(new_node, node3) # logger.debug(new_graph.to_json()) new_model = MyModel(config=config, graph=new_graph) self.copy_weight(model, new_model) # Way 1 # w, b = new_model.get_layers(node2.name)[0].get_weights() # w, b = Net2Net.rand_weight_like(w) # new_model.get_layers(node2.name)[0].set_weights((w, b)) # way 2 do nothing # way 3 divided by 2 # w, b = new_model.get_layers(node2.name)[0].get_weights() # w, b = Net2Net.rand_weight_like(w) # w = w / 2 # b = b / 2 # new_model.get_layers(node2.name)[0].set_weights((w, b)) return new_model
def main(_): sess = tf.compat.v1.Session() model = MyModel(sess,model_configs) if args.mode == "train": x_train, y_train, _,_ = load_data(args.data_dir) model.train(x_train, y_train,200) elif args.mode == "test": # Testing on public testing dataset _, _, x_test, y_test = load_data(args.data_dir) model.evaluate(x_test, y_test) elif args.mode == "predict": # Predicting and storing results on private testing dataset x_test = load_testing_images(args.data_dir) predictions = model.predict_prob(x_test) np.save("../predictions.npy", predictions)
import numpy as np from matplotlib import pyplot as plt from Model import MyModel from DataLoader import load_data, train_valid_split, load_testing_images from Configure import model_configs, training_configs import csv parser = argparse.ArgumentParser() parser.add_argument("mode", help="train, test or predict") parser.add_argument("data_dir", help="path to the data") parser.add_argument("--result_dir", help="path to save the results") args = parser.parse_args() if __name__ == '__main__': print(args.mode, args.data_dir) model = MyModel(model_configs) # model.load() if args.mode == 'train': x_train, y_train, x_test, y_test = load_data(args.data_dir) x_train, y_train, x_valid, y_valid = train_valid_split( x_train, y_train) train_stats = model.train(x_train, y_train, training_configs, x_valid, y_valid) w = csv.writer( open( os.path.join(model_configs["save_dir"], model_configs['name']) + ".csv", "w")) for key, val in train_stats.items(): w.writerow([key, val]) score, loss = model.evaluate(x_test, y_test)
def wider_group_conv2d(self, model, layer_name, new_width, config): # modify graph new_graph = model.graph.copy() node = new_graph.get_nodes(layer_name)[0] assert node.type == 'Group', 'must wide a group layer' node.config['filters'] = new_width new_model = MyModel(config=config, graph=new_graph) self.copy_weight(model, new_model) filter_num = int(new_width / node.config['group_num']) for i in range(node.config['group_num']): if i == node.config['group_num'] - 1: # last group filter_num = new_width - i * int( (new_width / node.config['group_num'])) layer_name_conv_1 = layer_name + '_conv2d_' + str(i) + '_1' layer_name_conv_2 = layer_name + '_conv2d_' + str(i) + '_2' # use keras get_layer function w_conv1, b_conv1 = model.model.get_layer( layer_name_conv_1).get_weights() w_conv2, b_conv2 = model.model.get_layer( layer_name_conv_2).get_weights() new_w_conv1, new_b_conv1, new_w_conv2 = Net2Net._wider_conv2d_weight( w_conv1, b_conv1, w_conv2, filter_num, "net2wider") new_model.model.get_layer(layer_name_conv_1).set_weights( [new_w_conv1, new_b_conv1]) # new_model.model.get_layer(layer_name_conv_2).set_weights([new_w_conv2, b_conv2]) if i == 0: w_conv_group = new_w_conv2 b_conv_group = b_conv2 else: w_conv_group = np.concatenate((w_conv_group, new_w_conv2), axis=3) b_conv_group = np.concatenate((b_conv_group, b_conv2), axis=0) # find group layer's next conv layer w_conv2, b_conv2 = model.get_layers( layer_name, next_layer=True, type=['Conv2D', 'Conv2D_Pooling', 'Group'])[0].get_weights() new_w_conv_group, new_b_conv_group, new_w_conv2 = Net2Net._wider_conv2d_weight( w_conv_group, b_conv_group, w_conv2, new_width, "net2wider") for i in range(node.config['group_num']): layer_name_conv_2 = layer_name + '_conv2d_' + str(i) + '_2' if i == node.config['group_num'] - 1: new_model.model.get_layer(layer_name_conv_2).set_weights([ new_w_conv_group[:, :, :, i * filter_num:], new_b_conv_group[i * filter_num:] ]) else: new_model.model.get_layer(layer_name_conv_2).set_weights([ new_w_conv_group[:, :, :, i * filter_num:(i + 1) * filter_num], new_b_conv_group[i * filter_num:(i + 1) * filter_num] ]) new_model.get_layers(layer_name, next_layer=True, type=['Conv2D', 'Conv2D_Pooling', 'Group' ])[0].set_weights([new_w_conv2, b_conv2]) return new_model
def main(miRNA_Disease_Association, disease_feature, disease_graph1, disease_graph2, disease_graph3, miRNA_feature, miRNA_graph1, miRNA_graph2, miRNA_graph3): adjProcess = adjTrainTestSplit(miRNA_Disease_Association) graph_train_kfold, graph_test_kfold = adjProcess.split_graph(KFold, SEED) auc_kfold = [] aupr_kfold = [] mean_tpr = 0.0 # 用来记录画平均ROC曲线的信息 mean_fpr = np.linspace(0, 1, 100) for i in range(KFold): print("Using {} th fold dataset.".format(i + 1)) graph_train = graph_train_kfold[i] graph_test = graph_test_kfold[i] # m = graph_train.shape[0] # n = graph_train.shape[1] # eval_coord = [(i, j) for i in range(m) for j in range(n)] # train_edge_x, train_edge_y = graph_train.nonzero() # one_index = list(zip(train_edge_x, train_edge_y)) # zero_index = set(eval_coord) - set(set(zip(train_edge_x, train_edge_y))) # zero_index = list(zero_index) adj_traget = torch.FloatTensor(graph_train) model = MyModel(disease_feature, disease_graph1, disease_graph2, disease_graph3, miRNA_feature, miRNA_graph1, miRNA_graph2, miRNA_graph3) model.cuda() obj = Myloss(adj_traget.cuda()) optimizer = torch.optim.Adam(model.parameters(), lr=LR, amsgrad=True, weight_decay=GAMA) evaluator = Evaluator(graph_train, graph_test) #obj_test = Myloss(torch.FloatTensor(graph_test).cuda()) for j in range(EPOCH): model.train() optimizer.zero_grad() Y_hat, m_x0, m_x1, m_x2, m_x3, d_x0, d_x1, d_x2, d_x3 = model() loss = obj.cal_loss(Y_hat, m_x0.cuda(), m_x1.cuda(), m_x2.cuda(), m_x3.cuda(), d_x0.cuda(), d_x1.cuda(), d_x2.cuda(), d_x3.cuda(), ALPHA, BETA, GAMA) loss = loss.cuda() # loss = obj.cal_loss(Y_hat,one_index,zero_index) #loss = obj.cal_loss(Y_hat,one_index,zero_index,m_x0,m_x1,m_x2,m_x3,d_x0, d_x1, d_x2,d_x3) loss.backward() optimizer.step() need_early_stop_check = j > TOLERANCE_EPOCH and abs( (loss.item() - last_loss) / last_loss) < STOP_THRESHOLD if (j % EVAL_INTER == 0) or need_early_stop_check or j + 1 >= EPOCH: t = time.time() model.eval() with torch.no_grad(): Y_hat, m_x0, m_x1, m_x2, m_x3, d_x0, d_x1, d_x2, d_x3 = model( ) #test_loss = obj_test.cal_loss(Y_hat, m_x0, m_x1, m_x2, m_x3, d_x0, d_x1, d_x2, d_x3,ALPHA,BETA) # Y_hat = torch.sigmoid(Y_hat) # eval_coord = [(i, j) for i in range(m) for j in range(n)] # test_edge_x, test_edge_y = graph_test.nonzero() # test_one_index = list(zip(test_edge_x, test_edge_y)) # #test_zero_index = set(eval_coord) - set(set(zip(test_edge_x, test_edge_y))) # test_zero_index = list(test_zero_index) # #test_loss = obj_test.cal_loss(Y_hat, test_one_index, test_zero_index) auc_test, aupr_test, fpr, tpr = evaluator.eval(Y_hat.cpu()) print("Epoch:", '%04d' % (j + 1), "train_loss=", "{:0>9.5f}".format(loss.item()), "test_auc=", "{:.5f}".format(auc_test), "test_aupr=", "{:.5f}".format(aupr_test), "time=", "{:.2f}".format(time.time() - t)) if need_early_stop_check or j + 1 >= EPOCH: auc_kfold.append(auc_test) aupr_kfold.append(aupr_test) mean_tpr += np.interp(mean_fpr, fpr, tpr) mean_tpr[0] = 0.0 if need_early_stop_check: print("Early stopping...") else: print("Arrived at the last Epoch...") break last_loss = loss.item() torch.cuda.empty_cache() print("\nOptimization Finished!") mean_tpr /= KFold mean_tpr[-1] = 1.0 np.save("../Data/Result/mean_tpr.npy", mean_tpr) mean_auc = sum(auc_kfold) / len(auc_kfold) mean_aupr = sum(aupr_kfold) / len(aupr_kfold) print("mean_auc:{0:.3f},mean_aupr:{1:.3f}".format(mean_auc, mean_aupr))
shuffle=True, num_workers=2) loss_function = nn.NLLLoss() test_dataloader = DataLoader(LegalDataset(train=False), batch_size=config.batch_size, shuffle=False, num_workers=2) learning_rate = 0.001 def to_cuda(param_list): return [x.cuda() for x in param_list] model = MyModel().cuda() optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate) for i in range(config.num_epoch): total_loss = 0. model.train() for batch_data in tqdm.tqdm(train_dataloader, desc='Epoch %3d' % (i + 1)): curr_fact, term, law, accu = to_cuda(batch_data) optimizer.zero_grad() # print('term', term) # print('law', law) # print('accu', accu) pred1, pred2, pred3 = model(curr_fact) accu_loss = loss_function(pred1, accu) law_loss = loss_function(pred2, law) term_loss = loss_function(pred3, term) loss = accu_loss + law_loss + term_loss
model_l = [["Conv2D", 'Conv2D1', { 'filters': 64 }], ["MaxPooling2D", 'maxpooling2d1', {}], ["Group", 'Group1', { 'filters': 120, 'group_num': 2 }], ["Conv2D_Pooling", 'Conv2D_Pooling_1', { 'filters': 139 }], ["Conv2D", 'Conv2D3', { 'filters': 10 }], ['GlobalMaxPooling2D', 'GlobalMaxPooling2D1', {}], ['Activation', 'Activation1', { 'activation_type': 'softmax' }]] graph = MyGraph(model_l) before_model = MyModel(config, graph) before_model.comp_fit_eval() net2net = Net2Net() model = net2net.wider_group_conv2d(before_model, layer_name='Group1', new_width=174, config=config.copy('wider_group')) model.comp_fit_eval() ''' model = net2net.deeper(before_model, config=config.copy('deeper1')) model = net2net.deeper_conv2d(before_model, layer_name='Conv2D2', config=config.copy('deeper1')) model = net2net.wider_conv2d(model, layer_name='Conv2D2', width_ratio=2, config=config.copy('wide')) model.comp_fit_eval()
from TFrecord import generator_TFrecord from data_split import ds_func from train import training from Model import MyModel data_dir = '/content/gdrive/My Drive/Dataset' train_dir = data_dir + '/1. Original Images/a. Training Set/' test_dir = data_dir + '/1. Original Images/b. Testing Set/' gt_dir = data_dir + '/2. Groundtruths/' train_label_dir = gt_dir + 'a. IDRiD_Disease Grading_Training Labels.csv' test_label_dir = gt_dir + 'b. IDRiD_Disease Grading_Testing Labels.csv' generator_TFrecord(train_dir, train_label_dir, 'train.tfrecords', oversampling=True) generator_TFrecord(test_dir, test_label_dir, 'test.tfrecords', oversampling=False) batch_size = 8 prefetch_size = 1000 train_ds, val_ds = ds_func('train.tfrecords', batch_size, prefetch_size, split=True) test_ds = ds_func('test.tfrecords', batch_size, prefetch_size, split=False) model = MyModel() training(train_ds, val_ds, test_ds, model, EPOCHS=10)
optimizer.setup(model) #Updater updater = training.StandardUpdater(train_iter, optimizer, device=gpu_id) # Trainer trainer = training.Trainer(updater, (max_epoch, 'epoch'), out='{}_result'.format(model_object.__class__.__name__)) # Evaluator class TestModeEvaluator(extensions.Evaluator): def evaluate(self): model = self.get_target('main') model.train = False ret = super(TestModeEvaluator, self).evaluate() model.train = True return ret trainer.extend(extensions.LogReport()) trainer.extend(TestModeEvaluator(test_iter, model, device=gpu_id)) trainer.extend(extensions.PrintReport(['epoch', 'main/loss', 'main/accuracy', 'validation/main/loss', 'validation/main/accuracy', 'elapsed_time'])) trainer.extend(extensions.PlotReport(['main/loss', 'validation/main/loss'], x_key='epoch', file_name='loss.png')) trainer.extend(extensions.PlotReport(['main/accuracy', 'validation/main/accuracy'], x_key='epoch', file_name='accuracy.png')) trainer.run() del trainer return model model = train(MyModel(10), gpu_id=-1)
# import torch import os, argparse import numpy as np from Model import MyModel from DataLoader import load_data, train_valid_split, load_testing_images from Configure import model_configs, training_configs parser = argparse.ArgumentParser() parser.add_argument("--mode", help="train, test or predict") parser.add_argument("--data_dir", help="path to the data") parser.add_argument("--test_file", help="path to the test file") parser.add_argument("--save_dir", help="path to save the results") args = parser.parse_args() if __name__ == '__main__': model = MyModel(model_configs, training_configs) if args.mode == 'train': x_train, y_train, x_test, y_test = load_data(args.data_dir) x_train, y_train, x_valid, y_valid = train_valid_split( x_train, y_train) model.train(x_train, y_train, x_valid, y_valid) model.save_weights( os.path.join(args.save_dir, model_configs["version"], "")) model.evaluate(x_test, y_test) elif args.mode == 'test': # Testing on public testing dataset model.load_weights( os.path.join(args.save_dir, model_configs["version"], ""))
from Model import MyModel from DataLoader import load_data, train_valid_split, load_testing_images from Configure import model_configs, training_configs import utils parser = argparse.ArgumentParser() parser.add_argument("mode", help="train, test or predict") parser.add_argument("data_dir", help="path to the data") parser.add_argument("--save_dir", help="path to save the results") parser.add_argument("--resume_checkpoint", help=".pth checkpoint file to resume") parser.add_argument("--checkpoint", help=".pth checkpoint file to use for evaluation") args = parser.parse_args() device = 'cuda' if torch.cuda.is_available() else 'cpu' if __name__ == '__main__': model = MyModel(model_configs) if args.mode == 'train': print('----- training mode ----') train,test,orig_trainset = load_data(args.data_dir,train_aug=training_configs['train_augmentation']) # augment the train data with config train,valid = train_valid_split(train,orig_trainset,train_ratio=1) if args.resume_checkpoint is not None: checkpoint = torch.load('../saved_models/' + args.resume_checkpoint) epoch,accuracy_type,prev_accuracy = (checkpoint[k] for k in ['epoch','accuracy_type','accuracy']) print('RESUME---> Loading model from Epoch %d with %s Accuracy %f' %(epoch,accuracy_type,prev_accuracy)) else: checkpoint = None model.train(train, training_configs,valid=None,test=test,checkpoint=checkpoint) # note test data is used only to evaluate model performance during training model.evaluate(test) elif args.mode == 'test':