def spectrumToArrays(device, params, img_path, output_folder, file_name): g = tf.Graph() content_image = ut.read_image(img_path) with g.device(device), g.as_default(), tf.Session(graph=g, config=tf.ConfigProto(allow_soft_placement=True)) as sess: print "Load content values..." image = tf.constant(content_image) model = models.getModel(image, params) content_image_y_val = [sess.run(y_l) for y_l in model.y()] # sess.run(y_l) is a constant numpy array for i in range(len(content_image_y_val)): output_path = output_folder + layers_names[i] + "/" if not os.path.exists(output_path): os.makedirs(output_path) dirr = os.path.dirname(output_path + file_name) if not os.path.exists(dirr): os.makedirs(dirr) np.save(output_path + file_name, content_image_y_val[i]) #cleanUp del image del content_image del model del content_image_y_val del g
def __init__(self, mdlParams): super(CNN_GRU, self).__init__() # Some necessary vars self.crop_number = mdlParams['multiCropTrain'] self.cell_type = mdlParams['cell_type'] self.cnn_output_point = mdlParams['CNN_Output_Point'] self.aux_classifier = mdlParams['aux_classifier'] # CNN first,up to feature vector self.cnn = models.getModel(mdlParams['model_type_cnn'])() if 'Dense' in mdlParams['model_type_cnn']: if self.cnn_output_point == 'end': self.cnn_features = self.cnn.features elif self.cnn_output_point == 'transition3': self.cnn_features = nn.Sequential( *list(self.cnn.features.children())[:10]) elif 'InceptionV3' in mdlParams['model_type_cnn']: if self.cnn_output_point == 'end': self.cnn_features = nn.Sequential( self.cnn.Conv2d_1a_3x3, self.cnn.Conv2d_2a_3x3, self.cnn.Conv2d_2b_3x3, nn.MaxPool2d(kernel_size=3, stride=2), self.cnn.Conv2d_3b_1x1, self.cnn.Conv2d_4a_3x3, nn.MaxPool2d(kernel_size=3, stride=2), self.cnn.Mixed_5b, self.cnn.Mixed_5c, self.cnn.Mixed_5d, self.cnn.Mixed_6a, self.cnn.Mixed_6b, self.cnn.Mixed_6c, self.cnn.Mixed_6d, self.cnn.Mixed_6e, self.cnn.Mixed_7a, self.cnn.Mixed_7b, self.cnn.Mixed_7c) else: if self.cnn_output_point == 'end': self.cnn_features = nn.Sequential(self.cnn.layer0, self.cnn.layer1, self.cnn.layer2, self.cnn.layer3, self.cnn.layer4) # The GRU if self.cell_type == 'A': self.gru = nn.GRU(mdlParams['CNN_Features'], mdlParams['GRU_FM_Hidden'], mdlParams['GRU_num_layers'], batch_first=True, bidirectional=mdlParams['bidirectional']) # The classifier if mdlParams['bidirectional']: self.classifier = nn.Linear(2 * mdlParams['GRU_FM_Hidden'], mdlParams['numClasses']) else: self.classifier = nn.Linear(mdlParams['GRU_FM_Hidden'], mdlParams['numClasses']) if self.aux_classifier: num_ftrs = self.cnn.classifier.in_features self.cnn.classifier = nn.Linear(num_ftrs, mdlParams['numClasses'])
def main(): train = pd.read_json("train.json") #test = pd.read_json("test.json") X_band_1 = np.array([ np.array(band).astype(np.float32).reshape(75, 75) for band in train["band_1"] ]) X_band_2 = np.array([ np.array(band).astype(np.float32).reshape(75, 75) for band in train["band_2"] ]) X_train = np.concatenate([ X_band_1[:, :, :, np.newaxis], X_band_2[:, :, :, np.newaxis], ((X_band_1 + X_band_2) / 2)[:, :, :, np.newaxis] ], axis=-1) target_train = train['is_iceberg'] file_path = ".model_weights.hdf5" #X_valid, y_train_cv, y_valid = train_test_split(X_train, target_train, random_state=1, train_size=0.75) X_train_cv, X_valid, y_train_cv, y_valid = GroupShuffleStratifiedS_angle( X_train, target_train, train.inc_angle) #create and set model gmodel = getModel() mypotim = Adam(lr=0.001, beta_1=0.9, beta_2=0.999, epsilon=1e-08, decay=0.0) gmodel.compile(loss='binary_crossentropy', optimizer=mypotim, metrics=['accuracy']) #train our model gmodel = train_model_(gmodel, X_train_cv, y_train_cv, X_valid, y_valid, file_path) #download our best weights from file, evaluate our model on 25% train data gmodel.load_weights(filepath=file_path) score = gmodel.evaluate(X_valid, y_valid, verbose=1) print('Test loss:', score[0]) print('Test accuracy:', score[1]) '''
def main(start_epoch=0, end_epoch=1000, save_frequency=50, mode='train'): print(f'Using device {device}') assert mode in ('train', 'test') assert save_frequency > 0 ds = data.RubikDataset(60000, maxScrambles) dl = DataLoader(ds, batch_size=64, num_workers=16) if start_epoch == 0: net = models.getModel(mul=2).to(device) else: net = torch.load(getModelPath(start_epoch), map_location=device) if mode == 'train': train(net, dl, start_epoch, end_epoch, save_frequency) if mode in ('train', 'test'): sum, numSolves = 0, 50 for i in range(numSolves): sum += testSolve(net, scrambles=random.randint(1000, 1099)) print(f'Average solution steps: {sum / numSolves}')
def __init__(self, mdlParams): super(CNN_GRU_TP, self).__init__() # Some necessary vars self.crop_number = mdlParams['multiCropTrain'] self.cell_type = mdlParams['cell_type'] self.cnn_output_point = mdlParams['CNN_Output_Point'] # CNN first,up to feature vector self.cnn = models.getModel(mdlParams['model_type_cnn'])() if self.cnn_output_point == 'end': self.cnn_features = self.cnn.features elif self.cnn_output_point == 'transition3': self.cnn_features = nn.Sequential( *list(self.cnn.features.children())[:10]) # The GRU if self.cell_type == 'A': self.gru = nn.GRU(mdlParams['CNN_Features'], mdlParams['GRU_FM_Hidden'], mdlParams['GRU_num_layers'], batch_first=True, bidirectional=mdlParams['bidirectional']) # The second CNN for global, make this better self.cnn_global = tvmodels.densenet121(pretrained=True) if self.cnn_output_point == 'end': self.cnn_features_global = self.cnn_global.features elif self.cnn_output_point == 'transition3': self.cnn_features_global = nn.Sequential( *list(self.cnn_global.features.children())[:10]) # The GRU # The classifier if mdlParams['bidirectional']: self.classifier = nn.Linear( 2 * mdlParams['GRU_FM_Hidden'] + mdlParams['CNN_Features'], mdlParams['numClasses']) else: self.classifier = nn.Linear( mdlParams['GRU_FM_Hidden'] + mdlParams['CNN_Features'], mdlParams['numClasses'])
with tf.device('gpu:%d' % mdlParams['numGPUs'][i]): with tf.name_scope('tower_%d' % i) as scope: # Batches for tower modelVars['X_' + str(i)], modelVars['Tar_' + str(i)], modelVars[ 'Inds_' + str(i)] = modelVars['iterator'].get_next() #summaries.append(tf.summary.image('model input',modelVars['X_'+str(i)][:,:,:,0:],max_outputs=5)) print("Input", modelVars['X_' + str(i)], modelVars['X_' + str(i)].get_shape(), "Output", modelVars['Tar_' + str(i)], modelVars['Tar_' + str(i)].get_shape()) # Build graph, put all variables on CPU with slim.arg_scope([slim.model_variable, slim.variable], device='/cpu:0'): model_function = models.getModel( mdlParams, placeholders) modelVars['pred_' + str(i)] = model_function( modelVars['X_' + str(i)]) print("Pred", modelVars['pred_' + str(i)], modelVars['pred_' + str(i)].get_shape()) # Build loss #modelVars['loss_'+str(i)] = tf.losses.mean_squared_error(labels=modelVars['Tar_'+str(i)], predictions=modelVars['pred_'+str(i)], scope=scope, loss_collection=tf.GraphKeys.LOSSES) modelVars['loss_' + str(i)] = tf.reduce_mean( tf.square( tf.subtract(modelVars['Tar_' + str(i)], modelVars['pred_' + str(i)]))) tf.add_to_collection(tf.GraphKeys.LOSSES, modelVars['loss_' + str(i)]) # Reuse tf.get_variable_scope().reuse_variables() # Get relevant variables for training
def trainModel(modelname, X_train, y_train, config): info("Training a {0} estimator".format(modelname), ind=0) info("X data is {0}".format(getDim(X_train)), ind=2) info("y data is {0}".format(getDim(y_train)), ind=2) problemType = config['problem'] info("This is a {0} problem".format(problemType), ind=2) modelData = getModelData(config, modelname) tuneForParams = True refitModel = False goodModel = True if modelData is not None: if modelData.get('tune') is False: tuneForParams = False if modelData.get('fit') is True: tuneForParams = False if modelData.get('cv') is True: tuneForParams = False if modelData.get('refit') is True: refitModel = True if modelData.get('error') is True: goodModel = False else: info("No model parameters were given. Using default {0} estimator".format(modelname), ind=4) tuneForParams = False if goodModel is False: error("Model {0} is no good and will not run it.".format(modelname)) return None ################################################################# # Get Model ################################################################# retval = getModel(config, modelname) ################################################################# # Tune Parameters ################################################################# estimator = retval['estimator'] params = retval['params'] if tuneForParams: tuneResults = tuneModel(modelname, estimator, params, X_train, y_train, config) estimator = tuneResults['estimator'] params = tuneResults['params'] if refitModel: try: estimator.set_params(probability=True) info("Set probability to True for model refit", ind=4) except: info("Could not set probability to True for model refit") info("Re-fitting for {0} model parameters with probability".format(modelname), ind=4) estimator = estimator.fit(X_train, y_train) info("Finished re-fitting {0} model parameters with probability".format(modelname), ind=4) else: if estimator is not None: info("Fitting for {0} model parameters".format(modelname), ind=2) estimator = estimator.fit(X_train, y_train) info("Finished fitting {0} model parameters".format(modelname), ind=4) else: error("No model with name {0} was trained".format(modelname)) return estimator
def main(): # add configuration file # Dictionary for model configuration mdlParams = {} # Import machine config pc_cfg = importlib.import_module('pc_cfgs.' + sys.argv[1]) mdlParams.update(pc_cfg.mdlParams) # Import model config model_cfg = importlib.import_module('cfgs.' + sys.argv[2]) mdlParams_model = model_cfg.init(mdlParams) mdlParams.update(mdlParams_model) # Indicate training mdlParams['trainSetState'] = 'train' # Path name from filename mdlParams['saveDirBase'] = mdlParams['saveDir'] + sys.argv[2] # Set visible devices if 'gpu' in sys.argv[3]: mdlParams['numGPUs'] = [[ int(s) for s in re.findall(r'\d+', sys.argv[3]) ][-1]] cuda_str = "" for i in range(len(mdlParams['numGPUs'])): cuda_str = cuda_str + str(mdlParams['numGPUs'][i]) if i is not len(mdlParams['numGPUs']) - 1: cuda_str = cuda_str + "," print("Devices to use:", cuda_str) os.environ["CUDA_VISIBLE_DEVICES"] = cuda_str # Specify val set to train for if len(sys.argv) > 4: mdlParams['cv_subset'] = [ int(s) for s in re.findall(r'\d+', sys.argv[4]) ] print("Training validation sets", mdlParams['cv_subset']) # Check if there is a validation set, if not, evaluate train error instead if 'valIndCV' in mdlParams or 'valInd' in mdlParams: eval_set = 'valInd' print("Evaluating on validation set during training.") else: eval_set = 'trainInd' print("No validation set, evaluating on training set during training.") # Check if there were previous ones that have alreary bin learned # prevFile = Path(mdlParams['saveDirBase'] + '/CV.pkl') prevFile = Path(mdlParams['saveDirBase'] + '\CV.pkl') # print(prevFile) if prevFile.exists(): print("Part of CV already done") # with open(mdlParams['saveDirBase'] + '/CV.pkl', 'rb') as f: with open(mdlParams['saveDirBase'] + '\CV.pkl', 'rb') as f: allData = pickle.load(f) else: allData = {} allData['f1Best'] = {} allData['sensBest'] = {} allData['specBest'] = {} allData['accBest'] = {} allData['waccBest'] = {} allData['aucBest'] = {} allData['convergeTime'] = {} allData['bestPred'] = {} allData['targets'] = {} # Take care of CV if mdlParams.get('cv_subset', None) is not None: cv_set = mdlParams['cv_subset'] else: cv_set = range(mdlParams['numCV']) for cv in cv_set: # Check if this fold was already trained already_trained = False if 'valIndCV' in mdlParams: mdlParams['saveDir'] = mdlParams['saveDirBase'] + '/CVSet' + str( cv) # mdlParams['saveDir'] = mdlParams['saveDirBase'] + '\CVSet' + str(cv) if os.path.isdir(mdlParams['saveDirBase']): if os.path.isdir(mdlParams['saveDir']): all_max_iter = [] for name in os.listdir(mdlParams['saveDir']): int_list = [int(s) for s in re.findall(r'\d+', name)] if len(int_list) > 0: all_max_iter.append(int_list[-1]) # if '-' + str(mdlParams['training_steps'])+ '.pt' in name: # print("Fold %d already fully trained"%(cv)) # already_trained = True all_max_iter = np.array(all_max_iter) if len(all_max_iter) > 0 and np.max( all_max_iter) >= mdlParams['training_steps']: print( "Fold %d already fully trained with %d iterations" % (cv, np.max(all_max_iter))) already_trained = True if already_trained: continue print("CV set", cv) # Reset model graph importlib.reload(models) # importlib.reload(torchvision) # Collect model variables modelVars = {} # print("here") modelVars['device'] = torch.device( "cuda:0" if torch.cuda.is_available() else "cpu") print(modelVars['device']) # Def current CV set mdlParams['trainInd'] = mdlParams['trainIndCV'] if 'valIndCV' in mdlParams: mdlParams['valInd'] = mdlParams['valIndCV'] # Def current path for saving stuff if 'valIndCV' in mdlParams: mdlParams['saveDir'] = mdlParams['saveDirBase'] + '/CVSet' + str( cv) # mdlParams['saveDir'] = mdlParams['saveDirBase'] + '\CVSet' + str(cv) else: mdlParams['saveDir'] = mdlParams['saveDirBase'] # Create basepath if it doesnt exist yet print(mdlParams['saveDir']) if not os.path.isdir(mdlParams['saveDirBase']): os.mkdir(mdlParams['saveDirBase']) # Check if there is something to load load_old = 0 if os.path.isdir(mdlParams['saveDir']): # Check if a checkpoint is in there print("i got here" + str(147)) if len([name for name in os.listdir(mdlParams['saveDir'])]) > 0: load_old = 1 print("Loading old model") else: # Delete whatever is in there (nothing happens) filelist = [ os.remove(mdlParams['saveDir'] + '/' + f) for f in os.listdir(mdlParams['saveDir']) ] # filelist = [os.remove(mdlParams['saveDir'] + '\\' + f) for f in os.listdir(mdlParams['saveDir'])] else: os.mkdir(mdlParams['saveDir']) # Save training progress in here save_dict = {} save_dict['acc'] = [] save_dict['loss'] = [] save_dict['wacc'] = [] save_dict['auc'] = [] save_dict['sens'] = [] save_dict['spec'] = [] save_dict['f1'] = [] save_dict['step_num'] = [] if mdlParams['print_trainerr']: save_dict_train = {} save_dict_train['acc'] = [] save_dict_train['loss'] = [] save_dict_train['wacc'] = [] save_dict_train['auc'] = [] save_dict_train['sens'] = [] save_dict_train['spec'] = [] save_dict_train['f1'] = [] save_dict_train['step_num'] = [] # Potentially calculate setMean to subtract if mdlParams['subtract_set_mean'] == 1: mdlParams['setMean'] = np.mean( mdlParams['images_means'][mdlParams['trainInd'], :], (0)) print("Set Mean", mdlParams['setMean']) if mdlParams['balance_classes'] == 9: # Only use official indicies for calculation print("Balance 9") indices_ham = mdlParams['trainInd'][mdlParams['trainInd'] < 25331] if mdlParams['numClasses'] == 9: class_weights_ = 1.0 / np.mean( mdlParams['labels_array'][indices_ham, :8], axis=0) # print("class before",class_weights_) class_weights = np.zeros([mdlParams['numClasses']]) class_weights[:8] = class_weights_ class_weights[-1] = np.max(class_weights_) else: class_weights = 1.0 / np.mean( mdlParams['labels_array'][indices_ham, :], axis=0) print("Current class weights", class_weights) if isinstance(mdlParams['extra_fac'], float): class_weights = np.power(class_weights, mdlParams['extra_fac']) else: class_weights = class_weights * mdlParams['extra_fac'] print("Current class weights with extra", class_weights) # Meta scaler if mdlParams.get('meta_features', None) is not None and mdlParams['scale_features']: mdlParams[ 'feature_scaler_meta'] = sklearn.preprocessing.StandardScaler( ).fit(mdlParams['meta_array'][mdlParams['trainInd'], :]) print("scaler mean", mdlParams['feature_scaler_meta'].mean_, "var", mdlParams['feature_scaler_meta'].var_) # Set up dataloaders num_workers = psutil.cpu_count(logical=False) # For train dataset_train = utils.ISICDataset(mdlParams, 'trainInd') # For val dataset_val = utils.ISICDataset(mdlParams, 'valInd') if mdlParams['multiCropEval'] > 0: modelVars['dataloader_valInd'] = DataLoader( dataset_val, batch_size=mdlParams['multiCropEval'], shuffle=False, num_workers=num_workers, pin_memory=True) else: modelVars['dataloader_valInd'] = DataLoader( dataset_val, batch_size=mdlParams['batchSize'], shuffle=False, num_workers=num_workers, pin_memory=True) modelVars['dataloader_trainInd'] = DataLoader( dataset_train, batch_size=mdlParams['batchSize'], shuffle=True, num_workers=num_workers, pin_memory=True, drop_last=True) # print("Setdiff",np.setdiff1d(mdlParams['trainInd'],mdlParams['trainInd'])) # Define model modelVars['model'] = models.getModel(mdlParams)() # Load trained model if mdlParams.get('meta_features', None) is not None: # Find best checkpoint files = glob(mdlParams['model_load_path'] + '/CVSet' + str(cv) + '/*') # files = glob(mdlParams['model_load_path'] + '\CVSet' + str(cv) + '\*') global_steps = np.zeros([len(files)]) # print("files",files) for i in range(len(files)): # Use meta files to find the highest index if 'best' not in files[i]: continue if 'checkpoint' not in files[i]: continue # Extract global step nums = [int(s) for s in re.findall(r'\d+', files[i])] global_steps[i] = nums[-1] # Create path with maximum global step found chkPath = mdlParams['model_load_path'] + '/CVSet' + str( cv) + '/checkpoint_best-' + str(int( np.max(global_steps))) + '.pt' # chkPath = mdlParams['model_load_path'] + '\CVSet' + str(cv) + '\checkpoint_best-' + str( # int(np.max(global_steps))) + '.pt' print("Restoring lesion-trained CNN for meta data training: ", chkPath) # Load state = torch.load(chkPath) # Initialize model curr_model_dict = modelVars['model'].state_dict() for name, param in state['state_dict'].items(): # print(name,param.shape) if isinstance(param, nn.Parameter): # backwards compatibility for serialized parameters param = param.data if curr_model_dict[name].shape == param.shape: curr_model_dict[name].copy_(param) else: print("not restored", name, param.shape) # Define classifier layer if 'Dense' in mdlParams['model_type']: if mdlParams['input_size'][0] != 224: modelVars['model'] = utils.modify_densenet_avg_pool( modelVars['model']) # print(modelVars['model']) num_ftrs = modelVars['model'].classifier.in_features modelVars['model'].classifier = nn.Linear(num_ftrs, mdlParams['numClasses']) # print(modelVars['model']) elif 'dpn' in mdlParams['model_type']: num_ftrs = modelVars['model'].classifier.in_channels modelVars['model'].classifier = nn.Conv2d(num_ftrs, mdlParams['numClasses'], [1, 1]) # modelVars['model'].add_module('real_classifier',nn.Linear(num_ftrs, mdlParams['numClasses'])) # print(modelVars['model']) elif 'efficient' in mdlParams['model_type'] and ('0' in mdlParams['model_type'] or '1' in mdlParams['model_type'] \ or '2' in mdlParams['model_type'] or '3' in mdlParams['model_type']): num_ftrs = modelVars['model'].classifier.in_features modelVars['model'].classifier = nn.Linear(num_ftrs, mdlParams['numClasses']) elif 'efficient' in mdlParams['model_type']: num_ftrs = modelVars['model']._fc.in_features modelVars['model'].classifier = nn.Linear(num_ftrs, mdlParams['numClasses']) elif 'wsl' in mdlParams['model_type'] or 'Resnet' in mdlParams[ 'model_type'] or 'Inception' in mdlParams['model_type']: # Do nothing, output is prepared num_ftrs = modelVars['model'].fc.in_features modelVars['model'].fc = nn.Linear(num_ftrs, mdlParams['numClasses']) else: num_ftrs = modelVars['model'].last_linear.in_features modelVars['model'].last_linear = nn.Linear(num_ftrs, mdlParams['numClasses']) # Take care of meta case if mdlParams.get('meta_features', None) is not None: # freeze cnn first if mdlParams['freeze_cnn']: # deactivate all for param in modelVars['model'].parameters(): param.requires_grad = False if 'wsl' in mdlParams['model_type'] or 'Resnet' in mdlParams[ 'model_type'] or 'Inception' in mdlParams['model_type']: # Activate classifier layer for param in modelVars['model'].fc.parameters(): param.requires_grad = True elif ('efficient' in mdlParams['model_type'] and ('0' in mdlParams['model_type'] or '1' in mdlParams['model_type'] \ or '2' in mdlParams['model_type'] or '3' in mdlParams['model_type'])) or 'Dense' in mdlParams['model_type']: # Activate classifier layer for param in modelVars['model'].classifier.parameters(): param.requires_grad = True elif 'efficient' in mdlParams['model_type']: #Activate classifier layer for param in modelVars['model']._fc.parameters(): param.requires_grad = True else: # Activate classifier layer for param in modelVars['model'].last_linear.parameters(): param.requires_grad = True else: # mark cnn parameters for param in modelVars['model'].parameters(): param.is_cnn_param = True # unmark fc for param in modelVars['model']._fc.parameters(): param.is_cnn_param = False # modify model modelVars['model'] = models.modify_meta(mdlParams, modelVars['model']) # Mark new parameters for param in modelVars['model'].parameters(): if not hasattr(param, 'is_cnn_param'): param.is_cnn_param = False # multi gpu support if len(mdlParams['numGPUs']) > 1: modelVars['model'] = nn.DataParallel(modelVars['model']) modelVars['model'] = modelVars['model'].cuda() # summary(modelVars['model'], modelVars['model'].input_size)# (mdlParams['input_size'][2], mdlParams['input_size'][0], mdlParams['input_size'][1])) modelVars['criterion'] = nn.CrossEntropyLoss( weight=torch.cuda.FloatTensor(class_weights.astype(np.float32))) if mdlParams.get('meta_features', None) is not None: #True for meta data if mdlParams['freeze_cnn']: modelVars['optimizer'] = optim.Adam( filter(lambda p: p.requires_grad, modelVars['model'].parameters()), lr=mdlParams['learning_rate_meta']) # sanity check for param in filter(lambda p: p.requires_grad, modelVars['model'].parameters()): print(param.name, param.shape) else: modelVars['optimizer'] = optim.Adam( [{ 'params': filter(lambda p: not p.is_cnn_param, modelVars['model'].parameters()), 'lr': mdlParams['learning_rate_meta'] }, { 'params': filter(lambda p: p.is_cnn_param, modelVars['model'].parameters()), 'lr': mdlParams['learning_rate'] }], lr=mdlParams['learning_rate']) else: modelVars['optimizer'] = optim.Adam( modelVars['model'].parameters(), lr=mdlParams['learning_rate']) # Decay LR by a factor of 0.2 every 25 epochs modelVars['scheduler'] = lr_scheduler.StepLR( modelVars['optimizer'], step_size=mdlParams['lowerLRAfter'], gamma=1 / np.float32(mdlParams['LRstep'])) # Define softmax modelVars['softmax'] = nn.Softmax(dim=1) # Set up training # loading from checkpoint if load_old: # Find last, not last best checkpoint files = glob(mdlParams['saveDir'] + '/*') # files = glob(mdlParams['saveDir'] + '\*') global_steps = np.zeros([len(files)]) for i in range(len(files)): # Use meta files to find the highest index if 'best' in files[i]: continue if 'checkpoint-' not in files[i]: continue # Extract global step nums = [int(s) for s in re.findall(r'\d+', files[i])] global_steps[i] = nums[-1] # Create path with maximum global step found chkPath = mdlParams['saveDir'] + '/checkpoint-' + str( int(np.max(global_steps))) + '.pt' # chkPath = mdlParams['saveDir'] + '\checkpoint-' + str(int(np.max(global_steps))) + '.pt' print("Restoring: ", chkPath) # Load state = torch.load(chkPath) # Initialize model and optimizer modelVars['model'].load_state_dict(state['state_dict']) modelVars['optimizer'].load_state_dict(state['optimizer']) start_epoch = state['epoch'] + 1 mdlParams['valBest'] = state.get('valBest', 1000) mdlParams['lastBestInd'] = state.get('lastBestInd', int(np.max(global_steps))) else: start_epoch = 1 mdlParams['lastBestInd'] = -1 # Track metrics for saving best model mdlParams['valBest'] = 1000 # Num batches numBatchesTrain = int( math.floor(len(mdlParams['trainInd']) / mdlParams['batchSize'])) print("Train batches", numBatchesTrain) # Run training start_time = time.time() print("Start training...") for step in range(start_epoch, mdlParams['training_steps'] + 1): # One Epoch of training print(step) if step >= mdlParams['lowerLRat'] - mdlParams['lowerLRAfter']: modelVars['scheduler'].step() modelVars['model'].train() for j, (inputs, labels, indices) in enumerate(modelVars['dataloader_trainInd']): # Run optimization if mdlParams.get('meta_features', None) is not None: inputs[0] = inputs[0].cuda() inputs[1] = inputs[1].cuda() else: inputs = inputs.cuda() labels = labels.cuda() # zero the parameter gradients modelVars['optimizer'].zero_grad() # forward # track history if only in train with torch.set_grad_enabled(True): if mdlParams.get('aux_classifier', False): outputs, outputs_aux = modelVars['model'](inputs) loss1 = modelVars['criterion'](outputs, labels) labels_aux = labels.repeat(mdlParams['multiCropTrain']) loss2 = modelVars['criterion'](outputs_aux, labels_aux) loss = loss1 + mdlParams[ 'aux_classifier_loss_fac'] * loss2 else: outputs = modelVars['model'](inputs) loss = modelVars['criterion'](outputs, labels) # backward + optimize only if in training phase loss.backward() modelVars['optimizer'].step() if step % mdlParams[ 'display_step'] == 0 or step == 1 or step == 101: # Calculate evaluation metrics if mdlParams['classification']: # Adjust model state modelVars['model'].eval() # Get metrics loss, accuracy, sensitivity, specificity, conf_matrix, f1, auc, waccuracy, predictions, targets, _ = utils.getErrClassification_mgpu( mdlParams, eval_set, modelVars) # Save in mat save_dict['loss'].append(loss) save_dict['acc'].append(accuracy) save_dict['wacc'].append(waccuracy) save_dict['auc'].append(auc) save_dict['sens'].append(sensitivity) save_dict['spec'].append(specificity) save_dict['f1'].append(f1) save_dict['step_num'].append(step) if os.path.isfile(mdlParams['saveDir'] + '/progression_' + eval_set + '.mat'): os.remove(mdlParams['saveDir'] + '/progression_' + eval_set + '.mat') io.savemat( mdlParams['saveDir'] + '/progression_' + eval_set + '.mat', save_dict) # if os.path.isfile(mdlParams['saveDir'] + '\progression_' + eval_set + '.mat'): # os.remove(mdlParams['saveDir'] + '\progression_' + eval_set + '.mat') # io.savemat(mdlParams['saveDir'] + '\progression_' + eval_set + '.mat', save_dict) eval_metric = -np.mean(waccuracy) # Check if we have a new best value if eval_metric < mdlParams['valBest']: mdlParams['valBest'] = eval_metric if mdlParams['classification']: allData['f1Best'][cv] = f1 allData['sensBest'][cv] = sensitivity allData['specBest'][cv] = specificity allData['accBest'][cv] = accuracy allData['waccBest'][cv] = waccuracy allData['aucBest'][cv] = auc oldBestInd = mdlParams['lastBestInd'] mdlParams['lastBestInd'] = step allData['convergeTime'][cv] = step # Save best predictions allData['bestPred'][cv] = predictions allData['targets'][cv] = targets # Write to File with open(mdlParams['saveDirBase'] + '/CV.pkl', 'wb') as f: # with open(mdlParams['saveDirBase'] + '\CV.pkl', 'wb') as f: pickle.dump(allData, f, pickle.HIGHEST_PROTOCOL) # Delte previously best model if os.path.isfile(mdlParams['saveDir'] + '/checkpoint_best-' + str(oldBestInd) + '.pt'): os.remove(mdlParams['saveDir'] + '/checkpoint_best-' + str(oldBestInd) + '.pt') # if os.path.isfile(mdlParams['saveDir'] + '\checkpoint_best-' + str(oldBestInd) + '.pt'): # os.remove(mdlParams['saveDir'] + '\checkpoint_best-' + str(oldBestInd) + '.pt') # Save currently best model state = { 'epoch': step, 'valBest': mdlParams['valBest'], 'lastBestInd': mdlParams['lastBestInd'], 'state_dict': modelVars['model'].state_dict(), 'optimizer': modelVars['optimizer'].state_dict() } torch.save( state, mdlParams['saveDir'] + '/checkpoint_best-' + str(step) + '.pt') # torch.save(state, mdlParams['saveDir'] + '\checkpoint_best-' + str(step) + '.pt') # If its not better, just save it delete the last checkpoint if it is not current best one # Save current model state = { 'epoch': step, 'valBest': mdlParams['valBest'], 'lastBestInd': mdlParams['lastBestInd'], 'state_dict': modelVars['model'].state_dict(), 'optimizer': modelVars['optimizer'].state_dict() } torch.save( state, mdlParams['saveDir'] + '/checkpoint-' + str(step) + '.pt') # torch.save(state, mdlParams['saveDir'] + '\checkpoint-' + str(step) + '.pt') # Delete last one if step == mdlParams['display_step']: lastInd = 1 else: lastInd = step - mdlParams['display_step'] if os.path.isfile(mdlParams['saveDir'] + '/checkpoint-' + str(lastInd) + '.pt'): os.remove(mdlParams['saveDir'] + '/checkpoint-' + str(lastInd) + '.pt') # if os.path.isfile(mdlParams['saveDir'] + '\checkpoint-' + str(lastInd) + '.pt'): # os.remove(mdlParams['saveDir'] + '\checkpoint-' + str(lastInd) + '.pt') # Duration so far duration = time.time() - start_time # Print if mdlParams['classification']: print("\n") print("Config:", sys.argv[2]) print('Fold: %d Epoch: %d/%d (%d h %d m %d s)' % (cv, step, mdlParams['training_steps'], int(duration / 3600), int(np.mod(duration, 3600) / 60), int(np.mod(np.mod(duration, 3600), 60))) + time.strftime("%d.%m.-%H:%M:%S", time.localtime())) print("Loss on ", eval_set, "set: ", loss, " Accuracy: ", accuracy, " F1: ", f1, " (best WACC: ", -mdlParams['valBest'], " at Epoch ", mdlParams['lastBestInd'], ")") print("Auc", auc, "Mean AUC", np.mean(auc)) print("Per Class Acc", waccuracy, "Weighted Accuracy", np.mean(waccuracy)) print("Sensitivity: ", sensitivity, "Specificity", specificity) print("Confusion Matrix") print(conf_matrix) # Potentially print train err if mdlParams['print_trainerr'] and 'train' not in eval_set: print("Evaluating training error") loss, accuracy, sensitivity, specificity, conf_matrix, f1, auc, waccuracy, predictions, targets, _ = utils.getErrClassification_mgpu( mdlParams, 'trainIndEval', modelVars) # Save in mat save_dict_train['loss'].append(loss) save_dict_train['acc'].append(accuracy) save_dict_train['wacc'].append(waccuracy) save_dict_train['auc'].append(auc) save_dict_train['sens'].append(sensitivity) save_dict_train['spec'].append(specificity) save_dict_train['f1'].append(f1) save_dict_train['step_num'].append(step) if os.path.isfile(mdlParams['saveDir'] + '/progression_trainInd.mat'): os.remove(mdlParams['saveDir'] + '/progression_trainInd.mat') # if os.path.isfile(mdlParams['saveDir'] + '\progression_trainInd.mat'): # os.remove(mdlParams['saveDir'] + '\progression_trainInd.mat') scipy.io.savemat( mdlParams['saveDir'] + '/progression_trainInd.mat', save_dict_train) # scipy.io.savemat(mdlParams['saveDir'] + '\progression_trainInd.mat', save_dict_train) print("Train loss: ", loss, "Accuracy:", accuracy, " F1: ", f1) print("Per Class Acc", waccuracy, "Weighted Accuracy", np.mean(waccuracy)) print("Sensitivity: ", sensitivity, "Specificity", specificity) print("Confusion Matrix") print(conf_matrix) # Free everything in modelvars modelVars.clear() # After CV Training: print CV results and save them print("Best F1:", allData['f1Best'][cv]) print("Best Sens:", allData['sensBest'][cv]) print("Best Spec:", allData['specBest'][cv]) print("Best Acc:", allData['accBest'][cv]) print("Best Per Class Accuracy:", allData['waccBest'][cv]) print("Best Weighted Acc:", np.mean(allData['waccBest'][cv])) print("Best AUC:", allData['aucBest'][cv]) print("Best Mean AUC:", np.mean(allData['aucBest'][cv])) print("Convergence Steps:", allData['convergeTime'][cv])
def __init__(self, config_path): self.image_config, self.model_config, self.run_config = LoadConfig( config_path=config_path).train_config() self.device = torch.device('cuda:%d' % self.run_config['device_ids'][0] if torch. cuda.is_available else 'cpu') self.model = getModel(self.model_config) os.makedirs(self.run_config['model_save_path'], exist_ok=True) self.run_config['num_workers'] = self.run_config['num_workers'] * len( self.run_config['device_ids']) self.train_set = Data(root=self.image_config['image_path'], phase='train', data_name=self.image_config['data_name'], img_mode=self.image_config['image_mode'], n_classes=self.model_config['num_classes'], size=self.image_config['image_size'], scale=self.image_config['image_scale']) self.valid_set = Data(root=self.image_config['image_path'], phase='valid', data_name=self.image_config['data_name'], img_mode=self.image_config['image_mode'], n_classes=self.model_config['num_classes'], size=self.image_config['image_size'], scale=self.image_config['image_scale']) self.className = self.valid_set.className self.train_loader = DataLoader( self.train_set, batch_size=self.run_config['batch_size'], shuffle=True, num_workers=self.run_config['num_workers'], pin_memory=True, drop_last=False) self.valid_loader = DataLoader( self.valid_set, batch_size=self.run_config['batch_size'], shuffle=True, num_workers=self.run_config['num_workers'], pin_memory=True, drop_last=False) train_params = self.model.parameters() self.optimizer = RAdam(train_params, lr=eval(self.run_config['lr']), weight_decay=eval( self.run_config['weight_decay'])) if self.run_config['swa']: self.optimizer = SWA(self.optimizer, swa_start=10, swa_freq=5, swa_lr=0.005) # 设置å¦ä¹ 率调节ç–ç•¥ self.lr_scheduler = utils.adjustLR.AdjustLr(self.optimizer) if self.run_config['use_weight_balance']: weight = utils.weight_balance.getWeight( self.run_config['weights_file']) else: weight = None self.Criterion = SegmentationLosses(weight=weight, cuda=True, device=self.device, batch_average=False) self.metric = utils.metrics.MetricMeter( self.model_config['num_classes'])
paramslstm = {'BATCH_SIZE': 512, 'GRAD_CLIP': 100, 'LEARNING_RATE': 0.01, 'NAME': 'LSTM', 'NUM_CLUST': 3, 'NUM_EPOCHS': 10, 'NUM_FEATURES': 8, 'N_HIDDEN': 64, 'SEQ_LENGTH': 4, 'TYPE': '2nd classifier'} # here in this example we are fetching a pre-trained model. Note # that all the saved models are in the ./models directory try: # print paramsold rnnModelold = getModel(paramsold, "rnnmodel-old") except: print "couldn't create the model... please correct the error" try: # print paramsold rnnModel = getModel(params, "newest") except: print "couldn't create the model... enter a valid filename" try: # print paramslstm lstmmodel = getModel(paramslstm, "lstmodel-old") except: print "couldn't create the model... please correct the error"
parser.add_argument('--numexs', help='Number of examples', required=True, type=int) parser.add_argument('--dataname', help='Data file name prefix', required=True) parser.add_argument('--modelname', help='modelname used in output', required=True) parser.add_argument('--bkgrd', help='sequence background global/feature', required=False, default='global') parser.add_argument('--task', help='task name', required=False, default='transmembrane-region') args = parser.parse_args() input_dim = len(dataset.AAs) + 1 # # Model Specific Settings - need to change these based on the model # NUMEXS = args.numexs DATA_NAME = args.dataname MODEL_NAME = args.modelname model = models.getModel(args.model, input_dim) background = args.bkgrd # path to save figures fig_base_path = '/home/gene245/cprobert/deep-psp/keras/figures/' + DATA_NAME # load training/test data X_train, X_test, y_train, y_test = dataset.getSplitDataset( args.task, num_exs=NUMEXS, bkgrd=background, max_len=100, min_len=10, test_size=0.1) model.compile(loss='categorical_crossentropy', optimizer='adagrad') weights_path = '/home/gene245/cprobert/deep-psp/keras/output/' + DATA_NAME + '_model_weights.hdf5'
assert (args.bkgrd in ['global', 'feature']) print 'using parameters:\n', vars(args) print '--loading dataset--' # load the input data X_train, X_test, y_train, y_test = dataset.getSplitDataset(task_name=args.task, num_exs=args.numexs, bkgrd=args.bkgrd, max_len=100, min_len=10, test_size=0.1) feature_dim = len(dataset.AAs) + 1 model = models.getModel(args.model, feature_dim) print '--compiling model--' model.compile(loss='categorical_crossentropy', optimizer='adagrad') # fit the model print '--fitting model--' fit_results = model.fit(X_train, y_train, nb_epoch=args.numepochs, batch_size=32, validation_split=0.1, shuffle=True, show_accuracy=True, verbose=2)
def initialize_model(cv): # Define model modelVars['model'] = models.getModel(params)() # Load trained model if params.get('meta_features', None) is not None: # Find best checkpoint files = (params['model_load_path'] / f'CVSet{cv}').glob('/*') global_steps = np.zeros([len(files)]) for i in range(len(files)): # Use meta files to find the highest index if 'best' not in files[i]: continue if 'checkpoint' not in files[i]: continue # Extract global step nums = [int(s) for s in re.findall(r'\d+', str(files[i]))] global_steps[i] = nums[-1] # Create path with maximum global step found chkPath = params[ 'model_load_path'] / f'CVSet{cv}' / 'checkpoint_best-{int(np.max(global_steps))}.pt' print("Restoring lesion-trained CNN for meta data training: ", chkPath) # Load state = torch.load(chkPath) # Initialize model curr_model_dict = modelVars['model'].state_dict() for name, param in state['state_dict'].items(): if isinstance(param, nn.Parameter): # backwards compatibility for serialized parameters param = param.data if curr_model_dict[name].shape == param.shape: curr_model_dict[name].copy_(param) else: print("not restored", name, param.shape) if 'Dense' in params['model_type']: if params['input_size'][0] != 224: modelVars['model'] = utils.modify_densenet_avg_pool( modelVars['model']) num_ftrs = modelVars['model'].classifier.in_features modelVars['model'].classifier = nn.Linear(num_ftrs, params['numClasses']) #print(modelVars['model']) elif 'dpn' in params['model_type']: num_ftrs = modelVars['model'].classifier.in_channels modelVars['model'].classifier = nn.Conv2d(num_ftrs, params['numClasses'], [1, 1]) elif 'efficient' in params['model_type']: # Do nothing, output is prepared num_ftrs = modelVars['model']._fc.in_features modelVars['model']._fc = nn.Linear(num_ftrs, params['numClasses']) elif 'wsl' in params['model_type']: num_ftrs = modelVars['model'].fc.in_features modelVars['model'].fc = nn.Linear(num_ftrs, params['numClasses']) else: num_ftrs = modelVars['model'].last_linear.in_features modelVars['model'].last_linear = nn.Linear(num_ftrs, params['numClasses']) # Take care of meta case if params.get('meta_features', None) is not None: # freeze cnn first if params['freeze_cnn']: # deactivate all for param in modelVars['model'].parameters(): param.requires_grad = False if 'efficient' in params['model_type']: # Activate fc for param in modelVars['model']._fc.parameters(): param.requires_grad = True elif 'wsl' in params['model_type']: # Activate fc for param in modelVars['model'].fc.parameters(): param.requires_grad = True else: # Activate fc for param in modelVars['model'].last_linear.parameters(): param.requires_grad = True else: # mark cnn parameters for param in modelVars['model'].parameters(): param.is_cnn_param = True # unmark fc for param in modelVars['model']._fc.parameters(): param.is_cnn_param = False # modify model modelVars['model'] = models.modify_meta(params, modelVars['model']) # Mark new parameters for param in modelVars['model'].parameters(): if not hasattr(param, 'is_cnn_param'): param.is_cnn_param = False # multi gpu support if params['numGPUs'] > 1: modelVars['model'] = nn.DataParallel(modelVars['model']) modelVars['model'] = modelVars['model'].cuda()
g = tf.Graph() content_image = ut.process_image(img) wanted_style = np.array([[0,1.0]]) with g.device(device), g.as_default(), tf.Session(graph=g, config=tf.ConfigProto(allow_soft_placement=True)) as sess: print "Load content values and calculate style and content softmaxes" #content cW = tf.constant(content_weights) cB = tf.constant(content_bias) #style sW = tf.constant(style_weights) sB = tf.constant(style_bias) wanted_style = tf.constant(wanted_style, tf.float32) image = tf.constant(content_image) model = models.getModel(image, params) pool2_image_val = sess.run(model.y()) #fc7_image_val = sess.run(model.y()) reshaped_pool2_image_val = tf.reshape(pool2_image_val, [-1, 401408]) #ReshapeToVector(pool2_image_val) content_values= tf.nn.softmax(tf.matmul(reshaped_pool2_image_val,cW) + cB) ##style_values= tf.nn.softmax(tf.matmul(pool2_image_val,sW) + sB) print "Generate noise" ##gen_image = tf.Variable(tf.truncated_normal(content_image.shape, stddev=20), trainable=True, name='gen_image') gen_image = tf.Variable(tf.constant(np.array(content_image, dtype=np.float32)), trainable=True, name='gen_image') ##gen_image = tf.Variable(tf.constant(np.array(style_image, dtype=np.float32)), trainable=True, name='gen_image') model_gen = models.getModel(gen_image, params) pool2_gen_image_val = model_gen.y()
def get_activates(args): if args.gpu >= 0: cuda.check_cuda_available() print('use GPU') else: print('use CPU only') xp = cuda.cupy if args.gpu >= 0 else np # cuda.cudnn_enabled = False # Prepare dataset val_list = dataio.load_image_list(args.val, args.root) val_size = len(val_list) assert val_size % args.val_batchsize == 0 # Prepare model model = models.getModel(args.arch) #from chainer.functions import caffe #model = caffe.CaffeFunction('./models/bvlc_alexnet.caffemodel') #model.insize=227 #model = caffe.CaffeFunction('./models/bvlc_googlenet.caffemodel') #model.insize=224 if model is None: raise ValueError('Invalid architecture name') if args.finetune and hasattr(model, 'load_param'): print('finetune') model.load_param() mean_image = None if hasattr(model, 'getMean'): mean_image = model.getMean() else: #mean_image = pickle.load(open(args.mean, 'rb')) mean_image = np.load(args.mean) #mean_image = np.ndarray((3, 256, 256), dtype=np.float32) #mean_image[0] = 104 #mean_image[1] = 117 #mean_image[2] = 123 assert mean_image is not None print(model.__class__.__name__) print('batchsize (validation) : ' + str(args.val_batchsize)) nowt = datetime.datetime.today() outdir = './' + args.arch if not os.path.exists(outdir): os.makedirs(outdir) #outdir = './results/' + args.arch + '_bs' + str(args.batchsize) + '_' + nowt.strftime("%Y%m%d-%H%M") #os.makedirs(outdir) #args.out = outdir + '/' + args.out + '_' + args.arch #args.outstate = outdir + '/' + args.outstate if args.gpu >= 0: cuda.get_device(args.gpu).use() model.to_gpu() # Init/Resume if args.initmodel: print('Load model from', args.initmodel) serializers.load_hdf5(args.initmodel, model) if args.resume: print('Load optimizer state from', args.resume) serializers.load_hdf5(args.resume, optimizer) # ------------------------------------------------------------------------------ # This example consists of three threads: data feeder, logger and trainer. # These communicate with each other via Queue. data_q = queue.Queue(maxsize=1) res_q = queue.Queue() ret = TrainResult() def feed_data(): # Data feeder i = 0 count = 0 val_x_batch = np.ndarray( (args.val_batchsize, 3, model.insize, model.insize), dtype=np.float32) val_y_batch = np.ndarray((args.val_batchsize, ), dtype=np.int32) val_batch_pool = [None] * args.val_batchsize pool = multiprocessing.Pool(args.loaderjob) data_q.put('val') j = 0 for path, label in val_list: val_batch_pool[j] = pool.apply_async( dataio.read_image, (path, model.insize, mean_image, True, False)) val_y_batch[j] = label j += 1 if j == args.val_batchsize: for k, x in enumerate(val_batch_pool): val_x_batch[k] = x.get() data_q.put((val_x_batch.copy(), val_y_batch.copy())) j = 0 pool.close() pool.join() data_q.put('end') def log_result(): # Logger testlogfilename = outdir + '/val.log' activatescsvfilename = outdir + '/acts_' + args.layer + '.csv' activatesnpyfilename = outdir + '/acts_' + args.layer + '.npy' train_count = 0 train_cur_loss = 0 train_cur_accuracy = 0 begin_at = time.time() val_begin_at = None result = None Ret = [ret] activates = None while True: result = res_q.get() if result == 'end': print(file=sys.stderr) break elif result == 'val': print(file=sys.stderr) train = False val_count = val_loss = val_accuracy = 0 val_begin_at = time.time() continue loss, accuracy, layer_activates = result if activates is None: activates = cuda.to_cpu(layer_activates) else: activates = np.r_[activates, cuda.to_cpu(layer_activates)] val_count += args.val_batchsize duration = time.time() - val_begin_at throughput = val_count / duration sys.stderr.write( '\rval {} batches ({} samples) time: {} ({} images/sec)'. format(val_count / args.val_batchsize, val_count, datetime.timedelta(seconds=duration), throughput)) val_loss += loss val_accuracy += accuracy #print('accuacy', accuracy) if val_count == val_size: mean_loss = val_loss * args.val_batchsize / val_size mean_error = 1 - val_accuracy * args.val_batchsize / val_size print(file=sys.stderr) print( json.dumps({ 'type': 'val', 'iteration': train_count, 'error': mean_error, 'loss': mean_loss })) with open(testlogfilename, 'a') as f: f.write( json.dumps({ 'type': 'val', 'iteration': train_count, 'error': mean_error, 'loss': mean_loss }) + '\n') Ret[0].val_loss = mean_loss Ret[0].val_error = mean_error sys.stdout.flush() print(activates.shape) #np.savetxt(activatescsvfilename, activates, delimiter=",") np.save(activatesnpyfilename, activates) Ret[0].activates = activates def train_loop(): # Trainer while True: while data_q.empty(): time.sleep(0.1) inp = data_q.get() if inp == 'end': # quit res_q.put('end') break elif inp == 'val': # start validation res_q.put('val') model.train = False continue model.train = False volatile = 'off' #if model.train else 'on' x = chainer.Variable(xp.asarray(inp[0]), volatile=volatile) t = chainer.Variable(xp.asarray(inp[1]), volatile=volatile) model(x, t) #fc8, = model(inputs={'data': x}, outputs=['fc8'], train=False) #model.loss = F.softmax_cross_entropy(fc8, t) #model.accuracy = F.accuracy(fc8, t) #y, = model(inputs={'data': x}, outputs=['loss3/classifier'], disable=['loss1/ave_pool', 'loss2/ave_pool'], train=False) #model.loss = F.softmax_cross_entropy(y, t) #model.accuracy = F.accuracy(y, t) variable = model.getLayerVariableFromLoss(args.layer) #print(model.layer2rank(args.layer)) #print(variable) ax = (2, 3) if len(variable.data.shape) == 4 else 1 layer_activates = variable.data.max(axis=ax) #layer_activates = np.arange(args.val_batchsize * 10).reshape((args.val_batchsize, 10)) #data = cuda.to_cpu(variable.data) #argmax = data.argmax(axis=(1)) #print(data.shape) #print(argmax) res_q.put((float(model.loss.data), float(model.accuracy.data), layer_activates)) del x, t, # Invoke threads feeder = threading.Thread(target=feed_data) feeder.daemon = True feeder.start() logger = threading.Thread(target=log_result) logger.daemon = True logger.start() train_loop() feeder.join() logger.join() # Save final model ret.outdir = outdir ret.valid = True return ret
kernel = np.ones((2, 2), np.uint8) kernel_1 = np.ones((5, 5), np.uint8) opening = cv2.morphologyEx(im_gray_ostu, cv2.MORPH_OPEN, kernel) opening_1 = cv2.morphologyEx(im_gray_ostu, cv2.MORPH_OPEN, kernel_1) closing = cv2.morphologyEx(im_gray_ostu, cv2.MORPH_CLOSE, kernel) opening_1 = np.abs(255 - opening_1) plt.imshow(opening_1) y_l, x = np.unique(np.where(opening_1 > 0)[0]), np.unique( np.where(opening_1 > 0)[1]) CONFIG_FILE = './config.cfg' settings = parseTrainingOptions(CONFIG_FILE) print settings model = getModel(settings) model.load_weights('./model.h5') print 'Loading model: ./model.h5' patch_size = 224 patches = [] flag = False heatmap = np.zeros((slide.level_dimensions[slide_level][1], slide.level_dimensions[slide_level][0])) hfp = open('heatmapFile.csv', 'w') batch_size = settings['batch_size'] #should be 32 counter = 0 batch = []
parser = argparse.ArgumentParser() parser.add_argument('-d', '--directory', type=str, required=True, help="base directory") args = parser.parse_args() nns = [] for root, dirs, files in os.walk(args.directory): for f in files: if f == "configs.txt": data = yaml.load(open(join(root, f))) data["version"] = root.split("/")[-1] #analyse model m = models.getModel(data["model_schema"], 34) nodes = [] for l in [i for i in m.layers if not "dropout" in i.name]: nodes.append(int(l.output.shape[1])) data["nodes"] = "-".join([str(i) for i in nodes]) data["n_weights"] = m.count_params() nns.append(data) nns = sorted(nns, key=lambda d: d["AUC"], reverse=True) for d in nns: print("-----------") print(d)
def __init__(self, mdlParams): super(CNN_FC, self).__init__() # Some necessary vars self.crop_number = mdlParams['multiCropTrain'] self.combine_features = mdlParams['combine_features'] self.cnn_output_point = mdlParams['CNN_Output_Point'] self.initial_attention = mdlParams['initial_attention'] if mdlParams.get('end_pool') is not None: self.end_pool = mdlParams['end_pool'] else: self.end_pool = False self.end_attention = mdlParams['end_attention'] # CNN first,up to feature vector self.cnn = models.getModel(mdlParams['model_type_cnn'])() #print(self.cnn) if 'Dense' in mdlParams['model_type_cnn']: if self.cnn_output_point == 'end': self.cnn_features = self.cnn.features elif self.cnn_output_point == 'transition3': self.cnn_features = nn.Sequential( *list(self.cnn.features.children())[:10]) elif 'InceptionV3' in mdlParams['model_type_cnn']: if self.cnn_output_point == 'end': self.cnn_features = nn.Sequential( self.cnn.Conv2d_1a_3x3, self.cnn.Conv2d_2a_3x3, self.cnn.Conv2d_2b_3x3, nn.MaxPool2d(kernel_size=3, stride=2), self.cnn.Conv2d_3b_1x1, self.cnn.Conv2d_4a_3x3, nn.MaxPool2d(kernel_size=3, stride=2), self.cnn.Mixed_5b, self.cnn.Mixed_5c, self.cnn.Mixed_5d, self.cnn.Mixed_6a, self.cnn.Mixed_6b, self.cnn.Mixed_6c, self.cnn.Mixed_6d, self.cnn.Mixed_6e, self.cnn.Mixed_7a, self.cnn.Mixed_7b, self.cnn.Mixed_7c) else: if self.cnn_output_point == 'end': self.cnn_features = nn.Sequential(self.cnn.layer0, self.cnn.layer1, self.cnn.layer2, self.cnn.layer3, self.cnn.layer4) #print(self.cnn_features) # The classifier if self.combine_features == 'add' or self.combine_features == 'conv1': self.classifier = nn.Linear(mdlParams['CNN_Features'], mdlParams['numClasses']) else: self.classifier = nn.Linear( self.crop_number * mdlParams['CNN_Features'], mdlParams['numClasses']) if self.combine_features == 'conv1': self.conv1x1 = nn.Conv1d(self.crop_number, 1, 1, 1) if self.initial_attention: if 'attention_size' in mdlParams: self.initial_attention_layer = SEPatchLayer_Pool( mdlParams['multiCropTrain'], mdlParams['attention_size'][0]) else: self.initial_attention_layer = SEPatchLayer( mdlParams['multiCropTrain']) if self.end_attention: if 'attention_size' in mdlParams: self.end_attention_layer = SEPatchLayer_Pool( mdlParams['multiCropTrain'], mdlParams['attention_size'][1]) else: self.end_attention_layer = SEPatchLayer( mdlParams['multiCropTrain'])
elif mdlParams['balance_classes'] == 9: # Only use HAM indicies for calculation indices_ham = mdlParams['trainInd'][mdlParams['trainInd'] < 10015] class_weights = 1.0/np.mean(mdlParams['labels_array'][indices_ham,:],axis=0) print("Current class weights",class_weights) # Set up dataloaders # For train dataset_train = utils.ISICDataset(mdlParams, 'trainInd') modelVars['dataloader_train'] = DataLoader(dataset_train, batch_size=mdlParams['batchSize'], shuffle=True, num_workers=8, pin_memory=True) # For val dataset_val = utils.ISICDataset(mdlParams, 'valInd') modelVars['dataloader_val'] = DataLoader(dataset_val, batch_size=mdlParams['multiCropEval'], shuffle=False, num_workers=8, pin_memory=True) #print("Setdiff",np.setdiff1d(mdlParams['trainInd'],mdlParams['trainInd'])) # Define model modelVars['model'] = models.getModel(mdlParams['model_type'])() # Original input size if 'Dense' not in mdlParams['model_type']: print("Original input size",modelVars['model'].input_size) #print(modelVars['model']) if 'Dense' in mdlParams['model_type']: num_ftrs = modelVars['model'].classifier.in_features modelVars['model'].classifier = nn.Linear(num_ftrs, mdlParams['numClasses']) print(modelVars['model']) elif 'dpn' in mdlParams['model_type']: num_ftrs = modelVars['model'].classifier.in_channels modelVars['model'].classifier = nn.Conv2d(num_ftrs,mdlParams['numClasses'],[1,1]) print(modelVars['model']) else: num_ftrs = modelVars['model'].last_linear.in_features modelVars['model'].last_linear = nn.Linear(num_ftrs, mdlParams['numClasses'])
'GRAD_CLIP': 100, 'LEARNING_RATE': 0.01, 'NAME': 'LSTM', 'NUM_CLUST': 3, 'NUM_EPOCHS': 10, 'NUM_FEATURES': 8, 'N_HIDDEN': 64, 'SEQ_LENGTH': 4, 'TYPE': '2nd classifier' } # here in this example we are fetching a pre-trained model. Note # that all the saved models are in the ./models directory try: # print paramsold rnnModelold = getModel(paramsold, "rnnmodel-old") except: print("couldn't create the model... please correct the error") try: # print paramsold rnnModel = getModel(params, "newest") except: print("couldn't create the model... enter a valid filename") try: # print paramslstm lstmmodel = getModel(paramslstm, "lstmodel-old") except: print("couldn't create the model... please correct the error")
host) args = vars(opt) print('------------ Options -------------') for k, v in sorted(args.items()): print('%s: %s' % (str(k), str(v))) print('-------------- End ----------------') expr_dir = os.path.join(opt.checkpoints_dir, opt.name) util.mkdirs(expr_dir) opt.file_name = os.path.join(expr_dir, 'log.txt') with open(opt.file_name, 'wt') as log_file: log_file.write('------------ Options -------------\n') for k, v in sorted(args.items()): log_file.write('%s: %s\n' % (str(k), str(v))) log_file.write('-------------- End ----------------\n') Model = getModel(opt) model = Model() ''' model.initialize(opt) print("model [%s] was created" % (model.name())) ''' MRFDataset = getDataset(opt) opt.set_type = 'train' dataset_train = MRFDataset() dataset_train.initialize(opt) dataloader_train = torch.utils.data.DataLoader(dataset_train, batch_size=opt.batchSize, shuffle=True, num_workers=int(opt.nThreads)) dataloader_train.dataset.patchSize = opt.patchSize
parser = argparse.ArgumentParser(description='Train a model') parser.add_argument('--model', help='The model to use. Either RNN or LSTM', required=True) parser.add_argument('--numexs', help='Number of examples', required=True, type=int) parser.add_argument('--dataname', help='Data file name prefix', required=True) parser.add_argument('--bkgrd', help='sequence background global/feature', required=False, default='global') parser.add_argument('--task', help='task name', required=False, default='transmembrane-region') args = parser.parse_args() input_dim = len(dataset.AAs) + 1 # # Model Specific Settings - need to change these based on the model # NUMEXS = args.numexs DATA_NAME = args.dataname model = models.getModel(args.model, input_dim) background = args.bkgrd # load training/test data X_train, X_test, y_train, y_test = dataset.getSplitDataset( args.task, num_exs=NUMEXS, bkgrd=background, max_len=100, min_len=10, test_size=0.1) model.compile(loss='categorical_crossentropy', optimizer='adagrad') weights_path = '/home/gene245/cprobert/deep-psp/keras/output/' + DATA_NAME + '_model_weights.hdf5' model.load_weights(weights_path) test_preds = model.predict(X_test) train_preds = model.predict(X_train)
def get_activated_patch(args): if args.gpu >= 0: cuda.check_cuda_available() print('use GPU') else: print('use CPU only') xp = cuda.cupy if args.gpu >= 0 else np # cuda.cudnn_enabled = False # Prepare dataset val_list = dataio.load_image_list(args.val, args.root) val_size = len(val_list) assert val_size % args.val_batchsize == 0 categories = pd.read_csv(args.categories, header=None) assert args.layeractivates is not None layeractivates = np.load(args.layeractivates) assert layeractivates.shape[0] == val_size map_size = layeractivates.shape[1] #indexes = np.argsort(a) df = pd.DataFrame(layeractivates) val_path_list = [v[0] for v in val_list] val_labelnum_list = [v[1] for v in val_list] val_label_list = [categories.ix[i[1],0] for i in val_list] df['path'] = val_path_list df['label'] = val_label_list df['labelnum'] = val_labelnum_list outdir = './' + args.arch + '/' + args.layer if not os.path.isdir(outdir): os.makedirs(outdir) # Prepare model model = models.getModel(args.arch) if model is None: raise ValueError('Invalid architecture name') if args.finetune and hasattr(model, 'load_param'): print ('finetune') model.load_param() mean_image = None if hasattr(model, 'getMean'): mean_image = model.getMean() else: #mean_image = pickle.load(open(args.mean, 'rb')) mean_image = np.load(args.mean) assert mean_image is not None print(model.__class__.__name__) print('batchsize (validation) : ' + str(args.val_batchsize)) val_image_pool = [None] * args.top impool = multiprocessing.Pool(args.loaderjob) nowt = datetime.datetime.today() #outdir = './results/' + args.arch + '_bs' + str(args.batchsize) + '_' + nowt.strftime("%Y%m%d-%H%M") #os.makedirs(outdir) #args.out = outdir + '/' + args.out + '_' + args.arch #args.outstate = outdir + '/' + args.outstate if args.gpu >= 0: cuda.get_device(args.gpu).use() model.to_gpu() # Init/Resume if args.initmodel: print('Load model from', args.initmodel) serializers.load_hdf5(args.initmodel, model) if args.resume: print('Load optimizer state from', args.resume) serializers.load_hdf5(args.resume, optimizer) for c in six.moves.range(map_size): #topacts = df.sort(i, ascending=False)[['path', 'label', i]].iloc[:args.top] topacts = df.sort_values(by=[c], ascending=False)[['path', 'label', 'labelnum', c]].iloc[:args.top] topacts.to_csv(outdir + '/' + "{0:0>4}".format(c) + '.csv', header=None, index=None) images = np.zeros((3, model.insize, model.insize * args.top)) #for n, path in enumerate(topacts['path']): # images[:,:,n*model.insize:(n+1)*model.insize] = \ # dataio.read_image(path, model.insize, None, True, False) val_x_batch = np.ndarray( (args.top, 3, model.insize, model.insize), dtype=np.float32) val_y_batch = np.ndarray((args.top,), dtype=np.int32) for n, path in enumerate(topacts['path']): val_image_pool[n] = impool.apply_async( dataio.read_image, (path, model.insize, mean_image, True, False)) for n, labelnum in enumerate(topacts['labelnum']): val_y_batch[n] = labelnum for k, im in enumerate(val_image_pool): val_x_batch[k] = im.get() # train model.train = False volatile = 'off' #if model.train else 'on' x = chainer.Variable(xp.asarray(val_x_batch), volatile=volatile) t = chainer.Variable(xp.asarray(val_y_batch), volatile=volatile) model(x, t) variable = model.getLayerVariableFromLoss(args.layer) ax = (2,3) if len(variable.data.shape) == 4 else 1 maxloc= variable.data.argmax(axis=ax) if c == 0: print(variable.data[0]) print(maxloc.shape) print(maxloc) for n, path in enumerate(topacts['path']): xmax, ymax = idx2loc(variable, maxloc[n, c]) pl, pr, pt, pb = get_patch_bounds(variable, xmax, ymax) if c == 0 and n == 0: print(xmax, ymax, ':', pl, pr, pt, pb) patch = dataio.read_image_patch(path, model.insize, pl, pr, pt, pb) if c == 0 and n == 0: print(patch, patch.shape) patchimg = Image.fromarray(np.uint8(patch[::-1].transpose(1, 2, 0))) patchimg.save(outdir + '/' + "{0:0>4}".format(c) + '_' + "{0:0>2}".format(n) + args.ext) #pilImg = Image.fromarray(np.uint8(images[::-1].transpose(1, 2, 0))) #pilImg.save(outdir + '/' + "{0:0>4}".format(i) + '.jpg', 'JPEG', quality=100, optimize=True) del x, t impool.close() impool.join() return # ------------------------------------------------------------------------------ # This example consists of three threads: data feeder, logger and trainer. # These communicate with each other via Queue. data_q = queue.Queue(maxsize=1) res_q = queue.Queue() ret = TrainResult() def feed_data(): # Data feeder i = 0 count = 0 val_x_batch = np.ndarray( (args.val_batchsize, 3, model.insize, model.insize), dtype=np.float32) val_y_batch = np.ndarray((args.val_batchsize,), dtype=np.int32) val_batch_pool = [None] * args.val_batchsize pool = multiprocessing.Pool(args.loaderjob) data_q.put('val') j = 0 for path, label in val_list: val_batch_pool[j] = pool.apply_async( dataio.read_image, (path, model.insize, mean_image, True, False)) val_y_batch[j] = label j += 1 if j == args.val_batchsize: for k, x in enumerate(val_batch_pool): val_x_batch[k] = x.get() data_q.put((val_x_batch.copy(), val_y_batch.copy())) j = 0 pool.close() pool.join() data_q.put('end') def log_result(): # Logger testlogfilename=args.out+'/val.log' activatescsvfilename=args.out+'/acts_'+args.layer+'.csv' activatesnpyfilename=args.out+'/acts_'+args.layer+'.npy' train_count = 0 train_cur_loss = 0 train_cur_accuracy = 0 begin_at = time.time() val_begin_at = None result = None Ret = [ret] activates=None while True: result = res_q.get() if result == 'end': print(file=sys.stderr) break elif result == 'val': print(file=sys.stderr) train = False val_count = val_loss = val_accuracy = 0 val_begin_at = time.time() continue loss, accuracy, max_activates = result if activates is None: activates = cuda.to_cpu(max_activates) else: activates = np.r_[activates, cuda.to_cpu(max_activates)] val_count += args.val_batchsize duration = time.time() - val_begin_at throughput = val_count / duration sys.stderr.write( '\rval {} batches ({} samples) time: {} ({} images/sec)' .format(val_count / args.val_batchsize, val_count, datetime.timedelta(seconds=duration), throughput)) val_loss += loss val_accuracy += accuracy #print('accuacy', accuracy) if val_count == val_size: mean_loss = val_loss * args.val_batchsize / val_size mean_error = 1 - val_accuracy * args.val_batchsize / val_size print(file=sys.stderr) print(json.dumps({'type': 'val', 'iteration': train_count, 'error': mean_error, 'loss': mean_loss})) with open(testlogfilename, 'a') as f: f.write(json.dumps({'type': 'val', 'iteration': train_count, 'error': mean_error, 'loss': mean_loss})+'\n') Ret[0].val_loss = mean_loss Ret[0].val_error = mean_error sys.stdout.flush() print(activates.shape) np.savetxt(activatescsvfilename, activates, delimiter=",") np.save(activatesnpyfilename, activates) Ret[0].activates = activates def train_loop(): # Trainer while True: while data_q.empty(): time.sleep(0.1) inp = data_q.get() if inp == 'end': # quit res_q.put('end') break elif inp == 'val': # start validation res_q.put('val') model.train = False continue model.train = False volatile = 'off' #if model.train else 'on' x = chainer.Variable(xp.asarray(inp[0]), volatile=volatile) t = chainer.Variable(xp.asarray(inp[1]), volatile=volatile) model(x, t) #fc8, = model(inputs={'data': x}, outputs=['fc8'], train=False) #model.loss = F.softmax_cross_entropy(fc8, t) #model.accuracy = F.accuracy(fc8, t) #y, = model(inputs={'data': x}, outputs=['loss3/classifier'], disable=['loss1/ave_pool', 'loss2/ave_pool'], train=False) #model.loss = F.softmax_cross_entropy(y, t) #model.accuracy = F.accuracy(y, t) variable = model.getLayerVariableFromLoss(args.layer) #print(model.layer2rank(args.layer)) #print(variable) ax = (2,3) if len(variable.data.shape) == 4 else 1 maxloc= variable.data.argmax(axis=ax) print(maxloc) #max_activates = np.arange(args.val_batchsize * 10).reshape((args.val_batchsize, 10)) #data = cuda.to_cpu(variable.data) #argmax = data.argmax(axis=(1)) #print(data.shape) #print(argmax) res_q.put((float(model.loss.data), float(model.accuracy.data), maxloc)) del x, t, # Invoke threads feeder = threading.Thread(target=feed_data) feeder.daemon = True feeder.start() logger = threading.Thread(target=log_result) logger.daemon = True logger.start() train_loop() feeder.join() logger.join() # Save final model ret.outdir = args.out ret.valid = True return ret
# For val dataset_val = utils.ISICDataset(params, 'valInd') if params['multiCropEval'] > 0: modelVars['dataloader_valInd'] = DataLoader(dataset_val, batch_size=params['multiCropEval'], shuffle=False, num_workers=num_workers, pin_memory=True) else: modelVars['dataloader_valInd'] = DataLoader(dataset_val, batch_size=params['batchSize'], shuffle=False, num_workers=num_workers, pin_memory=True) if params['balance_classes'] == 12 or params['balance_classes'] == 13: #print(np.argmax(params['labels_array'][params['trainInd'],:],1).size(0)) strat_sampler = utils.StratifiedSampler(params) modelVars['dataloader_trainInd'] = DataLoader(dataset_train, batch_size=params['batchSize'], sampler=strat_sampler, num_workers=num_workers, pin_memory=True) else: modelVars['dataloader_trainInd'] = DataLoader(dataset_train, batch_size=params['batchSize'], shuffle=True, num_workers=num_workers, pin_memory=True, drop_last=True) #print("Setdiff",np.setdiff1d(params['trainInd'],params['trainInd'])) # Define model modelVars['model'] = models.getModel(params)() # Load trained model if params.get('meta_features',None) is not None: # Find best checkpoint files = glob(params['model_load_path'] + '/CVSet' + str(cv) + '/*') global_steps = np.zeros([len(files)]) #print("files",files) for i in range(len(files)): # Use meta files to find the highest index if 'best' not in files[i]: continue if 'checkpoint' not in files[i]: continue # Extract global step nums = [int(s) for s in re.findall(r'\d+',files[i])] global_steps[i] = nums[-1]
modelVars = {} modelVars['device'] = torch.device("cuda:" + cuda_str.strip()) # For train dataset_train = utils.MetaAugDataset(mdlParams, 'trainInd', index=train_index) modelVars['dataloader_trainInd'] = DataLoader( dataset_train, batch_size=mdlParams['batchSize'], shuffle=True) # For val dataset_val = utils.MetaAugDataset(mdlParams, 'valInd', index=valid_index) modelVars['dataloader_valInd'] = DataLoader( dataset_val, batch_size=mdlParams['batchSize'], shuffle=False) # Define model modelVars['model'] = models.getModel(mdlParams)() # Load trained model if mdlParams.get('load_previous', False): # Find best checkpoint files = glob(mdlParams['model_load_path'] + '/*') global_steps = np.zeros([len(files)]) print("files", files) for i in range(len(files)): # Use meta files to find the highest index if 'best' not in files[i]: continue if 'checkpoint' not in files[i]: continue # Extract global step nums = [int(s) for s in re.findall(r'\d+', files[i])] global_steps[i] = nums[-1]
#!/usr/bin/env python import keras, models, losses m = models.getModel("CascadeNet-5", (8, 9, 1)) m.summary()
def train(model_name, optimizer_name, scheduler_name, lr, img_path, mask_path, names_path, epochs=10): model = models.getModel(model_name) model.build((None, None, None, 3)) print(model.summary()) scheduler = schedulers.getScheduler(scheduler_name, lr) optimizer = optimizers.getOptimizer(optimizer_name, scheduler) cce = tf.keras.losses.CategoricalCrossentropy() train_loss_metric = tf.keras.metrics.Mean() train_accuracy_metric = tf.keras.metrics.CategoricalAccuracy() test_loss_metric = tf.keras.metrics.Mean() test_accuracy_metric = tf.keras.metrics.CategoricalAccuracy() file_list = open(names_path, 'r') names = file_list.read().splitlines() file_list.close() current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") train_log_dir = 'logs/gradient_tape/' + current_time + '/train' test_log_dir = 'logs/gradient_tape/' + current_time + '/test' train_summary_writer = tf.summary.create_file_writer(train_log_dir) test_summary_writer = tf.summary.create_file_writer(test_log_dir) trainset, testval = ttsplit(names, train_size=0.9) test, val = ttsplit(testval, train_size=0.5) trainset = names print(names) total_step = 0 with tf.device('/device:GPU:0'): for epoch in range(epochs): for step_, batch in enumerate(trainset): total_step += 1 print(total_step) img, mask = utils.genData(batch, mask_path, img_path) with tf.GradientTape() as tape: mask_pred = model(img) loss = cce(mask, mask_pred) train_loss_metric.update_state(loss) train_accuracy_metric.update_state(mask, mask_pred) grads = tape.gradient(loss, model.trainable_weights) optimizer.apply_gradients(zip(grads, model.trainable_weights)) print(total_step) if step_ % 150 == 0: with train_summary_writer.as_default(): tf.summary.scalar('Training Loss', train_loss_metric.result(), step=total_step) tf.summary.scalar('Training Accuracy', train_accuracy_metric.result(), step=total_step) for step, batch in enumerate(val): img_val, mask_val = utils.genData( batch, mask_path, img_path) mask_pred_val = model(img_val) loss_val = cce(mask_val, mask_pred_val) print(loss_val) test_loss_metric.update_state(loss_val) test_accuracy_metric.update_state( mask_val, mask_pred_val) with test_summary_writer.as_default(): tf.summary.scalar('Validation Loss', test_loss_metric.result(), step=total_step) tf.summary.scalar('Validation Accuracy', test_accuracy_metric.result(), step=total_step) print('Epoch: ' + str(epoch) + ' | Batch: ' + str(step) + ' | Training Loss: ' + str(train_loss_metric.result().numpy()) + ' | Training Accuracy: ' + str(train_accuracy_metric.result().numpy())) print('Epoch: ' + str(epoch) + ' | Batch: ' + str(step) + ' | Validation Loss: ' + str(test_loss_metric.result().numpy()) + ' | Validation Accuracy: ' + str(test_accuracy_metric.result().numpy())) train_loss_metric.reset_states() train_accuracy_metric.reset_states() test_loss_metric.reset_states() test_accuracy_metric.reset_states()
batch_size = 64 epochs = 6 y_target = np.zeros((y_train.shape[0], len(ALL), y_train.shape[1])) y_test_pred = np.zeros((X_test.shape[0], len(ALL) * fold, y_train.shape[1])) n = 0 for name in DL: # print('='*80) seed = SEED * (n + 1) kfold = list( KFold(n_splits=fold, random_state=seed, shuffle=True).split(X_train, y_train)) for i, (train_index, val_index) in enumerate(kfold): X, y, val_X, val_y = X_train[train_index], y_train[ train_index], X_train[val_index], y_train[val_index] util.seed_everything(seed + i) model = models.getModel(param, name) # if i == 0: print(model.summary()) filepath = param['subject_ckp_path'] + name + '-' + str(i + 1) if not os.path.exists(filepath): reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=1, min_lr=0.0001, verbose=2) checkpoint = ModelCheckpoint(filepath, monitor='val_loss', verbose=2, save_best_only=True, mode='min') model.fit(X, y,