def trainCNN_on_Bulbul_architecture(): #CNN Training parameters activation = 'elu' init_type = 'xavier' batch_size = 32 epochs = 200 #0 # Regularizer parameters use_lr_decay = False #set this flag for LR decay wDecayFlag = False #whether to perform L2 weight decay or not lossPenalty = 0.001 # Using lambda=0.001 . applyBatchNorm = False deviceId = "/gpu:0" # Adam parameters optimizer_type = 'adam' b1 = 0.9 b2 = 0.999 epsilon = 0.1 momentum = 0.95 dropout1 = 0.6 #for input to first FC layer dropout3 = 0.5 #for intermediate layer input dropout2 = [0.5, 0.6] lambdas = [0.0005, 0.001] architectures = [ 2 ] # birds architecture 1, to make it unified (check model.py for definition) trainingSize = [1] #in seconds #lr= 0.0005 learning_rates = [0.0005, 0.0003, 0.0001, 0.005] targets = 2 fftSize = 256 specType = 'mag_spec' padding = True # Used following paths since I moved the scripts in git and used link so that code are synchronised spectrogramPath = '/homes/bc305/myphd/stage2/deeplearning.experiment1/spectrograms/' tensorboardPath = '/homes/bc305/myphd/stage2/deeplearning.experiment1/CNN3/tensorflow_log_dir/' modelPath = '/homes/bc305/myphd/stage2/deeplearning.experiment1/CNN3/models/' for duration in trainingSize: print('Now loading the data !!') outPath = spectrogramPath + specType + '/' + str( fftSize) + 'FFT/' + str(duration) + 'sec/' mean_std_file = outPath + 'train/mean_std.npz' # Load training data, labels and perform norm tD = dataset.load_data(outPath + 'train/') tL = dataset.get_labels_according_to_targets(trainP, targets) if not os.path.exists(mean_std_file): print('Computing Mean_std file ..') dataset.compute_global_norm(tD, mean_std_file) print('Shape of labels: ', tL.shape) #tD = dataset.normalise_data(tD,mean_std_file,'utterance') # utterance level tD = dataset.normalise_data(tD, mean_std_file, 'global_mv') # global # Load dev data, labels and perform norm devD = dataset.load_data(outPath + 'dev/') devL = dataset.get_labels_according_to_targets(devP, targets) #devD = dataset.normalise_data(devD,mean_std_file,'utterance') devD = dataset.normalise_data(devD, mean_std_file, 'global_mv') ### We are training on TRAIN set and validating on DEV set t_data = tD t_labels = tL v_data = devD v_labels = devL for dropout in dropout2: architecture = architectures[0] for lr in learning_rates: hyp_str = '_cnnModel' + str( architecture) + '_keepProb_0.6_' + str(dropout) + str( dropout3) + 'lr' + str(lr) log_dir = tensorboardPath + '/birdsArch_max2000epochsTEMP/' + hyp_str model_save_path = modelPath + '/birdsArch_max2000epochsTEMP/' + hyp_str logfile = model_save_path + '/training.log' figDirectory = model_save_path makeDirectory(model_save_path) print('Training model with ' + str(duration) + ' sec data and cnnModel' + str(architecture)) tLoss, vLoss, tAcc, vAcc = model.train( architecture, fftSize, padding, duration, t_data, t_labels, v_data, v_labels, activation, lr, use_lr_decay, epsilon, b1, b2, momentum, optimizer_type, dropout1, dropout, dropout3, model_save_path, log_dir, logfile, wDecayFlag, lossPenalty, applyBatchNorm, init_type, epochs, batch_size, targets) #plot_2dGraph('#Epochs', 'Avg CE Loss', tLoss,vLoss,'train_ce','val_ce', figDirectory+'/loss.png') #plot_2dGraph('#Epochs', 'Avg accuracy', tAcc,vAcc,'train_acc','val_acc',figDirectory+'/acc.png') plot_2dGraph('#Epochs', 'Val loss and accuracy', vLoss, vAcc, 'val_loss', 'val_acc', figDirectory + '/v_ls_acc.png')
def trainCNN_on_Sparrow_architecture(): #CNN Training parameters activation = 'elu' #'elu' init_type = 'xavier' batch_size = 32 epochs = 1000 # Regularizer parameters use_lr_decay = False #set this flag for LR decay wDecayFlag = False #whether to perform L2 weight decay or not lossPenalty = 0.001 # Using lambda=0.001 . applyBatchNorm = False deviceId = "/gpu:0" # Adam parameters optimizer_type = 'adam' b1 = 0.9 b2 = 0.999 epsilon = 0.1 #1e-08 is the default momentum = 0.95 dropout1 = 1.0 #for input to first FC layer dropout2 = 1.0 #for intermediate layer input dropouts = [0.5, 0.4, 0.3, 0.2, 0.1] #,0.6] lambdas = [0.0005, 0.001] architectures = [ 3 ] # birds architecture sparrow, to make it unified (check model.py for definition) trainingSize = [1] #in seconds learning_rates = [0.0001, 0.00008] targets = 2 fftSize = 256 specType = 'mag_spec' padding = False for duration in trainingSize: print('Now loading the data !!') outPath = '../../spectrograms/' + specType + '/' + str( fftSize) + 'FFT/' + str(duration) + 'sec/' mean_std_file = outPath + 'train/mean_std.npz' # Load training data, labels and perform norm tD = dataset.load_data(outPath + 'train/') tL = dataset.get_labels_according_to_targets(trainP, targets) dataset.compute_global_norm(tD, mean_std_file) print('Shape of labels: ', tL.shape) #tD = dataset.normalise_data(tD,mean_std_file,'utterance') # utterance level tD = dataset.normalise_data(tD, mean_std_file, 'global_mv') # global #print('Norm td: max and min are ', np.max(tD)) # Load dev data, labels and perform norm devD = dataset.load_data(outPath + 'dev/') devL = dataset.get_labels_according_to_targets(devP, targets) #devD = dataset.normalise_data(devD,mean_std_file,'utterance') #print('first Norm dev: max and min are ', np.max(devD)) devD = dataset.normalise_data(devD, mean_std_file, 'global_mv') #print('Norm dev: max and min are ', np.max(devD)) trainSize = str( duration) + 'sec' ##may be change this in model.py also ! ### We are training on TRAIN set and validating on DEV set t_data = tD t_labels = tL v_data = devD v_labels = devL for dropout in dropouts: architecture = architectures[0] for lr in learning_rates: #hyp_str ='cnn'+str(architecture)+'_keepProb_1.0_' + str(dropout)+str(dropout3)+'lr'+str(lr) hyp_str = 'sparrow' + '_keep_' + str( dropout) + '_' + 'lr' + str(lr) + '_' + str( activation) + '_' + 'fft' + str(fftSize) log_dir = '../tensorflow_log_dir/sparrowArch/' + hyp_str model_save_path = '../models/sparrowArch/' + hyp_str logfile = model_save_path + '/training.log' figDirectory = model_save_path makeDirectory(model_save_path) print('Training model with ' + str(duration) + ' sec data and cnnModel' + str(architecture)) tLoss, vLoss, tAcc, vAcc = model.train( architecture, fftSize, padding, trainSize, t_data, t_labels, v_data, v_labels, activation, lr, use_lr_decay, epsilon, b1, b2, momentum, optimizer_type, dropout, dropout1, dropout2, model_save_path, log_dir, logfile, wDecayFlag, lossPenalty, applyBatchNorm, init_type, epochs, batch_size, targets) #plot_2dGraph('#Epochs', 'Avg CE Loss', tLoss,vLoss,'train_ce','val_ce', figDirectory+'/loss.png') #plot_2dGraph('#Epochs', 'Avg accuracy', tAcc,vAcc,'train_acc','val_acc',figDirectory+'/acc.png') plot_2dGraph('#Epochs', 'Val loss and accuracy', vLoss, vAcc, 'val_loss', 'val_acc', figDirectory + '/v_ls_acc.png')
def trainCNN_on_trainData(): #CNN Training parameters activation = 'mfm' #choose activation: mfm,elu, relu, mfsoftmax, tanh ? init_type='xavier' #'truncated_normal' #'xavier' #or 'truncated_normal' batch_size = 32 epochs = 2000 # Regularizer parameters use_lr_decay=False #set this flag for LR decay wDecayFlag = False #whether to perform L2 weight decay or not lossPenalty = 0.001 # Using lambda=0.001 . applyBatchNorm = False deviceId = "/gpu:0" # Adam parameters optimizer_type = 'adam' b1=0.9 b2=0.999 epsilon=0.1 momentum=0.95 #dropout1=1.0 #for input to first FC layer #dropout2=1.0 #for intermediate layer input drops=[1.0] # No dropout lambdas = [0.001] architectures = [1] trainingSize = [1] #in seconds lr= 0.0001 targets=2 #specType='mag_spec' #lets try loading mag_spec #inputTypes=['mel_spec'] #'mag_spec' #'cqt_spec' ## Running on Hepworth ! #inputTypes=['mag_spec'] # Not Run yet #inputTypes=['cqt_spec'] # Not Run yet inputTypes=['mel_spec','mag_spec','cqt_spec'] #CQT may throw error during scoring in model.py padding=True augment = True trainPercentage=0.8 #Each training epoch will see only 80% of the original data at random ! valPercentage=0.3 #30% only used at random for validation if augment: spectrogramPath='/homes/bc305/myphd/stage2/deeplearning.experiment1/spectrograms_augmented/1sec_shift/' else: spectrogramPath='/homes/bc305/myphd/stage2/deeplearning.experiment1/spectrograms/' # Used following paths since I moved the scripts in git and used link so that code are synchronised tensorboardPath = '/homes/bc305/myphd/stage2/deeplearning.experiment1/CNN3/tensorflow_log_dir/' modelPath = '/homes/bc305/myphd/stage2/deeplearning.experiment1/CNN3/models_augmented/' #for duration in trainingSize: duration=1 for specType in inputTypes: if specType == 'mel_spec' or specType == 'cqt_spec': fftSize=512 else: fftSize=256 print('Now loading the data with FFT size: ', fftSize) outPath = spectrogramPath +specType + '/' +str(fftSize)+ 'FFT/' + str(duration)+ 'sec/' mean_std_file = outPath+'train/mean_std.npz' # Load training data, labels and perform norm tD,tL = dataset.load_data(outPath+'train/') tL = dataset.get_labels_according_to_targets(tL, targets) assert(len(tD)==len(tL)) if not os.path.exists(mean_std_file): print('Computing Mean_std file ..') dataset.compute_global_norm(tD,mean_std_file) tD = dataset.normalise_data(tD,mean_std_file,'utterance') # utterance level tD = dataset.normalise_data(tD,mean_std_file,'global_mv') # global # Load dev data, labels and perform norm devD,devL = dataset.load_data(outPath+'dev/') devL = dataset.get_labels_according_to_targets(devL, targets) assert(len(devD)==len(devL)) devD = dataset.normalise_data(devD,mean_std_file,'utterance') devD = dataset.normalise_data(devD,mean_std_file,'global_mv') ### We are training on TRAIN set and validating on DEV set t_data = tD t_labels = tL v_data = devD v_labels = devL print('Training model on ', specType) for dropout in drops: # dropout 1.0 and 0.5 to all inputs of DNN architecture = architectures[0] #penalty=0.001 #this is not used thought at the moment for penalty in lambdas: #for targets in target_list: hyp_str='arch'+str(architecture)+'_keep'+str(dropout)+'_'+str(specType)+'_targets'+str(targets) log_dir = tensorboardPath+ '/model1_max2000epochs_1/'+ hyp_str model_save_path = modelPath + '/model1_max2000epochs_1/'+ hyp_str logfile = model_save_path+'/training.log' figDirectory = model_save_path makeDirectory(model_save_path) tLoss,vLoss,tAcc,vAcc=model.train(specType,architecture,fftSize,padding,duration,t_data,t_labels, v_data,v_labels,activation,lr,use_lr_decay,epsilon,b1,b2,momentum, optimizer_type,dropout,dropout,dropout,model_save_path,log_dir, logfile,wDecayFlag,penalty,applyBatchNorm,init_type,epochs,batch_size, targets,augment,trainPercentage,valPercentage) #plot_2dGraph('#Epochs', 'Avg CE Loss', tLoss,vLoss,'train_ce','val_ce', figDirectory+'/loss.png') #plot_2dGraph('#Epochs', 'Avg accuracy', tAcc,vAcc,'train_acc','val_acc',figDirectory+'/acc.png') plot_2dGraph('#Epochs', 'Val loss and accuracy', vLoss,vAcc,'val_loss','val_acc',figDirectory+'/v_ls_acc.png')
def trainCNN_on_trainData(): #CNN Training parameters activation = 'mfm' #'elu' init_type = 'xavier' batch_size = 32 epochs = 2000 #0 # Regularizer parameters use_lr_decay = False #set this flag for LR decay wDecayFlag = False #whether to perform L2 weight decay or not lossPenalty = 0.001 # Using lambda=0.001 . applyBatchNorm = False deviceId = "/gpu:0" # Adam parameters optimizer_type = 'adam' b1 = 0.9 b2 = 0.999 epsilon = 0.1 momentum = 0.95 dropout3 = 1.0 #In russian arch there are only FC1 and output layer, so this not needed. #dropout1=[0.3,0.2,0.1] # we ran this originally ! we will look into this later #dropout2=[0.7,0.5,0.4,0.3,0.2] # for dropout1=0.3, we ran all combination dropout1 = [0.2, 0.1] dropout2 = [0.4, 0.2] #lambdas = [0.0005, 0.001] #dropout1=[1.0] #dropout2=[1.0] #dropout3=1.0 architectures = [5] # Russian Architecture is 5 trainingSize = [4] #in seconds lr = 0.0001 targets = 2 fftSize = 2048 specType = 'mag_spec' padding = True # Used following paths since I moved the scripts in git and used link so that code are synchronised spectrogramPath = '/homes/bc305/myphd/stage2/deeplearning.experiment1/spectrograms/' tensorboardPath = '/homes/bc305/myphd/stage2/deeplearning.experiment1/CNN3/tensorflow_log_dir/' modelPath = '/homes/bc305/myphd/stage2/deeplearning.experiment1/CNN3/models/' for duration in trainingSize: print('Now loading the data !!') outPath = spectrogramPath + specType + '/' + str( fftSize) + 'FFT/' + str(duration) + 'sec/' mean_std_file = outPath + 'train/mean_std.npz' # Load training data, labels and perform norm tD = dataset.load_data(outPath + 'train/') tL = dataset.get_labels_according_to_targets(trainP, targets) if not os.path.exists(mean_std_file): print('Computing Mean_std file ..') dataset.compute_global_norm(tD, mean_std_file) print('Shape of labels: ', tL.shape) #tD = dataset.normalise_data(tD,mean_std_file,'utterance') # utterance level tD = dataset.normalise_data(tD, mean_std_file, 'global_mv') # global # Load dev data, labels and perform norm devD = dataset.load_data(outPath + 'dev/') devL = dataset.get_labels_according_to_targets(devP, targets) #devD = dataset.normalise_data(devD,mean_std_file,'utterance') devD = dataset.normalise_data(devD, mean_std_file, 'global_mv') ### We are training on TRAIN set and validating on DEV set t_data = tD t_labels = tL v_data = devD v_labels = devL for dropout in dropout1: architecture = architectures[0] for drop in dropout2: hyp_str = 'arch' + str(architecture) + '_keep_' + str( dropout) + '_' + str(drop) + '_' + str(duration) + 'sec' log_dir = tensorboardPath + '/rusCNN_max2000epochs/' + hyp_str model_save_path = modelPath + '/rusCNN_max2000epochs/' + hyp_str logfile = model_save_path + '/training.log' figDirectory = model_save_path makeDirectory(model_save_path) print('Training model with ' + str(duration) + ' sec data and cnnModel' + str(architecture)) tLoss, vLoss, tAcc, vAcc = model.train( architecture, fftSize, padding, duration, t_data, t_labels, v_data, v_labels, activation, lr, use_lr_decay, epsilon, b1, b2, momentum, optimizer_type, dropout, drop, dropout3, model_save_path, log_dir, logfile, wDecayFlag, lossPenalty, applyBatchNorm, init_type, epochs, batch_size, targets) #plot_2dGraph('#Epochs', 'Avg CE Loss', tLoss,vLoss,'train_ce','val_ce', figDirectory+'/loss.png') #plot_2dGraph('#Epochs', 'Avg accuracy', tAcc,vAcc,'train_acc','val_acc',figDirectory+'/acc.png') plot_2dGraph('#Epochs', 'Val loss and accuracy', vLoss, vAcc, 'val_loss', 'val_acc', figDirectory + '/v_ls_acc.png')