def netFeatVald(net, vald_data, loss_fn):
###############
### Validation
###############
epo_val_loss = 0
val_batch_count = 0
start_time_val = time.time()
all_predictions = []
all_labels = []
trdata, trlabels = vald_data
net.eval()
for batch in iterate_minibatches_torch(trdata,
args.sgd_batch_size,
trlabels,
shuffle=False,
last_batch=True):
inputs, targets = batch
if use_cuda:
indata = Variable(inputs, volatile=True).cuda()
lbdata = Variable(targets, volatile=True).cuda()
else:
indata = Variable(inputs, volatile=True)
lbdata = Variable(targets, volatile=True)
batch_pred = net(indata)
batch_val_loss = loss_fn(batch_pred, lbdata)
epo_val_loss += batch_val_loss.data[0]
val_batch_count += 1
inres = batch_pred.data.cpu().numpy().tolist()
all_predictions.extend(inres)
all_labels.extend(lbdata.data.cpu().numpy().tolist())
#print "{} Set-Batch Validation done in {} seconds. Validation Loss {} ".format(i, time.time() - start_time_val, epo_val_loss)
epo_val_loss = epo_val_loss / val_batch_count
all_predictions = np.array(all_predictions)
all_labels = np.array(all_labels)
aps = metric.compute_AP_all_class(all_labels, all_predictions)
aucs = metric.compute_AUC_all_class(all_labels, all_predictions)
aps_ranked = metric.compute_AP_my_all_class(all_labels, all_predictions)
return aps, aucs, aps_ranked, epo_val_loss
def netVald(net, validation_input_directory, loss_fn):
###############
### Validation
###############
epo_val_loss = 0
val_batch_count = 0
start_time_val = time.time()
all_predictions = []
all_labels = []
validation_set_file_list = []
net.eval()
for i in range(len(args.val_test_list)):
validation_set_file_list.append(args.train_test_split_path + '/b_' +
str(args.val_test_list[i]) +
'_validation.list')
validation_files = read_file(validation_set_file_list[i])
feat_dim = int(args.val_test_list[i])
if len(validation_files) == 0:
continue
for batch in dataloader(validation_input_directory,
validation_files,
args.sgd_batch_size,
feat_dim,
shuffle=False,
last_batch=True):
inputs, targets = batch
if targets is None:
print 'No Target given to minibatch iterator. Using a dummy target with two outputs'
targets = np.zeros((inputs.shape[0], 2))
indata = torch.Tensor(inputs)
lbdata = torch.Tensor(targets)
if use_cuda:
indata = Variable(indata, volatile=True).cuda()
lbdata = Variable(lbdata, volatile=True).cuda()
else:
indata = Variable(indata, volatile=True)
lbdata = Variable(lbdata, volatile=True)
batch_pred = net(indata)
batch_val_loss = loss_fn(batch_pred, lbdata)
epo_val_loss += batch_val_loss.data[0]
val_batch_count += 1
inres = batch_pred.data.cpu().numpy().tolist()
all_predictions.extend(inres)
all_labels.extend(lbdata.data.cpu().numpy().tolist())
print "{} Set-Batch Validation done in {} seconds. Validation Loss {} ".format(
i,
time.time() - start_time_val, epo_val_loss)
epo_val_loss = epo_val_loss / val_batch_count
all_predictions = np.array(all_predictions)
all_labels = np.array(all_labels)
aps = metric.compute_AP_all_class(all_labels, all_predictions)
aucs = metric.compute_AUC_all_class(all_labels, all_predictions)
aps_ranked = metric.compute_AP_my_all_class(all_labels, all_predictions)
print "Val aps ranked " + str(aps_ranked[-1]), "Val APS " + str(
aps[-1]), "Val AUC " + str(aucs[-1])
return aps, aucs, aps_ranked, epo_val_loss
def setupClassifier(testing_input_directory,classCount,spec_count,segment_length,learning_rate,output_path,model_path):
sys.stdout = c1
testing_set_file_list = []
for i in range(len(args.val_test_list)):
testing_set_file_list.append(args.train_test_split_path + '/b_' + str(args.val_test_list[i]) + '_testing.list')
testing_set = [[] for x in range(len(args.val_test_list))]
for i in range(len(args.val_test_list)):
testing_set[i] = read_file(testing_set_file_list[i],testing_set[i])
# 864.txt, 768.txt, 672.txt, 576.txt, 480.txt, 288.txt
split_point= [10,1,1,1,1,1]
length_testing = []
for i in range(len(args.val_test_list)):
length_testing.append(len(testing_set[i]))
length_testing_sum = np.cumsum(length_testing)
net = NET(classCount)
net.xavier_init()
#optimizer = optim.SGD(net.parameters(), lr=args.learning_rate, weight_decay=5e-4,momentum=args.momentum)
optimizer = optim.Adam(net.parameters(), lr=args.learning_rate)
loss_fn = nn.BCELoss()
if use_cuda:
net.cuda()
net = torch.nn.DataParallel(net, device_ids=range(torch.cuda.device_count()))
load_model(net,args.model_path)
###############
### Testing
###############
i = 0
epo_test_loss = 0
test_batch_count = 0
start_time_test = time.time()
all_predictions = np.random.rand(1,classCount)
all_labels = np.zeros((1,classCount))
net.eval()
j = 0
while(i<sum(length_testing)):
if (j==0):
index1 = retrieve_index(i,length_testing_sum)
print index1
print "i - " + str(index1) + " length testing sum " + str(length_testing_sum[index1])
length = length_testing[index1]
batch_size = length/split_point[index1]
feat_dim = int(args.val_test_list[index1])
testing_files = testing_set[index1]
random.shuffle(testing_files)
X_test,Y_test=dataloader(testing_input_directory,testing_files,j,batch_size,feat_dim)
print "Test Data is loaded"
print "Value of J is = " + str(j)
print "Value of I is = " + str(i)
print "Value of length = " + str(length)
if (j+batch_size) < length:
j = j+batch_size
i = i+batch_size
else:
j = 0
i = length_testing_sum[index1]
print "At length testing sum"
print i
for batch in iterate_minibatches_torch(X_test,args.sgd_batch_size,Y_test,shuffle=True,last_batch=True):
indata,lbdata = batch
if use_cuda:
indata = Variable(indata).cuda()
lbdata = Variable(lbdata).cuda()
else:
indata = Variable(indata)
lbdata = Variable(lbdata)
batch_pred = net(indata)
batch_test_loss = loss_fn(batch_pred,lbdata)
epo_test_loss += batch_test_loss.data[0]
test_batch_count += 1
inres = batch_pred.data.cpu().numpy()
all_predictions = np.concatenate((all_predictions,inres),axis=0)
all_labels = np.concatenate((all_labels,lbdata.data.cpu().numpy()),axis=0)
print test_batch_count
print "{} Set-Batch Testing done in {} seconds. Testing Loss {}".format(i, time.time() - start_time_test, epo_test_loss)
print "Length of all prediction - Test " + str(len(all_predictions))
all_predictions = all_predictions[1:,:]
all_labels = all_labels[1:,:]
aps = metric.compute_AP_all_class(all_labels,all_predictions)
aucs = metric.compute_AUC_all_class(all_labels,all_predictions)
aps_ranked = metric.compute_AP_my_all_class(all_labels,all_predictions)
print "Test APS " + str(aps[-1])
print "Test aps ranked " + str(aps_ranked[-1])
print "Test AUCS " + str(aucs[-1])
filename = 'metrics_testing_' + str(args.run_number) + 'aps.txt'
filename1 = 'metrics_testing_' + str(args.run_number) + 'aps_ranked.txt'
filename2 = 'metrics_testing_' + str(args.run_number) + 'aucs.txt'
np.save(filename,aps)
np.save(filename1,aps_ranked)
np.save(filename2,aucs)
epo_test_loss = epo_test_loss/test_batch_count
print "Testing done in {} seconds. Testing Loss {}".format(time.time() - start_time_test,epo_test_loss)
def setupClassifier(training_input_directory, validation_input_directory,
testing_input_directory, classCount, spec_count,
segment_length, learning_rate, output_path):
sys.stdout = c1
net = NET(classCount)
net.xavier_init()
#optimizer = optim.SGD(net.parameters(), lr=args.learning_rate, weight_decay=5e-4,momentum=args.momentum)
optimizer = optim.Adam(net.parameters(), lr=args.learning_rate)
scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.87)
loss_fn = nn.BCELoss()
if use_cuda:
net.cuda()
net = torch.nn.DataParallel(net,
device_ids=range(
torch.cuda.device_count()))
if args.preloading:
load_model(net, args.preloading_model_path, False)
else:
if args.preloading:
load_model_cpu(net, args.preloading_model_path, False)
epoch_best = 0
aps_best = 0
model_path = ''
for epoch in range(args.num_epochs):
print "Current epoch is " + str(epoch)
i = 0
j = 0
epo_train_loss = 0
batch_count = 0
start_time = time.time()
scheduler.step()
#############
### Training
#############
net.train()
random.shuffle(args.train_list)
training_set_file_list = []
for i in range(len(args.train_list)):
training_set_file_list.append(args.train_test_split_path + '/b_' +
str(args.train_list[i]) +
'_training.list')
training_files = read_file(training_set_file_list[i])
feat_dim = int(args.train_list[i])
if len(training_files) == 0:
continue
for batch in dataloader(training_input_directory,
training_files,
args.sgd_batch_size,
feat_dim,
shuffle=True,
last_batch=True):
inputs, targets = batch
if targets is None:
print 'No Target given to minibatch iterator. Using a dummy target with two outputs'
targets = np.zeros((inputs.shape[0], 2))
indata = torch.Tensor(inputs)
lbdata = torch.Tensor(targets)
if use_cuda:
indata = Variable(indata).cuda()
lbdata = Variable(lbdata).cuda()
else:
indata = Variable(indata)
lbdata = Variable(lbdata)
optimizer.zero_grad()
batch_pred = net(indata)
batch_train_loss = loss_fn(batch_pred, lbdata)
batch_train_loss.backward()
optimizer.step()
epo_train_loss += batch_train_loss.data[0]
batch_count += 1
print batch_count
print "{} Set-Batch training done in {} seconds. Train Loss {} ".format(
i,
time.time() - start_time, epo_train_loss)
epo_train_loss = epo_train_loss / batch_count
print "{} Training done in {} seconds. Training Loss {}".format(
epoch,
time.time() - start_time, epo_train_loss)
###############
### Validation
###############
i = 0
epo_val_loss = 0
val_batch_count = 0
start_time_val = time.time()
all_predictions = []
all_labels = []
validation_set_file_list = []
net.eval()
for i in range(len(args.val_test_list)):
validation_set_file_list.append(args.train_test_split_path +
'/b_' +
str(args.val_test_list[i]) +
'_validation.list')
validation_files = read_file(validation_set_file_list[i])
feat_dim = int(args.val_test_list[i])
if len(validation_files) == 0:
continue
for batch in dataloader(validation_input_directory,
validation_files,
args.sgd_batch_size,
feat_dim,
shuffle=False,
last_batch=True):
inputs, targets = batch
if targets is None:
print 'No Target given to minibatch iterator. Using a dummy target with two outputs'
targets = np.zeros((inputs.shape[0], 2))
indata = torch.Tensor(inputs)
lbdata = torch.Tensor(targets)
if use_cuda:
indata = Variable(indata, volatile=True).cuda()
lbdata = Variable(lbdata, volatile=True).cuda()
else:
indata = Variable(indata, volatile=True)
lbdata = Variable(lbdata, volatile=True)
batch_pred = net(indata)
batch_val_loss = loss_fn(batch_pred, lbdata)
epo_val_loss += batch_val_loss.data[0]
val_batch_count += 1
inres = batch_pred.data.cpu().numpy().tolist()
all_predictions.extend(inres)
all_labels.extend(lbdata.data.cpu().numpy().tolist())
print val_batch_count
print "{} Set-Batch Validation done in {} seconds. Validation Loss {} ".format(
i,
time.time() - start_time_val, epo_val_loss)
epo_val_loss = epo_val_loss / val_batch_count
print len(all_labels)
all_predictions = np.array(all_predictions)
all_labels = np.array(all_labels)
aps = metric.compute_AP_all_class(all_labels, all_predictions)
aucs = metric.compute_AUC_all_class(all_labels, all_predictions)
aps_ranked = metric.compute_AP_my_all_class(all_labels,
all_predictions)
print "Epoch number " + str(epoch)
print "Val aps ranked " + str(aps_ranked[-1])
print "Val APS " + str(aps[-1])
print "Val AUC " + str(aucs[-1])
#print aps
#print aps_ranked
filename = os.path.join(
'output_' + str(args.run_number), 'metrics_validation_' +
str(args.run_number) + '_' + str(epoch) + '_aps.txt')
filename1 = os.path.join(
'output_' + str(args.run_number), 'metrics_validation_' +
str(args.run_number) + '_' + str(epoch) + '_aps_ranked.txt')
filename2 = os.path.join(
'output_' + str(args.run_number), 'metrics_validation_' +
str(args.run_number) + '_' + str(epoch) + '_aucs.txt')
if not os.path.isdir(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
np.save(filename, aps)
np.save(filename1, aps_ranked)
np.save(filename2, aucs)
if aps[-1] > aps_best:
aps_best = aps[-1]
epoch_best = epoch
if not os.path.exists('output_' + str(args.run_number)):
os.mkdir('output_' + str(args.run_number))
torch.save(
net.state_dict(),
'output_' + str(args.run_number) + '/model_path.' +
str(args.run_number) + '_' + str(epoch) + '.pkl')
model_path = 'output_' + str(
args.run_number) + '/model_path.' + str(
args.run_number) + '_' + str(epoch) + '.pkl'
print "{} Validation done in {} seconds. Validation Loss {}".format(
epoch,
time.time() - start_time_val, epo_val_loss)
if use_cuda:
load_model(net, model_path)
else:
load_model_cpu(net, model_path)
###############
### Testing
###############
i = 0
epo_test_loss = 0
test_batch_count = 0
start_time_test = time.time()
all_predictions = []
all_labels = []
net.eval()
j = 0
testing_set_file_list = []
for i in range(len(args.val_test_list)):
testing_set_file_list.append(args.train_test_split_path + '/b_' +
str(args.val_test_list[i]) +
'_testing.list')
testing_files = read_file(testing_set_file_list[i])
feat_dim = int(args.val_test_list[i])
if len(testing_files) == 0:
continue
for batch in dataloader(testing_input_directory,
testing_files,
args.sgd_batch_size,
feat_dim,
shuffle=False,
last_batch=True):
inputs, targets = batch
if targets is None:
print 'No Target given to minibatch iterator. Using a dummy target with two outputs'
targets = np.zeros((inputs.shape[0], 2))
indata = torch.Tensor(inputs)
lbdata = torch.Tensor(targets)
if use_cuda:
indata = Variable(indata, volatile=True).cuda()
lbdata = Variable(lbdata, volatile=True).cuda()
else:
indata = Variable(indata, volatile=True)
lbdata = Variable(lbdata, volatile=True)
batch_pred = net(indata)
batch_test_loss = loss_fn(batch_pred, lbdata)
epo_test_loss += batch_test_loss.data[0]
test_batch_count += 1
inres = batch_pred.data.cpu().numpy().tolist()
all_predictions.extend(inres)
all_labels.extend(lbdata.data.cpu().numpy().tolist())
print test_batch_count
print "{} Set-Batch Testing done in {} seconds. Testing Loss {} ".format(
i,
time.time() - start_time_test, epo_test_loss)
epo_test_loss = epo_test_loss / test_batch_count
print "Length of all prediction - Test " + str(len(all_predictions))
print len(all_labels)
all_predictions = np.array(all_predictions)
all_labels = np.array(all_labels)
aps = metric.compute_AP_all_class(all_labels, all_predictions)
aucs = metric.compute_AUC_all_class(all_labels, all_predictions)
aps_ranked = metric.compute_AP_my_all_class(all_labels, all_predictions)
print "Test APS " + str(aps[-1])
print "Test aps ranked " + str(aps_ranked[-1])
print "Test AUCS " + str(aucs[-1])
filename = os.path.join(
'output_' + str(args.run_number), 'metrics_testing_' +
str(args.run_number) + '_' + str(epoch_best) + '_aps.txt')
filename1 = os.path.join(
'output_' + str(args.run_number), 'metrics_testing_' +
str(args.run_number) + '_' + str(epoch_best) + '_aps_ranked.txt')
filename2 = os.path.join(
'output_' + str(args.run_number), 'metrics_testing_' +
str(args.run_number) + '_' + str(epoch_best) + '_aucs.txt')
np.save(filename, aps)
np.save(filename1, aps_ranked)
np.save(filename2, aucs)
epo_test_loss = epo_test_loss / test_batch_count
print "{} Testing done in {} seconds. Testing Loss {}".format(
epoch,
time.time() - start_time_test, epo_test_loss)
def netFeatVald(net, vald_data, loss_fn, epoch):
###############
### Validation
###############
epo_val_loss = 0
val_batch_count = 0
start_time_val = time.time()
all_predictions1 = []
all_predictions2 = []
all_labels = []
net.eval()
if args.div_apply[0] == 'Y':
div_type = args.div_apply.split('+')[1]
div_factor = float(args.div_apply.split('+')[2])
if len(args.div_apply.split('+')) > 3:
step_factors = args.div_apply.split('+')[3]
step = int(step_factors.split('|')[0])
factor = float(step_factors.split('|')[1])
div_factor = div_factor * (factor**(epoch // step))
for batch in vald_data:
inputs1, inputs2, targets = batch
indata1 = torch.Tensor(inputs1)
indata2 = torch.Tensor(inputs2)
lbdata = torch.Tensor(targets)
if use_cuda:
indata1 = Variable(indata1, volatile=True).cuda()
indata2 = Variable(indata2, volatile=True).cuda()
lbdata = Variable(lbdata, volatile=True).cuda()
else:
indata1 = Variable(indata1, volatile=True)
indata2 = Variable(indata2, volatile=True)
lbdata = Variable(lbdata, volatile=True)
batch_pred1, batch_pred2 = net(indata1, indata2)
batch_val_loss1 = loss_fn(batch_pred1, lbdata)
batch_val_loss2 = loss_fn(batch_pred2, lbdata)
batch_val_loss = batch_val_loss1 + batch_val_loss2
if args.div_apply[0] == 'Y':
batch_val_loss3 = loss_div_outs(batch_pred1, batch_pred2, div_type)
batch_val_loss = batch_val_loss + div_factor * batch_val_loss3
epo_val_loss += batch_val_loss.data[0]
val_batch_count += 1
inres1 = batch_pred1.data.cpu().numpy().tolist()
inres2 = batch_pred2.data.cpu().numpy().tolist()
all_predictions1.extend(inres1)
all_predictions2.extend(inres2)
all_labels.extend(lbdata.data.cpu().numpy().tolist())
#print "{} Set-Batch Validation done in {} seconds. Validation Loss {} ".format(i, time.time() - start_time_val, epo_val_loss)
epo_val_loss = epo_val_loss / val_batch_count
all_predictions1 = np.array(all_predictions1)
all_predictions2 = np.array(all_predictions2)
all_predictions = (all_predictions1 + all_predictions2) / 2.0
all_labels = np.array(all_labels)
aps1 = metric.compute_AP_all_class(all_labels, all_predictions1)
aucs1 = metric.compute_AUC_all_class(all_labels, all_predictions1)
aps_ranked1 = metric.compute_AP_my_all_class(all_labels, all_predictions1)
aps2 = metric.compute_AP_all_class(all_labels, all_predictions2)
aucs2 = metric.compute_AUC_all_class(all_labels, all_predictions2)
aps_ranked2 = metric.compute_AP_my_all_class(all_labels, all_predictions2)
aps = metric.compute_AP_all_class(all_labels, all_predictions)
aucs = metric.compute_AUC_all_class(all_labels, all_predictions)
aps_ranked = metric.compute_AP_my_all_class(all_labels, all_predictions)
return aps, aucs, aps_ranked, aps1, aucs1, aps_ranked1, aps2, aucs2, aps_ranked2, epo_val_loss
def setupClassifier(training_input_directory, validation_input_directory,
testing_input_directory, classCount, spec_count,
segment_length, learning_rate, output_path):
sys.stdout = c1
training_set_file_list = []
testing_set_file_list = []
validation_set_file_list = []
for i in range(len(args.train_list)):
training_set_file_list.append(args.train_test_split_path + '/b_' +
str(args.train_list[i]) +
'_training.list')
for i in range(len(args.val_test_list)):
testing_set_file_list.append(args.train_test_split_path + '/b_' +
str(args.val_test_list[i]) +
'_testing.list')
for i in range(len(args.val_test_list)):
validation_set_file_list.append(args.train_test_split_path + '/b_' +
str(args.val_test_list[i]) +
'_validation.list')
training_set = [[] for x in range(len(args.train_list))]
testing_set = [[] for x in range(len(args.val_test_list))]
validation_set = [[] for x in range(len(args.val_test_list))]
for i in range(len(args.train_list)):
training_set[i] = read_file(training_set_file_list[i], training_set[i])
for i in range(len(args.val_test_list)):
validation_set[i] = read_file(validation_set_file_list[i],
validation_set[i])
for i in range(len(args.val_test_list)):
testing_set[i] = read_file(testing_set_file_list[i], testing_set[i])
# 864.txt, 768.txt, 672.txt, 576.txt, 480.txt, 288.txt
split_point = [10, 1, 1, 1, 1, 1]
#batch sizes 2592, 2304, 2016, 1728, 1440, 1152, 864, 576, 288
split_point_training = [
10, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20
]
length_training = []
for i in range(len(args.train_list)):
length_training.append(len(training_set[i]))
length_training_sum = np.cumsum(length_training)
length_testing = []
for i in range(len(args.val_test_list)):
length_testing.append(len(testing_set[i]))
length_testing_sum = np.cumsum(length_testing)
length_validation = []
for i in range(len(args.val_test_list)):
length_validation.append(len(validation_set[i]))
length_validation_sum = np.cumsum(length_validation)
net = NET(classCount)
net.xavier_init()
optimizer = optim.Adam(net.parameters(), lr=args.learning_rate)
loss_fn = nn.BCELoss()
if use_cuda:
net.cuda()
net = torch.nn.DataParallel(net,
device_ids=range(
torch.cuda.device_count()))
#net = torch.nn.DataParallel(net, device_ids=[0,1])
for epoch in range(args.num_epochs):
print "Current epoch is " + str(epoch)
i = 0
j = 0
epo_train_loss = 0
batch_count = 0
start_time = time.time()
#############
### Training
#############
print sum(length_training)
net.train()
while (i < sum(length_training)):
if (j == 0):
index1 = retrieve_index(i, length_training_sum)
print "INDEX 1 is "
print index1
print "i - " + str(index1) + " length training sum " + str(
length_training_sum[index1])
length = length_training[index1]
batch_size = length / split_point_training[index1]
training_files = training_set[index1]
random.shuffle(training_files)
feat_dim = int(args.train_list[index1])
starttime1 = time.time()
X_train, Y_train = dataloader(training_input_directory,
training_files, j, batch_size,
feat_dim)
print "Train Data is loaded"
print "Value of J is = " + str(j)
print "Value of I is = " + str(i)
print "Value of length = " + str(length)
duration = time.time() - starttime1
print "Loading data took = " + str(duration)
if (j + batch_size) < length:
j = j + batch_size
i = i + batch_size
else:
j = 0
i = length_training_sum[index1]
print i
for batch in iterate_minibatches_torch(X_train,
args.sgd_batch_size,
Y_train,
shuffle=True,
last_batch=True):
indata, lbdata = batch
if use_cuda:
indata = Variable(indata).cuda()
lbdata = Variable(lbdata).cuda()
else:
indata = Variable(indata)
lbdata = Variable(lbdata)
optimizer.zero_grad()
batch_pred = net(indata)
batch_train_loss = loss_fn(batch_pred, lbdata)
batch_train_loss.backward()
optimizer.step()
epo_train_loss += batch_train_loss.data[0]
batch_count += 1
print batch_count
print "{} Set-Batch training done in {} seconds. Train Loss {} ".format(
i,
time.time() - start_time, epo_train_loss)
epo_train_loss = epo_train_loss / batch_count
print "{} Training done in {} seconds. Training Loss {}".format(
epoch,
time.time() - start_time, epo_train_loss)
if not os.path.exists('model_' + str(args.run_number)):
os.mkdir('model_' + str(args.run_number))
torch.save(
net.state_dict(), 'model_' + str(args.run_number) +
'/model_path.' + str(args.run_number) + '_' + str(epoch) + '.pkl')
###############
### Validation
###############
i = 0
epo_val_loss = 0
val_batch_count = 0
start_time_val = time.time()
all_predictions = np.random.rand(1, classCount)
all_labels = np.zeros((1, classCount))
net.eval()
j = 0
while (i < sum(length_validation)):
if (j == 0):
index1 = retrieve_index(i, length_validation_sum)
print index1
print "i - " + str(
index1) + " length validation sum " + str(
length_validation_sum[index1])
length = length_validation[index1]
batch_size = length / split_point[index1]
feat_dim = int(args.val_test_list[index1])
validation_files = validation_set[index1]
random.shuffle(validation_files)
X_val, Y_val = dataloader(validation_input_directory,
validation_files, j, batch_size,
feat_dim)
print "Validation Data is loaded"
print "Value of J is = " + str(j)
print "Value of I is = " + str(i)
print "Value of length = " + str(length)
if (j + batch_size) < length:
j = j + batch_size
i = i + batch_size
else:
j = 0
i = length_validation_sum[index1]
print "At length validation sum"
print i
for batch in iterate_minibatches_torch(X_val,
args.sgd_batch_size,
Y_val,
shuffle=True,
last_batch=True):
indata, lbdata = batch
if use_cuda:
indata = Variable(indata, volatile=True).cuda()
lbdata = Variable(lbdata, volatile=True).cuda()
else:
indata = Variable(indata, volatile=True)
lbdata = Variable(lbdata, volatile=True)
batch_pred = net(indata)
batch_val_loss = loss_fn(batch_pred, lbdata)
epo_val_loss += batch_val_loss.data[0]
val_batch_count += 1
inres = batch_pred.data.cpu().numpy()
all_predictions = np.concatenate((all_predictions, inres),
axis=0)
all_labels = np.concatenate(
(all_labels, lbdata.data.cpu().numpy()), axis=0)
print val_batch_count
print "{} Set-Batch Validation done in {} seconds. Validation Loss {} ".format(
i,
time.time() - start_time_val, epo_val_loss)
epo_val_loss = epo_val_loss / val_batch_count
all_predictions = all_predictions[1:, :]
all_labels = all_labels[1:, :]
print len(all_labels)
aps = metric.compute_AP_all_class(all_labels, all_predictions)
aucs = metric.compute_AUC_all_class(all_labels, all_predictions)
aps_ranked = metric.compute_AP_my_all_class(all_labels,
all_predictions)
print "Epoch number " + str(epoch)
print "Val aps ranked " + str(aps_ranked[-1])
print "Val APS " + str(aps[-1])
print "Val AUC " + str(aucs[-1])
filename = os.path.join(
'metrics_' + str(args.run_number), 'metrics_validation_' +
str(args.run_number) + '_' + str(epoch) + '_aps.txt')
filename1 = os.path.join(
'metrics_' + str(args.run_number), 'metrics_validation_' +
str(args.run_number) + '_' + str(epoch) + '_aps_ranked.txt')
filename2 = os.path.join(
'metrics_' + str(args.run_number), 'metrics_validation_' +
str(args.run_number) + '_' + str(epoch) + '_aucs.txt')
if not os.path.isdir(os.path.dirname(filename)):
os.makedirs(os.path.dirname(filename))
np.save(filename, aps)
np.save(filename1, aps_ranked)
np.save(filename2, aucs)
print "{} Validation done in {} seconds. Validation Loss {}".format(
epoch,
time.time() - start_time_val, epo_val_loss)
###############
### Testing
###############
i = 0
epo_test_loss = 0
test_batch_count = 0
start_time_test = time.time()
all_predictions = np.random.rand(1, classCount)
all_labels = np.zeros((1, classCount))
net.eval()
j = 0
while (i < sum(length_testing)):
if (j == 0):
index1 = retrieve_index(i, length_testing_sum)
print index1
print "i - " + str(index1) + " length testing sum " + str(
length_testing_sum[index1])
length = length_testing[index1]
batch_size = length / split_point[index1]
feat_dim = int(args.val_test_list[index1])
testing_files = testing_set[index1]
random.shuffle(testing_files)
X_test, Y_test = dataloader(testing_input_directory, testing_files, j,
batch_size, feat_dim)
print "Test Data is loaded"
print "Value of J is = " + str(j)
print "Value of I is = " + str(i)
print "Value of length = " + str(length)
if (j + batch_size) < length:
j = j + batch_size
i = i + batch_size
else:
j = 0
i = length_testing_sum[index1]
print "At length testing sum"
print i
for batch in iterate_minibatches_torch(X_test,
args.sgd_batch_size,
Y_test,
shuffle=True,
last_batch=True):
indata, lbdata = batch
if use_cuda:
indata = Variable(indata, volatile=True).cuda()
lbdata = Variable(lbdata, volatile=True).cuda()
else:
indata = Variable(indata, volatile=True)
lbdata = Variable(lbdata, volatile=True)
batch_pred = net(indata)
batch_test_loss = loss_fn(batch_pred, lbdata)
epo_test_loss += batch_test_loss.data[0]
test_batch_count += 1
inres = batch_pred.data.cpu().numpy()
all_predictions = np.concatenate((all_predictions, inres), axis=0)
all_labels = np.concatenate(
(all_labels, lbdata.data.cpu().numpy()), axis=0)
print test_batch_count
print "{} Set-Batch Testing done in {} seconds. Testing Loss {}".format(
i,
time.time() - start_time_test, epo_test_loss)
print "Length of all prediction - Test " + str(len(all_predictions))
all_predictions = all_predictions[1:, :]
all_labels = all_labels[1:, :]
aps = metric.compute_AP_all_class(all_labels, all_predictions)
aucs = metric.compute_AUC_all_class(all_labels, all_predictions)
aps_ranked = metric.compute_AP_my_all_class(all_labels, all_predictions)
print "Test APS " + str(aps[-1])
print "Test aps ranked " + str(aps_ranked[-1])
print "Test AUCS " + str(aucs[-1])
filename = 'metrics_testing_' + str(args.run_number) + 'aps.txt'
filename1 = 'metrics_testing_' + str(args.run_number) + 'aps_ranked.txt'
filename2 = 'metrics_testing_' + str(args.run_number) + 'aucs.txt'
np.save(filename, aps)
np.save(filename1, aps_ranked)
np.save(filename2, aucs)
epo_test_loss = epo_test_loss / test_batch_count
print "{} Testing done in {} seconds. Testing Loss {}".format(
epoch,
time.time() - start_time_test, epo_test_loss)