def cc():
print 'reading npy...'
data = np.load('../data/1st.npy')
feature_data = np.load('dnn_feature.npy')
train_order = np.load('../data/train.npy')
validation_order = np.load('../data/validation.npy')
test_order = np.load('../data/test.npy')
train_nlcd = get_data.get_feature(feature_data, train_order)
train_label = get_data.get_label(data, train_order)
test_nlcd = get_data.get_feature(feature_data, test_order)
test_label = get_data.get_label(data, test_order)
print 'chaining'
# Fit an ensemble of logistic regression classifier chains and take the
# take the average prediction of all the chains.
chains = []
for i in range(10):
chains.append(
ClassifierChain(LogisticRegression(),
order='random',
random_state=i))
#chains.append(ClassifierChain(LogisticRegression()))
#chains.append(ClassifierChain(LogisticRegression(), order=range(100), random_state=i))
f**k = 0
for chain in chains:
print f**k + 1
chain.fit(train_nlcd, train_label)
f**k += 1
print 'testing'
# Y_pred_chains = np.array([chain.predict(X_test) for chain in
# chains])
# # chain_jaccard_scores = [jaccard_similarity_score(Y_test, Y_pred_chain >= .5)
# for Y_pred_chain in Y_pred_chains]
# Y_pred_ensemble = Y_pred_chains.mean(axis=0)
# ensemble_jaccard_score = jaccard_similarity_score(Y_test,
# Y_pred_ensemble >= .5)
# model_scores = [ovr_jaccard_score] + chain_jaccard_scores
# model_scores.append(ensemble_jaccard_score)
scores = []
for chain in chains:
pre = chain.predict_proba(test_nlcd)
#np.save('pre.npy',pre)
chain_score = log_likelihood(test_label, pre)
print chain_score
scores.append(chain_score)
scores = np.array(scores)
print 'mean:'
print np.mean(scores)
def cc():
print 'reading npy...'
data = np.load('../data/1st.npy')
feature_data = np.load('dnn_feature.npy')
train_order = np.load('../data/train.npy')
validation_order = np.load('../data/validation.npy')
test_order = np.load('../data/test.npy')
train_nlcd = get_data.get_feature(feature_data,train_order)
train_label = get_data.get_label(data,train_order)
test_nlcd = get_data.get_feature(feature_data,test_order)
test_label = get_data.get_label(data,test_order)
print 'chaining'
# Fit an ensemble of logistic regression classifier chains and take the
# take the average prediction of all the chains.
chains = []
for i in range(10):
chains.append(ClassifierChain(LogisticRegression(), order='random', random_state=i))
#chains.append(ClassifierChain(LogisticRegression()))
#chains.append(ClassifierChain(LogisticRegression(), order=range(100), random_state=i))
f**k = 0
for chain in chains:
print f**k+1
chain.fit(train_nlcd, train_label)
f**k+=1
print 'testing'
# Y_pred_chains = np.array([chain.predict(X_test) for chain in
# chains])
# # chain_jaccard_scores = [jaccard_similarity_score(Y_test, Y_pred_chain >= .5)
# for Y_pred_chain in Y_pred_chains]
# Y_pred_ensemble = Y_pred_chains.mean(axis=0)
# ensemble_jaccard_score = jaccard_similarity_score(Y_test,
# Y_pred_ensemble >= .5)
# model_scores = [ovr_jaccard_score] + chain_jaccard_scores
# model_scores.append(ensemble_jaccard_score)
scores = []
for chain in chains:
pre = chain.predict_proba(test_nlcd)
#np.save('pre.npy',pre)
chain_score = log_likelihood(test_label,pre)
print chain_score
scores.append(chain_score)
scores = np.array(scores)
print 'mean:'
print np.mean(scores)
def test_step():
print 'Testing...'
# all_ce_loss = 0
# all_output = []
# all_label = []
all_feature = []
batch_size = 23
for i in range(int(len(train_order)/batch_size)):
input_image = get_data.get_jpg_test(train_jpg,train_order[batch_size*i:batch_size*(i+1)])/128.0
input_label = get_data.get_label(train_label,train_order[batch_size*i:batch_size*(i+1)])
feed_dict={}
feed_dict[hg.input_image]=input_image
feed_dict[hg.input_label]=input_label
feed_dict[hg.keep_prob]=1.0
feature = sess.run(hg.feature,feed_dict)
# all_ce_loss += ce_loss
# for i in output:
# all_output.append(i)
# for i in input_label:
# all_label.append(i)
all_feature.append(feature)
# all_output = np.array(all_output)
# all_label = np.array(all_label)
#average_precision = average_precision_score(all_label,all_output)
all_feature = np.array(all_feature)
all_feature = np.reshape(all_feature,(-1,256))
np.save('train_feature.npy',all_feature)
def test_step():
print 'Testing...'
all_z = []
batch_size = 23
for i in range(int(len(test_order)/batch_size)):
input_image = get_data.get_jpg_test(test_jpg,test_order[batch_size*i:batch_size*(i+1)])/128.0
input_label = get_data.get_label(test_label,test_order[batch_size*i:batch_size*(i+1)])
feed_dict={}
feed_dict[hg.input_image]=input_image
feed_dict[hg.input_label]=input_label
feed_dict[hg.keep_prob]=1.0
z_miu= sess.run(hg.output,feed_dict)
all_z.append(z_miu)
all_z = np.array(all_z)
all_z = np.reshape(all_z,(-1,17))
np.save('all_output.npy',all_z)
def validation_step(current_step):
print 'Validating...'
all_ce_loss = 0
all_l2_loss = 0
all_total_loss = 0
all_output = []
all_label = []
for i in range(int(len(validation_order) / 18)):
input_image = get_data.get_jpg_test(
jpg_list, validation_order[18 * i:18 * (i + 1)]) / 128.0
input_label = get_data.get_label(
data, validation_order[18 * i:18 * (i + 1)])
input_nlcd = get_data.get_nlcd(
data, validation_order[18 * i:18 * (i + 1)])
feed_dict = {}
feed_dict[hg.input_image] = input_image
feed_dict[hg.input_nlcd] = input_nlcd
feed_dict[hg.input_label] = input_label
feed_dict[hg.keep_prob] = 1.0
ce_loss, l2_loss, total_loss, output = sess.run(
[hg.ce_loss, hg.l2_loss, hg.total_loss, hg.output], feed_dict)
all_ce_loss += ce_loss
all_l2_loss += l2_loss
all_total_loss += total_loss
for i in output:
all_output.append(i)
for i in input_label:
all_label.append(i)
all_output = np.array(all_output)
all_label = np.array(all_label)
auc = roc_auc_score(all_label, all_output)
ce_loss = all_ce_loss / 283.0
l2_loss = all_l2_loss / 283.0
total_loss = all_total_loss / 283.0
all_output = np.reshape(all_output, (-1))
all_label = np.reshape(all_label, (-1))
ap = average_precision_score(all_label, all_output)
time_str = datetime.datetime.now().isoformat()
tempstr = "{}: auc {:g}, ap {:g}, ce_loss {:g}, l2_loss {:g}, total_loss {:g}".format(
time_str, auc, ap, ce_loss, l2_loss, total_loss)
print(tempstr)
summary_writer.add_summary(MakeSummary('validation/auc', auc),
current_step)
summary_writer.add_summary(MakeSummary('validation/ap', ap),
current_step)
summary_writer.add_summary(MakeSummary('validation/ce_loss', ce_loss),
current_step)
return ce_loss
def test_step():
print 'Testing...'
all_recon_loss = 0
all_kl_loss = 0
all_vae_loss = 0
all_l2_loss = 0
all_total_loss = 0
all_output = []
all_label = []
for i in range(int(len(test_order)/18)):
input_image = get_data.get_jpg_test(jpg_list,test_order[18*i:18*(i+1)])/128.0
input_nlcd = get_data.get_nlcd(data,test_order[18*i:18*(i+1)])
input_label = get_data.get_label(data,test_order[18*i:18*(i+1)])
feed_dict={}
feed_dict[hg.input_nlcd]=input_nlcd
feed_dict[hg.input_image]=input_image
feed_dict[hg.input_label]=input_label
feed_dict[hg.keep_prob]=1.0
recon_loss,kl_loss,vae_loss,l2_loss,total_loss,output= sess.run([hg.recon_loss,hg.kl_loss,hg.vae_loss,hg.l2_loss,hg.total_loss,hg.output],feed_dict)
all_recon_loss += recon_loss
all_kl_loss += kl_loss
all_vae_loss += vae_loss
all_l2_loss += l2_loss
all_total_loss += total_loss
for i in output:
all_output.append(i)
for i in input_label:
all_label.append(i)
all_output = np.array(all_output)
all_label = np.array(all_label)
auc = roc_auc_score(all_label,all_output)
recon_loss = all_recon_loss/283.0
kl_loss = all_kl_loss/283.0
vae_loss = all_vae_loss/283.0
l2_loss = all_l2_loss/283.0
total_loss = all_total_loss/283.0
all_output=np.reshape(all_output,(-1))
all_label=np.reshape(all_label,(-1))
ap = average_precision_score(all_label,all_output)
time_str = datetime.datetime.now().isoformat()
tempstr = "{}: auc {:g}, ap {:g}, recon_loss {:g}, kl_loss {:g}, vae_loss {:g}, l2_loss {:g}, total_loss {:g}".format(time_str, auc, ap, recon_loss, kl_loss, vae_loss, l2_loss, total_loss)
print(tempstr)
auc = roc_auc_score(all_label,all_output)
print "new_auc {:g}".format(auc)
def test_step():
print 'Testing...'
all_recon_loss = 0
all_kl_loss = 0
all_vae_loss = 0
all_l2_loss = 0
all_total_loss = 0
all_output = []
all_label = []
for i in range(int(len(test_order)/18)):
input_image = get_data.get_jpg_test(jpg_list,test_order[18*i:18*(i+1)])
input_nlcd = get_data.get_nlcd(data,test_order[18*i:18*(i+1)])
input_label = get_data.get_label(data,test_order[18*i:18*(i+1)])
feed_dict={}
feed_dict[hg.input_nlcd]=input_nlcd
feed_dict[hg.input_image]=input_image
feed_dict[hg.input_label]=input_label
feed_dict[hg.keep_prob]=1.0
recon_loss,kl_loss,vae_loss,l2_loss,total_loss,output= sess.run([hg.recon_loss,hg.kl_loss,hg.vae_loss,hg.l2_loss,hg.total_loss,hg.output],feed_dict)
all_recon_loss += recon_loss
all_kl_loss += kl_loss
all_vae_loss += vae_loss
all_l2_loss += l2_loss
all_total_loss += total_loss
for i in output:
all_output.append(i)
for i in input_label:
all_label.append(i)
all_output = np.array(all_output)
all_label = np.array(all_label)
auc = roc_auc_score(all_label,all_output)
recon_loss = all_recon_loss/283.0
kl_loss = all_kl_loss/283.0
vae_loss = all_vae_loss/283.0
l2_loss = all_l2_loss/283.0
total_loss = all_total_loss/283.0
all_output=np.reshape(all_output,(-1))
all_label=np.reshape(all_label,(-1))
ap = average_precision_score(all_label,all_output)
time_str = datetime.datetime.now().isoformat()
tempstr = "{}: auc {:g}, ap {:g}, recon_loss {:g}, kl_loss {:g}, vae_loss {:g}, l2_loss {:g}, total_loss {:g}".format(time_str, auc, ap, recon_loss, kl_loss, vae_loss, l2_loss, total_loss)
print(tempstr)
auc = roc_auc_score(all_label,all_output)
print "new_auc {:g}".format(auc)
def test_step():
print 'Testing...'
all_ce_loss = 0
all_output = []
all_label = []
batch_size = 18
for i in range(int(len(test_order) / batch_size)):
input_image = get_data.get_jpg_test(
jpg_list, test_order[batch_size * i:batch_size *
(i + 1)]) / 128.0
input_label = get_data.get_label(
data, test_order[batch_size * i:batch_size * (i + 1)])
input_nlcd = get_data.get_nlcd(
data, test_order[batch_size * i:batch_size * (i + 1)])
feed_dict = {}
feed_dict[hg.input_image] = input_image
feed_dict[hg.input_label] = input_label
feed_dict[hg.input_nlcd] = input_nlcd
feed_dict[hg.keep_prob] = 1.0
ce_loss, output = sess.run([hg.ce_loss, hg.output], feed_dict)
all_ce_loss += ce_loss
for i in output:
all_output.append(i)
for i in input_label:
all_label.append(i)
all_output = np.array(all_output)
all_label = np.array(all_label)
#average_precision = average_precision_score(all_label,all_output)
loglike = all_ce_loss / (int(len(test_order) / batch_size))
np.save('output.npy', all_output)
np.save('label.npy', all_label)
auc = roc_auc_score(all_label, all_output)
#loglike = log_likelihood(all_label,all_output)
time_str = datetime.datetime.now().isoformat()
tempstr = "{}: auc {:g}, log_likelihood {:g}".format(
time_str, auc, loglike)
print(tempstr)
all_output = np.reshape(all_output, (-1))
all_label = np.reshape(all_label, (-1))
ap = average_precision_score(all_label, all_output)
auc_2 = roc_auc_score(all_label, all_output)
print 'ap:' + str(ap)
print 'auc_2:' + str(auc_2)
def test_step():
print 'Testing...'
all_ce_loss = 0
all_l2_loss = 0
all_total_loss = 0
all_output = []
all_label = []
for i in range(int(len(test_order)/18)):
input_nlcd = get_data.get_nlcd(data,test_order[18*i:18*(i+1)])
input_label = get_data.get_label(data,test_order[18*i:18*(i+1)])
feed_dict={}
feed_dict[hg.input_nlcd]=input_nlcd
feed_dict[hg.input_label]=input_label
feed_dict[hg.keep_prob]=1.0
ce_loss,l2_loss,total_loss,output= sess.run([hg.ce_loss,hg.l2_loss,hg.total_loss,hg.output],feed_dict)
all_ce_loss += ce_loss
all_l2_loss += l2_loss
all_total_loss += total_loss
for i in output:
all_output.append(i)
for i in input_label:
all_label.append(i)
all_output = np.array(all_output)
all_label = np.array(all_label)
auc = roc_auc_score(all_label,all_output)
ce_loss = all_ce_loss/283.0
l2_loss = all_l2_loss/283.0
total_loss = all_total_loss/283.0
all_output=np.reshape(all_output,(-1))
all_label=np.reshape(all_label,(-1))
ap = average_precision_score(all_label,all_output)
np.save('all_output_1wavg.npy',all_output)
np.save('all_label.npy',all_label)
time_str = datetime.datetime.now().isoformat()
tempstr = "{}: auc {:g}, ap {:g}, ce_loss {:g}, l2_loss {:g}, total_loss {:g}".format(time_str, auc, ap, ce_loss, l2_loss, total_loss)
print(tempstr)
auc = roc_auc_score(all_label,all_output)
print "new_auc {:g}".format(auc)
def test_step(test_idx, name="Test"):
print('{}...'.format(name))
all_nll_loss = 0
all_l2_loss = 0
all_c_loss = 0
all_total_loss = 0
all_indiv_prob = []
all_label = []
all_indiv_max = []
sigma = []
real_batch_size = min(FLAGS.testing_size, len(test_idx))
N_test_batch = int((len(test_idx) - 1) / real_batch_size) + 1
for i in range(N_test_batch):
if i % 20 == 0:
print("%.1f%% completed" % (i * 100.0 / N_test_batch))
start = real_batch_size * i
end = min(real_batch_size * (i + 1), len(test_idx))
input_feat = get_data.get_feat(data, test_idx[start:end])
input_label = get_data.get_label(data, test_idx[start:end])
feed_dict = {}
feed_dict[classifier.input_feat] = input_feat
feed_dict[classifier.input_label] = input_label
feed_dict[classifier.keep_prob] = 1.0
nll_loss, l2_loss, c_loss, total_loss, indiv_prob, covariance = sess.run([classifier.nll_loss, classifier.l2_loss, classifier.c_loss, \
classifier.total_loss, classifier.indiv_prob, classifier.covariance], feed_dict)
all_nll_loss += nll_loss * (end - start)
all_l2_loss += l2_loss * (end - start)
all_c_loss += c_loss * (end - start)
all_total_loss += total_loss * (end - start)
if (all_indiv_prob == []):
all_indiv_prob = indiv_prob
else:
all_indiv_prob = np.concatenate((all_indiv_prob, indiv_prob))
if (all_label == []):
all_label = input_label
else:
all_label = np.concatenate((all_label, input_label))
nll_loss = all_nll_loss / len(test_idx)
l2_loss = all_l2_loss / len(test_idx)
c_loss = all_c_loss / len(test_idx)
total_loss = all_total_loss / len(test_idx)
return all_indiv_prob, all_label
def validation_step(current_step):
print 'Validating...'
all_ce_loss = 0
all_l2_loss = 0
all_total_loss = 0
all_output = []
all_label = []
valid_batch = 23
for i in range(int(len(validation_order)/valid_batch)):
input_image = get_data.get_jpg_test(validation_jpg,validation_order[valid_batch*i:valid_batch*(i+1)])/128.0
input_label = get_data.get_label(validation_label,validation_order[valid_batch*i:valid_batch*(i+1)])
feed_dict={}
feed_dict[hg.input_image]=input_image
feed_dict[hg.input_label]=input_label
feed_dict[hg.keep_prob]=1.0
ce_loss,l2_loss,total_loss,output= sess.run([hg.ce_loss,hg.l2_loss,hg.total_loss,hg.output],feed_dict)
all_ce_loss += ce_loss
all_l2_loss += l2_loss
all_total_loss += total_loss
for i in output:
all_output.append(i)
for i in input_label:
all_label.append(i)
all_output = np.array(all_output)
all_label = np.array(all_label)
#auc = roc_auc_score(all_label,all_output)
ce_loss = all_ce_loss/(4048/valid_batch)
l2_loss = all_l2_loss/(4048/valid_batch)
total_loss = all_total_loss/(4048/valid_batch)
all_output=np.reshape(all_output,(-1))
all_label=np.reshape(all_label,(-1))
ap = average_precision_score(all_label,all_output)
auc = roc_auc_score(all_label,all_output)
time_str = datetime.datetime.now().isoformat()
tempstr = "{}: auc {:g}, ap {:g}, ce_loss {:g}, l2_loss {:g}, total_loss {:g}".format(time_str, auc, ap, ce_loss, l2_loss, total_loss)
print(tempstr)
summary_writer.add_summary(MakeSummary('validation/auc',auc),current_step)
summary_writer.add_summary(MakeSummary('validation/ap',ap),current_step)
summary_writer.add_summary(MakeSummary('validation/ce_loss',ce_loss),current_step)
return ce_loss
def test_step():
print 'Testing...'
all_recon_loss = 0
all_output = []
all_label = []
batch_size = FLAGS.batch_size
for i in range(int(len(test_order) / batch_size)):
input_nlcd = get_data.get_nlcd(
data, test_order[batch_size * i:batch_size * (i + 1)])
input_image = get_data.get_jpg_test(
jpg_list, test_order[batch_size * i:batch_size *
(i + 1)]) / 128.0
input_label = get_data.get_label(
data, test_order[batch_size * i:batch_size * (i + 1)])
feed_dict = {}
feed_dict[hg.input_nlcd] = input_nlcd
feed_dict[hg.input_image] = input_image
feed_dict[hg.input_label] = input_label
feed_dict[hg.keep_prob] = 1.0
output = sess.run(hg.output, feed_dict)
for i in output:
all_output.append(i)
for i in input_label:
all_label.append(i)
#recon_loss = all_recon_loss/(5094.0/batch_size)
all_output = np.array(all_output)
all_label = np.array(all_label)
# auc = roc_auc_score(all_label,all_output)
# all_output=np.reshape(all_output,(-1))
# all_label=np.reshape(all_label,(-1))
# ap = average_precision_score(all_label,all_output)
# time_str = datetime.datetime.now().isoformat()
# new_auc = roc_auc_score(all_label,all_output)
# tempstr = "{}: auc {:g}, ap {:g}, recon_loss {:g}, new_auc {:g}".format(time_str, auc, ap, recon_loss, new_auc)
# print(tempstr)
all_output = np.reshape(all_output, (-1, 100))
all_label = np.reshape(all_label, (-1, 100))
return all_output, all_label
def test_step():
print 'Testing...'
all_ce_loss = 0
all_output = []
all_label = []
batch_size = 18
for i in range(int(len(test_order)/batch_size)):
input_image = get_data.get_jpg_test(jpg_list,test_order[batch_size*i:batch_size*(i+1)])/128.0
input_label = get_data.get_label(data,test_order[batch_size*i:batch_size*(i+1)])
input_nlcd = get_data.get_nlcd(data,test_order[batch_size*i:batch_size*(i+1)])
feed_dict={}
feed_dict[hg.input_image]=input_image
feed_dict[hg.input_label]=input_label
feed_dict[hg.input_nlcd]=input_nlcd
feed_dict[hg.keep_prob]=1.0
ce_loss,output= sess.run([hg.ce_loss,hg.output],feed_dict)
all_ce_loss += ce_loss
for i in output:
all_output.append(i)
for i in input_label:
all_label.append(i)
all_output = np.array(all_output)
all_label = np.array(all_label)
#average_precision = average_precision_score(all_label,all_output)
loglike = all_ce_loss/(int(len(test_order)/batch_size))
np.save('output.npy',all_output)
np.save('label.npy',all_label)
auc = roc_auc_score(all_label,all_output)
#loglike = log_likelihood(all_label,all_output)
time_str = datetime.datetime.now().isoformat()
tempstr = "{}: auc {:g}, log_likelihood {:g}".format(time_str, auc,loglike)
print(tempstr)
all_output=np.reshape(all_output,(-1))
all_label=np.reshape(all_label,(-1))
ap = average_precision_score(all_label,all_output)
auc_2 = roc_auc_score(all_label,all_output)
print 'ap:'+str(ap)
print 'auc_2:'+str(auc_2)
def validation_step():
print 'Validating...'
all_ce_loss = 0
all_l2_loss = 0
all_total_loss = 0
all_output = []
all_label = []
for i in range(int(len(validation_order) / 18)):
input_nlcd = get_data.get_nlcd(
data, validation_order[18 * i:18 * (i + 1)])
input_label = get_data.get_label(
data, validation_order[18 * i:18 * (i + 1)])
feed_dict = {}
feed_dict[hg.input_nlcd] = input_nlcd
feed_dict[hg.input_label] = input_label
feed_dict[hg.keep_prob] = 1.0
ce_loss, l2_loss, total_loss, output = sess.run(
[hg.ce_loss, hg.l2_loss, hg.total_loss, hg.output], feed_dict)
all_ce_loss += ce_loss
all_l2_loss += l2_loss
all_total_loss += total_loss
for i in output:
all_output.append(i)
for i in input_label:
all_label.append(i)
all_output = np.array(all_output)
all_label = np.array(all_label)
#average_precision = average_precision_score(all_label,all_output)
auc = roc_auc_score(all_label, all_output)
ce_loss = all_ce_loss / 283.0
l2_loss = all_l2_loss / 283.0
total_loss = all_total_loss / 283.0
time_str = datetime.datetime.now().isoformat()
tempstr = "{}: auc {:g}, ce_loss {:g}, l2_loss {:g}, total_loss {:g}".format(
time_str, auc, ce_loss, l2_loss, total_loss)
print(tempstr)
all_output = np.reshape(all_output, (-1))
all_label = np.reshape(all_label, (-1))
ap = average_precision_score(all_label, all_output)
print 'ap:' + str(ap)
return ce_loss
def test_step():
print 'testing...'
# all_ce_loss = 0
# all_l2_loss = 0
# all_total_loss = 0
# all_output = []
# all_label = []
all_feature = []
batch_size = 17 * 3
for i in range(int(len(test_order) / batch_size)):
input_image = get_data.get_jpg_test(
jpg_list, test_order[batch_size * i:batch_size * (i + 1)])
input_label = get_data.get_label(
data, test_order[batch_size * i:batch_size * (i + 1)])
feed_dict = {}
feed_dict[hg.input_image] = input_image
feature = sess.run(hg.feature, feed_dict)
for i in feature:
all_feature.append(i)
# for i in input_label:
# all_label.append(i)
# all_output = np.array(all_output)
# all_label = np.array(all_label)
# #average_precision = average_precision_score(all_label,all_output)
# np.save('output.npy',all_output)
# np.save('label.npy',all_label)
# auc = roc_auc_score(all_label,all_output)
# loglike = log_likelihood(all_label,all_output)
# time_str = datetime.datetime.now().isoformat()
# tempstr = "{}: auc {:g}, log_likelihood {:g}".format(time_str, auc, loglike)
# print(tempstr)
# all_output=np.reshape(all_output,(-1))
# all_label=np.reshape(all_label,(-1))
# ap = average_precision_score(all_label,all_output)
# auc_2 = roc_auc_score(all_label,all_output)
# print 'ap:'+str(ap)
# print 'auc_2:'+str(auc_2)
np.save('resnet50_feature.npy', all_feature)
def test_step():
print 'Testing...'
all_recon_loss = 0
all_output = []
all_label = []
batch_size = FLAGS.batch_size
for i in range(int(len(test_order)/batch_size)):
input_nlcd = get_data.get_nlcd(data,test_order[batch_size*i:batch_size*(i+1)])
input_image = get_data.get_jpg_test(jpg_list,test_order[batch_size*i:batch_size*(i+1)])
input_label = get_data.get_label(data,test_order[batch_size*i:batch_size*(i+1)])
feed_dict={}
feed_dict[hg.input_nlcd]=input_nlcd
feed_dict[hg.input_image]=input_image
feed_dict[hg.input_label]=input_label
feed_dict[hg.keep_prob]=1.0
output= sess.run(hg.output,feed_dict)
for i in output:
all_output.append(i)
for i in input_label:
all_label.append(i)
#recon_loss = all_recon_loss/(5094.0/batch_size)
all_output = np.array(all_output)
all_label = np.array(all_label)
# auc = roc_auc_score(all_label,all_output)
# all_output=np.reshape(all_output,(-1))
# all_label=np.reshape(all_label,(-1))
# ap = average_precision_score(all_label,all_output)
# time_str = datetime.datetime.now().isoformat()
# new_auc = roc_auc_score(all_label,all_output)
# tempstr = "{}: auc {:g}, ap {:g}, recon_loss {:g}, new_auc {:g}".format(time_str, auc, ap, recon_loss, new_auc)
# print(tempstr)
all_output=np.reshape(all_output,(-1,100))
all_label=np.reshape(all_label,(-1,100))
return all_output,all_label
def test_step():
print 'testing...'
# all_ce_loss = 0
# all_l2_loss = 0
# all_total_loss = 0
# all_output = []
# all_label = []
all_feature = []
batch_size=17*3
for i in range(int(len(test_order)/batch_size)):
input_image = get_data.get_jpg_test(jpg_list,test_order[batch_size*i:batch_size*(i+1)])
input_label = get_data.get_label(data,test_order[batch_size*i:batch_size*(i+1)])
feed_dict={}
feed_dict[hg.input_image]=input_image
feature = sess.run(hg.feature,feed_dict)
for i in feature:
all_feature.append(i)
# for i in input_label:
# all_label.append(i)
# all_output = np.array(all_output)
# all_label = np.array(all_label)
# #average_precision = average_precision_score(all_label,all_output)
# np.save('output.npy',all_output)
# np.save('label.npy',all_label)
# auc = roc_auc_score(all_label,all_output)
# loglike = log_likelihood(all_label,all_output)
# time_str = datetime.datetime.now().isoformat()
# tempstr = "{}: auc {:g}, log_likelihood {:g}".format(time_str, auc, loglike)
# print(tempstr)
# all_output=np.reshape(all_output,(-1))
# all_label=np.reshape(all_label,(-1))
# ap = average_precision_score(all_label,all_output)
# auc_2 = roc_auc_score(all_label,all_output)
# print 'ap:'+str(ap)
# print 'auc_2:'+str(auc_2)
np.save('resnet50_feature.npy',all_feature)
def test_step():
print 'testing...'
all_ce_loss = 0
all_l2_loss = 0
all_total_loss = 0
all_output = []
all_label = []
for i in range(int(len(test_order) / 18)):
input_image = get_data.get_jpg_test(
jpg_list, test_order[18 * i:18 * (i + 1)])
input_label = get_data.get_label(data,
test_order[18 * i:18 * (i + 1)])
feed_dict = {}
feed_dict[hg.input_image] = input_image
output = sess.run(hg.output, feed_dict)
for i in output:
all_output.append(i)
for i in input_label:
all_label.append(i)
all_output = np.array(all_output)
all_label = np.array(all_label)
#average_precision = average_precision_score(all_label,all_output)
np.save('output.npy', all_output)
np.save('label.npy', all_label)
auc = roc_auc_score(all_label, all_output)
loglike = log_likelihood(all_label, all_output)
time_str = datetime.datetime.now().isoformat()
tempstr = "{}: auc {:g}, log_likelihood {:g}".format(
time_str, auc, loglike)
print(tempstr)
all_output = np.reshape(all_output, (-1))
all_label = np.reshape(all_label, (-1))
ap = average_precision_score(all_label, all_output)
auc_2 = roc_auc_score(all_label, all_output)
print 'ap:' + str(ap)
print 'auc_2:' + str(auc_2)
def test_step():
print 'testing...'
all_output = []
all_label = []
for i in range(int(len(test_order)/18)):
input_nlcd = get_data.get_nlcd(data,test_order[18*i:18*(i+1)])
input_label = get_data.get_label(data,test_order[18*i:18*(i+1)])
feed_dict={}
feed_dict[hg.input_nlcd]=input_nlcd
feed_dict[hg.input_label]=input_label
feed_dict[hg.keep_prob]=1.0
output= sess.run(hg.output,feed_dict)
for i in output:
all_output.append(i)
for i in input_label:
all_label.append(i)
all_output = np.array(all_output)
all_label = np.array(all_label)
#average_precision = average_precision_score(all_label,all_output)
np.save('output.npy',all_output)
np.save('label.npy',all_label)
auc = roc_auc_score(all_label,all_output,average='micro')
loglike = log_likelihood(all_label,all_output)
time_str = datetime.datetime.now().isoformat()
tempstr = "{}: auc {:g}, log_likelihood {:g}".format(time_str, auc,loglike)
print(tempstr)
all_output=np.reshape(all_output,(-1))
all_label=np.reshape(all_label,(-1))
ap = average_precision_score(all_label,all_output)
auc_2 = roc_auc_score(all_label,all_output)
print 'ap:'+str(ap)
print 'auc_2:'+str(auc_2)
def test_step():
print 'Testing...'
# names = ['forest','human','ocean']
all_sample_z = []
batch_size = 17
# for i in range(len(orders)):
# input_image = get_data.get_jpg_test(jpg_list,orders[i])/128.0
# input_nlcd = get_data.get_nlcd(data,orders[i])
# input_label = get_data.get_label(data,orders[i])
for i in range(int(len(orders) / batch_size)):
input_image = get_data.get_jpg_test(
jpg_list, orders[batch_size * i:batch_size * (i + 1)]) / 128.0
input_nlcd = get_data.get_nlcd(
data, orders[batch_size * i:batch_size * (i + 1)])
input_label = get_data.get_label(
data, orders[batch_size * i:batch_size * (i + 1)])
feed_dict = {}
feed_dict[hg.input_nlcd] = input_nlcd
feed_dict[hg.input_image] = input_image
feed_dict[hg.input_label] = input_label
feed_dict[hg.keep_prob] = 1.0
sample_z = sess.run(hg.sample_z, feed_dict)
all_sample_z.append(sample_z)
all_sample_z = np.array(all_sample_z)
all_sample_z = np.reshape(all_sample_z, (-1, 100))
np.save('sample_z.npy', all_sample_z)
def test_step():
print 'Testing...'
# names = ['forest','human','ocean']
all_sample_z=[]
batch_size = 17
# for i in range(len(orders)):
# input_image = get_data.get_jpg_test(jpg_list,orders[i])/128.0
# input_nlcd = get_data.get_nlcd(data,orders[i])
# input_label = get_data.get_label(data,orders[i])
for i in range(int(len(orders)/batch_size)):
input_image = get_data.get_jpg_test(jpg_list,orders[batch_size*i:batch_size*(i+1)])
input_nlcd = get_data.get_nlcd(data,orders[batch_size*i:batch_size*(i+1)])
input_label = get_data.get_label(data,orders[batch_size*i:batch_size*(i+1)])
feed_dict={}
feed_dict[hg.input_nlcd]=input_nlcd
feed_dict[hg.input_image]=input_image
feed_dict[hg.input_label]=input_label
feed_dict[hg.keep_prob]=1.0
#sample_z= sess.run(hg.sample_z,feed_dict)
sample_z= sess.run(hg.condition_miu,feed_dict)
all_sample_z.append(sample_z)
all_sample_z = np.array(all_sample_z)
all_sample_z = np.reshape(all_sample_z,(-1,100))
np.save('condition_miu.npy',all_sample_z)
def test_step():
print 'Testing...'
# all_ce_loss = 0
# all_output = []
# all_label = []
all_feature = []
batch_size = 23
for i in range(int(len(train_order) / batch_size)):
input_image = get_data.get_jpg_test(
train_jpg, train_order[batch_size * i:batch_size *
(i + 1)]) / 128.0
input_label = get_data.get_label(
train_label, train_order[batch_size * i:batch_size * (i + 1)])
feed_dict = {}
feed_dict[hg.input_image] = input_image
feed_dict[hg.input_label] = input_label
feed_dict[hg.keep_prob] = 1.0
feature = sess.run(hg.feature, feed_dict)
# all_ce_loss += ce_loss
# for i in output:
# all_output.append(i)
# for i in input_label:
# all_label.append(i)
all_feature.append(feature)
# all_output = np.array(all_output)
# all_label = np.array(all_label)
#average_precision = average_precision_score(all_label,all_output)
all_feature = np.array(all_feature)
all_feature = np.reshape(all_feature, (-1, 256))
np.save('train_feature.npy', all_feature)
def test_step():
print 'Testing...'
all_recon_loss = 0
all_output = []
all_label = []
batch_size = FLAGS.batch_size
for i in range(int(len(test_order) / batch_size)):
input_image = get_data.get_jpg_test(
test_jpg, test_order[batch_size * i:batch_size *
(i + 1)]) / 128.0
input_label = get_data.get_label(
test_label, test_order[batch_size * i:batch_size * (i + 1)])
feed_dict = {}
feed_dict[hg.input_image] = input_image
feed_dict[hg.input_label] = input_label
feed_dict[hg.keep_prob] = 1.0
output = sess.run(hg.output, feed_dict)
for i in output:
all_output.append(i)
for i in input_label:
all_label.append(i)
all_output = np.array(all_output)
all_label = np.array(all_label)
all_output = np.reshape(all_output, (-1, 17))
all_label = np.reshape(all_label, (-1, 17))
return all_output, all_label
def test_step():
print 'Testing...'
all_recon_loss = 0
all_output = []
all_label = []
batch_size = FLAGS.batch_size
for i in range(int(len(test_order)/batch_size)):
input_image = get_data.get_jpg_test(test_jpg,test_order[batch_size*i:batch_size*(i+1)])/128.0
input_label = get_data.get_label(test_label,test_order[batch_size*i:batch_size*(i+1)])
feed_dict={}
feed_dict[hg.input_image]=input_image
feed_dict[hg.input_label]=input_label
feed_dict[hg.keep_prob]=1.0
output= sess.run(hg.output,feed_dict)
for i in output:
all_output.append(i)
for i in input_label:
all_label.append(i)
all_output = np.array(all_output)
all_label = np.array(all_label)
all_output=np.reshape(all_output,(-1,17))
all_label=np.reshape(all_label,(-1,17))
return all_output,all_label
def test_step():
print 'testing...'
all_output = []
for i in range(int(len(test_order) / 17)):
input_nlcd = get_data.get_nlcd(data,
test_order[17 * i:17 * (i + 1)])
input_label = get_data.get_label(data,
test_order[17 * i:17 * (i + 1)])
feed_dict = {}
feed_dict[hg.input_nlcd] = input_nlcd
feed_dict[hg.input_label] = input_label
feed_dict[hg.keep_prob] = 1.0
output = sess.run(hg.feature, feed_dict)
for i in output:
all_output.append(i)
all_output = np.array(all_output)
np.save('dnn_feature.npy', all_output)
def main(_):
print 'reading npy...'
data = np.load('../data/1st.npy')
train_order = np.load('../data/train.npy')
validation_order = np.load('../data/validation.npy')
one_epoch_iter = len(train_order)/FLAGS.batch_size
print 'reading finished'
sess = tf.Session()
print 'building network...'
hg = vae.vae(is_training=True)
global_step = tf.Variable(0,name='global_step',trainable=False)
learning_rate = tf.train.exponential_decay(FLAGS.learning_rate,global_step,0.5*FLAGS.max_epoch*len(train_order)/FLAGS.batch_size,0.1,staircase=True)
tf.summary.scalar('learning_rate', learning_rate)
optimizer = tf.train.AdamOptimizer(learning_rate)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op = optimizer.minimize(hg.total_loss,global_step=global_step)
merged_summary = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(FLAGS.summary_dir,sess.graph)
sess.run(tf.initialize_all_variables())
saver = tf.train.Saver(max_to_keep=5)
print 'building finished'
#def train_step(input_nlcd,input_label,smooth_ce_loss,smooth_l2_loss,smooth_total_loss):
def train_step(input_label,input_nlcd):
feed_dict={}
feed_dict[hg.input_nlcd]=input_nlcd
feed_dict[hg.input_label]=input_label
feed_dict[hg.keep_prob]=0.8
temp,step,ce_loss,l2_loss,total_loss,summary,output = sess.run([train_op,global_step,hg.ce_loss,hg.l2_loss,hg.total_loss,merged_summary,hg.output],feed_dict)
time_str = datetime.datetime.now().isoformat()
summary_writer.add_summary(summary,step)
# smooth_ce_loss+=ce_loss
# smooth_l2_loss+=l2_loss
# smooth_total_loss+=total_loss
# #average_precision = average_precision_score(input_label,output)
# if step%100==0:
# recon_loss=smooth_recon_loss/100.0
# kl_loss=smooth_kl_loss/100.0
# vae_loss=smooth_vae_loss/100.0
# l2_loss=smooth_l2_loss/100.0
# total_loss=smooth_total_loss/100.0
# tempstr = "{}: step {}, recon_loss {:g}, kl_loss {:g}, vae_loss {:g}, l2_loss {:g}, total_loss {:g}".format(time_str, step, recon_loss, kl_loss, vae_loss, l2_loss, total_loss)
# print(tempstr)
# return output,0.0,0.0,0.0,0.0,0.0
return output, ce_loss, l2_loss, total_loss
def validation_step():
print 'Validating...'
all_ce_loss = 0
all_l2_loss = 0
all_total_loss = 0
all_output = []
all_label = []
for i in range(int(len(validation_order)/18)):
input_nlcd = get_data.get_nlcd(data,validation_order[18*i:18*(i+1)])
input_label = get_data.get_label(data,validation_order[18*i:18*(i+1)])
feed_dict={}
feed_dict[hg.input_nlcd]=input_nlcd
feed_dict[hg.input_label]=input_label
feed_dict[hg.keep_prob]=1.0
ce_loss,l2_loss,total_loss,output= sess.run([hg.ce_loss,hg.l2_loss,hg.total_loss,hg.output],feed_dict)
all_ce_loss += ce_loss
all_l2_loss += l2_loss
all_total_loss += total_loss
for i in output:
all_output.append(i)
for i in input_label:
all_label.append(i)
all_output = np.array(all_output)
all_label = np.array(all_label)
#average_precision = average_precision_score(all_label,all_output)
auc = roc_auc_score(all_label,all_output)
ce_loss = all_ce_loss/283.0
l2_loss = all_l2_loss/283.0
total_loss = all_total_loss/283.0
# time_str = datetime.datetime.now().isoformat()
# tempstr = "{}: auc {:g}, ce_loss {:g}, l2_loss {:g}, total_loss {:g}".format(time_str, auc, ce_loss, l2_loss, total_loss)
# print(tempstr)
# all_output=np.reshape(all_output,(-1))
# all_label=np.reshape(all_label,(-1))
# ap = average_precision_score(all_label,all_output)
# print 'ap:'+str(ap)
# return ce_loss
all_output=np.reshape(all_output,(-1))
all_label=np.reshape(all_label,(-1))
ap = average_precision_score(all_label,all_output)
time_str = datetime.datetime.now().isoformat()
tempstr = "{}: auc {:g}, ap {:g}, ce_loss {:g}, l2_loss {:g}, total_loss {:g}".format(time_str, auc, ap, ce_loss, l2_loss, total_loss)
print(tempstr)
summary_writer.add_summary(MakeSummary('validation/auc',auc),current_step)
summary_writer.add_summary(MakeSummary('validation/ap',ap),current_step)
summary_writer.add_summary(MakeSummary('validation/ce_loss',ce_loss),current_step)
return ce_loss
best_vae_loss = 10000
best_iter = 0
smooth_ce_loss=0.0
smooth_l2_loss=0.0
smooth_total_loss=0.0
temp_label=[]
temp_output=[]
for one_epoch in range(FLAGS.max_epoch):
print('epoch '+str(one_epoch+1)+' starts!')
np.random.shuffle(train_order)
for i in range(int(len(train_order)/float(FLAGS.batch_size))):
start = i*FLAGS.batch_size
end = (i+1)*FLAGS.batch_size
input_nlcd = get_data.get_nlcd(data,train_order[start:end])
input_label = get_data.get_label(data,train_order[start:end])
# output,smooth_ce_loss,smooth_l2_loss,smooth_total_loss = train_step(input_nlcd,input_label,smooth_ce_loss,smooth_l2_loss,smooth_total_loss)
# temp_label.append(input_label)
# temp_output.append(output)
# current_step = tf.train.global_step(sess,global_step)
# if current_step%100==0:
# temp_output = np.reshape(np.array(temp_output),(-1))
# temp_label = np.reshape(np.array(temp_label),(-1))
# average_precision = average_precision_score(temp_label,temp_output)
# print 'pr_area:'+str(average_precision)
# temp_output=[]
# temp_label=[]
output, ce_loss, l2_loss, total_loss = train_step(input_label,input_nlcd)
smooth_ce_loss+=ce_loss
smooth_l2_loss+=l2_loss
smooth_total_loss+=total_loss
temp_label.append(input_label)
temp_output.append(output)
current_step = tf.train.global_step(sess,global_step)
if current_step%10==0:
ce_loss=smooth_ce_loss/10.0
l2_loss=smooth_l2_loss/10.0
total_loss=smooth_total_loss/10.0
temp_output = np.reshape(np.array(temp_output),(-1))
temp_label = np.reshape(np.array(temp_label),(-1))
ap = average_precision_score(temp_label,temp_output)
temp_output = np.reshape(temp_output,(-1,100))
temp_label = np.reshape(temp_label,(-1,100))
try:
auc = roc_auc_score(temp_label,temp_output)
except ValueError:
print 'ytrue error for auc'
else:
time_str = datetime.datetime.now().isoformat()
tempstr = "{}: step {}, auc {:g}, ap {:g}, ce_loss {:g}, l2_loss {:g}, total_loss {:g}".format(time_str, current_step, auc, ap, ce_loss, l2_loss, total_loss)
print(tempstr)
summary_writer.add_summary(MakeSummary('train/auc',auc),current_step)
summary_writer.add_summary(MakeSummary('train/ap',ap),current_step)
temp_output=[]
temp_label=[]
smooth_ce_loss = 0
smooth_l2_loss = 0
smooth_total_loss = 0
if current_step%int(one_epoch_iter*FLAGS.save_epoch)==0:
vae_loss = validation_step()
if vae_loss<best_vae_loss:
print 'currently the ce_loss is over the previous best one!!!'
best_vae_loss=vae_loss
best_iter = current_step
print 'saving model'
path = saver.save(sess,FLAGS.model_dir+'fc_model',global_step=current_step)
print 'have saved model to '+path
print 'training has been finished !'
print 'the best vae_loss on validation is '+str(best_vae_loss)
print 'the best checkpoint is '+str(best_iter)
def main(_):
print 'reading npy...'
train_jpg = np.load('../amazon_data/train_image64.npy')
train_order = range(len(train_jpg))
train_label = np.load('../amazon_data/train_label.npy')
validation_jpg = np.load('../amazon_data/validation_image64.npy')
validation_order = range(len(validation_jpg))
validation_label = np.load('../amazon_data/validation_label.npy')
one_epoch_iter = len(train_order)/FLAGS.batch_size
print 'reading finished'
sess = tf.Session()
print 'building network...'
hg = fc.fc(is_training=True)
global_step = tf.Variable(0,name='global_step',trainable=False)
learning_rate = tf.train.exponential_decay(FLAGS.learning_rate,global_step,0.5*FLAGS.max_epoch*len(train_order)/FLAGS.batch_size,1.0,staircase=True)
tf.summary.scalar('learning_rate', learning_rate)
optimizer = tf.train.AdamOptimizer(learning_rate)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op = optimizer.minimize(hg.total_loss,global_step=global_step)
merged_summary = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(FLAGS.summary_dir,sess.graph)
sess.run(tf.initialize_all_variables())
saver = tf.train.Saver(max_to_keep=5)
print 'building finished'
def train_step(input_image,input_label):
feed_dict={}
feed_dict[hg.input_image]=input_image
feed_dict[hg.input_label]=input_label
feed_dict[hg.keep_prob]=0.5
temp,step,ce_loss,l2_loss,total_loss,summary,output= sess.run([train_op,global_step,hg.ce_loss,hg.l2_loss,hg.total_loss,merged_summary,hg.output],feed_dict)
summary_writer.add_summary(summary,step)
return output,ce_loss,l2_loss,total_loss
def validation_step(current_step):
print 'Validating...'
all_ce_loss = 0
all_l2_loss = 0
all_total_loss = 0
all_output = []
all_label = []
valid_batch = 23
for i in range(int(len(validation_order)/valid_batch)):
input_image = get_data.get_jpg_test(validation_jpg,validation_order[valid_batch*i:valid_batch*(i+1)])/128.0
input_label = get_data.get_label(validation_label,validation_order[valid_batch*i:valid_batch*(i+1)])
feed_dict={}
feed_dict[hg.input_image]=input_image
feed_dict[hg.input_label]=input_label
feed_dict[hg.keep_prob]=1.0
ce_loss,l2_loss,total_loss,output= sess.run([hg.ce_loss,hg.l2_loss,hg.total_loss,hg.output],feed_dict)
all_ce_loss += ce_loss
all_l2_loss += l2_loss
all_total_loss += total_loss
for i in output:
all_output.append(i)
for i in input_label:
all_label.append(i)
all_output = np.array(all_output)
all_label = np.array(all_label)
#auc = roc_auc_score(all_label,all_output)
ce_loss = all_ce_loss/(4048/valid_batch)
l2_loss = all_l2_loss/(4048/valid_batch)
total_loss = all_total_loss/(4048/valid_batch)
all_output=np.reshape(all_output,(-1))
all_label=np.reshape(all_label,(-1))
ap = average_precision_score(all_label,all_output)
auc = roc_auc_score(all_label,all_output)
time_str = datetime.datetime.now().isoformat()
tempstr = "{}: auc {:g}, ap {:g}, ce_loss {:g}, l2_loss {:g}, total_loss {:g}".format(time_str, auc, ap, ce_loss, l2_loss, total_loss)
print(tempstr)
summary_writer.add_summary(MakeSummary('validation/auc',auc),current_step)
summary_writer.add_summary(MakeSummary('validation/ap',ap),current_step)
summary_writer.add_summary(MakeSummary('validation/ce_loss',ce_loss),current_step)
return ce_loss
best_ce_loss = 10000
best_iter = 0
smooth_ce_loss = 0
smooth_l2_loss = 0
smooth_total_loss = 0
temp_label=[]
temp_output=[]
for one_epoch in range(FLAGS.max_epoch):
print('epoch '+str(one_epoch+1)+' starts!')
np.random.shuffle(train_order)
for i in range(int(len(train_order)/float(FLAGS.batch_size))):
start = i*FLAGS.batch_size
end = (i+1)*FLAGS.batch_size
input_image = get_data.get_jpg_train(train_jpg,train_order[start:end])/128.0
input_label = get_data.get_label(train_label,train_order[start:end])
output,ce_loss,l2_loss,total_loss = train_step(input_image,input_label)
smooth_ce_loss+=ce_loss
smooth_l2_loss+=l2_loss
smooth_total_loss+=total_loss
temp_label.append(input_label)
temp_output.append(output)
current_step = tf.train.global_step(sess,global_step)
#print current_step
if current_step%10==0:
ce_loss=smooth_ce_loss/10.0
l2_loss=smooth_l2_loss/10.0
total_loss=smooth_total_loss/10.0
temp_output = np.reshape(np.array(temp_output),(-1))
temp_label = np.reshape(np.array(temp_label),(-1))
ap = average_precision_score(temp_label,temp_output)
# temp_output = np.reshape(temp_output,(-1,17))
# temp_label = np.reshape(temp_label,(-1,17))
try:
auc = roc_auc_score(temp_label,temp_output)
except ValueError:
print 'ytrue error for auc'
else:
time_str = datetime.datetime.now().isoformat()
tempstr = "{}: step {}, auc {:g}, ap {:g}, ce_loss {:g}, l2_loss {:g}, total_loss {:g}".format(time_str, current_step, auc, ap,ce_loss, l2_loss, total_loss)
print(tempstr)
summary_writer.add_summary(MakeSummary('train/auc',auc),current_step)
summary_writer.add_summary(MakeSummary('train/ap',ap),current_step)
temp_output=[]
temp_label=[]
smooth_ce_loss = 0
smooth_l2_loss = 0
smooth_total_loss = 0
if current_step%int(one_epoch_iter*FLAGS.save_epoch)==0:
ce_loss = validation_step(current_step)
if ce_loss<best_ce_loss:
print 'currently the validation ce_loss is less the previous best one!!!'
best_ce_loss=ce_loss
best_iter=current_step
print 'saving model'
path = saver.save(sess,FLAGS.model_dir+'fc_model',global_step=current_step)
print 'have saved model to '+path
print 'warmup training has been finished !'
print 'the best model iter is '+str(best_iter)
print 'the best ce_loss on validation is '+str(best_ce_loss)
def validation_step():
print 'Validating...'
all_recon_loss = 0
all_kl_loss = 0
all_vae_loss = 0
all_l2_loss = 0
all_total_loss = 0
all_output = []
all_label = []
for i in range(int(len(validation_order)/18)):
input_image = get_data.get_jpg_test(jpg_list,validation_order[18*i:18*(i+1)])
input_nlcd = get_data.get_nlcd(data,validation_order[18*i:18*(i+1)])
input_label = get_data.get_label(data,validation_order[18*i:18*(i+1)])
feed_dict={}
feed_dict[hg.input_nlcd]=input_nlcd
feed_dict[hg.input_label]=input_label
feed_dict[hg.input_image]=input_image
feed_dict[hg.keep_prob]=1.0
recon_loss,kl_loss,vae_loss,l2_loss,total_loss,output= sess.run([hg.recon_loss,hg.kl_loss,hg.vae_loss,hg.l2_loss,hg.total_loss,hg.output],feed_dict)
all_recon_loss += recon_loss
all_kl_loss += kl_loss
all_vae_loss += vae_loss
all_l2_loss += l2_loss
all_total_loss += total_loss
for i in output:
all_output.append(i)
for i in input_label:
all_label.append(i)
all_output = np.array(all_output)
all_label = np.array(all_label)
#average_precision = average_precision_score(all_label,all_output)
auc = roc_auc_score(all_label,all_output)
recon_loss = all_recon_loss/283.0
kl_loss = all_kl_loss/283.0
vae_loss = all_vae_loss/283.0
l2_loss = all_l2_loss/283.0
total_loss = all_total_loss/283.0
all_output=np.reshape(all_output,(-1))
all_label=np.reshape(all_label,(-1))
ap = average_precision_score(all_label,all_output)
time_str = datetime.datetime.now().isoformat()
tempstr = "{}: auc {:g}, ap {:g}, recon_loss {:g}, kl_loss {:g}, vae_loss {:g}, l2_loss {:g}, total_loss {:g}".format(time_str, auc, ap, recon_loss, kl_loss, vae_loss, l2_loss, total_loss)
print(tempstr)
summary_writer.add_summary(MakeSummary('validation/auc',auc),current_step)
summary_writer.add_summary(MakeSummary('validation/ap',ap),current_step)
summary_writer.add_summary(MakeSummary('validation/recon_loss',recon_loss),current_step)
summary_writer.add_summary(MakeSummary('validation/kl_loss',kl_loss),current_step)
summary_writer.add_summary(MakeSummary('validation/vae_loss',vae_loss),current_step)
return vae_loss
def main(_):
print 'reading npy...'
data = np.load('../1st.npy')
jpg_list = np.load('128bin.npy')
train_order = np.load('../train.npy')
validation_order = np.load('../validation.npy')
one_epoch_iter = len(train_order)/FLAGS.batch_size
print 'reading finished'
sess = tf.Session()
print 'building network...'
hg = vae.vae(is_training=True)
global_step = tf.Variable(0,name='global_step',trainable=False)
learning_rate = tf.train.exponential_decay(FLAGS.learning_rate,global_step,0.5*FLAGS.max_epoch*len(train_order)/FLAGS.batch_size,1.0,staircase=True)
tf.summary.scalar('learning_rate', learning_rate)
optimizer = tf.train.AdamOptimizer(learning_rate)
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op = optimizer.minimize(hg.total_loss,global_step=global_step)
merged_summary = tf.summary.merge_all()
summary_writer = tf.summary.FileWriter(FLAGS.summary_dir,sess.graph)
sess.run(tf.initialize_all_variables())
saver = tf.train.Saver(max_to_keep=None)
print 'building finished'
#def train_step(input_nlcd,input_label,smooth_ce_loss,smooth_l2_loss,smooth_total_loss):
def train_step(input_label,input_nlcd,input_image):
feed_dict={}
feed_dict[hg.input_nlcd]=input_nlcd
feed_dict[hg.input_label]=input_label
feed_dict[hg.input_image]=input_image
feed_dict[hg.keep_prob]=0.8
temp,step,recon_loss,kl_loss,vae_loss,l2_loss,total_loss,summary,output = sess.run([train_op,global_step,hg.recon_loss,hg.kl_loss,hg.vae_loss,hg.l2_loss,hg.total_loss,merged_summary,hg.output],feed_dict)
time_str = datetime.datetime.now().isoformat()
summary_writer.add_summary(summary,step)
return output, recon_loss, kl_loss, vae_loss, l2_loss, total_loss
def validation_step():
print 'Validating...'
all_recon_loss = 0
all_kl_loss = 0
all_vae_loss = 0
all_l2_loss = 0
all_total_loss = 0
all_output = []
all_label = []
for i in range(int(len(validation_order)/18)):
input_image = get_data.get_jpg_test(jpg_list,validation_order[18*i:18*(i+1)])
input_nlcd = get_data.get_nlcd(data,validation_order[18*i:18*(i+1)])
input_label = get_data.get_label(data,validation_order[18*i:18*(i+1)])
feed_dict={}
feed_dict[hg.input_nlcd]=input_nlcd
feed_dict[hg.input_label]=input_label
feed_dict[hg.input_image]=input_image
feed_dict[hg.keep_prob]=1.0
recon_loss,kl_loss,vae_loss,l2_loss,total_loss,output= sess.run([hg.recon_loss,hg.kl_loss,hg.vae_loss,hg.l2_loss,hg.total_loss,hg.output],feed_dict)
all_recon_loss += recon_loss
all_kl_loss += kl_loss
all_vae_loss += vae_loss
all_l2_loss += l2_loss
all_total_loss += total_loss
for i in output:
all_output.append(i)
for i in input_label:
all_label.append(i)
all_output = np.array(all_output)
all_label = np.array(all_label)
#average_precision = average_precision_score(all_label,all_output)
auc = roc_auc_score(all_label,all_output)
recon_loss = all_recon_loss/283.0
kl_loss = all_kl_loss/283.0
vae_loss = all_vae_loss/283.0
l2_loss = all_l2_loss/283.0
total_loss = all_total_loss/283.0
all_output=np.reshape(all_output,(-1))
all_label=np.reshape(all_label,(-1))
ap = average_precision_score(all_label,all_output)
time_str = datetime.datetime.now().isoformat()
tempstr = "{}: auc {:g}, ap {:g}, recon_loss {:g}, kl_loss {:g}, vae_loss {:g}, l2_loss {:g}, total_loss {:g}".format(time_str, auc, ap, recon_loss, kl_loss, vae_loss, l2_loss, total_loss)
print(tempstr)
summary_writer.add_summary(MakeSummary('validation/auc',auc),current_step)
summary_writer.add_summary(MakeSummary('validation/ap',ap),current_step)
summary_writer.add_summary(MakeSummary('validation/recon_loss',recon_loss),current_step)
summary_writer.add_summary(MakeSummary('validation/kl_loss',kl_loss),current_step)
summary_writer.add_summary(MakeSummary('validation/vae_loss',vae_loss),current_step)
return vae_loss
best_vae_loss = 10000
best_iter = 0
smooth_recon_loss=0.0
smooth_kl_loss=0.0
smooth_vae_loss=0.0
smooth_l2_loss=0.0
smooth_total_loss=0.0
temp_label=[]
temp_output=[]
for one_epoch in range(FLAGS.max_epoch):
print('epoch '+str(one_epoch+1)+' starts!')
np.random.shuffle(train_order)
for i in range(int(len(train_order)/float(FLAGS.batch_size))):
start = i*FLAGS.batch_size
end = (i+1)*FLAGS.batch_size
input_image = get_data.get_jpg_test(jpg_list,train_order[start:end])
input_nlcd = get_data.get_nlcd(data,train_order[start:end])
input_label = get_data.get_label(data,train_order[start:end])
output, recon_loss, kl_loss, vae_loss, l2_loss, total_loss = train_step(input_label,input_nlcd,input_image)
smooth_recon_loss+=recon_loss
smooth_kl_loss+=kl_loss
smooth_vae_loss+=vae_loss
smooth_l2_loss+=l2_loss
smooth_total_loss+=total_loss
temp_label.append(input_label)
temp_output.append(output)
current_step = tf.train.global_step(sess,global_step)
if current_step%10==0:
recon_loss=smooth_recon_loss/10.0
kl_loss=smooth_kl_loss/10.0
vae_loss=smooth_vae_loss/10.0
l2_loss=smooth_l2_loss/10.0
total_loss=smooth_total_loss/10.0
temp_output = np.reshape(np.array(temp_output),(-1))
temp_label = np.reshape(np.array(temp_label),(-1))
ap = average_precision_score(temp_label,temp_output)
temp_output = np.reshape(temp_output,(-1,100))
temp_label = np.reshape(temp_label,(-1,100))
try:
auc = roc_auc_score(temp_label,temp_output)
except ValueError:
print 'ytrue error for auc'
else:
time_str = datetime.datetime.now().isoformat()
tempstr = "{}: step {}, auc {:g}, ap {:g}, recon_loss {:g}, kl_loss {:g}, vae_loss {:g}, l2_loss {:g}, total_loss {:g}".format(time_str, current_step, auc, ap, recon_loss, kl_loss, vae_loss, l2_loss, total_loss)
print(tempstr)
summary_writer.add_summary(MakeSummary('train/auc',auc),current_step)
summary_writer.add_summary(MakeSummary('train/ap',ap),current_step)
temp_output=[]
temp_label=[]
smooth_recon_loss = 0
smooth_kl_loss = 0
smooth_vae_loss = 0
smooth_l2_loss = 0
smooth_total_loss = 0
if current_step%int(one_epoch_iter*FLAGS.save_epoch)==0:
vae_loss = validation_step()
if vae_loss<best_vae_loss:
print 'currently the vae_loss is over the previous best one!!!'
best_vae_loss=vae_loss
best_iter = current_step
print 'saving model'
path = saver.save(sess,FLAGS.model_dir+'fc_model',global_step=current_step)
print 'have saved model to '+path
print 'training has been finished !'
print 'the best vae_loss on validation is '+str(best_vae_loss)
print 'the best checkpoint is '+str(best_iter)
def validation_step():
print 'Validating...'
all_recon_loss = 0
all_kl_loss = 0
all_vae_loss = 0
all_l2_loss = 0
all_total_loss = 0
all_output = []
all_label = []
for i in range(int(len(validation_order)/18)):
input_nlcd = get_data.get_nlcd(data,validation_order[18*i:18*(i+1)])
input_label = get_data.get_label(data,validation_order[18*i:18*(i+1)])
feed_dict={}
feed_dict[hg.input_nlcd]=input_nlcd
feed_dict[hg.input_label]=input_label
feed_dict[hg.keep_prob]=1.0
recon_loss,kl_loss,vae_loss,l2_loss,total_loss,output= sess.run([hg.recon_loss,hg.kl_loss,hg.vae_loss,hg.l2_loss,hg.total_loss,hg.output],feed_dict)
all_recon_loss += recon_loss
all_kl_loss += kl_loss
all_vae_loss += vae_loss
all_l2_loss += l2_loss
all_total_loss += total_loss
for i in output:
all_output.append(i)
for i in input_label:
all_label.append(i)
all_output = np.array(all_output)
all_label = np.array(all_label)
#average_precision = average_precision_score(all_label,all_output)
auc = roc_auc_score(all_label,all_output)
recon_loss = all_recon_loss/283.0
kl_loss = all_kl_loss/283.0
vae_loss = all_vae_loss/283.0
l2_loss = all_l2_loss/283.0
total_loss = all_total_loss/283.0
# time_str = datetime.datetime.now().isoformat()
# tempstr = "{}: auc {:g}, ce_loss {:g}, l2_loss {:g}, total_loss {:g}".format(time_str, auc, ce_loss, l2_loss, total_loss)
# print(tempstr)
# all_output=np.reshape(all_output,(-1))
# all_label=np.reshape(all_label,(-1))
# ap = average_precision_score(all_label,all_output)
# print 'ap:'+str(ap)
# return ce_loss
all_output=np.reshape(all_output,(-1))
all_label=np.reshape(all_label,(-1))
ap = average_precision_score(all_label,all_output)
time_str = datetime.datetime.now().isoformat()
tempstr = "{}: auc {:g}, ap {:g}, recon_loss {:g}, kl_loss {:g}, vae_loss {:g}, l2_loss {:g}, total_loss {:g}".format(time_str, auc, ap, recon_loss, kl_loss, vae_loss, l2_loss, total_loss)
print(tempstr)
summary_writer.add_summary(MakeSummary('validation/auc',auc),current_step)
summary_writer.add_summary(MakeSummary('validation/ap',ap),current_step)
summary_writer.add_summary(MakeSummary('validation/recon_loss',recon_loss),current_step)
summary_writer.add_summary(MakeSummary('validation/kl_loss',kl_loss),current_step)
summary_writer.add_summary(MakeSummary('validation/vae_loss',vae_loss),current_step)
return vae_loss
def validation_step(sess,
hg,
data,
merged_summary,
summary_writer,
valid_idx,
global_step,
mode='val'):
print('%s...' % mode)
all_nll_loss = 0
all_l2_loss = 0
all_c_loss = 0
all_total_loss = 0
all_indiv_prob = []
all_label = []
real_batch_size = min(FLAGS.batch_size, len(valid_idx))
for i in range(int((len(valid_idx) - 1) / real_batch_size) + 1):
start = real_batch_size * i
end = min(real_batch_size * (i + 1), len(valid_idx))
input_feat = get_data.get_feat(data, valid_idx[start:end])
input_label = get_data.get_label(data, valid_idx[start:end])
feed_dict = {}
feed_dict[hg.input_feat] = input_feat
feed_dict[hg.input_label] = input_label
feed_dict[hg.keep_prob] = 1.0
nll_loss, l2_loss, c_loss, total_loss, indiv_prob = sess.run(
[hg.nll_loss, hg.l2_loss, hg.c_loss, hg.total_loss, hg.indiv_prob],
feed_dict)
all_nll_loss += nll_loss * (end - start)
all_l2_loss += l2_loss * (end - start)
all_c_loss += c_loss * (end - start)
all_total_loss += total_loss * (end - start)
for i in indiv_prob:
all_indiv_prob.append(i)
for i in input_label:
all_label.append(i)
# collect all predictions and ground-truths
all_indiv_prob = np.array(all_indiv_prob)
all_label = np.array(all_label)
nll_loss = all_nll_loss / len(valid_idx)
l2_loss = all_l2_loss / len(valid_idx)
c_loss = all_c_loss / len(valid_idx)
total_loss = all_total_loss / len(valid_idx)
best_val_metrics = None
if mode == 'val':
for threshold in THRESHOLDS:
val_metrics = evals.compute_metrics(all_indiv_prob,
all_label,
threshold,
all_metrics=True)
if best_val_metrics == None:
best_val_metrics = {}
for metric in METRICS:
best_val_metrics[metric] = val_metrics[metric]
else:
for metric in METRICS:
if 'FDR' in metric:
best_val_metrics[metric] = min(
best_val_metrics[metric], val_metrics[metric])
else:
best_val_metrics[metric] = max(
best_val_metrics[metric], val_metrics[metric])
time_str = datetime.datetime.now().isoformat()
acc, ha, ebf1, maf1, mif1 = best_val_metrics['ACC'], best_val_metrics[
'HA'], best_val_metrics['ebF1'], best_val_metrics[
'maF1'], best_val_metrics['miF1']
# nll_coeff: BCE coeff, lambda_1
# c_coeff: Ranking loss coeff, lambda_2
# l2_coeff: weight decay
print("**********************************************")
print(
"%s results: %s\nacc=%.6f\tha=%.6f\texam_f1=%.6f, macro_f1=%.6f, micro_f1=%.6f\nnll_loss=%.6f\tl2_loss=%.6f\tc_loss=%.6f\ttotal_loss=%.6f"
%
(mode, time_str, acc, ha, ebf1, maf1, mif1, nll_loss * FLAGS.nll_coeff,
l2_loss * FLAGS.l2_coeff, c_loss * FLAGS.c_coeff, total_loss))
print("**********************************************")
current_step = sess.run(global_step) #get the value of global_step
summary_writer.add_summary(MakeSummary('%s/nll_loss' % mode, nll_loss),
current_step)
summary_writer.add_summary(MakeSummary('%s/l2_loss' % mode, l2_loss),
current_step)
summary_writer.add_summary(MakeSummary('%s/c_loss' % mode, c_loss),
current_step)
summary_writer.add_summary(MakeSummary('%s/total_loss' % mode, total_loss),
current_step)
summary_writer.add_summary(MakeSummary('%s/macro_f1' % mode, maf1),
current_step)
summary_writer.add_summary(MakeSummary('%s/micro_f1' % mode, mif1),
current_step)
summary_writer.add_summary(MakeSummary('%s/exam_f1' % mode, ebf1),
current_step)
summary_writer.add_summary(MakeSummary('%s/acc' % mode, acc), current_step)
summary_writer.add_summary(MakeSummary('%s/ha' % mode, ha), current_step)
return nll_loss, best_val_metrics
def main(_):
print('reading npy...')
np.random.seed(19940423) # set the random seed of numpy
data = np.load(FLAGS.data_dir) #load data from the data_dir
train_idx = np.load(FLAGS.train_idx) #load the indices of the training set
valid_idx = np.load(
FLAGS.valid_idx) #load the indices of the validation set
test_idx = np.load(FLAGS.test_idx)
labels = get_data.get_label(
data, train_idx) #load the labels of the training set
print("min:", np.amin(labels))
print("max:", np.amax(labels))
print("positive label rate:", np.mean(
labels)) #print the rate of the positive labels in the training set
param_setting = "lr-{}_lr-decay_{:.2f}_lr-times_{:.1f}_nll-{:.2f}_l2-{:.2f}_c-{:.2f}".format(
FLAGS.learning_rate, FLAGS.lr_decay_ratio, FLAGS.lr_decay_times,
FLAGS.nll_coeff, FLAGS.l2_coeff, FLAGS.c_coeff)
build_path(FLAGS.summary_dir + param_setting)
build_path('model/model_{}/{}'.format(FLAGS.dataname, param_setting))
one_epoch_iter = len(
train_idx
) / FLAGS.batch_size # compute the number of iterations in each epoch
print("one_epoch_iter:", one_epoch_iter)
print('reading completed')
# config the tensorflow
session_config = tf.compat.v1.ConfigProto()
session_config.gpu_options.allow_growth = True
sess = tf.compat.v1.Session(config=session_config)
print('showing the parameters...\n')
# print all the hyper-parameters in the current training
for key in FLAGS:
print("%s\t%s" % (key, FLAGS[key].value))
print()
print('building network...')
#building the model
hg = model.MODEL(is_training=True)
global_step = tf.Variable(0, name='global_step', trainable=False)
learning_rate = tf.compat.v1.train.exponential_decay(
FLAGS.learning_rate,
global_step,
one_epoch_iter * (FLAGS.max_epoch / FLAGS.lr_decay_times),
FLAGS.lr_decay_ratio,
staircase=True)
#log the learning rate
tf.compat.v1.summary.scalar('learning_rate', learning_rate)
#use the Adam optimizer
optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate)
reset_optimizer_op = tf.compat.v1.variables_initializer(
optimizer.variables())
#set training update ops/backpropagation
var_x_encoder = tf.compat.v1.trainable_variables('feat_encoder')
update_ops = tf.compat.v1.get_collection(tf.compat.v1.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
if FLAGS.resume:
train_op = optimizer.minimize(hg.total_loss,
var_list=var_x_encoder,
global_step=global_step)
else:
train_op = optimizer.minimize(hg.total_loss,
global_step=global_step)
merged_summary = tf.compat.v1.summary.merge_all(
) # gather all summary nodes together
summary_writer = tf.compat.v1.summary.FileWriter(
FLAGS.summary_dir + param_setting + "/",
sess.graph) #initialize the summary writer
sess.run(tf.compat.v1.global_variables_initializer()
) # initialize the global variables in tensorflow
saver = tf.compat.v1.train.Saver(
max_to_keep=FLAGS.max_keep) #initializae the model saver
if FLAGS.saved_ckpt != "":
saver.restore(sess, FLAGS.saved_ckpt)
print('building finished')
#initialize several
best_loss = 1e10
best_iter = 0
best_macro_f1 = 0.
best_micro_f1 = 0.
# smooth means average. Every batch has a mean loss value w.r.t. different losses
smooth_nll_loss = 0.0 # label encoder decoder cross entropy loss
smooth_nll_loss_x = 0.0 # feature encoder decoder cross entropy loss
smooth_l2_loss = 0.0 # weights regularization
smooth_c_loss = 0.0 # label encoder decoder ranking loss
smooth_c_loss_x = 0.0 # feature encoder decoder ranking loss
smooth_kl_loss = 0.0 # kl divergence
smooth_total_loss = 0.0 # total loss
smooth_macro_f1 = 0.0 # macro_f1 score
smooth_micro_f1 = 0.0 # micro_f1 score
best_macro_f1 = 0.0 # best macro f1 for ckpt selection in validation
best_micro_f1 = 0.0 # best micro f1 for ckpt selection in validation
best_acc = 0.0 # best subset acc for ckpt selction in validation
temp_label = []
temp_indiv_prob = []
best_test_metrics = None
# training the model
for one_epoch in range(FLAGS.max_epoch):
print('epoch ' + str(one_epoch + 1) + ' starts!')
np.random.shuffle(train_idx) # random shuffle the training indices
for i in range(int(len(train_idx) / float(FLAGS.batch_size))):
start = i * FLAGS.batch_size
end = (i + 1) * FLAGS.batch_size
input_feat = get_data.get_feat(
data, train_idx[start:end]) # get the NLCD features
input_label = get_data.get_label(
data, train_idx[start:end]) # get the prediction labels
#train the model for one step and log the training loss
indiv_prob, nll_loss, nll_loss_x, l2_loss, c_loss, c_loss_x, kl_loss, total_loss, macro_f1, micro_f1 = train_step(
sess, hg, merged_summary, summary_writer, input_label,
input_feat, train_op, global_step)
smooth_nll_loss += nll_loss
smooth_nll_loss_x += nll_loss_x
smooth_l2_loss += l2_loss
smooth_c_loss += c_loss
smooth_c_loss_x += c_loss_x
smooth_kl_loss += kl_loss
smooth_total_loss += total_loss
smooth_macro_f1 += macro_f1
smooth_micro_f1 += micro_f1
temp_label.append(input_label) #log the labels
temp_indiv_prob.append(
indiv_prob
) #log the individual prediction of the probability on each label
current_step = sess.run(global_step) #get the value of global_step
lr = sess.run(learning_rate)
summary_writer.add_summary(MakeSummary('learning_rate', lr),
current_step)
if current_step % FLAGS.check_freq == 0: #summarize the current training status and print them out
nll_loss = smooth_nll_loss / float(FLAGS.check_freq)
nll_loss_x = smooth_nll_loss_x / float(FLAGS.check_freq)
l2_loss = smooth_l2_loss / float(FLAGS.check_freq)
c_loss = smooth_c_loss / float(FLAGS.check_freq)
c_loss_x = smooth_c_loss_x / float(FLAGS.check_freq)
kl_loss = smooth_kl_loss / float(FLAGS.check_freq)
total_loss = smooth_total_loss / float(FLAGS.check_freq)
macro_f1 = smooth_macro_f1 / float(FLAGS.check_freq)
micro_f1 = smooth_micro_f1 / float(FLAGS.check_freq)
temp_indiv_prob = np.reshape(np.array(temp_indiv_prob), (-1))
temp_label = np.reshape(np.array(temp_label), (-1))
temp_indiv_prob = np.reshape(temp_indiv_prob,
(-1, FLAGS.label_dim))
temp_label = np.reshape(temp_label, (-1, FLAGS.label_dim))
time_str = datetime.datetime.now().isoformat()
print(
"step=%d %s\nlr=%.6f\nmacro_f1=%.6f, micro_f1=%.6f\nnll_loss=%.6f\tnll_loss_x=%.6f\tl2_loss=%.6f\nc_loss=%.6f\tc_loss_x=%.6f\tkl_loss=%.6f\ntotal_loss=%.6f\n"
%
(current_step, time_str, lr, macro_f1, micro_f1,
nll_loss * FLAGS.nll_coeff, nll_loss_x * FLAGS.nll_coeff,
l2_loss * FLAGS.l2_coeff, c_loss * FLAGS.c_coeff,
c_loss_x * FLAGS.c_coeff, kl_loss, total_loss))
temp_indiv_prob = []
temp_label = []
smooth_nll_loss = 0
smooth_nll_loss_x = 0
smooth_l2_loss = 0
smooth_c_loss = 0
smooth_c_loss_x = 0
smooth_kl_loss = 0
smooth_total_loss = 0
smooth_macro_f1 = 0
smooth_micro_f1 = 0
if current_step % int(
one_epoch_iter *
FLAGS.save_epoch) == 0: #exam the model on validation set
print("--------------------------------")
# exam the model on validation set
current_loss, val_metrics = validation_step(
sess, hg, data, merged_summary, summary_writer, valid_idx,
global_step, 'val')
macro_f1, micro_f1 = val_metrics['maF1'], val_metrics['miF1']
# select the best checkpoint based on some metric on the validation set
# here we use macro F1 as the selection metric but one can use others
if val_metrics['maF1'] > best_macro_f1:
print(
'macro_f1:%.6f, micro_f1:%.6f, nll_loss:%.6f, which is better than the previous best one!!!'
% (macro_f1, micro_f1, current_loss))
best_loss = current_loss
best_iter = current_step
print('saving model')
saved_model_path = saver.save(sess,
FLAGS.model_dir +
param_setting + '/model',
global_step=current_step)
print('have saved model to ', saved_model_path)
print()
if FLAGS.write_to_test_sh:
ckptFile = open(
FLAGS.test_sh_path.replace('ebird',
FLAGS.dataname), "r")
command = []
for line in ckptFile:
arg_lst = line.strip().split(' ')
for arg in arg_lst:
if 'model/model_{}/lr-'.format(
FLAGS.dataname) in arg:
command.append(
'model/model_{}/{}/model-{}'.format(
FLAGS.dataname, param_setting,
best_iter))
else:
command.append(arg)
ckptFile.close()
ckptFile = open(
FLAGS.test_sh_path.replace('ebird',
FLAGS.dataname), "w")
ckptFile.write(" ".join(command) + "\n")
ckptFile.close()
best_macro_f1 = max(best_macro_f1, val_metrics['maF1'])
best_micro_f1 = max(best_micro_f1, val_metrics['miF1'])
best_acc = max(best_acc, val_metrics['ACC'])
print("--------------------------------")
def validation_step():
print 'Validating...'
all_recon_loss = 0
all_kl_loss = 0
all_vae_loss = 0
all_l2_loss = 0
all_total_loss = 0
all_output = []
all_label = []
valid_batch = 23
for i in range(int(len(validation_order) / valid_batch)):
input_image = get_data.get_jpg_test(
validation_jpg,
validation_order[valid_batch * i:valid_batch * (i + 1)])
input_label = get_data.get_label(
validation_label,
validation_order[valid_batch * i:valid_batch * (i + 1)])
feed_dict = {}
feed_dict[hg.input_label] = input_label
feed_dict[hg.input_image] = input_image
feed_dict[hg.keep_prob] = 1.0
recon_loss, kl_loss, vae_loss, l2_loss, total_loss, output = sess.run(
[
hg.recon_loss, hg.kl_loss, hg.vae_loss, hg.l2_loss,
hg.total_loss, hg.output
], feed_dict)
all_recon_loss += recon_loss
all_kl_loss += kl_loss
all_vae_loss += vae_loss
all_l2_loss += l2_loss
all_total_loss += total_loss
for i in output:
all_output.append(i)
for i in input_label:
all_label.append(i)
all_output = np.array(all_output)
all_label = np.array(all_label)
#average_precision = average_precision_score(all_label,all_output)
recon_loss = all_recon_loss / (4048 / valid_batch)
kl_loss = all_kl_loss / (4048 / valid_batch)
vae_loss = all_vae_loss / (4048 / valid_batch)
l2_loss = all_l2_loss / (4048 / valid_batch)
total_loss = all_total_loss / (4048 / valid_batch)
all_output = np.reshape(all_output, (-1))
all_label = np.reshape(all_label, (-1))
ap = average_precision_score(all_label, all_output)
auc = roc_auc_score(all_label, all_output)
time_str = datetime.datetime.now().isoformat()
tempstr = "{}: auc {:g}, ap {:g}, recon_loss {:g}, kl_loss {:g}, vae_loss {:g}, l2_loss {:g}, total_loss {:g}".format(
time_str, auc, ap, recon_loss, kl_loss, vae_loss, l2_loss,
total_loss)
print(tempstr)
summary_writer.add_summary(MakeSummary('validation/auc', auc),
current_step)
summary_writer.add_summary(MakeSummary('validation/ap', ap),
current_step)
summary_writer.add_summary(
MakeSummary('validation/recon_loss', recon_loss), current_step)
summary_writer.add_summary(MakeSummary('validation/kl_loss', kl_loss),
current_step)
summary_writer.add_summary(
MakeSummary('validation/vae_loss', vae_loss), current_step)
return vae_loss