# img_size=[]
#
# for name in names:
# img_path = os.path.join(path, name)
# a = Image.open(img_path)
# img_size.append(a.size)
# image = a.resize((256, 256), Image.ANTIALIAS)
#
# images[names.index(name), :, :, :] = np.array(image)
#
#
graph=tf.get_default_graph()
# writer = tf.summary.FileWriter('load', graph=graph)
# writer.flush()
batch_images, batch_labels = load_batch_data.batch_samples(1, '/home/weic/project/linux/train.tfrecords', True)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord, sess=sess1)
for ite in range(100):
print(ite)
images, labels = sess1.run([batch_images, batch_labels])
#打印所有变量
# for op in graph.get_operations():
# print(op.name,' ')
input=graph.get_tensor_by_name('input/input_images:0')
# loss=graph.get_tensor_by_name('loss/cross_entropy_loss:0')
output=graph.get_tensor_by_name('inference/output:0')
test=sess1.run(output,feed_dict={input:images})
#img=np.sum(test[0],axis=0)
# plt.matshow(img)
def train(lr):
print(learning_rate)
#global_step=tf.train.get_or_create_global_step()
global_step=tf.Variable(0,trainable=False)
print('==create model==')
#with tf.device(device):
with tf.name_scope('input'):
input_image = tf.placeholder(tf.float32, [None, 256, 256, 3], name='input_images')
labels = tf.placeholder(tf.float32, [None, 16, 64, 64], name='labels')
print('--input done--')
with tf.name_scope('inference'):
logits=resnet_model.model(input_image)
print('--inference done--')
with tf.name_scope('loss'):
diff=tf.subtract(logits,labels)
train_loss=tf.reduce_mean(tf.nn.l2_loss(diff,name='l2loss'))
#train_loss=tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels= labels),name='cross_entropy_loss')
print('--loss done--')
#with tf.device(CPU):
with tf.name_scope('accuracy'):
joint_accur=cal_acc(logits,labels,batch_size)
# with tf.name_scope('lr'):
# decay_lr=tf.train.exponential_decay(lr,global_step,decay_steps,decay_rate,staircase,name='learning_rate')#指数式衰减
with tf.name_scope('saver'):
saver=tf.train.Saver()
#with tf.device(device):
with tf.name_scope('train'):
with tf.name_scope('optimizer'):
opti=tf.train.RMSPropOptimizer(learning_rate)
train_op=opti.minimize(train_loss,global_step=global_step)
print('--minimize done--')
init=tf.global_variables_initializer()
print('--init done--')
#with tf.device(CPU):
tf.summary.scalar('loss',train_loss,collections=['train'])
#tf.summary.scalar('learning_rate',decay_lr,collections=['train'])
for i in range(gene_hm.nPoints):
tf.summary.scalar(str(i),joint_accur[i],collections=['train','test'])
merged_summary_train=tf.summary.merge_all('train')
merged_summary_test=tf.summary.merge_all('test')
train_list=list()
val_list=list()
train_list.append(['learning_rate',learning_rate,'training_epoch',Num_Epoch,
'batch_size',batch_size,
])
val_list.append(['val_step','val_loss'])
train_list.append(['train_step','train_loss'])
exma_list=[]
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
config = tf.ConfigProto(allow_soft_placement=True,gpu_options=gpu_options)
config.gpu_options.allow_growth = True
batch_images, batch_labels = load_batch_data.batch_samples(batch_size, filename,True)
val_images, val_labels = load_batch_data.batch_samples(batch_size, filename, True)
with tf.name_scope('Session'):
#with tf.device(device):
sess=tf.Session(config=config)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
sess.run(init)
ckpt=tf.train.get_checkpoint_state('./model/')
if ckpt and ckpt.model_checkpoint_path:
print('load model')
saver.restore(sess,ckpt.model_checkpoint_path )
train_writer = tf.summary.FileWriter('log/train', graph=tf.get_default_graph())
val_writer=tf.summary.FileWriter('log/val')
print('Start train')
with tf.name_scope('training'):
for epoch in range(Num_Epoch):
cost=0.0
val_cost=0.0
print('* * * *第%d个Epoch* * * *'%(epoch+1))
beginTime=time.time()
for i in range(epochSize):
example, l = sess.run([batch_images, batch_labels])
if np.any(np.isnan(example)):
print('no images')
continue
if np.any(np.isnan(l)):
print('no label')
continue
exma_list.append(example.shape)
nor_images = processing.image_normalization(example) # 归一化图像
label = gene_hm.batch_genehm(batch_size, l) # heatmap label
train_step = sess.run(global_step)
if (i+1)%step_to_save==0:
_, loss, summary = sess.run([train_op, train_loss, merged_summary_train],
feed_dict={input_image: nor_images, labels: label})
train_writer.add_summary(summary,train_step)
saver.save(sess,os.path.join(os.getcwd(),'model/model%d.ckpt'%(train_step)))
#print(save_path)
else:
_, loss,output = sess.run([train_op, train_loss,logits],
feed_dict={input_image: nor_images, labels: label})
for i in range(batch_size):
plt.matshow(np.sum(output[0], axis=0))
plt.show()
#csv
train_list.append([train_step,loss])
cost+=loss
print('第%d个batch的loss%f'%(i+1,loss))
epoch_cost=cost/(epochSize)
print('* *第%d个epoch的loss%f* *' % (epoch + 1, epoch_cost))
oneEpoch = time.time()
print('one epoch cost time:', str(oneEpoch - beginTime))
print('* '*20)
valBegin = time.time()
for j in range(valIters):
example, l = sess.run([val_images, val_labels])
if np.any(np.isnan(example)):
print('no images')
continue
if np.any(np.isnan(l)):
print('no label')
continue
nor_images = processing.image_normalization(example) # 归一化图像
label = gene_hm.batch_genehm(batch_size, l) # heatmap label
_, v_loss = sess.run([train_op, train_loss],
feed_dict={input_image: nor_images, labels: label})
val_summaries = sess.run(merged_summary_test, feed_dict={input_image: nor_images, labels: label})
val_step=epoch*10+j+1
val_writer.add_summary(val_summaries, val_step )
val_list.append([val_step,v_loss])
val_cost+=v_loss
val_loss=val_cost/(valIters)
print('val cost time:',str(time.time()-valBegin))
print('val loss:',val_loss)
print('* ' * 20)
train_file = open('result/train.csv', 'w', newline='')
val_file = open('result/val.csv', 'w', newline='')
ex_file=open('result/ex.csv','w',newline='')
train_csv_writer = csv.writer(train_file, dialect='excel')
val_csv_writer = csv.writer(val_file, dialect='excel')
ex_csv_writer=csv.writer(ex_file,dialect='excel')
for raw in train_list:
train_csv_writer.writerow(raw)
for line in val_list:
val_csv_writer.writerow(line)
train_file.close()
val_file.close()
for ex in exma_list:
ex_csv_writer.writerow(ex)
# print('loss',loss)
coord.request_stop()
coord.join(threads)
val_writer.flush()
train_writer.flush()
val_writer.close()
train_writer.close()
def main():
global_step = tf.Variable(0, trainable=False)
print('*' * 10, 'create model', '*' * 10)
with tf.name_scope('input'):
inp = tf.placeholder(tf.float32, [batch_size, ImageSize, ImageSize, 3],
name='pl_input')
label = tf.placeholder(tf.float32, [batch_size, Points, 2],
name='pl_label')
with tf.name_scope('model'):
out_model = create_train_model.create_model(inp, Points)
with tf.name_scope('loss'):
diff = tf.subtract(out_model, label, name='sbu_label')
l2_loss = tf.nn.l2_loss(diff, name='l2_loss')
f_loss = tf.div(tf.cast(tf.sqrt(l2_loss * 2), tf.float32), ImageSize)
with tf.name_scope('optimizer'):
if change_lr:
lr = tf.train.exponential_decay(learning_rate,
global_step,
decay_steps,
decay_rate,
staircase=True,
name='learning_rate') #指数式衰减
else:
lr = learning_rate
opt = tf.train.AdamOptimizer(lr)
opti_min = opt.minimize(l2_loss, global_step=global_step)
print('*' * 10, 'Done', '*' * 10)
# 添加summary量
with tf.name_scope('summary'):
tf.summary.scalar('lr', lr, collections=['every_step'])
tf.summary.scalar('loss', l2_loss, collections=['every_step'])
train_summaries = tf.summary.merge_all('every_step')
val_summarise = tf.summary.merge_all('epoch_step')
#定义加载一个batch数据的tensor
images, labels = load_batch_data.batch_samples(batch_size,
data_file,
shuffle=True)
saver = tf.train.Saver()
init = tf.global_variables_initializer()
with tf.Session() as sess:
sess.run(init)
#启动内存队列，加载图片到内存
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord, sess=sess)
#查看是否有之前保存的模型，有加载
ckpt = tf.train.get_checkpoint_state(model_dir)
if ckpt and ckpt.model_checkpoint_path:
print('have model ,load model ,continue train')
saver.restore(sess, ckpt.model_checkpoint_path)
else:
print('first run ,create model')
#保存模型图结构
train_file_writer = tf.summary.FileWriter(train_log_dir,
graph=tf.get_default_graph())
val_file_writer = tf.summary.FileWriter(epoch_log_dir)
#IFSaveMate=True #if save meta info
loss = []
for epoch in range(nEpoch):
print('这是第%d个epoch' % (epoch + 1))
e_loss = 0
for batch in range(numBatch):
print('这是第%d个batch' % (batch + 1))
#加载batch数据
b_images, b_labels = sess.run([images, labels])
# test_diff,test_label,test_output=sess.run([diff,label,out_model],feed_dict={inp: b_images, label: b_labels})
# print('label',test_label)
# print('out_model',test_output)
# print('diff',test_diff)
if batch % save_stage == 0:
train_step = sess.run(global_step)
_, b_loss, str_summary, floss = sess.run(
[opti_min, l2_loss, train_summaries, f_loss],
feed_dict={
inp: b_images,
label: b_labels
})
#写入训练的总结信息
train_file_writer.add_summary(str_summary, train_step)
else:
_, b_loss, floss = sess.run([opti_min, l2_loss, f_loss],
feed_dict={
inp: b_images,
label: b_labels
})
print('loss值为：%f' % (b_loss))
loss.append(floss)
e_loss += b_loss
break
print('epoch loss is :%f' % (e_loss / numBatch))
if epoch % save_stage == 0:
# 保存模型
saver.save(
sess,
os.path.join(os.getcwd(), model_dir,
'model%d.ckpt' % ((epoch + 1) * epoch)))
#写入一个epoch的汇总信息
e_images, e_labels = sess.run([images, labels])
str_epoch_summary = sess.run(val_summarise,
feed_dict={
inp: e_images,
label: e_labels
})
val_file_writer.add_summary(str_epoch_summary, epoch)
coord.request_stop()
coord.join(threads)
train_file_writer.flush()
val_file_writer.flush()
train_file_writer.close()
val_file_writer.close()
with open('./loss.txt', 'w') as f:
for l in loss:
f.write(str(l) + '\n')