# START TRAINING
training_duration = 0.0
cost = 0.0
save_no = 0
if not outputOnly:
try:
print('\n*****TRAINING STARTED*****\n')
print('(stop with ctrl-c)')
avgCost = 0
startTime = time.time()
epochTime = startTime
lastSave = 1
lastCost = 1e10
for epoch in range(trainingEpochs):
batch_xs, batch_ys = tiCr.selectRandomTiles(batchSize)
_, cost, summary = sess.run([optimizer, costFunc, lossTrain],
feed_dict={
x: batch_xs,
y_true: batch_ys,
training: True
})
# save model
if ((cost < lastCost)
or alwaysSave) and (lastSave >= saveInterval):
saver.save(sess, test_path + 'model_%04d.ckpt' % save_no)
save_no += 1
lastSave = 1
lastCost = cost
print('Saved Model with cost %f.' % cost)
# ---------------------------------------------
# training
if not outputOnly:
# manually reshape data to remove third spatial dimension
dataSize = tiCr.tile_data['outputs_train'].shape[0]
#print( "Data sizes "+ format(dataSize) +", inputs " + format( tiCr.tile_data['inputs_train'].shape) + ", outputs " + format( tiCr.tile_data['outputs_train'].shape) ) # 16x16, 64x64
tiCr.tile_data['inputs_train'] = np.reshape( tiCr.tile_data['inputs_train'], [dataSize,tileSizeLow,tileSizeLow, n_inputChannels] )
tiCr.tile_data['outputs_train'] = np.reshape( tiCr.tile_data['outputs_train'], [dataSize,tileSizeHigh,tileSizeHigh,1 ] )
startTime = time.time()
if 1:
lastSave = 1
save_no = 0
trainingEpochs = int(trainingEpochs/kerasChunk)
for epoch in range(trainingEpochs):
batch_xs, batch_ys = tiCr.selectRandomTiles(batchSize*kerasChunk)
batch_xs = np.reshape( batch_xs, [batchSize*kerasChunk,tileSizeLow,tileSizeLow, n_inputChannels] )
batch_ys = np.reshape( batch_ys, [batchSize*kerasChunk,tileSizeHigh,tileSizeHigh, 1] )
# ...go!
hist = model.fit( batch_xs, batch_ys, batch_size=batchSize, epochs=1, validation_split=0.05 )
# save model
doSave = False
if (lastSave >= saveInterval) or epoch == (trainingEpochs-1): doSave = True
if doSave:
mpath = test_path + 'model_%04d.kkpt' % save_no
model.save_weights(mpath)
print('Saved weights ' + mpath )
save_no += 1