# START TRAINING training_duration = 0.0 cost = 0.0 save_no = 0 if not outputOnly: try: print('\n*****TRAINING STARTED*****\n') print('(stop with ctrl-c)') avgCost = 0 startTime = time.time() epochTime = startTime lastSave = 1 lastCost = 1e10 for epoch in range(trainingEpochs): batch_xs, batch_ys = tiCr.selectRandomTiles(batchSize) _, cost, summary = sess.run([optimizer, costFunc, lossTrain], feed_dict={ x: batch_xs, y_true: batch_ys, training: True }) # save model if ((cost < lastCost) or alwaysSave) and (lastSave >= saveInterval): saver.save(sess, test_path + 'model_%04d.ckpt' % save_no) save_no += 1 lastSave = 1 lastCost = cost print('Saved Model with cost %f.' % cost)
# --------------------------------------------- # training if not outputOnly: # manually reshape data to remove third spatial dimension dataSize = tiCr.tile_data['outputs_train'].shape[0] #print( "Data sizes "+ format(dataSize) +", inputs " + format( tiCr.tile_data['inputs_train'].shape) + ", outputs " + format( tiCr.tile_data['outputs_train'].shape) ) # 16x16, 64x64 tiCr.tile_data['inputs_train'] = np.reshape( tiCr.tile_data['inputs_train'], [dataSize,tileSizeLow,tileSizeLow, n_inputChannels] ) tiCr.tile_data['outputs_train'] = np.reshape( tiCr.tile_data['outputs_train'], [dataSize,tileSizeHigh,tileSizeHigh,1 ] ) startTime = time.time() if 1: lastSave = 1 save_no = 0 trainingEpochs = int(trainingEpochs/kerasChunk) for epoch in range(trainingEpochs): batch_xs, batch_ys = tiCr.selectRandomTiles(batchSize*kerasChunk) batch_xs = np.reshape( batch_xs, [batchSize*kerasChunk,tileSizeLow,tileSizeLow, n_inputChannels] ) batch_ys = np.reshape( batch_ys, [batchSize*kerasChunk,tileSizeHigh,tileSizeHigh, 1] ) # ...go! hist = model.fit( batch_xs, batch_ys, batch_size=batchSize, epochs=1, validation_split=0.05 ) # save model doSave = False if (lastSave >= saveInterval) or epoch == (trainingEpochs-1): doSave = True if doSave: mpath = test_path + 'model_%04d.kkpt' % save_no model.save_weights(mpath) print('Saved weights ' + mpath ) save_no += 1