def main(nb_epoch):
# get Option
GPU = Option.GPU
batchSize = Option.batchSize
pathLog = '../log/' + Option.Time + '(' + Option.Notes + ')' + '.txt'
Log.Log(pathLog, 'w+', 1) # set log file
print(time.strftime('%Y-%m-%d %X', time.localtime()), '\n')
print(open('Option.py').read())
# get data
numThread = 4*len(GPU)
assert batchSize % len(GPU) == 0, ('batchSize must be divisible by number of GPUs')
with tf.device('/cpu:0'):
batchTrainX,batchTrainY,batchTestX,batchTestY,numTrain,numTest,label = getData.loadData(Option.dataSet,batchSize,numThread)
batchNumTrain = int(numTrain / batchSize)
batchNumTest = int(numTest / 100)
optimizer = Option.optimizer
global_step = tf.get_variable('global_step', [], dtype=tf.int32, initializer=tf.constant_initializer(0), trainable=False)
Net = []
# on my machine, alexnet does not fit multi-GPU training
# for single GPU
with tf.device('/gpu:%d' % GPU[0]):
Net.append(NN.NN(batchTrainX, batchTrainY, training=True, global_step=global_step))
lossTrainBatch, errorTrainBatch = Net[-1].build_graph()
update_op = tf.get_collection(tf.GraphKeys.UPDATE_OPS) # batchnorm moving average update ops (not used now)
# since we quantize W at the beginning and the update delta_W is quantized,
# there is no need to quantize W every iteration
# we just clip W after each iteration for speed
update_op += Net[0].W_clip_op
gradTrainBatch = optimizer.compute_gradients(lossTrainBatch)
gradTrainBatch_quantize = quantizeGrads(gradTrainBatch)
with tf.control_dependencies(update_op):
train_op = optimizer.apply_gradients(gradTrainBatch_quantize, global_step=global_step)
tf.get_variable_scope().reuse_variables()
Net.append(NN.NN(batchTestX, batchTestY, training=False))
_, errorTestBatch = Net[-1].build_graph()
showVariable()
# Build an initialization operation to run below.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
config.log_device_placement = False
sess = Option.sess = tf.InteractiveSession(config=config)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(max_to_keep=None)
# Start the queue runners.
tf.train.start_queue_runners(sess=sess)
def getErrorTest():
errorTest = 0.
for i in tqdm(range(batchNumTest),desc = 'Test', leave=False):
errorTest += sess.run([errorTestBatch])[0]
errorTest /= batchNumTest
return errorTest
if Option.loadModel is not None:
print('Loading model from %s ...' % Option.loadModel),
saver.restore(sess, Option.loadModel)
print('Finished'),
errorTestBest = getErrorTest()
print('Test:', errorTestBest)
else:
# at the beginning, we discrete W
sess.run([Net[0].W_q_op])
print('\nOptimization Start!\n')
lossTotal_ = []
errorTotal_ = []
errorTest_ = []
#####################################################################################################################################################################
gradH_ = []
gradW_ = []
gradWq_ = []
for epoch in range(nb_epoch):
# check lr_schedule
if len(Option.lr_schedule) / 2:
if epoch == Option.lr_schedule[0]:
Option.lr_schedule.pop(0)
lr_new = Option.lr_schedule.pop(0)
if lr_new == 0:
print('Optimization Ended!')
exit(0)
lr_old = sess.run(Option.lr)
sess.run(Option.lr.assign(lr_new))
print('lr: {} -> {}'.format(lr_old, lr_new))
print('Epoch: {}'.format(epoch)),
lossTotal = 0.
errorTotal = 0
t0 = time.time()
for batchNum in tqdm(range(batchNumTrain), desc='Epoch: %03d' % epoch, leave=False, smoothing=0.1):
if Option.debug is False:
_, loss_delta, error_delta = sess.run([train_op, lossTrainBatch, errorTrainBatch])
else:
_, loss_delta, error_delta, H, W, W_q, gradH, gradW, gradW_q=\
sess.run([train_op, lossTrainBatch, errorTrainBatch, Net[0].H, Net[0].W, Net[0].W_q, Net[0].gradsH, Net[0].gradsW, gradTrainBatch_quantize])
lossTotal += loss_delta
errorTotal += error_delta
lossTotal /= batchNumTrain
errorTotal /= batchNumTrain
print('Loss: {} Train: {}'.format(lossTotal, errorTotal)),
# get test error
errorTest = getErrorTest()
print('Test: {} FPS: {}'.format(errorTest, numTrain / (time.time() - t0))),
if epoch == 0:
errorTestBest = errorTest
if errorTest < errorTestBest:
if Option.saveModel is not None:
saver.save(sess, Option.saveModel)
print('S'),
if errorTest < errorTestBest:
errorTestBest = errorTest
print('BEST')
lossTotal_.append(lossTotal)
errorTotal_.append(errorTotal)
errorTest_.append(errorTest)
if epoch%5 == 0:
dd = {}
dd['lossTotal'] = lossTotal_
dd['errorTotal'] = errorTotal_
dd['errorTest'] = errorTest_
filename = '../savedata/temp_savedata.pkl'
with open(filename, 'wb') as f:
pickle.dump(dd, f)
# _, loss_delta, error_delta, H, W, W_q, gradH, gradW, gradW_q=\
# sess.run([train_op, lossTrainBatch, errorTrainBatch, Net[0].H, Net[0].W, Net[0].W_q, Net[0].gradsH, Net[0].gradsW, gradTrainBatch_quantize])
return lossTotal_, errorTotal_, errorTest_ #, H, W, W_q, gradH, gradW, gradW_q
def main():
# get Option
GPU = Option.GPU
batchSize = Option.batchSize
pathLog = '../log/' + Option.Time + '(' + Option.Notes + ')' + '.txt'
Log.Log(pathLog, 'w+', 1) # set log file
print time.strftime('%Y-%m-%d %X', time.localtime()), '\n'
print open('Option.py').read()
# get data
numThread = 4 * len(GPU)
assert batchSize % len(GPU) == 0, (
'batchSize must be divisible by number of GPUs')
with tf.device('/cpu:0'):
batchTrainX,batchTrainY,batchTestX,batchTestY,numTrain,numTest,label =\
getData.loadData(Option.dataSet,batchSize,numThread,Option.validNum)
batchNumTrain = numTrain / batchSize
batchNumTest = numTest / 100
optimizer = Option.optimizer
global_step = tf.get_variable('global_step', [],
dtype=tf.int32,
initializer=tf.constant_initializer(0),
trainable=False)
Net = []
# for single GPU
with tf.device('/gpu:%d' % GPU[0]):
Net.append(
NN.NN(batchTrainX,
batchTrainY,
training=True,
global_step=global_step))
lossTrainBatch, errorTrainBatch = Net[-1].build_graph()
update_op = tf.get_collection(
tf.GraphKeys.UPDATE_OPS) # batchnorm moving average update ops
update_op += Net[0].W_clip_op
gradTrainBatch = quantizeGrads(
optimizer.compute_gradients(lossTrainBatch))
with tf.control_dependencies(update_op):
train_op = optimizer.apply_gradients(gradTrainBatch,
global_step=global_step)
tf.get_variable_scope().reuse_variables()
Net.append(NN.NN(batchTestX, batchTestY, training=False))
_, errorTestBatch = Net[-1].build_graph()
# Build an initialization operation to run below.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
config.log_device_placement = False
sess = Option.sess = tf.InteractiveSession(config=config)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(max_to_keep=None)
# Start the queue runners.
tf.train.start_queue_runners(sess=sess)
def getErrorTest():
errorTest = 0.
for i in tqdm(xrange(batchNumTest), desc='Test', leave=False):
errorTest += sess.run([errorTestBatch])[0]
errorTest /= batchNumTest
return errorTest
if Option.loadModel is not None:
print 'Loading model from %s ...' % Option.loadModel,
saver.restore(sess, Option.loadModel)
print 'Finished',
errorTestBest = getErrorTest()
print 'Test: %.4f ' % (errorTestBest)
sess.run([Net[0].W_q_op])
print "\nOptimization Start!\n"
for epoch in xrange(1000):
# check lr_schedule
if len(Option.lr_schedule) / 2:
if epoch == Option.lr_schedule[0]:
Option.lr_schedule.pop(0)
lr_new = Option.lr_schedule.pop(0)
if lr_new == 0:
print 'Optimization Ended!'
exit(0)
lr_old = sess.run(Option.lr)
sess.run(Option.lr.assign(lr_new))
print 'lr: %f -> %f' % (lr_old, lr_new)
print 'Epoch: %03d ' % (epoch),
lossTotal = 0.
errorTotal = 0
t0 = time.time()
for batchNum in tqdm(xrange(batchNumTrain),
desc='Epoch: %03d' % epoch,
leave=False,
smoothing=0.1):
_, loss_delta, error_delta = sess.run(
[train_op, lossTrainBatch, errorTrainBatch])
# _, loss_delta, error_delta, H, W, W_q, gradsH, gradsW, gradW_q=\
# sess.run([train_op, lossTrainBatch, errorTrainBatch, Net[0].H, Net[0].W, Net[0].W_q, Net[0].gradsH, Net[0].gradsW, gradTrainBatch])
lossTotal += loss_delta
errorTotal += error_delta
lossTotal /= batchNumTrain
errorTotal /= batchNumTrain
print 'Loss: %.6f Train: %.4f' % (lossTotal, errorTotal),
# get test error
errorTest = getErrorTest()
print 'Test: %.4f FPS: %d' % (errorTest, numTrain /
(time.time() - t0)),
if epoch == 0:
errorTestBest = errorTest
if errorTest < errorTestBest:
if Option.saveModel is not None:
saver.save(sess, Option.saveModel)
print 'S',
if errorTest < errorTestBest:
errorTestBest = errorTest
print 'BEST',
print ''
def main():
# get Option
GPU = Option.GPU
batchSize = Option.batchSize
pathLog = os.path.join('../log', Option.modelName)
os.system('mkdir -p ' + pathLog)
logFile = os.path.join(pathLog, 'train.txt')
fLog = open(logFile, 'w+')
sys.stdout = Log.Log(fLog, sys.stdout) # set log file
print('>>> Path to log file ' + pathLog)
print('>>> Start at ' + time.strftime('%Y-%m-%d %X', time.localtime()) +
'\n')
open('Option.py').read()
# get data
numThread = 4 * len(GPU)
assert batchSize % len(GPU) == 0, (
'batchSize must be divisible by number of GPUs')
with tf.device('/cpu:0'):
batchTrainX,batchTrainY,batchTestX,batchTestY,numTrain,numTest,label =\
getData.loadData(Option.dataSet, batchSize, numThread)
batchNumTrain = numTrain / batchSize
batchNumTest = numTest / 100
optimizer = Option.optimizer
global_step = tf.get_variable('global_step', [],
dtype=tf.int32,
initializer=tf.constant_initializer(0),
trainable=False)
Net = []
# on my machine, alexnet does not fit multi-GPU training
# for single GPU
with tf.device('/gpu:%d' % GPU[0]):
Net.append(
NN.NN(batchTrainX,
batchTrainY,
training=True,
global_step=global_step))
lossTrainBatch, errorTrainBatch = Net[-1].build_graph()
update_op = tf.get_collection(
tf.GraphKeys.UPDATE_OPS
) # batchnorm moving average update ops (not used now)
# since we quantize W at the beginning and the update delta_W is quantized,
# there is no need to quantize W every iteration
# we just clip W after each iteration for speed
update_op += Net[0].W_clip_op
gradTrainBatch = optimizer.compute_gradients(lossTrainBatch)
gradTrainBatch_quantize = quantizeGrads(gradTrainBatch)
with tf.control_dependencies(update_op):
train_op = optimizer.apply_gradients(gradTrainBatch_quantize,
global_step=global_step)
tf.get_variable_scope().reuse_variables()
Net.append(NN.NN(batchTestX, batchTestY, training=False))
_, errorTestBatch = Net[-1].build_graph()
w_max = []
showVariable(debug=Option.debug)
# Build an initialization operation to run below.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
config.log_device_placement = False
sess = Option.sess = tf.InteractiveSession(config=config)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(max_to_keep=None)
# Start the queue runners.
tf.train.start_queue_runners(sess=sess)
#----
def getErrorTest():
errorTest = 0.
for i in tqdm(range(int(batchNumTest)), desc='Test', leave=False):
errorTest += sess.run([errorTestBatch])[0]
errorTest /= batchNumTest
return errorTest
#---- Resume
if Option.loadModel is not None:
if os.path.isfile(Option.loadModel):
print('>>> Loading model from %s ...' % Option.loadModel, end='')
saver.restore(sess, Option.loadModel)
print(' finished')
errorTestBest = getErrorTest()
print('>>> Test error %.3f' % errorTestBest)
else:
print('>>> Model does not exist for restoring')
sys.exit()
else:
# at the beginning, we discrete W
sess.run([Net[0].W_q_op])
#---- Derive LR
print('>>> Optimization starting ...')
for epoch in range(Option.epoches):
# check lr_schedule
if len(Option.lr_schedule) / 2:
if epoch == Option.lr_schedule[0]:
Option.lr_schedule.pop(0)
lr_new = Option.lr_schedule.pop(0)
if lr_new == 0:
print(' - optimization ended')
exit(0)
lr_old = sess.run(Option.lr)
sess.run(Option.lr.assign(lr_new))
print('>>> Learning rate lr: %f -> %f' % (lr_old, lr_new))
#--- Training
lossTotal = 0.0
errorTotal = 0.0
t0 = time.time()
for batchNum in tqdm(range(int(batchNumTrain)),
desc='Epoch: {:3d}'.format(epoch),
leave=False,
smoothing=0.1):
if Option.debug is False:
_, loss_delta, error_delta = sess.run(
[train_op, lossTrainBatch, errorTrainBatch])
else:
_, loss_delta, error_delta, H, W, W_q, gradH, gradW, gradW_q = \
sess.run([train_op, lossTrainBatch, errorTrainBatch, Net[0].H, Net[0].W, Net[0].W_q, Net[0].gradsH, Net[0].gradsW, gradTrainBatch_quantize])
lossTotal += loss_delta
errorTotal += error_delta
lossTotal /= batchNumTrain
errorTotal /= batchNumTrain
#for i in W_q:
# print i
print('>>>> Epoch: %3d: ' % (epoch), end='')
print('Total loss %12.4f, Train error %.4f, ' %
(lossTotal, errorTotal),
end='')
# get test error
errorTest = getErrorTest()
print('Test error %.4f, FPS: %4d' % (errorTest, numTrain /
(time.time() - t0)),
end='')
if epoch == 0:
errorTestBest = errorTest
if errorTest < errorTestBest:
if Option.saveModel is not None:
saver.save(sess, Option.saveModel)
print(', Saved', end='')
if errorTest < errorTestBest:
errorTestBest = errorTest
print(', Best test error %.4f' % errorTestBest, end='')
print('')
print('>>> Configuration')
print(' - Dataset %s' % Option.dataSet)
print(' - Number of epoches %d' % Option.epoches)
print(' - Batch size %d' % Option.batchSize)
print(' - batchNumTrain %d' % batchNumTrain)
print(' - batchNumTest %d' % batchNumTest)
print(' - modelName %s' % Option.modelName)
print(' - lr_schedule %s' % str(Option.lr_schedule))
print(' - loss function type %s' % Option.lossFunc)
print(' - Quantization')
print(' + bit width of weights %d' % Option.bitsW)
print(' + bit width of activations %d' % Option.bitsA)
print(' + bit width of gradients %d' % Option.bitsG)
print(' + bit width of errrors %d' % Option.bitsE)
def main():
# get Option
GPU = Option.GPU
batchSize = Option.batchSize
pathLog = '../log/' + Option.Time + '(' + Option.Notes + ')' + '.txt'
Log.Log(pathLog, 'w+', 1) # set log file
print time.strftime('%Y-%m-%d %X', time.localtime()), '\n'
print open('Option.py').read()
# get data
numThread = 4 * len(GPU)
assert batchSize % len(GPU) == 0, (
'batchSize must be divisible by number of GPUs')
with tf.device('/cpu:0'):
batchTrainX,batchTrainY,batchTestX,batchTestY,numTrain,numTest,label =\
getData.loadData(Option.dataSet,batchSize,numThread)
batchNumTrain = numTrain / batchSize
batchNumTest = numTest / 100
optimizer = Option.optimizer
global_step = tf.get_variable('global_step', [],
dtype=tf.int32,
initializer=tf.constant_initializer(0),
trainable=False)
Net = []
# on my machine, alexnet does not fit multi-GPU training
# for single GPU
with tf.device('/gpu:%d' % GPU[0]):
Net.append(NN.NN(batchTestX, batchTestY, training=False))
_, errorTestBatch = Net[-1].build_graph()
showVariable()
# Build an initialization operation to run below.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
config.log_device_placement = False
sess = Option.sess = tf.InteractiveSession(config=config)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(max_to_keep=None)
# Start the queue runners.
tf.train.start_queue_runners(sess=sess)
def getErrorTest():
errorTest = 0.
for i in tqdm(xrange(batchNumTest), desc='Test', leave=False):
errorTest += sess.run([errorTestBatch])[0]
errorTest /= batchNumTest
return errorTest
if Option.loadModel is not None:
print 'Loading model from %s ...' % Option.loadModel,
saver.restore(sess, Option.loadModel)
print 'Finished',
errorTestBest = getErrorTest()
print 'Test Error:', errorTestBest
H, W = sess.run([Net[0].input_array, Net[0].W_b])
hardware_estimation(H, W, Option.bitsW, Option.bitsA)
else:
print "No saved model"
def main():
# get Option
GPU = Option.GPU
batchSize = Option.batchSize
pathLog = '../log/' + Option.Time + '(exp1)' + '.txt'
# f = open(pathLog, 'w')
# sys.stdout = f
Log.Log(pathLog, 'w+', 1) # set log file
print
time.strftime('%Y-%m-%d %X', time.localtime()), '\n'
print
open('Option.py').read()
# get data
numThread = 4 * len(GPU)
assert batchSize % len(GPU) == 0, ('batchSize must be divisible by number of GPUs')
with tf.device('/cpu:0'):
batchTrainX, batchTrainY, batchTestX, batchTestY, numTrain, numTest, label = \
getData.loadData(Option.dataSet, batchSize, numThread)
batchNumTrain = numTrain / batchSize
batchNumTest = numTest / 100
optimizer = Option.optimizer
global_step = tf.get_variable('global_step', [], dtype=tf.int32, initializer=tf.constant_initializer(0),
trainable=False)
Net = []
# on my machine, alexnet does not fit multi-GPU training
# for single GPU
with tf.device('/gpu:%d' % GPU[0]):
Net.append(ResNet.NN(batchTrainX, batchTrainY, training=True, global_step=global_step))
lossTrainBatch, errorTrainBatch = Net[-1].build_graph()
update_op = tf.get_collection(tf.GraphKeys.UPDATE_OPS) # batchnorm moving average update ops (not used now)
# since we quantize W at the beginning and the update delta_W is quantized,
# there is no need to quantize W every iteration
# we just clip W after each iteration for speed
update_op += Net[0].W_clip_op
gradTrainBatch = optimizer.compute_gradients(lossTrainBatch)
gradTrainBatch_quantize = quantizeGrads(gradTrainBatch)
with tf.control_dependencies(update_op):
train_op = optimizer.apply_gradients(gradTrainBatch_quantize, global_step=global_step)
tf.get_variable_scope().reuse_variables()
Net.append(ResNet.NN(batchTestX, batchTestY, training=False))
_, errorTestBatch = Net[-1].build_graph()
showVariable()
# Build an initialization operation to run below.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
config.log_device_placement = False
sess = Option.sess = tf.InteractiveSession(config=config)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(max_to_keep=None)
# Start the queue runners.
tf.train.start_queue_runners(sess=sess)
def getErrorTest():
errorTest = 0.
for i in tqdm(range(batchNumTest), desc='Test', leave=False):
errorTest += sess.run([errorTestBatch])[0]
errorTest /= batchNumTest
return errorTest
if Option.loadModel is not None:
print
'Loading model from %s ...' % Option.loadModel,
saver.restore(sess, Option.loadModel)
print
'Finished',
errorTestBest = getErrorTest()
print
'Test:', errorTestBest
else:
# at the beginning, we discrete W
sess.run([Net[0].W_q_op])
print
"\nOptimization Start!\n"
tmp_lr_schedule = [0, 1./8, 5, 0]
for epoch in range(1000):
# check lr_schedule
if len(Option.lr_schedule) / 2:
if epoch == tmp_lr_schedule[0]:
tmp_lr_schedule.pop(0)
lr_new = tmp_lr_schedule.pop(0)
if lr_new == 0:
print('Optimization Ended!')
break
lr_old = sess.run(Option.lr)
sess.run(Option.lr.assign(lr_new))
print
'lr: %f -> %f' % (lr_old, lr_new)
print
'Epoch: %03d ' % (epoch),
lossTotal = 0.
errorTotal = 0
t0 = time.time()
for batchNum in tqdm(range(batchNumTrain), desc='Epoch: %03d' % epoch, leave=False, smoothing=0.1):
if Option.debug is False:
_, loss_delta, error_delta = sess.run([train_op, lossTrainBatch, errorTrainBatch])
else:
_, loss_delta, error_delta, H, W, W_q, gradH, gradW, gradW_q = \
sess.run([train_op, lossTrainBatch, errorTrainBatch, Net[0].H, Net[0].W, Net[0].W_q, Net[0].gradsH,
Net[0].gradsW, gradTrainBatch_quantize])
lossTotal += loss_delta
errorTotal += error_delta
lossTotal /= batchNumTrain
errorTotal /= batchNumTrain
print
'Loss: %.4f Train: %.4f' % (lossTotal, errorTotal),
# get test error
errorTest = getErrorTest()
print
'Test: %.4f FPS: %d' % (errorTest, numTrain / (time.time() - t0)),
if epoch == 0:
errorTestBest = errorTest
# if errorTest < errorTestBest:
# if Option.saveModel is not None:
# saver.save(sess, Option.saveModel)
# print
# 'S',
if errorTest < errorTestBest:
errorTestBest = errorTest
print
'BEST',
print
""
save_np(sess)
print('End of Training')
def main():
# get Option
GPU = Option.GPU
batchSize = Option.batchSize # batchSize = 128
pathLog = '../log/' + Option.Time + '(' + Option.Notes + ')' + '.txt'
Log.Log(pathLog, 'w+', 1) # set log file
print time.strftime('%Y-%m-%d %X', time.localtime()), '\n'
print open('Option.py').read()
# get data
numThread = 4*len(GPU) # len(GPU) = 1
assert batchSize % len(GPU) == 0, ('batchSize must be divisible by number of GPUs') # 128 = 32 * 4
with tf.device('/cpu:0'): # 첫번째 CPU사용.
batchTrainX,batchTrainY,batchTestX,batchTestY,numTrain,numTest,label =\
getData.loadData(Option.dataSet,batchSize,numThread)
batchNumTrain = numTrain / batchSize # batchSize = 128, numTrain =
batchNumTest = numTest / 100
optimizer = Option.optimizer
# global_step이란 변수를 0으로 초기화하여 생성.
global_step = tf.get_variable('global_step', [], dtype=tf.int32, initializer=tf.constant_initializer(0), trainable=False)
Net = []
# CPU와 GPU를 동시에 지원하게하면 GPU에 먼저 배치.
# on my machine, alexnet does not fit multi-GPU training
# for single GPU
with tf.device('/gpu:%d' % GPU[0]): # GPU[0] = 0
Net.append(NN.NN(batchTrainX, batchTrainY, training=True, global_step=global_step))
lossTrainBatch, errorTrainBatch = Net[-1].build_graph()
update_op = tf.get_collection(tf.GraphKeys.UPDATE_OPS) # batchnorm moving average update ops (not used now)
# batchnorm에서 test할때 사용할 moving_average와 moving_var이 train할때 계산을 해줘야하는데,
# train과정에서 moving_average와 moving_var이 직접 호출되지 않기에 train 하는것과 별도로
# 이것들에 대한 op를 실행해줘서 업데이트를 해줘야한다(갱신연산)
# since we quantize W at the beginning and the update delta_W is quantized,
# there is no need to quantize W every iteration
# we just clip W after each iteration for speed
update_op += Net[0].W_clip_op
gradTrainBatch = optimizer.compute_gradients(lossTrainBatch)
gradTrainBatch_quantize = quantizeGrads(gradTrainBatch)
#train_op update_ops 연산이 끝난 후에 연산되게 함.
with tf.control_dependencies(update_op):
train_op = optimizer.apply_gradients(gradTrainBatch_quantize, global_step=global_step)
tf.get_variable_scope().reuse_variables() # 변수를 재사용하기 위한 플래그
Net.append(NN.NN(batchTestX, batchTestY, training=False))
_, errorTestBatch = Net[-1].build_graph()
showVariable()
# 변수 생성,초기화,저장.
# Build an initialization operation to run below.
config = tf.ConfigProto()
config.gpu_options.allow_growth = True # GPU 메모리 증가 허용(요구되는 만큼의 GPU만 할당)
config.allow_soft_placement = True # 명시된 디바이스가 없을경우 TF가 자동으로 디바이스 선택.
config.log_device_placement = False # 하지만 디바이스 명시하지 않음
sess = Option.sess = tf.InteractiveSession(config=config)
sess.run(tf.global_variables_initializer())
saver = tf.train.Saver(max_to_keep=None) # 최대 모델 수 정하지 않고 저장.
# Start the queue runners.
tf.train.start_queue_runners(sess=sess)
def getErrorTest():
errorTest = 0.
for i in tqdm(xrange(batchNumTest),desc = 'Test', leave=False):
errorTest += sess.run([errorTestBatch])[0]
errorTest /= batchNumTest
return errorTest
if Option.loadModel is not None:
print 'Loading model from %s ...' % Option.loadModel,
saver.restore(sess, Option.loadModel)
print 'Finished',
errorTestBest = getErrorTest()
print 'Test:', errorTestBest
else: # True이므로
# at the beginning, we discrete W
sess.run([Net[0].W_q_op])
print "\nOptimization Start!\n"
for epoch in xrange(1000):
# check lr_schedule
if len(Option.lr_schedule) / 2:
if epoch == Option.lr_schedule[0]: #epoch이 0이면 [0]을 뽑아내고
Option.lr_schedule.pop(0)
lr_new = Option.lr_schedule.pop(0)
if lr_new == 0:
print 'Optimization Ended!' #
exit(0)
lr_old = sess.run(Option.lr)
sess.run(Option.lr.assign(lr_new))
print 'lr: %f -> %f' % (lr_old, lr_new)
print 'Epoch: %03d ' % (epoch),
lossTotal = 0.
errorTotal = 0
t0 = time.time()
for batchNum in tqdm(xrange(batchNumTrain), desc='Epoch: %03d' % epoch, leave=False, smoothing=0.1):
if Option.debug is False:
_, loss_delta, error_delta = sess.run([train_op, lossTrainBatch, errorTrainBatch])
else:
_, loss_delta, error_delta, H, W, W_q, gradH, gradW, gradW_q=\
sess.run([train_op, lossTrainBatch, errorTrainBatch, Net[0].H, Net[0].W, Net[0].W_q, Net[0].gradsH, Net[0].gradsW, gradTrainBatch_quantize])
lossTotal += loss_delta
errorTotal += error_delta
lossTotal /= batchNumTrain
errorTotal /= batchNumTrain
print 'Loss: %.4f Train: %.4f' % (lossTotal, errorTotal),
# get test error
errorTest = getErrorTest()
print 'Test: %.4f FPS: %d' % (errorTest, numTrain / (time.time() - t0)),
if epoch == 0:
errorTestBest = errorTest
if errorTest < errorTestBest:
if Option.saveModel is not None:
saver.save(sess, Option.saveModel)
print 'S',
if errorTest < errorTestBest:
errorTestBest = errorTest
print 'BEST',
print ''
