def train():
grasp_loader = GraspLoader.csv_loader(data_path=PARAMS.csv_path,
csv_filename=PARAMS.csv_filename,
save_folder=PARAMS.save_folder,
is_saved=True,
resize_image_size=PARAMS.image_size,
train_subject_list=PARAMS.train_list,
val_subject_list=PARAMS.val_list,
test_subject_list=PARAMS.test_list,
label_order=label_order,
batch_size=PARAMS.batch_size,
trans_range=PARAMS.trans_range,
rotate_range=PARAMS.rotate_range,
max_hue_delta=PARAMS.max_hue_delta,
saturation_range=PARAMS.saturation_range,
max_bright_delta=PARAMS.max_bright_delta,
max_contrast_delta=PARAMS.max_contrast_delta,
is_training=True
)
next_element, training_init_op, validation_init_op, test_init_op = \
grasp_loader.initialization_dataset()
batch_image, batch_label = next_element
# Define Model
model = taxonomy_model.taxonomy_model(inputs=batch_image,
true_labels=batch_label,
input_size=PARAMS.image_size,
batch_size=PARAMS.batch_size,
taxonomy_nums=len(grasp_loader.label_order), #len(grasp_loader.classes_numbers),
taxonomy_classes=grasp_loader.classes_numbers,
resnet_version=PARAMS.resnet_version,
resnet_pretrained_path=PARAMS.resnet_path,
resnet_exclude=PARAMS.resnet_exclude,
trainable_scopes=PARAMS.trainable_scopes,
extra_global_feature=True,
taxonomy_loss=True,
learning_rate=PARAMS.learning_rate,
num_samples=len(grasp_loader.train_info),
beta=PARAMS.beta,
taxonomy_weights=[1.0, 1.0, 1.0, 1.0],
all_label=None,
all_value=None,
batch_weight_range=[1.0, 1.0],
optimizer=PARAMS.optimizer,
is_mode='train'
)
all_inputs, end_point, losses, eval_value, eval_update, eval_reset = \
model.build_model()
train_summary_op = model.get_summary_op()
test_summary_op = model.get_summary_op_test()
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
config.gpu_options.visible_device_list = PARAMS.gpu_num
# Create a saver to save and restore all the variables.
saver = tf.train.Saver()
with tf.Session(config=config) as sess:
now = datetime.datetime.now()
folder_log = './' + 'train_%s_%s_%s_%s_%s' % (now.year, now.month, now.day, now.hour, now.minute)
# folder_log = '.\\' + 'train_%s_%s_%s_%s_%s' % (now.year, now.month, now.day, now.hour, now.minute)
print('folder_log: ', folder_log)
if not os.path.exists(folder_log):
os.makedirs(folder_log + '/code')
# For windows
# os.system('copy .\\*.py %s' % (folder_log))
# For Linux
os.system('cp ./*.py %s/code' % (folder_log))
# remember to intiate both global and local variables for training and evaluation
sess.run([tf.global_variables_initializer(), tf.local_variables_initializer()])
# don't forget to insert these lines
model.resnet_restore(sess)
summary_writer = tf.summary.FileWriter('%s' % (folder_log), sess.graph)
total_step_num, test_step_num = 0, 0
best_acc = 0.0
for epoch in range(PARAMS.epochs):
current_time = datetime.datetime.now() # measure training time for each epoch
# initiate the batch extraction using tf.data.Dataset
sess.run(training_init_op)
while (True):
try:
# extract batch and training
update = sess.run(
{'all_inputs': all_inputs, 'all_outputs': end_point, 'update_op': model.update_flag,
'losses': losses, 'eval_update': eval_update,
'summary': train_summary_op},
feed_dict={
# inputs: images,
# true_labels: labels[:,int(-args.layer):],
model.resnet_training_flag: False,
model.vgg19_training_flag: True,
model.vgg_dropout: 0.5})
if total_step_num % PARAMS.print_freq==0:
# print(len(update['all_inputs']), len(update['all_outputs']))
print('losses:', update['losses'], 'accuracy (top1, top3):',
update['eval_update']['Accuracy_top1'], update['eval_update']['Accuracy_top3'])
# print('losses:', update['losses'])
summary_writer.add_summary(update['summary'], total_step_num)
summary_writer.flush()
total_step_num += 1
# imgs, lbls = sess.run([images, labels])
except tf.errors.OutOfRangeError:
break
print('Epoch %d done. ' % (epoch + 1))
print('Training time: {}'.format(datetime.datetime.now() - current_time))
# save model after each epoch
checkpoint_file = os.path.join(folder_log, 'Grasp.ckpt')
saver.save(sess, checkpoint_file)
print('Model has been saved.')
# reset all local variabels so that the streaming metrics reset new calculation
sess.run(eval_reset)
################################################################################
# Validate the model with val_dataset
current_time = datetime.datetime.now() # measure training time for each epoch
# initiate the batch extraction using tf.data.Dataset
sess.run(validation_init_op)
while (True):
try:
# extract batch and training
update = sess.run({'all_inputs': all_inputs, 'all_outputs': end_point,
'losses': losses, 'eval_update': eval_update,
'summary':test_summary_op
},
feed_dict={
# inputs: images,
# true_labels: labels[:,int(-args.layer):],
model.resnet_training_flag: False,
model.vgg19_training_flag: False,
model.vgg_dropout: 1.0})
summary_writer.add_summary(update['summary'], test_step_num)
summary_writer.flush()
test_step_num += 1
except tf.errors.OutOfRangeError:
break
print('Validation time: {}'.format(datetime.datetime.now() - current_time))
val_metrics = sess.run(eval_value)
print({name: val_metrics[name]['stage_0'] for name in val_metrics.keys()})
# convert result in to np array and save to file
val_metrics_arr = [ [name] + [val_metrics[name][stage] for stage in val_metrics[name].keys()]
for name in val_metrics.keys()]
metric_names = list(val_metrics.keys())
stages = ['Metrics'] + list(val_metrics[metric_names[0]].keys())
val_metrics_arr = [stages] + val_metrics_arr
with open(folder_log + '/val_evaluation.csv', "a") as output:
writer = csv.writer(output, lineterminator='\n')
writer.writerows([['Epoch %s'% epoch]])
writer.writerows(val_metrics_arr)
# reset all local variabels so that the streaming metrics reset new calculation
sess.run(eval_reset)
################################################################################
# Evaluate the model with test_dataset
current_time = datetime.datetime.now() # measure training time for each epoch
# initiate the batch extraction using tf.data.Dataset
sess.run(test_init_op)
while (True):
try:
# extract batch and training
update = sess.run({'all_inputs': all_inputs, 'all_outputs': end_point,
'losses': losses, 'eval_update': eval_update,
'summary': test_summary_op
},
feed_dict={
# inputs: images,
# true_labels: labels[:,int(-args.layer):],
model.resnet_training_flag: False,
model.vgg19_training_flag: False,
model.vgg_dropout: 1.0})
summary_writer.add_summary(update['summary'], test_step_num)
summary_writer.flush()
test_step_num += 1
except tf.errors.OutOfRangeError:
break
print('Test time: {}'.format(datetime.datetime.now() - current_time))
test_metrics = sess.run(eval_value)
print({name: test_metrics[name]['stage_0'] for name in test_metrics.keys()})
# convert result in to np array and save to file
test_metrics_arr = [[name] + [test_metrics[name][stage] for stage in test_metrics[name].keys()]
for name in test_metrics.keys()]
metric_names = list(test_metrics.keys())
stages = ['Metrics'] + list(test_metrics[metric_names[0]].keys())
test_metrics_arr = [stages] + test_metrics_arr
with open(folder_log + '/test_evaluation.csv', "a") as output:
writer = csv.writer(output, lineterminator='\n')
writer.writerows([['Epoch %s' % epoch]])
writer.writerows(test_metrics_arr)
# reset all local variabels so that the streaming metrics reset new calculation
sess.run(eval_reset)
if test_metrics['Accuracy_top1']['stage_%s'%(len(grasp_loader.label_order)-1)] > best_acc:
best_acc = test_metrics['Accuracy_top1']['stage_%s'%(len(grasp_loader.label_order)-1)]
# save trained model
checkpoint_file = os.path.join(folder_log, 'BestGraspResnet%s.ckpt' % PARAMS.resnet_version)
saver.save(sess, checkpoint_file)
def train():
images = np.random.randn(100, 224, 224, 3)
labels = np.random.randint(0, 10, 100).reshape((100, 1))
classes = [10]
batch_image = tf.constant(images, tf.float32)
batch_label = tf.constant(labels, tf.int64)
# Define Model
model = taxonomy_model.taxonomy_model(
inputs=batch_image,
true_labels=batch_label,
input_size=PARAMS.image_size,
batch_size=PARAMS.batch_size,
taxonomy_nums=1,
taxonomy_classes=classes,
resnet_version=PARAMS.resnet_version,
resnet_pretrained_path=PARAMS.resnet_path,
resnet_exclude=PARAMS.resnet_exclude,
trainable_scopes=PARAMS.trainable_scopes,
extra_global_feature=True,
taxonomy_loss=True,
learning_rate=PARAMS.learning_rate,
num_samples=100,
beta=PARAMS.beta,
taxonomy_weights=[1.0, 1.0, 1.0, 1.0],
all_label=None,
all_value=None,
batch_weight_range=[1.0, 1.0],
is_mode='train')
all_inputs, end_point, losses, eval_value, eval_update, eval_reset = \
model.build_model()
train_summary_op = model.get_summary_op()
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
config.gpu_options.visible_device_list = '1'
# Create a saver to save and restore all the variables.
saver = tf.train.Saver()
with tf.Session(config=config) as sess:
now = datetime.datetime.now()
# remember to intiate both global and local variables for training and evaluation
sess.run([
tf.global_variables_initializer(),
tf.local_variables_initializer()
])
# don't forget to insert these lines
model.resnet_restore(sess)
total_step_num = 0
best_acc = 0.0
for epoch in range(PARAMS.epochs):
current_time = datetime.datetime.now(
) # measure training time for each epoch
while (True):
try:
step_time = datetime.datetime.now()
# imgs, lbls = sess.run([batch_image, batch_label])
# extract batch and training
update = sess.run(
{
'all_inputs': all_inputs,
'all_outputs': end_point,
'update_op': model.update_flag,
'losses': losses,
'eval_update': eval_update,
'summary': train_summary_op
},
feed_dict={
# inputs: images,
# true_labels: labels[:,int(-args.layer):],
model.resnet_training_flag:
True,
model.vgg19_training_flag:
False,
model.vgg_dropout:
0.5
})
# print(len(update['all_inputs']), len(update['all_outputs']))
print('losses:', update['losses'], 'accuracy:',
update['eval_update']['Accuracy_top1'])
# print('losses:', update['losses'])
total_step_num += 1
print('Step training time: {}'.format(
datetime.datetime.now() - step_time))
except tf.errors.OutOfRangeError:
break
print('Epoch %d done. ' % (epoch + 1))
print('Training time: {}'.format(datetime.datetime.now() -
current_time))
# reset all local variabels so that the streaming metrics reset new calculation
sess.run(eval_reset)
def train():
grasp_loader = Grasp_csv_Loader.csv_loader(
data_path=PARAMS.csv_path,
csv_filename='test_annotated_data.csv',
save_folder=PARAMS.save_folder,
is_saved=True,
resize_image_size=PARAMS.image_size,
train_list=[0],
val_list=[1],
test_list=[1],
label_order=label_order,
batch_size=PARAMS.batch_size,
max_hue_delta=PARAMS.max_hue_delta,
saturation_range=PARAMS.saturation_range,
max_bright_delta=PARAMS.max_bright_delta,
max_contrast_delta=PARAMS.max_contrast_delta,
is_training=True)
print('dataset: ', len(grasp_loader.train_meaningful_jpg_names),
len(grasp_loader.val_meaningful_jpg_names),
len(grasp_loader.test_meaningful_jpg_names))
next_element, training_init_op, validation_init_op, test_init_op = \
grasp_loader.initialization_dataset()
batch_image, batch_label = next_element
# Define Model
model = taxonomy_model.taxonomy_model(
inputs=batch_image,
true_labels=batch_label,
input_size=PARAMS.image_size,
batch_size=PARAMS.batch_size,
taxonomy_nums=len(grasp_loader.classes_numbers),
taxonomy_classes=grasp_loader.classes_numbers,
resnet_version=PARAMS.resnet_version,
resnet_pretrained_path=PARAMS.resnet_path,
resnet_exclude=PARAMS.resnet_exclude,
trainable_scopes=PARAMS.trainable_scopes,
extra_global_feature=True,
taxonomy_loss=True,
learning_rate=PARAMS.learning_rate,
num_samples=len(grasp_loader.train_meaningful_jpg_names),
beta=PARAMS.beta,
taxonomy_weights=[1.0, 1.0, 1.0, 1.0],
all_label=None,
all_value=None,
batch_weight_range=[1.0, 1.0],
is_mode='train')
all_inputs, end_point, losses, eval_value, eval_update, eval_reset = \
model.build_model()
train_summary_op = model.get_summary_op()
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
config.gpu_options.visible_device_list = '1' #PARAMS.gpu_num
# Create a saver to save and restore all the variables.
saver = tf.train.Saver()
with tf.Session(config=config) as sess:
now = datetime.datetime.now()
# folder_log = './' + 'train_%s_%s_%s_%s_%s' % (now.year, now.month, now.day, now.hour, now.minute)
# # folder_log = '.\\' + 'train_%s_%s_%s_%s_%s' % (now.year, now.month, now.day, now.hour, now.minute)
# print('folder_log: ', folder_log)
# if not os.path.exists(folder_log):
# os.makedirs(folder_log + '/code')
#
# # For windows
# # os.system('copy .\\*.py %s' % (folder_log))
# # For Linux
# os.system('cp ./*.py %s/code' % (folder_log))
# remember to intiate both global and local variables for training and evaluation
sess.run([
tf.global_variables_initializer(),
tf.local_variables_initializer()
])
# don't forget to insert these lines
model.resnet_restore(sess)
# summary_writer = tf.summary.FileWriter('%s' % (folder_log), sess.graph)
total_step_num = 0
best_acc = 0.0
for epoch in range(PARAMS.epochs):
current_time = datetime.datetime.now(
) # measure training time for each epoch
# initiate the batch extraction using tf.data.Dataset
sess.run(training_init_op)
while (True):
try:
step_time = datetime.datetime.now()
imgs, lbls = sess.run([batch_image, batch_label])
# # extract batch and training
# update = sess.run(
# {'all_inputs': all_inputs, 'all_outputs': end_point, 'update_op': model.update_flag,
# 'losses': losses, 'eval_update': eval_update,
# 'summary': train_summary_op},
# feed_dict={
# # inputs: images,
# # true_labels: labels[:,int(-args.layer):],
# model.resnet_training_flag: True,
# model.vgg19_training_flag: False,
# model.vgg_dropout: 0.5})
#
# # print(len(update['all_inputs']), len(update['all_outputs']))
# print('losses:', update['losses'], 'accuracy:', update['eval_update']['Accuracy_top1'])
# # print('losses:', update['losses'])
# summary_writer.add_summary(update['summary'], total_step_num)
# summary_writer.flush()
total_step_num += 1
print('Step training time: {}'.format(
datetime.datetime.now() - step_time))
except tf.errors.OutOfRangeError:
break
print('Epoch %d done. ' % (epoch + 1))
print('Training time: {}'.format(datetime.datetime.now() -
current_time))
# reset all local variabels so that the streaming metrics reset new calculation
sess.run(eval_reset)
# Validate the model with val_dataset
# initiate the batch extraction using tf.data.Dataset
sess.run(validation_init_op)
while (True):
try:
# extract batch and training
update = sess.run(
{
'all_inputs': all_inputs,
'all_outputs': end_point,
'losses': losses,
'eval_update': eval_update
},
feed_dict={
# inputs: images,
# true_labels: labels[:,int(-args.layer):],
model.resnet_training_flag:
False,
model.vgg19_training_flag:
False,
model.vgg_dropout:
1.0
})
print('losses:', update['losses'], 'accuracy:',
update['eval_update']['Accuracy_top1'])
except tf.errors.OutOfRangeError:
break
val_metrics = sess.run(eval_value)
# convert result in to np array and save to file
val_metrics_arr = [[
val_metrics[name][stage] for stage in val_metrics[name].keys()
] for name in val_metrics.keys()]
print('results:\n', val_metrics_arr)
# with open(folder_log + '/val/Epoch_%s.csv' % (epoch), "a") as output:
# writer = csv.writer(output, lineterminator='\n')
# writer.writerows(val_metrics_arr)
# reset all local variabels so that the streaming metrics reset new calculation
sess.run(eval_reset)
def test(folder_log, modelname):
model_fullpath = os.path.join(folder_log, modelname)
if os.path.isfile(model_fullpath + '.meta') is False:
raise IsADirectoryError('No file at: ' + model_fullpath)
grasp_loader = GraspLoader.csv_loader(
data_path=PARAMS.csv_path,
csv_filename=PARAMS.csv_filename,
save_folder=PARAMS.save_folder,
is_saved=True,
resize_image_size=PARAMS.image_size,
train_list=PARAMS.train_list,
val_list=PARAMS.val_list,
test_list=PARAMS.test_list,
label_order=label_order,
batch_size=PARAMS.batch_size,
max_hue_delta=PARAMS.max_hue_delta,
saturation_range=PARAMS.saturation_range,
max_bright_delta=PARAMS.max_bright_delta,
max_contrast_delta=PARAMS.max_contrast_delta,
is_training=False)
grasp_loader.do_preprocessing = False # for evaluating the training set
next_element, training_init_op, validation_init_op, test_init_op = \
grasp_loader.initialization_dataset()
batch_image, batch_label = next_element
# Define Model
model = taxonomy_model.taxonomy_model(
inputs=batch_image,
true_labels=batch_label,
input_size=PARAMS.image_size,
batch_size=PARAMS.batch_size,
taxonomy_nums=len(
grasp_loader.label_order), #len(grasp_loader.classes_numbers),
taxonomy_classes=grasp_loader.classes_numbers,
resnet_version=PARAMS.resnet_version,
resnet_pretrained_path=PARAMS.resnet_path,
resnet_exclude=PARAMS.resnet_exclude,
trainable_scopes=PARAMS.trainable_scopes,
extra_global_feature=True,
taxonomy_loss=True,
learning_rate=PARAMS.learning_rate,
num_samples=len(grasp_loader.train_meaningful_jpg_names),
beta=PARAMS.beta,
taxonomy_weights=[1.0, 1.0, 1.0, 1.0],
all_label=None,
all_value=None,
batch_weight_range=[1.0, 1.0],
is_mode='train')
all_inputs, end_point, losses, eval_value, eval_update, eval_reset = \
model.build_model()
test_predictions = model.get_prediction_topk(k=1)
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
config.gpu_options.visible_device_list = '1' #PARAMS.gpu_num
# Create a saver to save and restore all the variables.
saver = tf.train.Saver()
with tf.Session(config=config) as sess:
# load model from checkpoint
saver = tf.train.Saver()
saver.restore(sess, model_fullpath)
# remember to intiate both global and local variables for training and evaluation
sess.run([tf.local_variables_initializer()])
# don't forget to insert these lines
# model.resnet_restore(sess)
total_step_num = 0
predicted_filename = folder_log + '/prediction_%s.csv' % (modelname)
with open(predicted_filename, "a") as output:
writer = csv.writer(output, lineterminator='\n')
writer.writerows([['id', 'predicted']])
current_time = datetime.datetime.now(
) # measure training time for each epoch
# Test the model with test_dataset
# initiate the batch extraction using tf.data.Dataset
sess.run(test_init_op) #training_init_op
while (True):
try:
# extract batch and training
update = sess.run(
{
'all_inputs': all_inputs,
'all_outputs': end_point,
'labels': batch_label,
'topk': test_predictions,
'eval_update': eval_update
},
feed_dict={
# inputs: images,
# true_labels: labels[:,int(-args.layer):],
model.resnet_training_flag:
False,
model.vgg19_training_flag:
False,
model.vgg_dropout:
1.0
})
print('Accuracy_top1:', update['eval_update']['Accuracy_top1'],
'Accuracy_top3:', update['eval_update']['Accuracy_top3'])
result_array = [[update['labels'][i], update['topk'][i]]
for i in range(len(update['labels']))]
with open(predicted_filename, "a") as output:
writer = csv.writer(output, lineterminator='\n')
writer.writerows(result_array)
except tf.errors.OutOfRangeError:
break
test_metrics = sess.run(eval_value)
print({
name: test_metrics[name]['stage_0']
for name in test_metrics.keys()
})
print('Evaluation time: ', datetime.datetime.now() - current_time)
# convert result in to np array and save to file
test_metrics_arr = [
[name] +
[test_metrics[name][stage] for stage in test_metrics[name].keys()]
for name in test_metrics.keys()
]
metric_names = list(test_metrics.keys())
stages = ['Metrics'] + list(test_metrics[metric_names[0]].keys())
test_metrics_arr = [stages] + test_metrics_arr
with open(folder_log + '/test_evaluation.csv', "a") as output:
writer = csv.writer(output, lineterminator='\n')
writer.writerows([[modelname]])
writer.writerows(test_metrics_arr)
# reset all local variabels so that the streaming metrics reset new calculation
sess.run(eval_reset)
def train():
grasp_loader = Grasp_csv_Loader_v2.csv_loader(
data_path=PARAMS.csv_path,
csv_filename=PARAMS.csv_filename,
save_folder=PARAMS.save_folder,
is_saved=True,
resize_image_size=PARAMS.image_size,
train_subject_list=PARAMS.train_list,
val_subject_list=PARAMS.val_list,
test_subject_list=PARAMS.test_list,
divide_by_ratio=True,
is_divided_saved=True,
divided_npz_name='divided_dataset.npz',
label_order=label_order,
batch_size=PARAMS.batch_size,
max_hue_delta=PARAMS.max_hue_delta,
saturation_range=PARAMS.saturation_range,
max_bright_delta=PARAMS.max_bright_delta,
max_contrast_delta=PARAMS.max_contrast_delta,
is_training=True)
next_element, training_init_op, validation_init_op, test_init_op = \
grasp_loader.initialization_dataset()
batch_image, batch_label = next_element
# Define Model
model = taxonomy_model.taxonomy_model(
inputs=batch_image,
true_labels=batch_label,
input_size=PARAMS.image_size,
batch_size=PARAMS.batch_size,
taxonomy_nums=len(grasp_loader.classes_numbers),
taxonomy_classes=grasp_loader.classes_numbers,
resnet_version=PARAMS.resnet_version,
resnet_pretrained_path=PARAMS.resnet_path,
resnet_exclude=PARAMS.resnet_exclude,
trainable_scopes=PARAMS.trainable_scopes,
extra_global_feature=True,
taxonomy_loss=True,
learning_rate=PARAMS.learning_rate,
num_samples=len(grasp_loader.train_info),
beta=PARAMS.beta,
taxonomy_weights=[1.0, 1.0, 1.0, 1.0, 1.0],
all_label=None,
all_value=None,
batch_weight_range=[1.0, 1.0],
optimizer=PARAMS.optimizer,
is_mode='train')
all_inputs, end_point, losses, eval_value, eval_update, eval_reset = \
model.build_model()
train_summary_op = model.get_summary_op()
test_summary_op = model.get_summary_op_test()
config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = 0.5
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
config.gpu_options.visible_device_list = PARAMS.gpu_num
# Create a saver to save and restore all the variables.
saver = tf.train.Saver()
with tf.Session(config=config) as sess:
now = datetime.datetime.now()
folder_log = './' + 'train_%s_%s_%s_%s_%s' % (
now.year, now.month, now.day, now.hour, now.minute)
# folder_log = '.\\' + 'train_%s_%s_%s_%s_%s' % (now.year, now.month, now.day, now.hour, now.minute)
print('folder_log: ', folder_log)
if not os.path.exists(folder_log):
os.makedirs(folder_log)
# For windows
# os.system('copy .\\*.py %s' % (folder_log))
# For Linux
os.system('cp ./*.py %s' % (folder_log))
# remember to intiate both global and local variables for training and evaluation
sess.run([
tf.global_variables_initializer(),
tf.local_variables_initializer()
])
# don't forget to insert these lines
model.resnet_restore(sess)
summary_writer = tf.summary.FileWriter('%s' % (folder_log), sess.graph)
total_step_num = 0
for epoch in range(PARAMS.epochs):
current_time = datetime.datetime.now(
) # measure training time for each epoch
# initiate the batch extraction using tf.data.Dataset
sess.run(training_init_op)
while (True):
try:
# extract batch and training
each_current_time = datetime.datetime.now()
update = sess.run(
{
'all_inputs': all_inputs,
'all_outputs': end_point,
'update_op': model.update_flag,
'losses': losses,
'eval_update': eval_update,
'summary': train_summary_op
},
feed_dict={
# inputs: images,
# true_labels: labels[:,int(-args.layer):],
model.resnet_training_flag:
False,
model.vgg19_training_flag:
True,
model.vgg_dropout:
0.5
})
# print(len(update['all_inputs']), len(update['all_outputs']))
print('losses:', update['losses'],
'accuracy (top1, top3):',
update['eval_update']['Accuracy_top1'],
update['eval_update']['Accuracy_top3'])
# print('losses:', update['losses'])
print('Each iteration time: {}'.format(
datetime.datetime.now() - each_current_time))
summary_writer.add_summary(update['summary'],
total_step_num)
summary_writer.flush()
total_step_num += 1
except tf.errors.OutOfRangeError:
break
print('Epoch %d done. ' % (epoch + 1))
print('Training time: {}'.format(datetime.datetime.now() -
current_time))
checkpoint_file = os.path.join(folder_log, 'Grasp.ckpt')
saver.save(sess, checkpoint_file, global_step=epoch)
# reset all local variabels so that the streaming metrics reset new calculation
sess.run(eval_reset)
if (epoch % PARAMS.epoch_decay) == 0:
current_time = datetime.datetime.now(
) # measure training time for each epoch
# initiate the batch extraction using tf.data.Dataset
sess.run(validation_init_op)
while (True):
try:
# extract batch and training
update = sess.run(
{
'all_inputs': all_inputs,
'all_outputs': end_point,
'losses': losses,
'eval_update': eval_update,
'summary': test_summary_op
},
feed_dict={
# inputs: images,
# true_labels: labels[:,int(-args.layer):],
model.resnet_training_flag:
False,
model.vgg19_training_flag:
False,
model.vgg_dropout:
1.0
})
# print(len(update['all_inputs']), len(update['all_outputs']))
print('losses:', update['losses'],
'accuracy (top1, top3):',
update['eval_update']['Accuracy_top1'],
update['eval_update']['Accuracy_top3'])
# print('losses:', update['losses'])
print('Each iteration time: {}'.format(
datetime.datetime.now() - current_time))
summary_writer.add_summary(update['summary'], epoch)
summary_writer.flush()
except tf.errors.OutOfRangeError:
break
print('Validation time: {}'.format(datetime.datetime.now() -
current_time))
# reset all local variabels so that the streaming metrics reset new calculation
sess.run(eval_reset)