def train(batch_size,
epochs,
lr_base,
lr_power,
weight_decay,
classes,
model_name,
train_file_path,
val_file_path,
data_dir,
label_dir,
target_size=None,
batchnorm_momentum=0.9,
resume_training=False,
class_weight=None,
dataset='VOC2012',
loss_fn=softmax_sparse_crossentropy_ignoring_last_label,
metrics=[sparse_accuracy_ignoring_last_label],
loss_shape=None,
label_suffix='.png',
data_suffix='.jpg',
ignore_label=255,
label_cval=255):
if target_size:
input_shape = target_size + (3, )
else:
input_shape = (None, None, 3)
batch_shape = (batch_size, ) + input_shape
###########################################################
current_dir = os.path.dirname(os.path.realpath(__file__))
save_path = os.path.join(current_dir, 'Models/' + model_name)
if os.path.exists(save_path) is False:
os.mkdir(save_path)
# ###############learning rate scheduler####################
def lr_scheduler(epoch, mode='power_decay'):
'''if lr_dict.has_key(epoch):
lr = lr_dict[epoch]
print 'lr: %f' % lr'''
if mode is 'power_decay':
# original lr scheduler
lr = lr_base * ((1 - float(epoch) / epochs)**lr_power)
if mode is 'exp_decay':
# exponential decay
lr = (float(lr_base)**float(lr_power))**float(epoch + 1)
# adam default lr
if mode is 'adam':
lr = 0.001
if mode is 'progressive_drops':
# drops as progression proceeds, good for sgd
if epoch > 0.9 * epochs:
lr = 0.0001
elif epoch > 0.75 * epochs:
lr = 0.001
elif epoch > 0.5 * epochs:
lr = 0.01
else:
lr = 0.1
print('lr: %f' % lr)
return lr
scheduler = LearningRateScheduler(lr_scheduler)
# ###################### make model ########################
checkpoint_path = os.path.join(save_path, 'checkpoint_weights.hdf5')
model = globals()[model_name](weight_decay=weight_decay,
input_shape=input_shape,
batch_momentum=batchnorm_momentum,
classes=classes)
# ###################### optimizer ########################
LR_mult_dict = {}
LR_mult_dict['colorization_conv1'] = 3
LR_mult_dict['colorization_conv2'] = 3
#LR_mult_dict['colorization_conv3']=10
#LR_mult_dict['colorization_conv4']=10
#optimizer = SGD(lr=lr_base, momentum=0.9)
optimizer = LR_SGD(lr=lr_base,
momentum=0.9,
decay=0.0,
nesterov=False,
multipliers=LR_mult_dict)
# optimizer = Nadam(lr=lr_base, beta_1 = 0.825, beta_2 = 0.99685)
model.compile(loss=softmax_sparse_crossentropy_ignoring_last_label,
optimizer=optimizer,
metrics=metrics)
if resume_training:
model.load_weights(checkpoint_path, by_name=True)
model.summary()
# lr_reducer = ReduceLROnPlateau(monitor=softmax_sparse_crossentropy_ignoring_last_label, factor=np.sqrt(0.1),
# cooldown=0, patience=15, min_lr=0.5e-6)
# early_stopper = EarlyStopping(monitor=sparse_accuracy_ignoring_last_label, min_delta=0.0001, patience=70)
# callbacks = [early_stopper, lr_reducer]
callbacks = [scheduler]
# ####################### tfboard ###########################
if K.backend() == 'tensorflow':
tensorboard = TensorBoard(log_dir=os.path.join(save_path, 'logs'),
histogram_freq=0,
write_graph=True)
callbacks.append(tensorboard)
# ################### checkpoint saver#######################
checkpoint = ModelCheckpoint(filepath=os.path.join(
save_path, 'checkpoint_weights.hdf5'),
save_weights_only=True) #.{epoch:d}
callbacks.append(checkpoint)
# set data generator and train
train_datagen = SegDataGenerator(
zoom_range=[1.0, 1.5],
zoom_maintain_shape=True,
crop_mode='random',
crop_size=target_size,
# pad_size=(505, 505),
rotation_range=5.,
shear_range=0,
horizontal_flip=True,
channel_shift_range=0.,
fill_mode='constant',
label_cval=label_cval)
def get_file_len(file_path):
fp = open(file_path)
lines = fp.readlines()
fp.close()
return len(lines)
# from Keras documentation: Total number of steps (batches of samples) to yield from generator before declaring one epoch finished
# and starting the next epoch. It should typically be equal to the number of unique samples of your dataset divided by the batch size.
steps_per_epoch = int(
np.ceil(get_file_len(train_file_path) / float(batch_size)))
history = model.fit_generator(
generator=train_datagen.flow_from_directory(
file_path=train_file_path,
data_dir=data_dir,
data_suffix=data_suffix,
label_dir=label_dir,
label_suffix=label_suffix,
classes=classes,
target_size=target_size,
color_mode='rgb',
batch_size=batch_size,
shuffle=True,
loss_shape=loss_shape,
ignore_label=ignore_label,
# save_to_dir='Images/'
),
steps_per_epoch=steps_per_epoch,
epochs=epochs,
callbacks=callbacks,
workers=4,
class_weight=class_weight)
model.save_weights(save_path + '/model.hdf5')
def train(batch_size, epochs, lr_base, lr_power, weight_decay, classes,
model_name, train_file_path, val_file_path,
data_dir, label_dir, target_size=None, batchnorm_momentum=0.9,
resume_training=False, class_weight=None, dataset='VOC2012',
loss_fn = softmax_sparse_crossentropy_ignoring_last_label,
metrics = [sparse_accuracy_ignoring_last_label],
loss_shape=None,
label_suffix='.png',
data_suffix='.jpg',
ignore_label=255,
label_cval=255):
if target_size:
input_shape = target_size + (3,)
else:
input_shape = (None, None, 3)
batch_shape = (batch_size,) + input_shape
###########################################################
current_dir = os.path.dirname(os.path.realpath(__file__))
save_path = os.path.join(current_dir, 'Models/' + model_name)
if os.path.exists(save_path) is False:
os.mkdir(save_path)
# ###############learning rate scheduler####################
def lr_scheduler(epoch, mode='power_decay'):
'''if lr_dict.has_key(epoch):
lr = lr_dict[epoch]
print 'lr: %f' % lr'''
if mode is 'power_decay':
# original lr scheduler
lr = lr_base * ((1 - float(epoch)/epochs) ** lr_power)
if mode is 'exp_decay':
# exponential decay
lr = (float(lr_base) ** float(lr_power)) ** float(epoch+1)
# adam default lr
if mode is 'adam':
lr = 0.001
if mode is 'progressive_drops':
# drops as progression proceeds, good for sgd
if epoch > 0.9 * epochs:
lr = 0.0001
elif epoch > 0.75 * epochs:
lr = 0.001
elif epoch > 0.5 * epochs:
lr = 0.01
else:
lr = 0.1
print('lr: %f' % lr)
return lr
scheduler = LearningRateScheduler(lr_scheduler)
# ###################### make model ########################
checkpoint_path = os.path.join(save_path, 'checkpoint_weights.hdf5')
model = globals()[model_name](weight_decay=weight_decay,
input_shape=input_shape,
batch_momentum=batchnorm_momentum,
classes=classes)
# ###################### optimizer ########################
optimizer = SGD(lr=lr_base, momentum=0.9)
# optimizer = Nadam(lr=lr_base, beta_1 = 0.825, beta_2 = 0.99685)
model.compile(loss=loss_fn,
optimizer=optimizer,
metrics=metrics)
if resume_training:
model.load_weights(checkpoint_path, by_name=True)
model_path = os.path.join(save_path, "model.json")
# save model structure
f = open(model_path, 'w')
model_json = model.to_json()
f.write(model_json)
f.close
img_path = os.path.join(save_path, "model.png")
# #vis_util.plot(model, to_file=img_path, show_shapes=True)
model.summary()
# lr_reducer = ReduceLROnPlateau(monitor=softmax_sparse_crossentropy_ignoring_last_label, factor=np.sqrt(0.1),
# cooldown=0, patience=15, min_lr=0.5e-6)
# early_stopper = EarlyStopping(monitor=sparse_accuracy_ignoring_last_label, min_delta=0.0001, patience=70)
# callbacks = [early_stopper, lr_reducer]
callbacks = [scheduler]
# ####################### tfboard ###########################
if K.backend() == 'tensorflow':
tensorboard = TensorBoard(log_dir=os.path.join(save_path, 'logs'), histogram_freq=10, write_graph=True)
callbacks.append(tensorboard)
# ################### checkpoint saver#######################
checkpoint = ModelCheckpoint(filepath=os.path.join(save_path, 'checkpoint_weights.hdf5'), save_weights_only=True)#.{epoch:d}
callbacks.append(checkpoint)
# set data generator and train
train_datagen = SegDataGenerator(zoom_range=[0.5, 2.0],
zoom_maintain_shape=True,
crop_mode='random',
crop_size=target_size,
# pad_size=(505, 505),
rotation_range=0.,
shear_range=0,
horizontal_flip=True,
channel_shift_range=20.,
fill_mode='constant',
label_cval=label_cval)
val_datagen = SegDataGenerator()
def get_file_len(file_path):
fp = open(file_path)
lines = fp.readlines()
fp.close()
return len(lines)
history = model.fit_generator(
generator=train_datagen.flow_from_directory(
file_path=train_file_path,
data_dir=data_dir, data_suffix=data_suffix,
label_dir=label_dir, label_suffix=label_suffix,
classes=classes,
target_size=target_size, color_mode='rgb',
batch_size=batch_size, shuffle=True,
loss_shape=loss_shape,
ignore_label=ignore_label,
# save_to_dir='Images/'
),
steps_per_epoch=get_file_len(train_file_path),
epochs=epochs,
callbacks=callbacks,
nb_worker=4,
validation_data=val_datagen.flow_from_directory(
file_path=val_file_path, data_dir=data_dir, data_suffix='.jpg',
label_dir=label_dir, label_suffix='.png',classes=classes,
target_size=target_size, color_mode='rgb',
batch_size=batch_size, shuffle=False
),
nb_val_samples = 64,
class_weight=class_weight
)
model.save_weights(save_path+'/model.hdf5')
data_suffix = '.jpg'
label_suffix = '.png'
classes = 21
target_size = (320, 320)
ignore_label = 255
label_cval = 255
loss_shape = None
batch_size = 1
train_datagen = SegDataGenerator(
featurewise_center=False,
featurewise_std_normalization=False,
# zoom_range=[0.5, 2.0],
zoom_maintain_shape=True,
crop_mode='none',
crop_size=target_size,
# pad_size=(505, 505),
rotation_range=0.,
shear_range=0,
horizontal_flip=False,
# channel_shift_range=20.,
fill_mode='constant',
label_cval=label_cval)
train_generator = train_datagen.flow_from_directory(
file_path=val_file_path,
data_dir=data_dir,
data_suffix=data_suffix,
label_dir=label_dir,
label_suffix=label_suffix,
classes=classes,
target_size=target_size,
def train(batch_size, epochs, lr_base, lr_power, weight_decay, classes,
model_name, train_file_path, val_file_path,
data_dir, label_dir, target_size=None, batchnorm_momentum=0.9,
resume_training=False, class_weight=None, dataset='VOC2012',
loss_fn=softmax_sparse_crossentropy_ignoring_last_label,
metrics=[sparse_accuracy_ignoring_last_label],
loss_shape=None,
label_suffix='.png',
data_suffix='.jpg',
ignore_label=255,
label_cval=255):
if target_size:
input_shape = target_size + (3,)
else:
input_shape = (None, None, 3)
batch_shape = (batch_size,) + input_shape
###########################################################
current_dir = os.path.dirname(os.path.realpath(__file__))
save_path = os.path.join(current_dir, 'Models/' + model_name)
if os.path.exists(save_path) is False:
os.mkdir(save_path)
# ###############learning rate scheduler####################
def lr_scheduler(epoch, mode='power_decay'):
'''if lr_dict.has_key(epoch):
lr = lr_dict[epoch]
print 'lr: %f' % lr'''
if mode is 'power_decay':
# original lr scheduler
lr = lr_base * ((1 - float(epoch)/epochs) ** lr_power)
if mode is 'exp_decay':
# exponential decay
lr = (float(lr_base) ** float(lr_power)) ** float(epoch+1)
# adam default lr
if mode is 'adam':
lr = 0.001
if mode is 'progressive_drops':
# drops as progression proceeds, good for sgd
if epoch > 0.9 * epochs:
lr = 0.0001
elif epoch > 0.75 * epochs:
lr = 0.001
elif epoch > 0.5 * epochs:
lr = 0.01
else:
lr = 0.1
print('lr: %f' % lr)
return lr
scheduler = LearningRateScheduler(lr_scheduler)
# ###################### make model ########################
checkpoint_path = os.path.join(save_path, 'checkpoint_weights.hdf5')
model = globals()[model_name](weight_decay=weight_decay,
input_shape=input_shape,
batch_momentum=batchnorm_momentum,
classes=classes)
# ###################### optimizer ########################
optimizer = SGD(lr=lr_base, momentum=0.9)
# optimizer = Nadam(lr=lr_base, beta_1 = 0.825, beta_2 = 0.99685)
model.compile(loss=loss_fn,
optimizer=optimizer,
metrics=metrics)
if resume_training:
model.load_weights(checkpoint_path, by_name=True)
model_path = os.path.join(save_path, "model.json")
# save model structure
f = open(model_path, 'w')
model_json = model.to_json()
f.write(model_json)
f.close
img_path = os.path.join(save_path, "model.png")
# #vis_util.plot(model, to_file=img_path, show_shapes=True)
model.summary()
# lr_reducer = ReduceLROnPlateau(monitor=softmax_sparse_crossentropy_ignoring_last_label, factor=np.sqrt(0.1),
# cooldown=0, patience=15, min_lr=0.5e-6)
# early_stopper = EarlyStopping(monitor=sparse_accuracy_ignoring_last_label, min_delta=0.0001, patience=70)
# callbacks = [early_stopper, lr_reducer]
callbacks = [scheduler]
# ####################### tfboard ###########################
if K.backend() == 'tensorflow':
tensorboard = TensorBoard(log_dir=os.path.join(save_path, 'logs'), histogram_freq=10, write_graph=True)
callbacks.append(tensorboard)
# ################### checkpoint saver#######################
checkpoint = ModelCheckpoint(filepath=os.path.join(save_path, 'checkpoint_weights.hdf5'), save_weights_only=True)#.{epoch:d}
callbacks.append(checkpoint)
# set data generator and train
train_datagen = SegDataGenerator(zoom_range=[0.5, 2.0],
zoom_maintain_shape=True,
crop_mode='random',
crop_size=target_size,
# pad_size=(505, 505),
rotation_range=0.,
shear_range=0,
horizontal_flip=True,
channel_shift_range=20.,
fill_mode='constant',
label_cval=label_cval)
val_datagen = SegDataGenerator()
def get_file_len(file_path):
fp = open(file_path)
lines = fp.readlines()
fp.close()
return len(lines)
# from Keras documentation: Total number of steps (batches of samples) to yield from generator before declaring one epoch finished
# and starting the next epoch. It should typically be equal to the number of unique samples of your dataset divided by the batch size.
steps_per_epoch = int(np.ceil(get_file_len(train_file_path) / float(batch_size)))
history = model.fit_generator(
generator=train_datagen.flow_from_directory(
file_path=train_file_path,
data_dir=data_dir, data_suffix=data_suffix,
label_dir=label_dir, label_suffix=label_suffix,
classes=classes,
target_size=target_size, color_mode='rgb',
batch_size=batch_size, shuffle=True,
loss_shape=loss_shape,
ignore_label=ignore_label,
# save_to_dir='Images/'
),
steps_per_epoch=steps_per_epoch,
epochs=epochs,
callbacks=callbacks,
workers=4,
# validation_data=val_datagen.flow_from_directory(
# file_path=val_file_path, data_dir=data_dir, data_suffix='.jpg',
# label_dir=label_dir, label_suffix='.png',classes=classes,
# target_size=target_size, color_mode='rgb',
# batch_size=batch_size, shuffle=False
# ),
# nb_val_samples = 64
class_weight=class_weight
)
model.save_weights(save_path+'/model.hdf5')
def train(batch_size, epochs, lr_base, lr_power, weight_decay, classes,
model_name, train_file_path, val_file_path,
data_dir, label_dir, target_size=None, batchnorm_momentum=0.9,
load_model=False, class_weight=None, dataset='VOC2012',
loss_fn = softmax_sparse_crossentropy_ignoring_last_label,
metrics = [sparse_accuracy_ignoring_last_label],
loss_shape=None,
label_suffix='.png',
data_suffix='.jpg',
ignore_label=255,
label_cval=255):
if target_size:
input_shape = target_size + (3,)
else:
input_shape = (None, None, 3)
batch_shape = (batch_size,) + input_shape
###########################################################
current_dir = os.path.dirname(os.path.realpath(__file__))
#save_path = os.path.join(current_dir, 'Models/' + model_name)
save_path = os.path.join(current_dir, run_name)
if os.path.exists(save_path) is False:
os.mkdir(save_path)
with open(os.path.join(save_path, 'conf.json'), 'w') as outfile:
json.dump(conf, outfile)
# ###############learning rate scheduler####################
def lr_scheduler(epoch, mode='power_decay'):
'''if lr_dict.has_key(epoch):
lr = lr_dict[epoch]
print 'lr: %f' % lr'''
if mode is 'power_decay':
# original lr scheduler
lr = lr_base * ((1 - float(epoch)/epochs) ** lr_power)
if mode is 'exp_decay':
# exponential decay
lr = (float(lr_base) ** float(lr_power)) ** float(epoch+1)
# adam default lr
if mode is 'adam':
lr = 0.001
if mode is 'progressive_drops':
# drops as progression proceeds, good for sgd
if epoch > 0.9 * epochs:
lr = 0.0001
elif epoch > 0.75 * epochs:
lr = 0.001
elif epoch > 0.5 * epochs:
lr = 0.01
else:
lr = 0.1
print('lr: %f' % lr)
return lr
scheduler = LearningRateScheduler(lr_scheduler)
# ###################### make model ########################
checkpoint_path = os.path.join(save_path, 'checkpoint_weights.hdf5')
model = globals()[model_name](weight_decay=weight_decay,
input_shape=input_shape,
batch_momentum=batchnorm_momentum,
classes=classes)
# ###################### optimizer ########################
optimizer = SGD(lr=lr_base, momentum=0.9)
# optimizer = Nadam(lr=lr_base, beta_1 = 0.825, beta_2 = 0.99685)
csv_logger = CSVLogger(
#'output/{}_fcn_vgg16.csv'.format(datetime.datetime.now().isoformat()))
'%s/tmp_fcn_vgg16_cosmic_small.csv' % run_name)
with open(conf.test_file_path) as f:
test_image_list = f.read().splitlines()
#print(test_image_list)
class SaveSampleImages(keras.callbacks.Callback):
def on_train_begin(self, logs={}):
self.batchnum=0
def on_epoch_end(self, batch, logs={}):
directory = '%s/samples/%i' % (run_name, self.batchnum)
if not os.path.exists(directory):
os.makedirs(directory)
model_name = 'AtrousFCN_Resnet50_16s'
#model_name = 'Atrous_DenseNet'
#model_name = 'DenseNet_FCN'
weight_file = 'checkpoint_weights.hdf5'
image_size = (320, 320)
log_file = os.path.join(directory, 'metrics.json')
results = inference_model(self.model, image_size, test_image_list,
conf.data_dir, conf.label_dir, run_name,
label_suffix=conf.label_suffix, data_suffix=conf.data_suffix,
target_size=target_size, log_file=log_file
)
for image, result in zip(test_image_list, results):
result.save('%s/%s.png' % (directory, image))
self.batchnum+=1
model.compile(loss=loss_fn,
optimizer=optimizer,
metrics=metrics)
if load_model:
model.load_weights(load_model)
model_path = os.path.join(save_path, "model.json")
# save model structure
f = open(model_path, 'w')
model_json = model.to_json()
f.write(model_json)
f.close()
img_path = os.path.join(save_path, "model.png")
# #vis_util.plot(model, to_file=img_path, show_shapes=True)
model.summary()
# lr_reducer = ReduceLROnPlateau(monitor=softmax_sparse_crossentropy_ignoring_last_label, factor=np.sqrt(0.1),
# cooldown=0, patience=15, min_lr=0.5e-6)
# early_stopper = EarlyStopping(monitor=sparse_accuracy_ignoring_last_label, min_delta=0.0001, patience=70)
# callbacks = [early_stopper, lr_reducer]
callbacks = [scheduler]
# ####################### tfboard ###########################
if K.backend() == 'tensorflow':
tensorboard = TensorBoard(log_dir=os.path.join(save_path, 'logs'), histogram_freq=10, write_graph=True)
callbacks.append(tensorboard)
# ################### checkpoint saver#######################
callbacks.append(csv_logger)
checkpoint = ModelCheckpoint(filepath=os.path.join(save_path, 'checkpoint_weights.hdf5'), save_weights_only=True)#.{epoch:d}
callbacks.append(checkpoint)
save_sample_images = SaveSampleImages()
callbacks.append(save_sample_images)
# set data generator and train
train_datagen = SegDataGenerator(zoom_range=[0.5, 2.0],
zoom_maintain_shape=True,
crop_mode='random',
crop_size=target_size,
# pad_size=(505, 505),
rotation_range=0.,
shear_range=0,
horizontal_flip=True,
channel_shift_range=20.,
fill_mode='constant',
label_cval=label_cval)
val_datagen = SegDataGenerator()
def get_file_len(file_path):
fp = open(file_path)
lines = fp.readlines()
fp.close()
return len(lines)
history = model.fit_generator(
train_datagen.flow_from_directory(
file_path=train_file_path,
data_dir=data_dir, data_suffix=data_suffix,
label_dir=label_dir, label_suffix=label_suffix,
classes=classes,
target_size=target_size, color_mode='rgb',
batch_size=batch_size, shuffle=True,
loss_shape=loss_shape,
ignore_label=ignore_label,
# save_to_dir='Images/'
),
2000,#get_file_len(train_file_path),
epochs=epochs,
callbacks=callbacks,
workers=12,
# validation_data=val_datagen.flow_from_directory(
# file_path=val_file_path, data_dir=data_dir, data_suffix='.jpg',
# label_dir=label_dir, label_suffix='.png',classes=classes,
# target_size=target_size, color_mode='rgb',
# batch_size=batch_size, shuffle=False
# ),
# nb_val_samples = 64
class_weight=class_weight
)
model.save_weights(save_path+'/model.hdf5')
def train(batch_size,
epochs,
lr_base,
lr_power,
weight_decay,
classes,
model_name,
train_file_path,
val_file_path,
data_dir,
label_dir,
target_size=None,
batchnorm_momentum=0.9,
loss_fn=softmax_crossentropy,
metrics=[],
label_cval=255):
#设置批文件大小
if target_size:
input_shape = target_size + (1, ) #之前给的是320*320
else:
input_shape = (None, None, 1)
batch_shape = (batch_size, ) + input_shape #每一个batch的大小
# ###############设置学习率随权重衰减####################
def lr_scheduler(epoch, mode='power_decay'): #设置学习率参数，注意默认参数是power_decay
if mode is 'power_decay':
# ** 在python里面表示幂运算,radius**3 表示radius的3次方
lr = lr_base * ((1 - float(epoch) / epochs)**lr_power)
if mode is 'exp_decay':
# exponential decay
lr = (float(lr_base)**float(lr_power))**float(epoch + 1)
# adam default lr
if mode is 'adam':
lr = 0.001
if mode is 'progressive_drops':
# drops as progression proceeds, good for sgd
if epoch > 0.9 * epochs:
lr = 0.0001
elif epoch > 0.75 * epochs:
lr = 0.001
elif epoch > 0.5 * epochs:
lr = 0.01
else:
lr = 0.1
print('回调一次：lr: %f' % lr)
return lr
scheduler = LearningRateScheduler(lr_scheduler) #设置学习率
#生成模型
model = globals()[model_name](
weight_decay=weight_decay, #正则化项目的系数
input_shape=input_shape, #每一张输入图片的大小
batch_momentum=batchnorm_momentum, #动量前面的系数
classes=classes) #glob是为了访问全局变量，并且修改这个model的参数
# 定义优化器，并且进行编译,这里使用带动量的SGD下降
optimizer = SGD(lr=lr_base, momentum=0.9)
# optimizer = Nadam(lr=lr_base, beta_1 = 0.825, beta_2 = 0.99685)
"""----------------------loss_fn用法---------------------------"""
# 可以通过传递预定义目标函数名字指定目标函数,也可以传递一个Theano/TensroFlow的符号函数作为目标函数，
# 该函数对每个数据点应该只返回一个标量值，并以下列两个参数为参数
#y_true：真实的数据标签，Theano/TensorFlow张量
#y_pred：预测值，与y_true相同shape的Theano/TensorFlow张量
"""----------------------metrics用法---------------------------"""
#性能评估模块提供了一系列用于模型性能评估的函数, 这些函数在模型编译时由metrics关键字设置
#性能评估函数类似与目标函数, 只不过请注意该性能的评估结果讲不会用于训练.
#可以通过字符串来使用域定义的性能评估函数
#y_true:真实标签,theano/tensorflow张量
#y_pred:预测值, 与y_true形式相同的theano/tensorflow张量
model.compile(loss=loss_fn, optimizer=optimizer, metrics=metrics)
# 设置model的保存路径
current_dir = os.path.dirname(os.path.realpath(__file__)) # 获得当前py文件所在的路径
save_path = os.path.join(current_dir,
'Models/' + model_name) # 给出文件model保存的路径
checkpoint_path = os.path.join(save_path, 'checkpoint_weights.hdf5')
if os.path.exists(save_path) is False:
os.mkdir(
save_path
) # /home/ye/桌面/我的tensorflow之旅/heart/Models/AtrousFCN_Resnet50_16s
#model.load_weights(checkpoint_path, by_name=True)#重新训练
# 保存model的结构
model_path = os.path.join(save_path, "model.json")
f = open(model_path, 'w')
model_json = model.to_json()
f.write(model_json)
f.close #将model的结构写入到文件里面
#draw_structor_to_png(save_path,model)#这一个是将网络结构图画出来的，老是装不上，弃疗了
callbacks = [scheduler] #设置学习率
# ####################### 保存结果进行可视化 ###########################
if K.backend() == 'tensorflow':
tensorboard = TensorBoard(log_dir=os.path.join(save_path, 'logs'),
write_graph=True)
callbacks.append(tensorboard)
#用于可视化
# ################### 每个epoch的结尾处，尝试一次保留权重#######################
checkpoint = ModelCheckpoint(filepath=os.path.join(
save_path, 'checkpoint_weights.hdf5'),
save_weights_only=True)
callbacks.append(checkpoint)
##################################################################################################
# losshistory = LossHistory()
# callbacks.append(losshistory)
###################################################################################################
# set data generator and train
train_datagen = SegDataGenerator() #初始化一些变量
val_datagen = SegDataGenerator() #获取验证集并且不剪取图片
#报错来自于这个地方，因为它找不到train.path
def get_file_len(file_path):
fp = open(file_path)
lines = fp.readlines()
fp.close()
return len(lines)
steps_per_epoch = int(
np.ceil(get_file_len(train_file_path) / float(batch_size)))
# 发生器产生的步骤总数（样品批次）。它通常应该等于数据集中样本的数量除以批量大小。
#第一个参数是生成函数
#generator：生成器函数，生成器的输出应该为：
# 一个形如（inputs，targets）的tuple
#一个形如（inputs, targets, sample_weight）的tuple。所有的返回值都应该包含相同数目的样本。生成器将无限在数据集上循环。每个epoch以经过模型的样本数达到samples_per_epoch时，记一个epoch结束
history = model.fit_generator(
generator=train_datagen.flow_from_directory(
file_path=train_file_path,
data_dir=data_dir,
label_dir=label_dir,
classes=classes,
target_size=target_size,
color_mode='grayscale',
batch_size=1,
shuffle=True,
),
steps_per_epoch=
steps_per_epoch, #当生成器返回steps_per_epoch次数据时计一个epoch结束，执行下一个epoch
epochs=epochs, #整数，数据迭代的轮数
callbacks=callbacks,
workers=4, #最大进程数量
validation_data=val_datagen.flow_from_directory(
file_path=val_file_path,
data_dir=data_dir,
label_dir=label_dir,
classes=classes,
target_size=target_size,
color_mode='grayscale',
batch_size=1,
shuffle=True,
),
)
model.save_weights(save_path + '/model.hdf5')
def train(batch_size,
epochs,
lr_base,
lr_power,
weight_decay,
classes,
model_name,
train_file_path,
val_file_path,
data_dir,
label_dir,
target_size=None,
batchnorm_momentum=0.9,
resume_training=False,
class_weight=None,
dataset='VOC2012',
loss_fn=softmax_sparse_crossentropy_ignoring_last_label,
metrics=[sparse_accuracy_ignoring_last_label],
loss_shape=None,
label_suffix='.png',
data_suffix='.jpg',
ignore_label=255,
label_cval=255):
if target_size:
input_shape = target_size + (3, )
else:
input_shape = (None, None, 3)
batch_shape = (batch_size, ) + input_shape
###########################################################
current_dir = os.path.dirname(os.path.realpath(__file__))
save_path = os.path.join(current_dir, 'Models/' + model_name)
if os.path.exists(save_path) is False:
os.mkdir(save_path)
# ###############learning rate scheduler####################
def lr_scheduler(epoch, mode='power_decay'):
'''if lr_dict.has_key(epoch):
lr = lr_dict[epoch]
print 'lr: %f' % lr'''
if mode is 'power_decay':
# original lr scheduler
lr = lr_base * ((1 - float(epoch) / epochs)**lr_power)
if mode is 'exp_decay':
# exponential decay
lr = (float(lr_base)**float(lr_power))**float(epoch + 1)
# adam default lr
if mode is 'adam':
lr = 0.001
if mode is 'progressive_drops':
# drops as progression proceeds, good for sgd
if epoch > 0.9 * epochs:
lr = 0.0001
elif epoch > 0.75 * epochs:
lr = 0.001
elif epoch > 0.5 * epochs:
lr = 0.01
else:
lr = 0.1
# print('lr: %f' % lr)
return lr
scheduler = LearningRateScheduler(lr_scheduler)
# ###################### make model ########################
checkpoint_path = os.path.join(save_path, 'checkpoint_weights.hdf5')
model = globals()[model_name](
weight_decay=weight_decay,
input_shape=input_shape,
batch_momentum=batchnorm_momentum,
classes=classes,
)
# ###################### optimizer ########################
optimizer = SGD(lr=lr_base, momentum=0.9)
#optimizer = Nadam(lr=lr_base, beta_1 = 0.825, beta_2 = 0.99685)
model.compile(loss=loss_fn, optimizer=optimizer, metrics=metrics)
if resume_training:
model.load_weights(checkpoint_path, by_name=True)
model_path = os.path.join(save_path, "model.json")
# save model structure
f = open(model_path, 'w')
model_json = model.to_json()
f.write(model_json)
f.close
img_path = os.path.join(save_path, "model.png")
# #vis_util.plot(model, to_file=img_path, show_shapes=True)
model.summary()
# lr_reducer = ReduceLROnPlateau(monitor=softmax_sparse_crossentropy_ignoring_last_label, factor=np.sqrt(0.1),
# cooldown=0, patience=15, min_lr=0.5e-6)
# early_stopper = EarlyStopping(monitor=sparse_accuracy_ignoring_last_label, min_delta=0.0001, patience=70)
# callbacks = [early_stopper, lr_reducer]
callbacks = [scheduler]
# ####################### tfboard ###########################
if K.backend() == 'tensorflow':
tensorboard = TensorBoard(log_dir=os.path.join(save_path, 'logs'),
histogram_freq=0,
write_graph=True)
callbacks.append(tensorboard)
# ################### checkpoint saver#######################
checkpoint = ModelCheckpoint(filepath=os.path.join(
save_path, 'checkpoint_weights.hdf5'),
save_weights_only=True) #.{epoch:d}
#print 'hi'
callbacks.append(checkpoint)
# set data generator and train
train_datagen = SegDataGenerator(
zoom_range=[0.5, 2.0],
zoom_maintain_shape=True,
crop_mode='random',
crop_size=target_size,
# pad_size=(505, 505),
rotation_range=0.,
shear_range=0,
horizontal_flip=True,
channel_shift_range=20.,
fill_mode='constant',
label_cval=label_cval)
val_datagen = SegDataGenerator()
train_datagen = SegDataGenerator()
def load_test_images_from_folder(folder):
images = []
for filename in os.listdir(folder):
#img = cv2.imread(os.path.join(folder, filename))
img = Image.open(os.path.join(folder, filename))
img = img.resize((320, 320))
img = np.array(img, dtype='uint32')
img = img[..., np.newaxis]
#print img.shape
#print img_to_array(img=img)
if img is not None:
images.append(img)
return images
def load_train_images_from_folder(folder):
images = []
for filename in os.listdir(folder):
img = cv2.imread(os.path.join(folder, filename))
img = cv2.resize(img, (512, 512))
#print img_to_array(img=img)
if img is not None:
images.append(img)
return images
def get_file_len(file_path):
fp = open(file_path)
lines = fp.readlines()
fp.close()
return len(lines)
def decode_segmap(label_mask, plot=False):
"""Decode segmentation class labels into a color image
Args:
label_mask (np.ndarray): an (M,N) array of integer values denoting
the class label at each spatial location.
plot (bool, optional): whether to show the resulting color image
in a figure.
Returns:
(np.ndarray, optional): the resulting decoded color image.
"""
label_colours = get_pascal_labels()
r = label_mask.copy()
g = label_mask.copy()
b = label_mask.copy()
for ll in range(0, 21):
r[label_mask == ll] = label_colours[ll, 0]
g[label_mask == ll] = label_colours[ll, 1]
b[label_mask == ll] = label_colours[ll, 2]
rgb = np.zeros((label_mask.shape[0], label_mask.shape[1], 3))
rgb[:, :, 0] = r
rgb[:, :, 1] = g
rgb[:, :, 2] = b
if plot:
plt.imshow(rgb)
plt.show()
else:
return rgb
def get_pascal_labels():
"""Load the mapping that associates pascal classes with label colors
Returns:
np.ndarray with dimensions (21, 3)
"""
return np.asarray([[0, 0, 0], [128, 0, 0], [0, 128, 0], [128, 128, 0],
[0, 0, 128], [128, 0, 128], [0, 128, 128],
[128, 128, 128], [64, 0, 0], [192, 0, 0],
[64, 128, 0], [192, 128, 0], [64, 0, 128],
[192, 0, 128], [64, 128, 128], [192, 128, 128],
[0, 64, 0], [128, 64, 0], [0, 192, 0],
[128, 192, 0], [0, 64, 128]])
def encode_segmap(mask):
"""Encode segmentation label images as pascal classes
Args:
mask (np.ndarray): raw segmentation label image of dimension
(M, N, 3), in which the Pascal classes are encoded as colours.
Returns:
(np.ndarray): class map with dimensions (M,N), where the value at
a given location is the integer denoting the class index.
"""
#print(mask.shape)
mask = mask.astype(int)
#print mask.shape
label_mask = np.zeros((mask.shape[0], mask.shape[1]), dtype=np.int16)
#print label_mask.shape
for ii, label in enumerate(get_pascal_labels()):
#print label.shape
#print (np.all(mask == label, axis=-1)).shape
label_mask[np.where(np.all(mask == label, axis=-1))[:2]] = ii
#print label_mask.shape
label_mask = label_mask.astype(int)
# print( label_mask.shape)
#print('label')
return label_mask
X = load_train_images_from_folder(
'../data_processed/new/JPEGImages_train/')
y = load_train_images_from_folder('../data_processed/SegmentationClass/')
# from Keras documentation: Total number of steps (batches of samples) to yield from generator before declaring one epoch finished
# and starting the next epoch. It should typically be equal to the number of unique samples of your dataset divided by the batch size.
# steps_per_epoch = int(np.ceil(get_file_len(train_file_path) / float(batch_size)))
# print(len(X))
# print(len(y))
#print('data loaded')
X = np.array(X)
y = np.array(y)
# print y.shape
y_en = np.zeros((y.shape[0], y.shape[1], y.shape[2]))
#print X.shape
#print type(X)
#print type(y)
# print((X[0]).shape)
#print(y[0].shape)
y_temp = np.zeros((512, 512))
#print(y[0,:,:,:].shape)
y[0, :, :, [0, 1, 2]] = y[0, :, :, [2, 1, 0]]
y_temp[:, :] = encode_segmap(y[0])
#print(y_temp.shape)
#print(y_temp)
#print((y_temp>0).sum())
y_orig = decode_segmap(y_temp)
#print(y_orig.shape)
y_orig[:, :, [0, 1, 2]] = y_orig[:, :, [2, 1, 0]]
cv2.imwrite('test.png', y_orig)
y[0, :, :, [2, 1, 0]] = y[0, :, :, [0, 1, 2]]
for i in range(y.shape[0]):
#print(y[i].shape)
y[i, :, :, [0, 1, 2]] = y[i, :, :, [2, 1, 0]]
y_en[i, :, :] = encode_segmap(y[i])
#y_temp[:,:]=y_en[i,:,:]
#print(np.sum(np.sum(y_temp>0)))
#y_orig[:,:,[0,1,2]]=y_orig[:,:,[2,1,0]]
#cv2.imwrite('test.png',y_orig)
# print y_en.shape
#print type()
#print y_en.shape
#y_enp=np.reshape(y_en,(320,320,1))
y_en = y_en[..., np.newaxis]
model.fit(X, y_en, epochs=100, batch_size=16, callbacks=callbacks)
'''
history = model.fit_generator(
generator=train_datagen.flow_from_directory(
file_path=train_file_path,
data_dir=data_dir, data_suffix=data_suffix,
label_dir=label_dir, label_suffix=label_suffix,
classes=classes,
target_size=target_size, color_mode='rgb',
batch_size=batch_size, shuffle=True,
loss_shape=loss_shape,
ignore_label=ignore_label,
# save_to_dir='Images/'
),
steps_per_epoch=steps_per_epoch,
epochs=epochs,
callbacks=callbacks,
workers=4,
# validation_data=val_datagen.flow_from_directory(
# file_path=val_file_path, data_dir=data_dir, data_suffix='.jpg',
# label_dir=label_dir, label_suffix='.png',classes=classes,
# target_size=target_size, color_mode='rgb',
# batch_size=batch_size, shuffle=False
# ),
# nb_val_samples = 64
class_weight=class_weight
)
'''
model.save_weights(save_path + '/model.hdf5')
from utils.loss_function import *
from utils.metrics import *
from utils.SegDataGenerator import *
import time
target_size = (320, 320)
label_cval = 100
loss_shape = None
ignore_label = 100
numImages = 100 #num images generated is numImages * 100
datagen = SegDataGenerator( #zoom_range=[0.5, 2.0],
zoom_maintain_shape=True,
#crop_mode='random',
crop_size=target_size,
# pad_size=(505, 505),
#rotation_range=60,
#shear_range=0,
#horizontal_flip=True,
#channel_shift_range=1.,
fill_mode='constant',
label_cval=label_cval,
)
train_file_path = os.path.expanduser(
'~/.keras/datasets/weedspic/lettuce/train.txt')
val_file_path = os.path.expanduser(
'~/.keras/datasets/weedspic/lettuce/validation.txt')
data_dir = os.path.expanduser('~/.keras/datasets/weedspic/lettuce/image')
label_dir = os.path.expanduser('~/.keras/datasets/weedspic/lettuce/label')
data_suffix = '.png'
label_suffix = '.png'
classes = 2
def train(batch_size,
nb_epoch,
lr_base,
lr_power,
weight_decay,
nb_classes,
model_name,
train_file_path,
val_file_path,
data_dir,
label_dir,
target_size=None,
batchnorm_momentum=0.9,
resume_training=False):
if target_size:
input_shape = target_size + (3, )
else:
input_shape = (None, None, 3)
batch_shape = (batch_size, ) + input_shape
###########################################################
current_dir = os.path.dirname(os.path.realpath(__file__))
save_path = os.path.join(current_dir, 'Models/' + model_name)
if os.path.exists(save_path) == False:
os.mkdir(save_path)
################learning rate scheduler####################
def lr_scheduler(epoch):
'''if lr_dict.has_key(epoch):
lr = lr_dict[epoch]
print 'lr: %f' % lr'''
lr = lr_base * ((1 - float(epoch) / nb_epoch)**lr_power)
print 'lr: %f' % lr
return lr
scheduler = LearningRateScheduler(lr_scheduler)
######################## tfboard ###########################
tfboard = TensorBoard(log_dir=os.path.join(save_path, 'logs'))
####################### make model ########################
checkpoint_path = os.path.join(save_path, 'checkpoint_weights.hdf5')
model = globals()[model_name](weight_decay=weight_decay,
input_shape=input_shape,
batch_momentum=batchnorm_momentum)
sgd = SGD(lr=lr_base, momentum=0.9)
model.compile(loss=softmax_sparse_crossentropy_ignoring_last_label,
optimizer=sgd,
metrics=[sparse_accuracy_ignoring_last_label])
if resume_training:
model.load_weights(checkpoint_path, by_name=True)
model_path = os.path.join(save_path, "model.json")
# save model structure
f = open(model_path, 'w')
model_json = model.to_json()
f.write(model_json)
f.close
img_path = os.path.join(save_path, "model.png")
vis_util.plot(model, to_file=img_path, show_shapes=True)
model.summary()
#################### checkpoint saver#######################
checkpoint = ModelCheckpoint(filepath=os.path.join(
save_path, 'checkpoint_weights.hdf5'),
save_weights_only=True) #.{epoch:d}
# set data generator and train
train_datagen = SegDataGenerator(
zoom_range=[0.5, 2.0],
zoom_maintain_shape=True,
crop_mode='random',
crop_size=target_size, #pad_size=(505, 505),
rotation_range=0.,
shear_range=0,
horizontal_flip=True,
channel_shift_range=20.,
fill_mode='constant',
label_cval=255)
def get_file_len(file_path):
fp = open(file_path)
lines = fp.readlines()
fp.close()
return len(lines)
history = model.fit_generator(
generator=train_datagen.flow_from_directory(
file_path=train_file_path,
data_dir=data_dir,
data_suffix='.jpg',
label_dir=label_dir,
label_suffix='.png',
nb_classes=nb_classes,
target_size=target_size,
color_mode='rgb',
batch_size=batch_size,
shuffle=True,
#save_to_dir='Images/'
),
samples_per_epoch=get_file_len(train_file_path),
nb_epoch=nb_epoch,
callbacks=[scheduler, checkpoint],
nb_worker=4,
#validation_data=val_datagen.flow_from_directory(
# file_path=val_file_path, data_dir=data_dir, data_suffix='.jpg',
# label_dir=label_dir, label_suffix='.png',nb_classes=nb_classes,
# target_size=target_size, color_mode='rgb',
# batch_size=batch_size, shuffle=False
#),
#nb_val_samples = 64
)
model.save_weights(save_path + '/model.hdf5')
def train(batch_size, nb_epoch, lr_dict, weight_decay, nb_classes, model_name, train_file_path, val_file_path,
data_dir, label_dir, target_size=None, resume_training=False):
if target_size:
input_shape = target_size + (3,)
else:
input_shape = (None, None, 3)
###########################################################
current_dir = os.path.dirname(os.path.realpath(__file__))
save_path = os.path.join(current_dir, model_name)
if os.path.exists(save_path) == False:
os.mkdir(save_path)
################learning rate scheduler####################
lr = 0.
def lr_scheduler(epoch):
global lr
if lr_dict.has_key(epoch):
lr = lr_dict[epoch]
print 'lr: %f' % lr
return lr
scheduler = LearningRateScheduler(lr_scheduler)
######################## tfboard ###########################
tfboard = TensorBoard(log_dir=os.path.join(save_path, 'logs'))
####################### make model ########################
checkpoint_path = os.path.join(save_path, 'checkpoint_weights.hdf5')
resume_from_epoch = 0
if os.path.isfile(checkpoint_path) and resume_training: # resume from last checkpoint
model_path = os.path.join(save_path, "model.json")
f = open(model_path, 'r')
model_json = f.read()
f.close
model = model_from_json(model_json, {'BilinearUpSampling2D': BilinearUpSampling2D})
sgd = SGD(lr=0.001, momentum=0.9)
model.compile(loss = softmax_sparse_crossentropy_ignoring_last_label, optimizer=sgd, metrics=[sparse_accuracy_ignoring_last_label])
model.load_weights(checkpoint_path)
with open(os.path.join(save_path, 'train_state.pkl')) as f:
resume_from_epoch, batch_size, nb_epoch, lr_dict, weight_decay, nb_classes = pickle.load(f)
print 'resuming from epoch %d'%resume_from_epoch
print lr_dict
else:
model = globals()[model_name](input_shape, weight_decay)
sgd = SGD(lr=0.001, momentum=0.9)
model.compile(loss = softmax_sparse_crossentropy_ignoring_last_label, optimizer=sgd, metrics=[sparse_accuracy_ignoring_last_label])
model_path = os.path.join(save_path, "model.json")
# save model structure
f = open(model_path, 'w')
model_json = model.to_json()
f.write(model_json)
f.close
img_path = os.path.join(save_path, "model.png")
vis_util.plot(model, to_file=img_path, show_shapes=True)
#################### checkpoint saver#######################
checkpoint = ModelCheckpoint(filepath=checkpoint_path, save_weights_only=True)
train_state = TrainingStateCheckpoint(os.path.join(save_path, 'train_state.pkl'), batch_size, nb_epoch, lr_dict, weight_decay, nb_classes, resume_from_epoch)
# set data generator and train
train_datagen = SegDataGenerator(channelwise_center=True, zoom_range=0.5, fill_mode='constant', horizontal_flip=True)
train_datagen.set_ch_mean(np.array([104.00699, 116.66877, 122.67892]))
val_datagen = SegDataGenerator(channelwise_center=True, fill_mode='constant')
val_datagen.set_ch_mean(np.array([104.00699, 116.66877, 122.67892]))
def get_file_len(file_path):
fp = open(file_path)
lines = fp.readlines()
fp.close( )
return len(lines)
history = model.fit_generator(
generator=train_datagen.flow_from_directory(
file_path=train_file_path, data_dir=data_dir, data_suffix='.jpg',
label_dir=label_dir, label_suffix='.png', nb_classes=nb_classes,
target_size=target_size, color_mode='rgb',
batch_size=batch_size, shuffle=True
),
samples_per_epoch=get_file_len(train_file_path),
nb_epoch=nb_epoch,
callbacks=[scheduler, tfboard, checkpoint, train_state],
validation_data=val_datagen.flow_from_directory(
file_path=val_file_path, data_dir=data_dir, data_suffix='.jpg',
label_dir=label_dir, label_suffix='.png', nb_classes=nb_classes,
target_size=target_size, color_mode='rgb',
batch_size=batch_size, shuffle=False
),
nb_val_samples=get_file_len(val_file_path)
)
model.save(save_path+'/final_model.hdf5')
def train(batch_size,
epochs,
lr_base,
lr_power,
weight_decay,
classes,
model_name,
train_file_path,
val_file_path,
data_dir,
label_dir,
target_size=None,
batchnorm_momentum=0.9,
resume_training=False,
class_weight=None,
dataset='VOC2012',
loss_fn=softmax_sparse_crossentropy_ignoring_last_label,
metrics=[sparse_accuracy_ignoring_last_label],
loss_shape=None,
label_suffix='.png',
data_suffix='.jpg',
ignore_label=255,
label_cval=255):
if target_size:
input_shape = target_size + (3,)
else:
input_shape = (None, None, 3)
batch_shape = (batch_size,) + input_shape
# PATH
current_dir = os.path.dirname(os.path.realpath(__file__))
save_path = os.path.join(current_dir, 'Models/' + model_name)
checkpoint_path = os.path.join(save_path, 'checkpoint_weights.hdf5')
img_path = os.path.join(save_path, "model.png")
if os.path.exists(save_path) is False:
os.mkdir(save_path)
# make model
model = globals()[model_name](weight_decay=weight_decay,
input_shape=input_shape,
batch_momentum=batchnorm_momentum,
classes=classes)
# compile model
optimizer = SGD(lr=lr_base, momentum=0.9)
model.compile(loss=loss_fn,
optimizer=optimizer,
metrics=metrics)
if resume_training:
model.load_weights(checkpoint_path, by_name=True)
model.summary()
# save model structure
model_path = os.path.join(save_path, "model.json")
with open(model_path, "w") as f:
model_json = model.to_json()
f.write(model_json)
# set callbacks
lrcb = LearningRateScheduler(lr_scheduler(epoch, mode='power_decay'))
tbcb = TensorBoard(log_dir=os.path.join(save_path, 'logs'), histogram_freq=10, write_graph=True)
cpcb = ModelCheckpoint(filepath=os.path.join(save_path, 'checkpoint_weights.hdf5'), save_weights_only=True)
callbacks = []
callbacks.append(lrcb)
callbacks.append(tbcb)
callbacks.append(cpcb)
# set data generator and train
train_datagen = SegDataGenerator(zoom_range=[0.5, 2.0],
zoom_maintain_shape=True,
crop_mode='random',
crop_size=target_size,
# pad_size=(505, 505),
rotation_range=0.,
shear_range=0,
horizontal_flip=True,
channel_shift_range=20.,
fill_mode='constant',
label_cval=label_cval)
val_datagen = SegDataGenerator()
with open(train_file_path) as fp:
train_file_length = fp.readlines()
steps_per_epoch = int(np.ceil(train_file_length / float(batch_size)))
# from Keras documentation: Total number of steps (batches of samples) to yield from generator before declaring one epoch finished
# and starting the next epoch. It should typically be equal to the number of unique samples of your dataset divided by the batch size.
# model fitting
history = model.fit_generator(
generator=train_datagen.flow_from_directory(
file_path=train_file_path,
data_dir=data_dir, data_suffix=data_suffix,
label_dir=label_dir, label_suffix=label_suffix,
classes=classes,
target_size=target_size, color_mode='rgb',
batch_size=batch_size, shuffle=True,
loss_shape=loss_shape,
ignore_label=ignore_label,
# save_to_dir='Images/'
),
steps_per_epoch=steps_per_epoch,
epochs=epochs,
callbacks=callbacks,
workers=4,
class_weight=class_weight
)
'''
