def main(_):
log_path = os.path.join(config.PATH, config.LOG_PATH)
checkpoint_dir = os.path.join(config.PATH, config.CHECKPOINT_PATH)
# 加载数据集
(x_train, y_train), (x_test, y_test) = load_fashion(
path=os.path.join(config.PATH, config.DATA_PATH))
x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255
x_train = tf.reshape(x_train, shape=(-1, 28, 28, 1))
x_test = tf.reshape(x_test, shape=(-1, 28, 28, 1))
y_train_ohe = tf.one_hot(y_train, depth=FLAGS.num_classes)
y_test_ohe = tf.one_hot(y_test, depth=FLAGS.num_classes)
buffer_size = x_train.numpy().shape[0]
n_batch = buffer_size // FLAGS.batch_size
resiud = buffer_size % FLAGS.batch_size
train_dataset = tf.data.Dataset.from_tensor_slices(
(x_train, y_train_ohe)).shuffle(1000)
train_dataset = train_dataset.batch(FLAGS.batch_size, drop_remainder=False)
#优化器
optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.learning_rate)
# 初始化模型
model = ResNet(block_list=[2, 2, 2], num_classes=FLAGS.num_classes)
save_kwargs = {
'model': model,
'optimizer': optimizer,
'global_step': tf.train.get_or_create_global_step()
}
# 定义callbacks
write_summary = WriteSummary(log_path=log_path)
model_checkpoint = ModelCheckpoint(checkpoint_dir=checkpoint_dir,
mode='min',
monitor='val_loss',
save_best_only=False,
epoch_per=3,
**save_kwargs)
early_stop = EarlyStopping(mode='min',
patience=FLAGS.early_stopping_rounds)
progressbar = ProgressBar(data_size=buffer_size,
n_batch=n_batch,
batch_size=FLAGS.batch_size,
resiud=resiud,
eval_name='acc',
loss_name='loss')
# 开始训练模型
model.fit(trainDataset=train_dataset,
valDataset=[x_test, y_test_ohe],
epochs=FLAGS.epochs,
optimizer=optimizer,
ModelCheckPoint=model_checkpoint,
progressbar=progressbar,
write_summary=write_summary,
early_stopping=early_stop)
def main(_):
log_path = os.path.join(config.PATH, config.LOG_PATH)
checkpoint_dir = os.path.join(config.PATH, config.CHECKPOINT_PATH)
# 加载数据集
(x_train, y_train), (x_test, y_test) = load_imdb(path=os.path.join(
config.PATH, config.DATA_PATH),
maxlen=FLAGS.maxlen,
num_words=FLAGS.num_words)
train_sequence = sequence.pad_sequences(x_train,
maxlen=FLAGS.maxlen,
dtype='int32',
padding='post',
truncating='post',
value=0)
test_sequence = sequence.pad_sequences(x_test,
maxlen=FLAGS.maxlen,
dtype='int32',
padding='post',
truncating='post',
value=0)
gc.collect()
buffer_size = len(train_sequence)
n_batch = buffer_size // FLAGS.batch_size
resiud = buffer_size % FLAGS.batch_size
train_dataset = tf.data.Dataset.from_tensor_slices(
(train_sequence, y_train)).shuffle(10000)
train_dataset = train_dataset.batch(FLAGS.batch_size, drop_remainder=False)
val = [tf.convert_to_tensor(test_sequence), tf.convert_to_tensor(y_test)]
# 优化器
optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.learning_rate)
# 初始化模型
model = BiLSTM(units=FLAGS.lstm_units,
output_dim=FLAGS.output_dim,
input_length=FLAGS.maxlen,
input_dim=FLAGS.num_words,
trainable=True)
save_kwargs = {
'model': model,
'optimizer': optimizer,
'global_step': tf.train.get_or_create_global_step()
}
# 定义callbacks
write_summary = WriteSummary(log_path=log_path)
model_checkpoint = ModelCheckpoint(checkpoint_dir=checkpoint_dir,
mode='min',
monitor='val_loss',
save_best_only=True,
epoch_per=None,
**save_kwargs)
early_stop = EarlyStopping(mode='min',
patience=FLAGS.early_stopping_rounds)
progressbar = ProgressBar(data_size=buffer_size,
n_batch=n_batch,
batch_size=FLAGS.batch_size,
resiud=resiud,
eval_name='acc',
loss_name='loss')
# 开始训练模型
model.fit(trainDataset=train_dataset,
valDataset=val,
epochs=FLAGS.epochs,
optimizer=optimizer,
ModelCheckPoint=model_checkpoint,
progressbar=progressbar,
write_summary=write_summary,
early_stopping=early_stop)
def main(_):
log_path = os.path.join(config.PATH, config.LOG_PATH)
checkpoint_dir = os.path.join(config.PATH, config.CHECKPOINT_PATH)
# 加载数据
(x_train, y_train), (x_test, y_test) = load_boston(path=os.path.join(
config.PATH, config.DATA_PATH),
test_split=0.2,
seed=0)
# 数据预处理，标准化
x_train, x_test = z_score(x_train, x_test)
# 如果全量数据不大，可以对全量数据进行shuffle
# 如果全量数据很大，直接全量数据shuffle会很慢，
# 则直接shuffle所需大小即可，一般都是10000
buffer_size = len(x_train)
resiud = buffer_size % FLAGS.batch_size
n_batch = buffer_size // FLAGS.batch_size # batch的次数
# 转换为dataset形式
train_dataset = tf.data.Dataset.from_tensor_slices(
(x_train, y_train)).shuffle(buffer_size)
train_dataset = train_dataset.batch(FLAGS.batch_size, drop_remainder=False)
# 优化器
optimizer = tf.train.GradientDescentOptimizer(
learning_rate=FLAGS.learning_rate)
# 初始化模型
model = Regressor()
save_kwargs = {
'model': model,
'optimizer': optimizer,
'global_step': tf.train.get_or_create_global_step()
}
# 定义callbacks
write_summary = WriteSummary(log_path=log_path)
model_checkpoint = ModelCheckpoint(checkpoint_dir=checkpoint_dir,
mode='min',
monitor='val_loss',
save_best_only=False,
epoch_per=3,
**save_kwargs)
early_stop = EarlyStopping(mode='min',
patience=FLAGS.early_stopping_rounds)
progressbar = ProgressBar(data_size=buffer_size,
n_batch=n_batch,
batch_size=FLAGS.batch_size,
resiud=resiud,
eval_name='r2',
loss_name='loss')
# 开始训练模型
model.fit(trainDataset=train_dataset,
valDataset=[
tf.convert_to_tensor(x_test),
tf.convert_to_tensor(y_test)
],
epochs=FLAGS.epochs,
optimizer=optimizer,
ModelCheckPoint=model_checkpoint,
progressbar=progressbar,
write_summary=write_summary,
early_stopping=early_stop)
def main(_):
log_path = os.path.join(config.PATH, config.LOG_PATH)
checkpoint_dir = os.path.join(config.PATH, config.CHECKPOINT_PATH)
# 加载数据
(x_train, y_train), (x_test, y_test) = load_mnist(
path=os.path.join(config.PATH, config.DATA_PATH))
# cpu下好像不能使用tf.float64
x_train = x_train.astype('float32') / 255.
x_test = x_test.astype('float32') / 255.
x_train = tf.reshape(x_train, shape=(-1, 28, 28, 1))
x_test = tf.reshape(x_test, shape=(-1, 28, 28, 1))
y_train_ohe = tf.one_hot(y_train, depth=FLAGS.num_classes)
y_test_ohe = tf.one_hot(y_test, depth=FLAGS.num_classes)
buffer_size = x_train.numpy().shape[0]
n_batch = buffer_size // FLAGS.batch_size
resiud = buffer_size % FLAGS.batch_size
train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train_ohe))
if FLAGS.data_aug:
'''
好像dataset里面不太好实用numpy形式（调用cv2）进行数据曾强处理
'''
c_i_dataset = train_dataset.map(
lambda img, label: [central_scale_images(img, scale=0.65), label])
r_b_dataset = train_dataset.map(
lambda img, label: [random_brightness_image(img), label])
train_dataset = train_dataset.concatenate(r_b_dataset)
train_dataset = train_dataset.concatenate(c_i_dataset)
buffer_size = sum([1 for _ in train_dataset])
train_dataset = train_dataset.shuffle(1000)
train_dataset = train_dataset.batch(FLAGS.batch_size, drop_remainder=False)
#优化器
optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.learning_rate)
# 初始化模型
model = CNN(num_classes=FLAGS.num_classes)
save_kwargs = {
'model': model,
'optimizer': optimizer,
'global_step': tf.train.get_or_create_global_step()
}
# 定义callbacks
write_summary = WriteSummary(log_path=log_path)
model_checkpoint = ModelCheckpoint(checkpoint_dir=checkpoint_dir,
mode='min',
monitor='val_loss',
save_best_only=True,
epoch_per=3,
**save_kwargs)
early_stop = EarlyStopping(mode='min',
patience=FLAGS.early_stopping_rounds)
progressbar = ProgressBar(data_size=buffer_size,
n_batch=n_batch,
batch_size=FLAGS.batch_size,
resiud=resiud,
eval_name='acc',
loss_name='loss')
# 开始训练模型
model.fit(trainDataset=train_dataset,
valDataset=[x_test, y_test_ohe],
epochs=FLAGS.epochs,
optimizer=optimizer,
ModelCheckPoint=model_checkpoint,
progressbar=progressbar,
write_summary=write_summary,
early_stopping=early_stop)
def main(_):
feature_path = os.path.join(config.PATH, config.FEATURES_PATH) # tfrecord数据路径
log_path = os.path.join(config.PATH, config.LOG_PATH) # 日志路径
checkpoint_dir = os.path.join(config.PATH, config.CHECKPOINT_PATH) # checkpoint路径
train_tfrecord = os.path.join(feature_path, 'train.tfrecords')
test_tfrecord = os.path.join(feature_path, 'test.tfrecords')
buffer_size = sum(1 for _ in tf.python_io.tf_record_iterator(train_tfrecord)) # 计算训练数据总数
n_batch = buffer_size // FLAGS.batch_size
resiud = buffer_size % FLAGS.batch_size
base_lr = FLAGS.max_learning_rate / 3
stepsize = 2 * n_batch
# 获取数据集
train_dataset,val_dataset = get_dataset(train_tfrecord=train_tfrecord,
val_tfrecord = test_tfrecord,
buffer_size=1000,
batch_size=FLAGS.batch_size)
#采用cyclical机制
learning_rate_tf = tf.Variable(base_lr) # 定义学习率张量
clr = CyclicLR(base_lr = base_lr,
init_lr=learning_rate_tf,
max_lr=FLAGS.max_learning_rate,
mode='triangular',
step_size=stepsize)
optimizer = tf.train.AdamOptimizer(learning_rate=clr.learning_rate,beta1=0.98,beta2=0.99)
# 初始化模型
model = DenseNet(k=FLAGS.growth_rate,
weight_decay=FLAGS.l2_regularization,
num_outputs=FLAGS.num_outputs,
units_per_block=FLAGS.units_per_block,
momentum=FLAGS.momentum,
epsilon=FLAGS.epsilon,
initial_pool=FLAGS.initial_pool)
save_kwargs = {
'model': model,
'optimizer': optimizer,
'global_step': tf.train.get_or_create_global_step()
}
# 定义callbacks
write_summary = WriteSummary(log_path=log_path)
model_checkpoint = ModelCheckpoint(checkpoint_dir=checkpoint_dir, mode='min', monitor='val_Loss',save_best_only=False, epoch_per=3,
**save_kwargs)
early_stop = EarlyStopping(mode='min',patience=FLAGS.early_stopping_rounds)
progressbar = ProgressBar(data_size=buffer_size, n_batch=n_batch, batch_size=FLAGS.batch_size, resiud=resiud,
eval_name='acc', loss_name='loss')
# 开始训练模型
model.fit(trainDataset=train_dataset,
valDataset=val_dataset,
epochs=FLAGS.epochs,
optimizer=optimizer,
ModelCheckPoint=model_checkpoint,
progressbar=progressbar,
write_summary=write_summary,
early_stopping=early_stop,
lr_schedule=clr)