def main(_): log_path = os.path.join(config.PATH, config.LOG_PATH) checkpoint_dir = os.path.join(config.PATH, config.CHECKPOINT_PATH) # 加载数据集 (x_train, y_train), (x_test, y_test) = load_fashion( path=os.path.join(config.PATH, config.DATA_PATH)) x_train = x_train.astype('float32') / 255 x_test = x_test.astype('float32') / 255 x_train = tf.reshape(x_train, shape=(-1, 28, 28, 1)) x_test = tf.reshape(x_test, shape=(-1, 28, 28, 1)) y_train_ohe = tf.one_hot(y_train, depth=FLAGS.num_classes) y_test_ohe = tf.one_hot(y_test, depth=FLAGS.num_classes) buffer_size = x_train.numpy().shape[0] n_batch = buffer_size // FLAGS.batch_size resiud = buffer_size % FLAGS.batch_size train_dataset = tf.data.Dataset.from_tensor_slices( (x_train, y_train_ohe)).shuffle(1000) train_dataset = train_dataset.batch(FLAGS.batch_size, drop_remainder=False) #优化器 optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.learning_rate) # 初始化模型 model = ResNet(block_list=[2, 2, 2], num_classes=FLAGS.num_classes) save_kwargs = { 'model': model, 'optimizer': optimizer, 'global_step': tf.train.get_or_create_global_step() } # 定义callbacks write_summary = WriteSummary(log_path=log_path) model_checkpoint = ModelCheckpoint(checkpoint_dir=checkpoint_dir, mode='min', monitor='val_loss', save_best_only=False, epoch_per=3, **save_kwargs) early_stop = EarlyStopping(mode='min', patience=FLAGS.early_stopping_rounds) progressbar = ProgressBar(data_size=buffer_size, n_batch=n_batch, batch_size=FLAGS.batch_size, resiud=resiud, eval_name='acc', loss_name='loss') # 开始训练模型 model.fit(trainDataset=train_dataset, valDataset=[x_test, y_test_ohe], epochs=FLAGS.epochs, optimizer=optimizer, ModelCheckPoint=model_checkpoint, progressbar=progressbar, write_summary=write_summary, early_stopping=early_stop)
def main(_): log_path = os.path.join(config.PATH, config.LOG_PATH) checkpoint_dir = os.path.join(config.PATH, config.CHECKPOINT_PATH) # 加载数据集 (x_train, y_train), (x_test, y_test) = load_imdb(path=os.path.join( config.PATH, config.DATA_PATH), maxlen=FLAGS.maxlen, num_words=FLAGS.num_words) train_sequence = sequence.pad_sequences(x_train, maxlen=FLAGS.maxlen, dtype='int32', padding='post', truncating='post', value=0) test_sequence = sequence.pad_sequences(x_test, maxlen=FLAGS.maxlen, dtype='int32', padding='post', truncating='post', value=0) gc.collect() buffer_size = len(train_sequence) n_batch = buffer_size // FLAGS.batch_size resiud = buffer_size % FLAGS.batch_size train_dataset = tf.data.Dataset.from_tensor_slices( (train_sequence, y_train)).shuffle(10000) train_dataset = train_dataset.batch(FLAGS.batch_size, drop_remainder=False) val = [tf.convert_to_tensor(test_sequence), tf.convert_to_tensor(y_test)] # 优化器 optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.learning_rate) # 初始化模型 model = BiLSTM(units=FLAGS.lstm_units, output_dim=FLAGS.output_dim, input_length=FLAGS.maxlen, input_dim=FLAGS.num_words, trainable=True) save_kwargs = { 'model': model, 'optimizer': optimizer, 'global_step': tf.train.get_or_create_global_step() } # 定义callbacks write_summary = WriteSummary(log_path=log_path) model_checkpoint = ModelCheckpoint(checkpoint_dir=checkpoint_dir, mode='min', monitor='val_loss', save_best_only=True, epoch_per=None, **save_kwargs) early_stop = EarlyStopping(mode='min', patience=FLAGS.early_stopping_rounds) progressbar = ProgressBar(data_size=buffer_size, n_batch=n_batch, batch_size=FLAGS.batch_size, resiud=resiud, eval_name='acc', loss_name='loss') # 开始训练模型 model.fit(trainDataset=train_dataset, valDataset=val, epochs=FLAGS.epochs, optimizer=optimizer, ModelCheckPoint=model_checkpoint, progressbar=progressbar, write_summary=write_summary, early_stopping=early_stop)
def main(_): log_path = os.path.join(config.PATH, config.LOG_PATH) checkpoint_dir = os.path.join(config.PATH, config.CHECKPOINT_PATH) # 加载数据 (x_train, y_train), (x_test, y_test) = load_boston(path=os.path.join( config.PATH, config.DATA_PATH), test_split=0.2, seed=0) # 数据预处理,标准化 x_train, x_test = z_score(x_train, x_test) # 如果全量数据不大,可以对全量数据进行shuffle # 如果全量数据很大,直接全量数据shuffle会很慢, # 则直接shuffle所需大小即可,一般都是10000 buffer_size = len(x_train) resiud = buffer_size % FLAGS.batch_size n_batch = buffer_size // FLAGS.batch_size # batch的次数 # 转换为dataset形式 train_dataset = tf.data.Dataset.from_tensor_slices( (x_train, y_train)).shuffle(buffer_size) train_dataset = train_dataset.batch(FLAGS.batch_size, drop_remainder=False) # 优化器 optimizer = tf.train.GradientDescentOptimizer( learning_rate=FLAGS.learning_rate) # 初始化模型 model = Regressor() save_kwargs = { 'model': model, 'optimizer': optimizer, 'global_step': tf.train.get_or_create_global_step() } # 定义callbacks write_summary = WriteSummary(log_path=log_path) model_checkpoint = ModelCheckpoint(checkpoint_dir=checkpoint_dir, mode='min', monitor='val_loss', save_best_only=False, epoch_per=3, **save_kwargs) early_stop = EarlyStopping(mode='min', patience=FLAGS.early_stopping_rounds) progressbar = ProgressBar(data_size=buffer_size, n_batch=n_batch, batch_size=FLAGS.batch_size, resiud=resiud, eval_name='r2', loss_name='loss') # 开始训练模型 model.fit(trainDataset=train_dataset, valDataset=[ tf.convert_to_tensor(x_test), tf.convert_to_tensor(y_test) ], epochs=FLAGS.epochs, optimizer=optimizer, ModelCheckPoint=model_checkpoint, progressbar=progressbar, write_summary=write_summary, early_stopping=early_stop)
def main(_): log_path = os.path.join(config.PATH, config.LOG_PATH) checkpoint_dir = os.path.join(config.PATH, config.CHECKPOINT_PATH) # 加载数据 (x_train, y_train), (x_test, y_test) = load_mnist( path=os.path.join(config.PATH, config.DATA_PATH)) # cpu下好像不能使用tf.float64 x_train = x_train.astype('float32') / 255. x_test = x_test.astype('float32') / 255. x_train = tf.reshape(x_train, shape=(-1, 28, 28, 1)) x_test = tf.reshape(x_test, shape=(-1, 28, 28, 1)) y_train_ohe = tf.one_hot(y_train, depth=FLAGS.num_classes) y_test_ohe = tf.one_hot(y_test, depth=FLAGS.num_classes) buffer_size = x_train.numpy().shape[0] n_batch = buffer_size // FLAGS.batch_size resiud = buffer_size % FLAGS.batch_size train_dataset = tf.data.Dataset.from_tensor_slices((x_train, y_train_ohe)) if FLAGS.data_aug: ''' 好像dataset里面不太好实用numpy形式(调用cv2)进行数据曾强处理 ''' c_i_dataset = train_dataset.map( lambda img, label: [central_scale_images(img, scale=0.65), label]) r_b_dataset = train_dataset.map( lambda img, label: [random_brightness_image(img), label]) train_dataset = train_dataset.concatenate(r_b_dataset) train_dataset = train_dataset.concatenate(c_i_dataset) buffer_size = sum([1 for _ in train_dataset]) train_dataset = train_dataset.shuffle(1000) train_dataset = train_dataset.batch(FLAGS.batch_size, drop_remainder=False) #优化器 optimizer = tf.train.AdamOptimizer(learning_rate=FLAGS.learning_rate) # 初始化模型 model = CNN(num_classes=FLAGS.num_classes) save_kwargs = { 'model': model, 'optimizer': optimizer, 'global_step': tf.train.get_or_create_global_step() } # 定义callbacks write_summary = WriteSummary(log_path=log_path) model_checkpoint = ModelCheckpoint(checkpoint_dir=checkpoint_dir, mode='min', monitor='val_loss', save_best_only=True, epoch_per=3, **save_kwargs) early_stop = EarlyStopping(mode='min', patience=FLAGS.early_stopping_rounds) progressbar = ProgressBar(data_size=buffer_size, n_batch=n_batch, batch_size=FLAGS.batch_size, resiud=resiud, eval_name='acc', loss_name='loss') # 开始训练模型 model.fit(trainDataset=train_dataset, valDataset=[x_test, y_test_ohe], epochs=FLAGS.epochs, optimizer=optimizer, ModelCheckPoint=model_checkpoint, progressbar=progressbar, write_summary=write_summary, early_stopping=early_stop)
def main(_): feature_path = os.path.join(config.PATH, config.FEATURES_PATH) # tfrecord数据路径 log_path = os.path.join(config.PATH, config.LOG_PATH) # 日志路径 checkpoint_dir = os.path.join(config.PATH, config.CHECKPOINT_PATH) # checkpoint路径 train_tfrecord = os.path.join(feature_path, 'train.tfrecords') test_tfrecord = os.path.join(feature_path, 'test.tfrecords') buffer_size = sum(1 for _ in tf.python_io.tf_record_iterator(train_tfrecord)) # 计算训练数据总数 n_batch = buffer_size // FLAGS.batch_size resiud = buffer_size % FLAGS.batch_size base_lr = FLAGS.max_learning_rate / 3 stepsize = 2 * n_batch # 获取数据集 train_dataset,val_dataset = get_dataset(train_tfrecord=train_tfrecord, val_tfrecord = test_tfrecord, buffer_size=1000, batch_size=FLAGS.batch_size) #采用cyclical机制 learning_rate_tf = tf.Variable(base_lr) # 定义学习率张量 clr = CyclicLR(base_lr = base_lr, init_lr=learning_rate_tf, max_lr=FLAGS.max_learning_rate, mode='triangular', step_size=stepsize) optimizer = tf.train.AdamOptimizer(learning_rate=clr.learning_rate,beta1=0.98,beta2=0.99) # 初始化模型 model = DenseNet(k=FLAGS.growth_rate, weight_decay=FLAGS.l2_regularization, num_outputs=FLAGS.num_outputs, units_per_block=FLAGS.units_per_block, momentum=FLAGS.momentum, epsilon=FLAGS.epsilon, initial_pool=FLAGS.initial_pool) save_kwargs = { 'model': model, 'optimizer': optimizer, 'global_step': tf.train.get_or_create_global_step() } # 定义callbacks write_summary = WriteSummary(log_path=log_path) model_checkpoint = ModelCheckpoint(checkpoint_dir=checkpoint_dir, mode='min', monitor='val_Loss',save_best_only=False, epoch_per=3, **save_kwargs) early_stop = EarlyStopping(mode='min',patience=FLAGS.early_stopping_rounds) progressbar = ProgressBar(data_size=buffer_size, n_batch=n_batch, batch_size=FLAGS.batch_size, resiud=resiud, eval_name='acc', loss_name='loss') # 开始训练模型 model.fit(trainDataset=train_dataset, valDataset=val_dataset, epochs=FLAGS.epochs, optimizer=optimizer, ModelCheckPoint=model_checkpoint, progressbar=progressbar, write_summary=write_summary, early_stopping=early_stop, lr_schedule=clr)