import argparse
import tensorflow as tf
from flyai.dataset import Dataset
from model import Model
# 数据获取辅助类
from path import MODEL_PATH
dataset = Dataset()
dataset.get_all_processor_data()
model = Model(dataset)
parser = argparse.ArgumentParser()
parser.add_argument("-e", "--EPOCHS", default=10, type=int, help="train epochs")
parser.add_argument("-b", "--BATCH", default=32, type=int, help="batch size")
args = parser.parse_args()
'''
使用tensorflow实现自己的算法
'''
# 定义命名空间
x = tf.placeholder(tf.float32, shape=[None, 200, 200, 3], name='input_x')
y = tf.placeholder(tf.int64, shape=[None], name='input_y')
keep_prob = tf.placeholder(tf.float32, name='keep_prob')
# 初始化权值
def weight_variable(shape, name):
initial = tf.truncated_normal(shape, stddev=0.1)
return tf.Variable(initial, name=name)
model_cnn.add(MaxPooling1D(1))
model_cnn.add(Flatten())
model_cnn.add(Dense(hidden_dim, activation='relu'))
model_cnn.add(Dropout(0.5))
model_cnn.add(Dense(numclass, activation='softmax'))
model_cnn.summary()
model_cnn.compile(loss='categorical_crossentropy',
optimizer=Adam(lr=learning_rate),
metrics=['acc'])
lr_level = 2
best_score_by_acc = 0.
best_score_by_loss = 999.
x_train, y_train, x_val, y_val = dataset.get_all_processor_data()
# x_val, y_val =dataset.get_all_validation_data()
for step in range(args.EPOCHS):
cur_step = str(step + 1) + "/" + str(args.EPOCHS)
print('\n步骤' + cur_step)
x_3, y_3, x_4, y_4 = [], [], [], []
for iters in range(numclass):
xx_tmp_train, yy_tmp_train, xx_tmp_val, yy_tmp_val = dataset_slice[
iters].next_batch()
# 合并3类train
x_3.append(xx_tmp_train)
y_3.append(yy_tmp_train)
x_4.append(xx_tmp_val)
y_4.append(yy_tmp_val)
args = parser.parse_args()
def eval(preds_prob, y_test):
preds = np.zeros(preds_prob.shape)
preds[preds_prob >= 0.5] = 1
train_accuracy = (preds == y_test).sum() / preds_prob.shape[0]
return train_accuracy
# 训练并评估模型
data = Dataset(epochs=args.EPOCHS, batch=args.BATCH,)
model = Model(data)
x, y, x_test, y_test = data.get_all_processor_data()
#
# validateNum = 30
# x_train = x[0:x.shape[0]-validateNum,:]
# y_train = y[0:y.shape[0]-validateNum]
# x_test = x[-validateNum:,:]
# y_test = y[-validateNum:]
x_train = x
y_train = y
print("the length of train data: %d" % data.get_train_length())
print("the length of x_train: %d" % x_train.shape[0])
print("the length of x_test: %d" % x_test.shape[0])
# the length of train data: 162
# the length of x_train: 162
项目的超参
'''
parser = argparse.ArgumentParser()
parser.add_argument("-e", "--EPOCHS", default=10,
type=int, help="train epochs")
parser.add_argument("-b", "--BATCH", default=32, type=int, help="batch size")
args = parser.parse_args()
'''
flyai库中的提供的数据处理方法
传入整个数据训练多少轮，每批次批大小
'''
dataset = Dataset(epochs=args.EPOCHS, batch=args.BATCH)
model = Model(dataset)
train_x, train_y, val_x, val_y = dataset.get_all_processor_data()
train_dataset = FlyAIDataSet(train_x, train_y)
val_dataset = FlyAIDataSet(val_x, val_y)
def collate_fn(data):
data.sort(key=lambda x: len(x[0]), reverse=True)
inputs, labels = zip(*data)
inputs_len = [len(item) for item in inputs if len(item) != 0]
inputs = [torch.Tensor(x) for x in inputs[:len(inputs_len)]]
inputs = rnn_utils.pad_sequence(inputs, batch_first=True)
labels = torch.LongTensor(labels[:len(inputs_len)])
return inputs, inputs_len, labels
train_loader = torch.utils.data.DataLoader(train_dataset, shuffle=True,
parser = argparse.ArgumentParser()
parser.add_argument("-e", "--EPOCHS", default=1, type=int, help="train epochs")
parser.add_argument("-b", "--BATCH", default=64, type=int, help="batch size")
args = parser.parse_args()
'''
flyai库中的提供的数据处理方法
传入整个数据训练多少轮，每批次批大小
'''
dataset = Dataset(epochs=args.EPOCHS, batch=args.BATCH)
vocab_size = Processor().getWordsCount()
# region 准备数据
allDataLength = dataset.get_train_length()
print('length of all dev data: %d' % allDataLength)
x, y, x_, y_ = dataset.get_all_processor_data()
# trainLen = (int)(95*allDataLength/100)
# x_train = x[0:trainLen]
# y_train = y[0:trainLen]
# x_val = x[trainLen:]
# y_val = y[trainLen:]
x_train = x
y_train = y
x_val = x_
y_val = y_
# endregion
myModel = Model(dataset)
