def train_vgg19(train_data, test_data):
print("begin train vgg19")
ctx = utils.try_all_gpus()
net = vgg19()
net.initialize(ctx=ctx, init=init.Xavier())
soft_max_cross = gluon.loss.SoftmaxCrossEntropyLoss()
trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': 0.05})
utils.train(train_data, test_data, net, soft_max_cross, trainer, ctx, num_epochs=10);
def train(train_augs, test_augs, learning_rate=.1):
batch_size = 128
num_epochs = 10
ctx = utils.try_all_gpus()
loss = gluon.loss.SoftmaxCrossEntropyLoss()
train_data, test_data = get_data(batch_size, train_augs, test_augs)
net = utils.resnet18(10)
net.initialize(ctx=ctx, init=init.Xavier())
net.hybridize()
trainer = gluon.Trainer(net.collect_params(), 'sgd',
{'learning_rate': learning_rate})
utils.train(train_data, test_data, net, loss, trainer, ctx, num_epochs)
def main(column, DIM_NUM):
'''
:params column 表示使用数据集中的哪部分语料
:params DIM_NUM 表示使用的词向量的维度
'''
Params = CNNParameter(column, DIM_NUM)
Params.load_dir()
ctx = utils.try_all_gpus()
csvfile = Params.train_file
vocabfile = Params.vocab_file
vocab = utils.read_vocab(vocabfile)
glove_embedding = text.embedding.CustomEmbedding(
pretrained_file_path=Params.embedding_file, vocabulary=vocab)
net = utils.TextCNN(vocab, DIM_NUM, Params.ngram_kernel_sizes,
Params.nums_channels, Params.num_outputs)
net.initialize(init.Xavier(), ctx=ctx)
# embedding_static 和 embedding_non_static 均使用预训练的词向量。
net.embedding_static.weight.set_data(glove_embedding.idx_to_vec)
#net.embedding_non_static.weight.set_data(glove_embedding.idx_to_vec)
# 训练中不更新 embedding_static 的词向量，即不更新 embedding_static 的模型参数。
net.embedding_static.collect_params().setattr('grad_req', 'null')
trainer = gluon.Trainer(net.collect_params(), 'adam',
{'learning_rate': Params.lr})
loss = gloss.SoftmaxCrossEntropyLoss()
trainSet, valSet = utils.select_sample_by_class(csvfile, ratio=0.85)
train_features, test_features, train_labels, test_labels = utils.read_dg_data(
trainSet, valSet, vocab, column, MAX_LEN=2500)
train_set = gdata.ArrayDataset(train_features, train_labels) #训练集
test_set = gdata.ArrayDataset(test_features, test_labels) #测试集
train_loader = gdata.DataLoader(train_set,
batch_size=Params.batch_size,
shuffle=True)
test_loader = gdata.DataLoader(test_set,
batch_size=Params.batch_size,
shuffle=False)
logging.info("开始训练cnn {} 文本分类模型...".format(column))
best_acc = utils.train(train_loader, test_loader, net, loss, trainer, ctx,
Params.num_epochs, column, Params.best_param_file)
logging.info("模型训练完成,最佳的acc:{:.4f} 开始测试...".format(best_acc))
net.load_parameters(Params.best_param_file, ctx=ctx)
f1 = utils.evaluate_valset(net, valSet, vocab, column)
best_file = os.path.join(Params.result_dir,
"rnn_{}_{:.4f}.csv".format(column, f1))
best_prob_file = os.path.join(Params.result_dir,
"rnn_{}_{:.4f}_prob.csv".format(column, f1))
logging.info("cnn网络在验证集最佳的f1_score:{:.4f}".format(f1))
logging.info("对数据进行测试")
textSet = pd.read_csv(Params.test_file)
y_probs = utils.predict_test_result(net, vocab, textSet, column, best_file)
logging.info("保存概率数据...")
utils.save_prob_file(y_probs, best_prob_file)
logging.info("保存完毕,请查看目录result.")
def main(column,DIM_NUM):
Params = RNNParameter(column,DIM_NUM)
Params.load_dir()
num_outputs = Params.num_outputs
lr = Params.lr
num_epochs = Params.num_epochs
batch_size = Params.batch_size
embed_size = DIM_NUM
num_hiddens = Params.num_hiddens
num_layers = Params.num_layers
bidirectional = Params.bidirectional
ctx = utils.try_all_gpus()
csvfile = Params.train_file
vocab = utils.read_vocab(Params.vocab_file)
glove_embedding = text.embedding.CustomEmbedding(pretrained_file_path=Params.embedding_file, vocabulary=vocab)
net = utils.BiRNN(vocab, embed_size, num_hiddens, num_layers, bidirectional,num_outputs)
net.initialize(init.Xavier(), ctx=ctx)
# 设置 embedding 层的 weight 为预训练的词向量。
net.embedding.weight.set_data(glove_embedding.idx_to_vec)
# 训练中不更新词向量（net.embedding 中的模型参数）。
net.embedding.collect_params().setattr('grad_req', 'null')
trainer = gluon.Trainer(net.collect_params(), 'sgd', {'learning_rate': lr})
loss = gloss.SoftmaxCrossEntropyLoss()
trainSet,valSet = utils.select_sample_by_class(csvfile,ratio=0.85)
train_features,test_features,train_labels,test_labels=utils.read_dg_data(trainSet,valSet,vocab,column,MAX_LEN=2500)
train_set = gdata.ArrayDataset(train_features, train_labels) #训练集
test_set = gdata.ArrayDataset(test_features, test_labels) #测试集
train_loader = gdata.DataLoader(train_set, batch_size=batch_size,shuffle=True)
test_loader = gdata.DataLoader(test_set, batch_size=batch_size, shuffle=False)
logging.info("开始训练rnn {}文本分类模型".format(column))
best_acc = utils.train(train_loader, test_loader, net, loss, trainer, ctx, num_epochs,column,Params.best_param_file)
logging.info("模型训练完成,最佳模型的acc:{:.4f} 开始测试.".format(best_acc))
net.load_parameters(Params.best_param_file,ctx=ctx)
f1= utils.evaluate_valset(net,valSet,vocab,column)
logging.info("rnn网络在验证集的f1_score:{:.4f}".format(f1))
logging.info("对数据进行测试")
textSet = pd.read_csv(os.path.join(Params.data_dir,'test_set.csv'))
y_probs = utils.predict_test_result(net,vocab,textSet,column,'result/rnn_{}_{:.4f}.csv'.format(column,f1))
logging.info("保存概率数据")
utils.save_prob_file(y_probs,'result/rnn_{}_{:.4f}_prob.csv'.format(column,f1))
logging.info("保存完毕,请查看目录result.")
action='store_true',
help='Use synchronize BN across devices.')
parser.add_argument(
'--no-wd',
action='store_true',
help=
'whether to remove weight decay on bias, and beta/gamma for batchnorm layers.'
)
args = parser.parse_args()
return args
data_path = '../traffic-matrices/'
num_epochs = 4096
batchsize = 4096
ctx = utils.try_all_gpus()
tms = utils.buildAllTMToATensor(xmlPath=data_path)
train_data = utils.train_dataloader(tms=tms)
test_data = utils.test_dataloader(tms=tms)
data_train = gdata.ArrayDataset(train_data[0], train_data[1])
data_test = gdata.ArrayDataset(test_data[0], test_data[1])
data_train_iter = gdata.DataLoader(dataset=data_train,
batch_size=batchsize,
shuffle=True)
data_test_iter = gdata.DataLoader(dataset=data_test,
batch_size=batchsize,
shuffle=False)
# for x, y in data_test_iter:
# print(x,y)
# break
import os
import get_net
import utils
import cv2, numpy as np
from mxnet import nd, image
MODEL_NAME = 'res34_cbam_parallel'
CTX = utils.try_all_gpus()
def load_model(
model,
arp_path='../param/res34_bcam_parallel_625_0.2043_0.945_9.74.params'):
net = model
import_path = arp_path
net.load_parameters(import_path)
net.collect_params().reset_ctx(CTX)
print('load {} finished on {}'.format(MODEL_NAME, CTX))
return net
def predict(img, net):
# X = image.imread(img)
X = nd.array(img)
X = normalize_image(X).as_in_context(CTX[0])
# X = (X.astype('float32') / 255).as_in_context(CTX[0])
X = X.transpose((2, 0, 1)).expand_dims(axis=0)
pred = nd.argmin(net(X), axis=1)
utils.save_kvstore(vars(args), os.path.join(path, 'config.json')))
start_epoch = 0
trainingfile = os.path.join(path, 'training.json')
utils.save_kvstore({'epoch': 0, 'lr': args.lr}, trainingfile)
logging.basicConfig(level=logging.INFO,
handlers=[
logging.StreamHandler(),
logging.FileHandler(os.path.join(path, "log.log"))
])
# config the conputation resources
if args.cpu:
ctxs = [mxnet.cpu()]
else:
ctxs = utils.try_all_gpus(args.num_gpus)
assert args.batch_size % len(ctxs) == 0, \
'Total batch size must be multiple of the number of devices'
m = args.batch_size // len(ctxs)
logging.info(
"Split batch samples (batch size={}) to {}, each device loaded {} samples"
.format(args.batch_size, ctxs, m))
# Set the random seed manually for reproducibility.
if args.seed:
np.random.seed(args.seed)
mxnet.random.seed(args.seed)
###############################################################################
# Load data
###############################################################################
