def _train(self):
print('....')
with tf.Session(config=tf.ConfigProto(
device_count={'GPU': 0})) as self.sess:
#with tf.Session() as self.sess:
self.sess.run(tf.global_variables_initializer())
allDataLength = len(self.train)
global_step = 0
print('total step:%d' %
(Config.epochs *
(int(allDataLength / Config.batch_size) + 1)))
for i in range(Config.epochs):
num_batchs = int(allDataLength / Config.batch_size) + 1
for j in range(num_batchs):
global_step += 1
now_batch = self._get_batch(self.train, j)
start = time.time()
batch_dict = {
self.ph['single_index']: now_batch[1],
self.ph['numerical_index']: now_batch[2],
self.ph['value']: now_batch[-1],
self.ph['label']: get_label(now_batch[0], 2),
self.ph['numerical_value']: now_batch[3],
self.train_phase: True
}
if Config.multi_features:
for idx, s in enumerate(Config.multi_features):
batch_dict[self.ph['multi_index_%s' %
s]] = now_batch[4]
batch_dict[self.ph['multi_value_%s' %
s]] = now_batch[5]
end = time.time()
start = time.time()
_out, _loss, _ = self.sess.run(
(self.softmax_output, self.loss, self.optimizer),
feed_dict=batch_dict)
end = time.time()
if global_step % 10 == 0:
__out, __loss, __ = self.sess.run(
(self.softmax_output, self.loss, self.optimizer),
feed_dict=self.valid_dict)
self.global_step.append(global_step)
self.global_train_auc.append(_loss)
self.global_valid_auc.append(__loss)
print(
'step:', global_step, 'train loss:', _loss,
'valid loss:', __loss, 'valid_auc:',
auc_score(__out, get_label(self.valid_batch[0], 2),
2))
def __init__(self):
self.total_emb, self.single_size, self.numerical_size, self.multi_size = _get_conf()
self.field_size = self.single_size + self.numerical_size + self.multi_size
self.embedding_length = self.field_size * Config.embedding_size
self._init_data()
self._init_placeholder()
self._init_Variable()
self._init_Model()
self.valid_batch = self._get_batch(self.valid, -1)
self.valid_label = get_label(self.valid_batch[0], 2)
self.valid_dict = {
self.ph['single_index']: self.valid_batch[1],
self.ph['numerical_index']: self.valid_batch[2],
self.ph['numerical_value']: self.valid_batch[3],
self.ph['value']: self.valid_batch[-1],
self.ph['label']: self.valid_label,
self.train_phase: False
}
if Config.multi_features:
for idx, s in enumerate(Config.multi_features):
self.valid_dict[self.ph['multi_index_%s' % s]] = self.valid_batch[4]
self.valid_dict[self.ph['multi_value_%s' % s]] = self.valid_batch[5]
self.global_step = []
self.global_train_auc = []
self.global_valid_auc = []
self._train()
self._save_loss()
def train(model):
print('begin to train model......')
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
all_data_length = len(train_data)
global_step = 0
print('total step:%d' % (conf.epochs *
(int(all_data_length / conf.batch_size) + 1)))
for i in range(conf.epochs):
num_batchs = int(all_data_length / conf.batch_size) + 1
for j in range(num_batchs):
global_step += 1
now_batch = tools.get_batch(train_data, j, single_size,
numerical_size, multi_size,
conf.batch_size,
conf.use_numerical_embedding)
batch_dict = {
model.ph['label']: tools.get_label(now_batch[0], 2),
model.ph['single_index']: now_batch[1],
model.ph['numerical_index']: now_batch[2],
model.ph['numerical_value']: now_batch[3],
model.ph['value']: now_batch[-1],
model.train_phase: True
}
if conf.MULTI_FEATURES:
for idx, s in enumerate(conf.MULTI_FEATURES):
batch_dict[model.ph['multi_index_%s' %
s]] = now_batch[4]
batch_dict[model.ph['multi_value_%s' %
s]] = now_batch[5]
_out, _loss, _ = sess.run(
[model.softmax_output, model.loss, model.optimizer],
feed_dict=batch_dict)
end = time.time()
if global_step % 10 == 0:
_out2, _loss2, _ = sess.run(
[model.softmax_output, model.loss, model.optimizer],
feed_dict=valid_dict)
print(
'step:', global_step, 'train loss:', _loss,
'valid loss:', _loss2, 'valid_auc:',
tools.auc_score(_out2,
tools.get_label(valid_batch[0], 2), 2))
def _train(self):
print('....')
#设置GPU用量
#config = tf.ConfigProto()
#config.gpu_options.per_process_gpu_memory_fraction = 0.9 # 占用GPU100%的显存
#session = tf.Session(config=config)
#设置最小的GPU使用量
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
print('start')
saver = tf.train.Saver() #保存模型
sess = tf.Session()
with tf.Session() as self.sess:
self.sess.run(tf.global_variables_initializer())
model_file = tf.train.get_checkpoint_state('ckpt/')
path = model_file.all_model_checkpoint_paths[model_i]
#print(path)
saver.restore(self.sess, path)
allDataLength = len(self.test)
num_batchs = int(allDataLength / Config.batch_size) + 1
print('total step:%d' % (num_batchs))
f = open('OUT/test_out.csv', 'w')
for j in range(num_batchs):
print(j)
#测试集
self.test_batch = self._get_batch(self.test, j)
self.test_label = get_label(self.test_batch[0], 2)
self.test_dict = {
self.ph['single_index']: self.test_batch[1],
self.ph['numerical_index']: self.test_batch[2],
self.ph['numerical_value']: self.test_batch[3],
self.ph['value']: self.test_batch[-1],
self.ph['label']: self.test_label,
self.train_phase: False
}
if Config.multi_features:
for idx, s in enumerate(Config.multi_features):
self.test_dict[self.ph['multi_index_%s' %
s]] = self.test_batch[4]
self.test_dict[self.ph['multi_value_%s' %
s]] = self.test_batch[5]
test_out = self.sess.run(self.softmax_output,
feed_dict=self.test_dict)
test_output = test_out.tolist()
f.write(str(test_output) + '\n')
f.close()
def train():
f_size = int(flags.img_size / 8)
encoder_length = f_size * f_size
with open(flags.train_txt) as f:
sample = [line.rstrip() for line in f]
sample = np.array(sample)
iteration = len(sample) // flags.batch_size
data = Dataset(sample)
tf.reset_default_graph()
train_graph = tf.Graph()
infer_graph = tf.Graph()
start = time.time()
with train_graph.as_default():
c, inputs = build_cnn(is_training=True,
batch_size=flags.batch_size,
img_size=flags.img_size,
channels=flags.channels)
deconv_outputs = build_deconv(True, c, flags.batch_size)
x = np.linspace(-0.5, 0.5, f_size)
x = np.tile(x, (f_size, 1))
y = np.transpose(x)
x = np.expand_dims(x, axis=2)
y = np.expand_dims(y, axis=2)
m = np.concatenate((x, y), axis=2)
m = np.expand_dims(m, axis=0)
m = np.repeat(m, flags.batch_size, axis=0)
m = tf.convert_to_tensor(m, np.float32)
encoder_outputs = tf.concat([c, m], -1)
encoder_outputs = tf.reshape(encoder_outputs,
shape=(-1, f_size * f_size, 258))
encoder_outputs = tf.transpose(encoder_outputs, [1, 0, 2])
train_op, loss , sample_ids,logits, decoder_inputs, \
target_labels, learning_rate,attention_weights_history,att_label,lamda,att_mask,input_seg= build_network(encoder_outputs,
False,
flags.batch_size,
flags.decoder_length,
flags.tgt_vocab_size,
flags.attn_num_hidden,
encoder_length,
flags.max_gradient_norm,
f_size,
flags.att_loss,
flags.img_size,
deconv_outputs
)
initializer = tf.global_variables_initializer()
train_saver = tf.train.Saver()
train_sess = tf.Session(graph=train_graph)
train_sess.run(initializer)
with infer_graph.as_default():
c_t, inputs_t = build_cnn(is_training=False,
batch_size=flags.batch_size,
img_size=flags.img_size,
channels=flags.channels)
deconv_outputs_t = build_deconv(False, c_t, flags.batch_size)
x_t = np.linspace(-0.5, 0.5, f_size)
x_t = np.tile(x_t, (f_size, 1))
y_t = np.transpose(x_t)
x_t = np.expand_dims(x_t, axis=2)
y_t = np.expand_dims(y_t, axis=2)
m_t = np.concatenate((x_t, y_t), axis=2)
m_t = np.expand_dims(m_t, axis=0)
m_t = np.repeat(m_t, flags.batch_size, axis=0)
m_t = tf.convert_to_tensor(m_t, np.float32)
encoder_outputs_t = tf.concat([c_t, m_t], -1)
encoder_outputs_t = tf.reshape(encoder_outputs_t,
shape=(-1, f_size * f_size, 258))
encoder_outputs_t = tf.transpose(encoder_outputs_t, [1, 0, 2])
_, _ , pred_ids,logits_t, decoder_inputs_t, \
_, _,_,_,_,_,_= build_network(encoder_outputs_t,
True,
flags.batch_size,
flags.decoder_length,
flags.tgt_vocab_size,
flags.attn_num_hidden,
encoder_length,
flags.max_gradient_norm,
f_size,
flags.att_loss,
flags.img_size,
deconv_outputs_t
)
infer_saver = tf.train.Saver()
infer_sess = tf.Session(graph=infer_graph)
# Training
la = 10
acc_log = 0
count = 0
lr = flags.learning_rate
for h in range(flags.epoch):
for i in range(iteration):
batch_train = data.next_batch(flags.batch_size)
path = []
texts = []
for line in batch_train:
path.append(line.split(' ')[0])
texts.append(line.split(' ')[1])
if flags.att_loss:
images, npy, mask, seg = load_img_label(
path, flags.img_size, flags.decoder_length)
else:
images = load_img(path, flags.img_size)
training_target_labels = get_label(texts, flags.decoder_length)
training_decoder_inputs = np.delete(training_target_labels,
-1,
axis=1)
training_decoder_inputs = np.c_[
np.zeros(training_decoder_inputs.shape[0]),
training_decoder_inputs].T
feed_dict = {
inputs: images,
decoder_inputs: training_decoder_inputs,
target_labels: training_target_labels,
learning_rate: lr
}
if flags.att_loss:
feed_dict[att_label] = npy
feed_dict[att_mask] = mask
feed_dict[input_seg] = seg[:, :, :, np.newaxis]
feed_dict[lamda] = la
_, loss_value, att = train_sess.run(
[train_op, loss, attention_weights_history],
feed_dict=feed_dict)
step = float(i + 1)
if step % flags.display_step == 0:
now = time.time()
print(step, now - start, loss_value)
start = now
if step % flags.eval_step == 0:
train_saver.save(train_sess, flags.save_dir)
model_file = tf.train.latest_checkpoint(
flags.save_dir.rsplit('/', 1)[0])
infer_saver.restore(infer_sess, model_file)
gt = []
predict = []
images = load_img(path, flags.img_size)
testing_decoder_inputs = np.zeros(
(flags.decoder_length, flags.batch_size), dtype=float)
feed_dict_t = {
inputs_t: images,
decoder_inputs_t: testing_decoder_inputs
}
q = infer_sess.run(pred_ids, feed_dict=feed_dict_t)
for j in range(flags.batch_size):
gt.append(texts[j])
ans = np.array(q).T[j]
pd = []
for c in ans:
if c != -1:
character = tools.idx_to_word[c]
if character != '<EOS>':
pd.append(character)
predict.append(''.join(pd))
correct = float(0)
cnt = 0
acc_s = 0
for l in range(len(gt)):
cnt = cnt + 1
if gt[l] == predict[l]:
correct = correct + 1
count = count + 1
acc_s = correct / cnt
if acc_s > acc_log:
acc_log = acc_s
count = 0
if count == (iteration // flags.eval_step):
lr = lr / 5
import conf
import tools
from xdeepfm_model import xDeepFM
single_size, numerical_size, multi_size = len(conf.SINGLE_FEATURES), len(
conf.NUMERICAL_FEATURES), len(conf.MULTI_FEATURES)
model = xDeepFM(single_size, numerical_size, multi_size)
train_data = tools.get_data(conf.train_instance_file)
valid_data = tools.get_data(conf.valid_instance_file)
test_data = tools.get_data(conf.test_instance_file)
valid_batch = tools.get_batch(valid_data, -1, single_size, numerical_size,
multi_size, conf.batch_size,
conf.use_numerical_embedding)
valid_dict = {
model.ph['label']: tools.get_label(valid_batch[0], 2),
model.ph['single_index']: valid_batch[1],
model.ph['numerical_index']: valid_batch[2],
model.ph['numerical_value']: valid_batch[3],
model.ph['value']: valid_batch[-1],
model.train_phase: False
}
if conf.MULTI_FEATURES:
for idx, s in enumerate(conf.MULTI_FEATURES):
valid_dict[model.ph['multi_index_%s' % s]] = valid_batch[4]
valid_dict[model.ph['multi_value_%s' % s]] = valid_batch[5]
def train(model):
print('begin to train model......')
def train():
with open(flags.train_txt) as f:
sample = [line.rstrip() for line in f]
sample = np.array(sample)
iteration = len(sample) // flags.batch_size
data = Dataset(sample)
tf.reset_default_graph()
train_graph = tf.Graph()
infer_graph = tf.Graph()
with train_graph.as_default():
encoder_outputs, inputs = build_cnn(True, flags.batch_size,
flags.height, flags.width,
flags.channels)
train_op, loss, sample_ids, logits, decoder_inputs, \
target_labels, learning_rate,keep_prob = build_network(encoder_outputs,
False,
flags.batch_size,
flags.decoder_length,
flags.tgt_vocab_size,
flags.attn_num_hidden,
flags.encoder_length,
flags.max_gradient_norm
)
initializer = tf.global_variables_initializer()
train_saver = tf.train.Saver()
train_sess = tf.Session(graph=train_graph)
train_sess.run(initializer)
with infer_graph.as_default():
encoder_outputs_t, inputs_t = build_cnn(False, flags.batch_size,
flags.height, flags.width,
flags.channels)
_, _, pred_ids, logits_t, decoder_inputs_t, \
_, _ ,keep_prob_t= build_network(encoder_outputs_t,
True,
flags.batch_size,
flags.decoder_length,
flags.tgt_vocab_size,
flags.attn_num_hidden,
flags.encoder_length,
flags.max_gradient_norm
)
infer_saver = tf.train.Saver()
infer_sess = tf.Session(graph=infer_graph)
# Training
start = time.time()
acc_log = 0
count = 0
lr = flags.learning_rate
for h in range(flags.epoch):
for i in range(iteration):
batch_train = data.next_batch(flags.batch_size)
path = []
texts = []
for line in batch_train:
path.append(line.split(' ')[0])
texts.append(line.split(' ')[1])
images = load_img(path, flags.height, flags.width)
training_target_labels = get_label(texts, flags.decoder_length)
training_decoder_inputs = np.delete(training_target_labels,
-1,
axis=1)
training_decoder_inputs = np.c_[
np.zeros(training_decoder_inputs.shape[0]),
training_decoder_inputs].T
feed_dict = {
inputs: images[:, :, :, np.newaxis],
decoder_inputs: training_decoder_inputs,
target_labels: training_target_labels,
learning_rate: lr,
keep_prob: 0.5
}
_, loss_value = train_sess.run([train_op, loss],
feed_dict=feed_dict)
step = float(i + 1)
if step % flags.display_step == 0:
now = time.time()
print(step, now - start, loss_value)
start = now
if step % flags.eval_step == 0:
train_saver.save(train_sess, flags.save_dir)
model_file = tf.train.latest_checkpoint(
flags.save_dir.rsplit('/', 1)[0])
infer_saver.restore(infer_sess, model_file)
gt = []
predict = []
images = load_img(path, flags.height, flags.width)
testing_decoder_inputs = np.zeros(
(flags.decoder_length, flags.batch_size), dtype=float)
feed_dict_t = {
inputs_t: images[:, :, :, np.newaxis],
decoder_inputs_t: testing_decoder_inputs,
keep_prob_t: 1
}
q = infer_sess.run(pred_ids, feed_dict=feed_dict_t)
for j in range(flags.batch_size):
gt.append(texts[j])
ans = np.array(q).T[j]
pd = []
for c in ans:
if c != -1:
character = tools.idx_to_word[c]
if character != '<EOS>':
pd.append(character)
predict.append(''.join(pd))
correct = float(0)
cnt = 0
acc_s = 0
for l in range(len(gt)):
cnt = cnt + 1
if gt[l] == predict[l]:
correct = correct + 1
count = count + 1
acc_s = correct / cnt
if acc_s > acc_log:
acc_log = acc_s
count = 0
if count == (iteration // flags.eval_step):
lr = lr / 5
prediction = tf.get_collection('pred_network')[0]
# loss = tf.get_collection('loss')[0]
loss = graph.get_tensor_by_name('loss:0')
total_emb, single_size, numerical_size, multi_size = _get_conf()
field_size = single_size + numerical_size + multi_size
embedding_length = field_size * Config.embedding_size
test = _get_data(Config.test_save_file)
test = test[:30000]
test_batch = _get_batch(test,
-1,
single_size=single_size,
numerical_size=numerical_size,
multi_size=multi_size)
test_label = get_label(test_batch[0], 2)
test_dict = {
ph['single_index']: test_batch[1],
ph['numerical_index']: test_batch[2],
ph['numerical_value']: test_batch[3],
ph['value']: test_batch[-1],
ph['label']: test_label,
train_phase: False
}
if Config.multi_features:
for idx, s in enumerate(Config.multi_features):
test_dict[ph['multi_index_%s' % s]] = test_batch[4]
test_dict[ph['multi_value_%s' % s]] = test_batch[5]
prediction, loss = sess.run((prediction, loss), feed_dict=test_dict)
def train():
from dataset import Dataset
from tools import get_label, load_train_img
import time
with open(flags.train_s_txt) as f:
sample = [line.rstrip() for line in f]
sample = np.array(sample)
data1 = Dataset(sample)
with open(flags.train_u_txt) as f:
sample = [line.rstrip() for line in f]
sample = np.array(sample)
data2 = Dataset(sample)
tf.reset_default_graph()
train_graph = tf.Graph()
with train_graph.as_default():
encoder_inputs, x2, inputs = build_cnn(training=True,batch_size=flags.batch_size, height=flags.height, width=flags.width, channels=flags.channels)
train_op, loss , sample_ids,logits, decoder_inputs, decoder_lengths, \
target_labels, keep_prob, prob = build_network(encoder_inputs,
is_training=True,
batch_size=flags.batch_size,
decoder_length=flags.decoder_length,
tgt_vocab_size=flags.tgt_vocab_size,
num_units=flags.num_units,
beam_width=flags.beam_width,
encoder_length=flags.encoder_length,
max_gradient_norm=flags.max_gradient_norm,
embedding_size=flags.embedding_size,
initial_learning_rate=flags.learning_rate)
initializer = tf.global_variables_initializer()
train_saver = tf.train.Saver()
train_sess = tf.Session(graph=train_graph)
train_sess.run(initializer)
start = time.time()
for i in range(flags.iteration):
train_batch = int(flags.batch_size/2)
batch_train1 = data1.next_batch(train_batch)
batch_train2 = data2.next_batch(train_batch)
batch_train = np.append(batch_train1,batch_train2)
np.random.shuffle(batch_train)
path = []
texts = []
label = []
for line in batch_train:
path.append(line.split(' ')[0])
texts.append(line.split(' ')[1])
label.append(line.split(' ')[2])
label = np.array(label).astype(np.int32)
images = load_train_img(path,flags.height,flags.width)
training_target_labels = get_label(texts,flags.decoder_length)
training_decoder_inputs = np.delete(training_target_labels, -1, axis=1)
training_decoder_inputs = np.column_stack([label, np.delete(training_target_labels, -1, axis=1)]).T
feed_dict = {inputs:images[:, :, :, np.newaxis],decoder_inputs:training_decoder_inputs,
decoder_lengths:np.ones((flags.batch_size), dtype=int) * flags.decoder_length,
target_labels:training_target_labels,keep_prob:0.8,prob:label}
_,loss_value = train_sess.run([train_op, loss], feed_dict=feed_dict)
step = float(i+1)
if step % flags.display_step == 0:
now = time.time()
print(step, now-start, loss_value)
start = now
if step % flags.save_step == 0:
train_saver.save(train_sess,flags.save_dir)
def train():
with open(FLAGS.train_txt) as f:
sample = [line.rstrip() for line in f]
sample = np.array(sample)
iteration = len(sample) // FLAGS.batch_size
data = Dataset(sample)
tf.reset_default_graph()
train_graph = tf.Graph()
infer_graph = tf.Graph()
with train_graph.as_default():
train_op, loss , sample_ids,logits, inputs, decoder_inputs, decoder_lengths, \
target_labels, keep_prob = build_network(is_training=True,
batch_size=FLAGS.batch_size,
height=FLAGS.height,
width=FLAGS.width,
channels=FLAGS.channels,
decoder_length=FLAGS.decoder_length,
tgt_vocab_size=FLAGS.tgt_vocab_size,
num_units=FLAGS.num_units,
beam_width=FLAGS.beam_width,
encoder_length=FLAGS.encoder_length,
max_gradient_norm=FLAGS.max_gradient_norm,
embedding_size=FLAGS.embedding_size,
initial_learning_rate=FLAGS.learning_rate)
initializer = tf.global_variables_initializer()
train_saver = tf.train.Saver()
train_sess = tf.Session(graph=train_graph)
train_sess.run(initializer)
with infer_graph.as_default():
_, _, pred_ids,pred_logits , inputs_t, decoder_inputs_t, decoder_lengths_t, \
_,keep_prob_t = build_network(is_training=False,
batch_size=FLAGS.batch_size,
height=FLAGS.height,
width=FLAGS.width,
channels=FLAGS.channels,
decoder_length=FLAGS.decoder_length,
tgt_vocab_size=FLAGS.tgt_vocab_size,
num_units=FLAGS.num_units,
beam_width=FLAGS.beam_width,
encoder_length=FLAGS.encoder_length,
max_gradient_norm=FLAGS.max_gradient_norm,
embedding_size=FLAGS.embedding_size,
initial_learning_rate=None)
infer_saver = tf.train.Saver()
infer_sess = tf.Session(graph=infer_graph)
start = time.time()
acc_log = 0
count = 0
lr = FLAGS.learning_rate
for h in range(FLAGS.epoch):
for i in range(iteration):
batch_train = data.next_batch(FLAGS.batch_size)
np.random.shuffle(batch_train)
path = []
texts = []
for line in batch_train:
path.append(line.split(' ')[0])
texts.append(line.split(' ')[1])
images = load_train_img(path, FLAGS.height, FLAGS.width)
training_target_labels = get_label(texts, FLAGS.decoder_length)
training_decoder_inputs = np.delete(training_target_labels,
-1,
axis=1)
training_decoder_inputs = np.c_[
np.zeros(training_decoder_inputs.shape[0]),
training_decoder_inputs].T
feed_dict = {
inputs: images[:, :, :, np.newaxis],
decoder_inputs: training_decoder_inputs,
decoder_lengths: np.ones(
(FLAGS.batch_size), dtype=int) * FLAGS.decoder_length,
target_labels: training_target_labels,
keep_prob: 0.8
}
_, loss_value = train_sess.run([train_op, loss],
feed_dict=feed_dict)
step = float(i)
if step % FLAGS.display_step == 0:
now = time.time()
print(step, now - start, loss_value)
start = now
if step % FLAGS.eval_step == 0:
train_saver.save(train_sess, FLAGS.save_dir)
model_file = tf.train.latest_checkpoint(
FLAGS.save_dir.rsplit('/', 1)[0])
infer_saver.restore(infer_sess, model_file)
gt = []
predict = []
images = load_img(path, FLAGS.height, FLAGS.width)
feed_dict_t = {inputs_t:images[:, :, :, np.newaxis],
decoder_lengths_t:np.ones((FLAGS.batch_size), \
dtype=int) * FLAGS.decoder_length,
keep_prob_t:1}
q = infer_sess.run(pred_ids, feed_dict=feed_dict_t)
for j in range(len(texts)):
gt.append(texts[j])
ans = q[j].T[0]
pd = []
for c in ans:
if c != -1:
character = tools.idx_to_word[c]
if character != '<EOS>':
pd.append(character)
predict.append(''.join(pd))
correct = float(0)
cnt = 0
acc_s = 0
for l in range(len(gt)):
cnt = cnt + 1
if gt[l] == predict[l]:
correct = correct + 1
acc_s = correct / cnt
if acc_s > acc_log:
acc_log = acc_s
count = 0
if count == (iteration // FLAGS.eval_step):
lr = lr / 5
# statistics = collections.defaultdict(list)
i = 0
for document in documents:
# line = ''
brand = document['brand']
texts = []
i += 1
if i > 200:
break
parameterList = document['parameterList'] # 商品介绍
for item in parameterList:
results = item.split('：')
label = tools.get_label(results[0])
texts.append(results[1])
print(','.join(texts))
# tableItem = document['tableItem'] # 规格与包装
# for item in tableItem:
# column1 = item['tableName']
#
# dlItems = item['dlItems']
# for dlItem in dlItems:
# column2 = dlItem[0]
# column3 = dlItem[1]
def _train(self):
print('....')
#设置GPU用量
#config = tf.ConfigProto()
#config.gpu_options.per_process_gpu_memory_fraction = 0.9 # 占用GPU100%的显存
#session = tf.Session(config=config)
#设置最小的GPU使用量
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
self.sess = tf.Session(config=config)
'''
#测试集
self.test_batch = self._get_batch(self.test, -1)
self.test_label = get_label(self.test_batch[0], 2)
self.test_dict = {
self.ph['single_index']: self.test_batch[1],
self.ph['numerical_index']: self.test_batch[2],
self.ph['numerical_value']: self.test_batch[3],
self.ph['value']: self.test_batch[-1],
self.ph['label']: self.test_label,
self.train_phase: False
}
if Config.multi_features:
for idx, s in enumerate(Config.multi_features):
self.test_dict[self.ph['multi_index_%s' % s]] = self.test_batch[4]
self.test_dict[self.ph['multi_value_%s' % s]] = self.test_batch[5]
'''
saver = tf.train.Saver(max_to_keep=10) #保存模型
with tf.Session() as self.sess:
self.sess.run(tf.global_variables_initializer())
allDataLength = len(self.train)
global_step = 0
print('total step:%d' %
(Config.epochs *
(int(allDataLength / Config.batch_size) + 1)))
for i in range(Config.epochs):
num_batchs = int(allDataLength / Config.batch_size) + 1
for j in range(num_batchs):
global_step += 1
now_batch = self._get_batch(self.train, j)
start = time.time()
batch_dict = {
self.ph['single_index']: now_batch[1],
self.ph['numerical_index']: now_batch[2],
self.ph['value']: now_batch[-1],
self.ph['label']: get_label(now_batch[0], 2),
self.ph['numerical_value']: now_batch[3],
self.train_phase: True
}
if Config.multi_features:
for idx, s in enumerate(Config.multi_features):
batch_dict[self.ph['multi_index_%s' %
s]] = now_batch[4]
batch_dict[self.ph['multi_value_%s' %
s]] = now_batch[5]
end = time.time()
start = time.time()
_out, _loss, _ = self.sess.run(
(self.softmax_output, self.loss, self.optimizer),
feed_dict=batch_dict) #训练集
end = time.time()
if global_step % 10 == 0:
#__out, __loss, __ = self.sess.run((self.softmax_output, self.loss, self.optimizer),feed_dict=self.valid_dict) #验证集
#print('step:',global_step,'train loss:',_loss,'valid loss:',__loss)
#if global_step % 50 == 0:
__out, __loss = self.sess.run(
(self.softmax_output, self.loss),
feed_dict=self.valid_dict) #验证集
auc, accuracy, precision, recall = auc_score(
__out, get_label(self.valid_batch[0], 2), 2)
print('step:', global_step, 'train loss:', _loss,
'valid loss:', __loss, 'valid_auc:', auc,
'accuracy:', accuracy, ' precision: ', precision,
' recall: ', recall)
#写入文件
with open('OUT/DeepFM_loss_result.csv', 'a') as f:
f.write('step: ' + str(global_step) +
' train loss: ' + str(_loss) +
' valid loss: ' + str(__loss) +
' valid_auc: ' + str(auc) + ' accuracy: ' +
str(accuracy) + ' precision: ' +
str(precision) + ' recall: ' +
str(recall) + '\n')
#每轮输出一次测试结果，并保存模型
saver.save(self.sess, 'ckpt/mnist.ckpt', global_step=i + 1)
'''
def plot_each_labels(labels,
predictions,
nb_labels,
title_lab='Expected',
title_pred='Predicted',
figsize=(10, 10),
cmap='viridis',
begin=0,
end=0):
'''
Plot Predictions image and Labels image for each label
params
----------
labels: Labels array of shape (n_pixels_row, n_pixels_col)
predictions: Predictions array of shape (n_pixels_row, n_pixels_col)
nb_labels: Number of labels
title_lab: Title of the labels images
title_pred: Title of the predictions images
figsize: Matplotlib figsize
cmap: Matplotlib cmap (change plot color)
begin: Plot labels from begin index (int)
end: Plot labels to 'nb_labels - end' index (int)
'''
# Plot each label from begin to end
for x in range(begin, nb_labels - end, 2):
plt.figure(figsize=figsize)
plt.subplot(4, 4, 1)
plt.imshow(get_label(labels, x + 1), cmap=cmap)
plt.title("{} label {}".format(title_lab, x + 1))
plt.subplot(4, 4, 2)
plt.imshow(get_label(predictions, x + 1), cmap=cmap)
plt.title("{} label {}".format(title_pred, x + 1))
plt.subplot(4, 4, 3)
plt.imshow(get_label(labels, x + 2), cmap=cmap)
plt.title("{} label {}".format(title_lab, x + 2))
plt.subplot(4, 4, 4)
plt.imshow(get_label(predictions, x + 2), cmap=cmap)
plt.title("{} label {}".format(title_pred, x + 2))
plt.show()
if nb_labels % 2 == 1 and end >= nb_labels:
plt.figure(figsize=(20, 20))
plt.subplot(4, 4, 1)
plt.imshow(get_label(labels, nb_labels), cmap=cmap)
plt.title("{} label {}".format(title_lab, nb_labels))
plt.subplot(4, 4, 2)
plt.imshow(get_label(predictions, nb_labels), cmap=cmap)
plt.title("{} label {}".format(title_pred, nb_labels))
plt.show()
