def get_attention_mask(self):
if not self.attention_FM and not self.use_AutoInt:
return
self.attention_masks = []
for bx, bx_dense, by in batcher(X_=self.ids_test,
y_=self.y_test,
batch_size=500,
hash_size=self.hash_size): #dense
if self.attention_FM:
self.attention_masks.append(
self.sess.run(self.normalize_att_score,
feed_dict={
self.ids: bx,
self.y: by,
self.dropout_keeprate_holder: 1.0
}))
else:
self.attention_masks.append(
self.sess.run(self.normalize_autoint_att_score,
feed_dict={
self.ids: bx,
self.y: by,
self.dropout_keeprate_holder: 1.0
}))
#return np.array(self.attention_masks)
return self.attention_masks
def get_attention_mask(self):
if not self.attention_FM:
return
self.attention_masks = []
for bx, by in batcher(self.ids_test,
self.y_test,
500,
hash_size=self.hash_size):
self.attention_masks.append(
self.sess.run(self.normalize_att_score,
feed_dict={
self.ids: bx,
self.y: by,
self.dropout_keeprate_holder: 1.0
}))
return np.array(self.attention_masks)
def main():
FLAGS = parse_args()
try:
triton_client = tritonclient.grpc.InferenceServerClient(url=FLAGS.url,
verbose=FLAGS.verbose)
except Exception as e:
print("channel creation failed: " + str(e))
sys.exit()
model_name = FLAGS.model_name
model_version = -1
print("Loading images")
image_data = load_images(FLAGS.img_dir if FLAGS.img_dir is not None else FLAGS.img,
max_images=FLAGS.batch_size * FLAGS.n_iter)
image_data = array_from_list(image_data)
print("Images loaded")
for batch in tqdm(utils.batcher(image_data, FLAGS.batch_size, n_iterations=FLAGS.n_iter),
desc="Inferring", total=FLAGS.n_iter):
inputs = generate_inputs(FLAGS.input_name, batch.shape, "UINT8")
outputs = generate_outputs(FLAGS.output_name)
# Initialize the data
inputs[0].set_data_from_numpy(batch)
# Test with outputs
results = triton_client.infer(model_name=model_name, inputs=inputs, outputs=outputs)
# Get the output arrays from the results
output0_data = results.as_numpy(FLAGS.output_name)
maxs = np.argmax(output0_data, axis=1)
if FLAGS.statistics:
for i in range(len(maxs)):
print("Sample ", i, " - label: ", maxs[i], " ~ ", output0_data[i, maxs[i]])
statistics = triton_client.get_inference_statistics(model_name="dali")
if len(statistics.model_stats) != 1:
print("FAILED: Inference Statistics")
sys.exit(1)
if FLAGS.statistics:
print(statistics)
print('PASS: infer')
def predict(self, ids_pred):
if self.hash_size is None:
for i, column in enumerate(ids_pred.columns):
if i >= 1:
ids_pred.loc[:, column] = ids_pred[column] + sum(
self.features_sizes[:i])
outputs = []
self.ids_pred = ids_pred
for bx in batcher(ids_pred,
None,
batch_size=self.batch_size,
hash_size=self.hash_size): #y=None
outputs.append(
self.sess.run(self.pred,
feed_dict={
self.ids: bx,
self.dropout_keeprate_holder: 1.0
}))
self.output = np.concatenate(outputs, axis=0) #.reshape((-1))
return self.output
def valRes(weightName, iter):
net.load(weightName, sess)
batcher = ut.batcher(LIST, sourcePath, BSZ, iter)
val_loss = []
val_accuracy = []
imgs, nlbls = batcher.get_batch()
_ = sess.run(data.fetchOp, feed_dict=data.getfeed(imgs))
for i in range(VAL_numBatches):
_ = sess.run(data.swapOp)
clbls = nlbls
imgs, nlbls = batcher.get_batch()
fdict = data.getfeed(imgs)
fdict[labels] = clbls
outs = sess.run([opt.accuracy, opt.loss, data.fetchOp],
feed_dict=fdict)
val_loss.append(outs[1])
val_accuracy.append(outs[0])
loss_val = np.mean(val_loss)
accuracy_val = np.mean(val_accuracy)
sys.stdout.write(" [%09d] Val loss = %.6f, Val accuracy = %.6f\n" %
(iter, loss_val, accuracy_val))
def predict(self, ids_pred, dense_pred=None):
# [bug fix]mutable prevention 19/06/27
start_time = time.time()
ids_pred = ids_pred.copy()
if self.hash_size is None:
for i, column in enumerate(ids_pred.columns):
if i >= 1:
ids_pred.loc[:, column] = ids_pred[column] + sum(
self.features_sizes[:i])
outputs = []
#self.ids_pred=ids_pred 0708 del
for bx, bx_dense in batcher(ids_pred,
y_=None,
X_dense=dense_pred,
batch_size=self.batch_size,
hash_size=self.hash_size): #y=None
if self.dense_features_size > 0:
outputs.append(
self.sess.run(self.pred,
feed_dict={
self.ids: bx,
self.dense_inputs: bx_dense,
self.dropout_keeprate_holder: 1.0
}))
else: #bx_dense==None
outputs.append(
self.sess.run(self.pred,
feed_dict={
self.ids: bx,
self.dropout_keeprate_holder: 1.0
}))
self.output = np.concatenate(outputs, axis=0) #.reshape((-1))
time_cost = time.time() - start_time
print("Time cost: %3f seconds, average epoch time:%2f ms" %
(time_cost, (time_cost / len(outputs)) * 1000))
return self.output
WEIGHT_DECAY = 0. LR = 0.01 MOM = 0.9 MAX_ITER = int(1.2e7) # Update BSZ, lr BSZ = BSZ // AVG_IT LR_E = LR / float(AVG_IT) # Check for saved weights saved = ut.ckpter(WTS_DIR + 'iter_*.model.npz') iter = saved.iter # Set up batching batcher = ut.batcher(LIST, BSZ, iter * AVG_IT) # Set up data prep data = ldr.trainload(BSZ) labels = tf.placeholder(shape=(BSZ, ), dtype=tf.int32) # Load model-def net = md.model(data.batch, train=True) # Load trainer-def opt = tr.train(net, labels, LR_E, MOM, WEIGHT_DECAY) # Start session sess = tf.Session() sess.run(tf.initialize_all_variables())
import cifar100 as cf import trainer as tr import model as md import utils as ut from params import * # Load dataset tset, vset = cf.get_tval() # Restore any saved weights saved = ut.ckpter(WTS_DIR + 'iter_*.model.npz') iter = saved.iter # Set up batching tbatch = ut.batcher(tset[0], tset[1], BSZ, iter) vbatch = ut.batcher(vset[0], vset[1], BSZ, 0, False) # Create placeholders data = tf.placeholder(shape=(BSZ, ) + tset[0].shape[1:], dtype=tset[0].dtype) labels = tf.placeholder(shape=(BSZ, ) + tset[1].shape[1:], dtype=tset[1].dtype) lr = tf.placeholder(shape=(), dtype=tf.float32) phase = tf.placeholder(shape=(), dtype=tf.int32) # Load model-def net = md.model(data, phase) # Load trainer-def opt = tr.train(net, labels, lr, MOM, WD)
img = np.clip(img, 0., 1.) * 255
img = np.uint8(img)
img = Image.fromarray(img)
return img
saved = ut.ckpter(args.WTS + 'iter_*.model.npz')
iter = saved.iter
std_num = len(args.STD.split(' '))
stds = [float(i) for i in args.STD.split(' ')]
# Start session
sess = tf.Session()
batcher = ut.batcher(args.LIST, 1)
data = tldr.testload(args.SIZE, args.BSIZE)
std_img = tf.placeholder(shape=[args.BSIZE * args.BSIZE], dtype=tf.float32)
std_denoise = tf.placeholder(shape=[args.BSIZE * args.BSIZE], dtype=tf.float32)
rnd = tf.placeholder(shape=[], dtype=tf.float32)
comp = tf.placeholder(shape=[], dtype=tf.float32)
if args.MD == 1:
net = md.model(data.batch, std_img, std_denoise)
elif args.MD == 2:
net = mdv2.model(data.batch, std_img, std_denoise)
elif args.MD == 3:
net = mdv3.model(data.batch, std_img, std_denoise)
elif args.MD == 4:
net = mdv4.model(data.batch, std_img, std_denoise)
elif args.MD == 5:
net = mdv5.model(data.batch, std_img, std_denoise)
# Ayan Chakrabarti <*****@*****.**>
import sys
import tensorflow as tf
import numpy as np
import cifar100 as cf
import trainer as tr
import model as md
import utils as ut
BSZ = 500
testset = cf.get_test()
# Set up batching
tbatch = ut.batcher(testset[0], testset[1], BSZ, 0, False)
# Create placeholders
data = tf.placeholder(shape=(BSZ, ) + testset[0].shape[1:],
dtype=testset[0].dtype)
# Load model-def
net = md.model(data)
# Start session
sess = tf.Session()
sys.stdout.write("Restoring from " + sys.argv[1] + "\n")
sys.stdout.flush()
net.load(sys.argv[1], sess)
# Test
nIter = testset[0].shape[0] // BSZ
def train(config):
# Store results
results = []
# Timestamp to name the model
timestamp = datetime.datetime.now().strftime("%y%m%d%M%S")
# Build the Folds
folds = utils.fold_iter(dataset=config.train,
k_folds=config.k_folds,
test_split=config.test_split)
# Iterate through the folds
for fold, (train, test) in enumerate(folds):
# Define the model
if config.model == "cnn":
model = CNN(config)
else:
model = RNN(config)
# Build the model
model.build()
# Fold metrics
fold_best, patience = 0, 0
result = []
print("\nFold ", fold + 1, "\n")
for ep in range(config.n_epochs):
# At the start of each epoch we shuffle the training set
train = utils.shuffle(train)
# Batch
batcher = utils.batcher(dataset=train,
batch_size=config.batch_size)
# For each batch
for batch in batcher:
# Pad the data such that every word and sentence has the same length
# for the sake of less code, tags in the classification task holds
# the labels
words, chars, labels = config.pad(dataset=batch)
# Build the feed dictionary
feed = config.feed(model,
words,
labels,
chars=chars,
dropout=config.dropout)
# Train
_, loss, acc = model.sess.run(
[model.train_op, model.loss, model.accuracy], feed)
# print(np.shape(model.features.eval(session=model.sess, feed_dict = feed)))
# print(np.shape(model.max_pool.eval(session=model.sess, feed_dict = feed)))
# Write the metrics
summary = "Epoch " + str(ep + 1) + " :\n\tLoss: " + str(loss)
# Score agasint test set
f1_acc = []
epoch_results = []
metrics = eval.evaluate(model=model, test_set=train, config=config)
epoch_results.append(metrics)
f1 = (2 * metrics["tpr"] * metrics["tnr"]) / (metrics["tpr"] +
metrics["tnr"])
# Write the metrics
summary += "\n\tTPR D1: " + \
format(metrics["tpr"], ".3f") + "\tTNR D1: " + format(metrics["tnr"],
".3f") + "\tF1 D1: " + format(f1, ".3f")
line = "\t" + str(metrics["tpr"]) + "\t" + \
str(metrics["tnr"]) + "\t" + str(f1)
# IF there is only one fold and no test split %, then no testing occurs
if not (config.k_folds == 1 and config.test_split == 0.0):
# Fold test subset
metrics = eval.evaluate(model=model,
test_set=test,
config=config)
epoch_results.append(metrics)
f1 = (2 * metrics["tpr"] * metrics["tnr"]) / (metrics["tpr"] +
metrics["tnr"])
# Write the metrics
summary += "\n\tTPR D1: " + \
format(metrics["tpr"], ".3f") + "\tTNR D1: " + format(metrics["tnr"],
".3f") + "\tF1 D1: " + format(f1, ".3f")
# Test sets
for i, _set in enumerate(config.test):
metrics = eval.evaluate(model=model,
test_set=_set,
config=config)
epoch_results.append(metrics)
# Calculate F-measure F1
f1 = (2 * metrics["tpr"] * metrics["tnr"]) / (metrics["tpr"] +
metrics["tnr"])
f1_acc.append(f1)
line += "\t" + str(metrics["tpr"]) + "\t" + \
str(metrics["tnr"]) + "\t" + str(f1)
# Write the metrics
summary += "\n\tTPR D" + \
str(i + 2) + ": " + format(metrics["tpr"], ".3f") + \
"\tTNR D" + str(i + 2) + ": " + format(metrics["tnr"], ".3f") + "\tF1 D" + str(
i + 2) + ": " + format(f1, ".3f")
# Average the F1 scores
f1_acc = f1_acc[0]
# Output current metrics
print(summary)
# Check if the model has improved
if f1_acc > fold_best:
result = epoch_results
fold_best = f1_acc
# Reset patience
patience = 0
# save the model
if not config.save == None:
model.class_save(fold, timestamp, config.save, ep)
else:
patience += 1
# After 3 epochs, if the model has not improved then we can stop
if patience > 300:
break
# Store all metrics
results.append(result)
# Print some metrics
for i, metrics in enumerate(result):
print("\nTest set", i + 1, "\t--acc: ",
format(metrics["accuracy"], '.3f'), " --tpr: ",
format(metrics["tpr"], '.3f'), " --tnr: ",
format(metrics["tnr"], '.3f'))
model.close_session()
line = eval.getstats(config, results)
print(line)
return line
parser.add_argument('--ITER',
type=int,
default=int(1e7),
help='total iterations')
args = parser.parse_args()
saved = ut.ckpter(args.WTS + 'iter_*.model.npz')
iter = saved.iter
std_num = len(args.STD.split(' '))
stds = np.zeros(BSZ)
for i in range(std_num):
stds[i * 16:(i + 1) * 16] = float(args.STD.split(' ')[i])
batcher = ut.batcher(args.LIST, BSZ, iter)
# Set up data prep
data = ldr.trainload(BSZ, IMG_SZ)
# Noise STD
noise_std = tf.placeholder(shape=[BSZ], dtype=tf.float32)
# Load model-def
net = md.model(data.batch, noise_std)
# Learning rate
lr = tf.placeholder(shape=[], dtype=tf.float32)
# Start session
sess = tf.Session()
parser.add_argument('--LOSSWEIGHT', type=float, default=0., help='comb loss')
args = parser.parse_args()
# Check for saved weights
saved = ut.ckpter(args.WTS + 'iter_*.model.npz')
iter = saved.iter
std_num = len(args.STD.split(' '))
stds = np.zeros(BSZ)
s_loss = []
for i in range(std_num):
s_loss.append([])
stds[i * 16:(i + 1) * 16] = float(args.STD.split(' ')[i])
# Set up batching
batcher_train = ut.batcher(args.LISTTRAIN, BSZ, iter)
# Set up data prep
data = ldr.trainload(BSZ, args.SIZE)
# Noise STD
std_img = tf.placeholder(shape=[BSZ], dtype=tf.float32)
std_denoise = tf.placeholder(shape=[BSZ], dtype=tf.float32)
# Load model-def
if args.MD == 1:
net = md.model(data.batch, std_img, std_denoise)
elif args.MD == 2:
net = mdv2.model(data.batch, std_img, std_denoise)
elif args.MD == 3:
net = mdv3.model(data.batch, std_img, std_denoise)
def train(config):
# Store results
results = []
# Timestamp to name the model
timestamp = datetime.datetime.now().strftime("%y%m%d%H%M%S")
# Build the Folds
folds = utils.fold_iter(dataset=config.train,
k_folds=config.k_folds,
test_split=config.test_split)
# Iterate through the folds
for fold, (train, test) in enumerate(folds):
# Define the model
if config.model == "cnn":
model = CNN(config)
else:
model = RNN(config)
# Build the model
model.build()
# Fold metrics
fold_best, patience = 0, 0
result = []
print("\nFold ", fold + 1, "\n")
for ep in range(config.n_epochs):
# At the start of each epoch we shuffle the training set
train = utils.shuffle(train)
# Batch
batcher = utils.batcher(dataset=train,
batch_size=config.batch_size)
# For each batch
for batch in batcher:
# Pad the data such that every word and sentence has the same length
# for the sake of less code, tags in the classification task holds
# the labels
words, tags, chars = config.pad(dataset=batch,
use_chars=config.use_chars)
# Build the feed dictionary
feed = config.feed(model,
words=words,
tags=tags,
chars=chars,
dropout=config.dropout,
use_chars=config.use_chars)
# Train
_, loss = model.sess.run([model.train_op, model.loss],
feed_dict=feed)
# print(np.shape(model.word_vectors.eval(session=model.sess,feed_dict=feed)))
# print(np.shape(model.conv.eval(session=model.sess,feed_dict=feed)))
# print(np.shape(model.h_flat.eval(session=model.sess,feed_dict=feed)))
# print((model.h_nodes))
# Write the metrics
summary = "Epoch " + str(ep + 1) + " :\n\tLoss: " + str(loss)
# Score agasint test set
f1 = []
epoch_results = []
# IF there is only one fold and no test split %, then no testing occurs
if not (config.k_folds == 1 and config.test_split == 0.0):
# Fold test subset
metrics = eval.evaluate(model=model,
test_set=test,
config=config)
epoch_results.append(metrics)
f1.append(metrics["f1"])
# Write the metrics
summary += "\n\tPrecision D1: " + \
format(metrics["precision"], ".3f") + "\tRecall D1: " + format(metrics["recall"],
".3f") + "\tF1 D1: " + format(metrics["f1"], ".3f")
# Test sets
for i, _set in enumerate(config.test):
metrics = eval.evaluate(model=model,
test_set=_set,
config=config)
epoch_results.append(metrics)
# Sum the F1s for averaging
f1.append(metrics["f1"])
# Write the metrics
summary += "\n\tPrecision D" + \
str(i + 2) + ": " + format(metrics["precision"], ".3f") + \
"\tRecall D" + str(i + 2) + ": " + format(metrics["recall"], ".3f") + "\tF1 D" + str(
i + 2) + ": " + format(metrics["f1"], ".3f")
# Output current metrics
print(summary)
# Average the F1 scores
#f1 = np.average(np.array(f1)) if len(f1) > 0 else 0
f1 = f1[0]
# Check if the model has improved
if f1 > fold_best:
result = epoch_results
fold_best = f1
# Reset patience
patience = 0
# save the model
if not config.save == None:
model.ner_save(fold, timestamp, config.save, ep)
else:
patience += 1
# After 3 epochs, if the model has not improved then we can stop
if patience > config.patience:
break
# Store all metrics
results.append(result)
# Print some metrics
for i, metrics in enumerate(result):
print("\nTest set", i + 1, "\t--f1: ",
format(metrics["f1"], '.3f'), " --acc: ",
format(metrics["accuracy"], '.3f'), " --precis: ",
format(metrics["precision"], '.3f'), " --recall: ",
format(metrics["recall"], '.3f'))
model.close_session()
line = eval.getstats(config=config, results=results)
return line
DISP_FREQ = 10 BSZ = 32 WEIGHT_DECAY = 0. LR = 0.001 MOM = 0.9 MAX_ITER = int(1.2e7) # Check for saved weights saved = ut.ckpter(WTS_DIR + 'iter_*.model.npz') iter = saved.iter # Set up batching batcher = ut.batcher(LIST, BSZ, iter) # Set up data prep data = ldr.trainload(BSZ) labels = tf.placeholder(shape=(BSZ, ), dtype=tf.int32) # Load model-def net = md.model(data.batch, train=True) # Load trainer-def opt = tr.train(net, labels, LR, MOM, WEIGHT_DECAY) # Start session sess = tf.Session() sess.run(tf.initialize_all_variables())
SAVE_FREQ = 1000 DISP_FREQ = 100 BSZ = 64 WEIGHT_DECAY = 0. LR = 0.0001 MOM = 0.9 MAX_ITER = int(1.2e7) # val VAL_FREQ = 100 VAL_numBatches = 20 # Check for saved weights saved = ut.ckpter(WTS_DIR + 'iter_*.model.npz') iter = saved.iter # Set up batching batcher = ut.batcher(LIST, sourcePath, BSZ, iter) batcher_val = ut.batcher(VAL, sourcePath, BSZ, iter) # Set up data prep data = ldr.trainload(BSZ) labels = tf.placeholder(shape=(BSZ, ), dtype=tf.int32) labels_val = tf.placeholder(shape=(BSZ, ), dtype=tf.int32) # Load model-def iftrain = tf.placeholder(tf.bool, shape=[]) net = md.model(data.batch, iftrain) # Load trainer-def opt = tr.train(net, labels, LR, MOM, WEIGHT_DECAY) # Start session
def fit(
self,
ids_train,
ids_test,
y_train,
y_test,
dense_train=None,
dense_test=None,
lr=0.001,
N_EPOCH=50,
batch_size=200,
early_stopping_rounds=20,
):
start_time = time.time()
#[bug fix]mutable prevention 19/06/27
ids_train = ids_train.copy()
ids_test = ids_test.copy()
self.batch_size = batch_size
#data preprocess:对ids的每个features，label encoder都要从上一个的末尾开始。函数输入时则保证每个都从0起.
if self.hash_size is None:
for i, column in enumerate(ids_train.columns):
if i >= 1:
ids_train.loc[:, column] = ids_train[column] + sum(
self.features_sizes[:i])
ids_test.loc[:, column] = ids_test[column] + sum(
self.features_sizes[:i])
if self.attention_FM or self.use_AutoInt: #储存为classs变量并用在get_attention里获取attention
self.ids_train, self.ids_test, self.y_train, self.y_test = ids_train, ids_test, y_train, y_test
self.ids = tf.placeholder(tf.int32, [None, self.fields])
self.dense_inputs = tf.placeholder(tf.float32,
[None, self.dense_features_size])
self.y = tf.placeholder(tf.float32, [None, 1])
self.L2_reg = 0
self.dropout_keeprate_holder = tf.placeholder(tf.float32)
embed_L2 = 0
if self.use_FM or self.use_MLP or self.use_AutoInt:
self.embedding, embed_L2 = self.Embedding(
self.ids, self.embedding_weights) #(None,fields,k)
if self.use_SE:
self.embeddingSE = self.SELayer(self.embedding, self.SE_weights)
self.pred = 0
if self.use_LR:
#bug detected. LR didn't keepdims
self.pred = self.LR(self.ids, self.w, self.b)
if self.use_MLR:
print("use Mix of LR.")
self.pred += self.MLR(self.ids, self.MLR_u, self.MLR_w)
#only one FM will be used.
if self.use_NFM:
print("use NFM")
self.pred += self.NFM(self.embedding, self.NFM_weights)
elif self.use_BiFM:
if self.use_SE:
cross_term = tf.concat([
self.Bilinear_FM(
self.embedding, self.bilinear_weights, se_emb=False),
self.Bilinear_FM(
self.embeddingSE, self.bilinear_weights, se_emb=True),
],
axis=-1) # N,c,2k
if self.use_FiBiNet:
print("use FiBiNet") #deep backend
cross_term = tf.reshape(
cross_term, [-1, self.c * self.k * 2]) #None,2ck
cross_term = tf.nn.relu(
tf.matmul(cross_term, self.FiBiNet_weights['W1']) +
self.FiBiNet_weights['b1'])
self.pred += (
tf.matmul(cross_term, self.FiBiNet_weights['W2']) +
self.FiBiNet_weights['b2'])
else:
print("use Fibifm")
self.pred += tf.expand_dims(tf.reduce_sum(cross_term,
axis=[1, 2]),
axis=1) # N,1
else:
print("use bifm")
cross_term = self.Bilinear_FM(self.embedding,
self.bilinear_weights,
se_emb=False) # N,c,k
self.pred += tf.expand_dims(tf.reduce_sum(cross_term,
axis=[1, 2]),
axis=1) # N,1
elif self.use_FM and not self.attention_FM and not self.use_CFM:
print("use FM")
if len(self.FM_ignore_interaction
) == 0: #if self.use_FM and self.FM_ignore_interaction==[]
self.pred += self.FM2(self.embedding)
if len(self.FM_ignore_interaction) > 0:
self.pred += self.FMDE(self.embedding)
elif self.use_FM and self.attention_FM:
print("use AFM")
afm_out, reg = self.AFM(self.embedding, self.AFM_weights)
self.pred += afm_out
self.L2_reg += reg
elif self.use_FM and self.use_CFM:
print("use CFM")
cfm_out, reg = self.CFM(self.embedding, self.CFM_weights)
self.pred += cfm_out
self.L2_reg += reg
if self.use_AutoInt:
self.y_deep = self.embedding
for _l in range(self.autoint_params['autoint_layers']):
self.y_deep = self.AutoInt(self.y_deep,
self.AutoInt_weights,
layer=_l) #N,f,d
self.pred += tf.matmul(
tf.reshape(self.y_deep,
shape=[
-1,
self.fields * self.autoint_d * self.autoint_head
]),
self.AutoInt_weights['W_out']) + self.AutoInt_weights['b_out']
if self.use_CrossNet_layers > 0: #combine crossnet with DNN
MLP_in = tf.reshape(self.embedding,
[-1, self.fields * self.k]) #(N,f*k)
if self.dense_features_size > 0:
MLP_in = tf.concat([MLP_in, self.dense_inputs],
axis=1) #(N,f*k+dense)
self.MLP_out = self.MLP(MLP_in,
self.weights,
self.bias,
return_pred=False) #(None,last_layers)
self.CrossNet_out = self.CrossNet(tf.expand_dims(
MLP_in, axis=-1), self.CrossNet_weights) #(None,f*k+d)
self.pred += tf.keras.layers.Dense(
1, use_bias=False,
activation=None)(tf.concat([self.MLP_out, self.CrossNet_out],
axis=1))
elif self.use_MLP: #并联dnn pred
MLP_in = tf.reshape(self.embedding,
[-1, self.fields * self.k]) #(N,f*k)
if self.dense_features_size > 0:
MLP_in = tf.concat([MLP_in, self.dense_inputs],
axis=1) #(N,f*k+dense)
self.pred += self.MLP(MLP_in, self.weights, self.bias)
assert self.pred is not None, "must have one predicion layer"
if self.loss_type == 'rmse':
self.loss = tf.sqrt(tf.reduce_mean(tf.square(self.y - self.pred)))
elif self.loss_type == 'mse':
self.loss = tf.reduce_mean(tf.square(self.y - self.pred))
elif self.loss_type in ['binary_crossentropy', 'binary', 'logloss']:
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(labels=self.y,
logits=self.pred))
else:
raise Exception("Loss type %s not supported" % self.loss_type)
#todo EMBEDL2 coef
self.loss += self.lambda_l2 * self.L2_reg #+ embed_L2*1e-5
self.optimizer = tf.train.AdamOptimizer(lr).minimize(self.loss)
if self.metric_type is not None:
assert self.metric_type == 'auc'
assert self.loss_type in [
'binary_crossentropy', 'binary', 'logloss'
]
#tf.auc mode: remove sklearn auc part
#self.loss=tf.metrics.auc(labels=self.y,predictions=tf.nn.sigmoid(self.pred))
self.sess = self._init_session()
self.sess.run(tf.global_variables_initializer())
self.sess.run(tf.local_variables_initializer())
cur_best_rounds = 0
is_greater_better = False if self.metric_type is None else True #默认Loss越小越好
cur_min_loss = 1e8 if not is_greater_better else -1e8
best_weights = {
v.name: v.eval(self.sess)
for v in tf.trainable_variables()
}
for epoch in range(N_EPOCH):
train_loss = 0.
y_preds_train = []
total_batches = int(ids_train.shape[0] / batch_size)
# id input + dense input
for bx, bx_dense, by in batcher(ids_train,
y_train,
X_dense=dense_train,
batch_size=batch_size,
hash_size=self.hash_size):
if self.dense_features_size > 0:
_, l = self.sess.run(
[self.optimizer, self.loss],
feed_dict={
self.ids: bx,
self.y: by,
self.dense_inputs: bx_dense,
self.dropout_keeprate_holder: self.dropout_keeprate
})
else:
_, l = self.sess.run(
[self.optimizer, self.loss],
feed_dict={
self.ids: bx,
self.y: by,
self.dropout_keeprate_holder: self.dropout_keeprate
})
train_loss += l #if not self.metric_type else l[1]
if self.metric_type:
y_preds_train.append(self.sess.run(self.pred,feed_dict={self.ids:bx,self.dense_inputs:bx_dense,self.dropout_keeprate_holder:1.0})) \
if self.dense_features_size>0 \
else y_preds_train.append(self.sess.run(self.pred,feed_dict={self.ids:bx,self.dropout_keeprate_holder:1.0}))
train_loss /= total_batches
if self.coldStartAvg:
print("Cold Start Averaging start") if epoch == 0 else None
self.coldStartAvgTool()
#todo movielens afm rounded
test_loss = 0.
y_preds = []
for bx, bx_dense, by in batcher(ids_test,
y_test,
X_dense=dense_test,
batch_size=batch_size,
hash_size=self.hash_size):
if self.dense_features_size > 0:
l = self.sess.run(self.loss,
feed_dict={
self.ids: bx,
self.y: by,
self.dense_inputs: bx_dense,
self.dropout_keeprate_holder: 1.0
})
else:
l = self.sess.run(self.loss,
feed_dict={
self.ids: bx,
self.y: by,
self.dropout_keeprate_holder: 1.0
})
test_loss += l #if not self.metric_type else l[1]
if self.metric_type:
y_preds.append(self.sess.run(self.pred,feed_dict={self.ids:bx,self.dense_inputs:bx_dense,self.dropout_keeprate_holder:1.0})) \
if self.dense_features_size>0 \
else y_preds.append(self.sess.run(self.pred,feed_dict={self.ids:bx,self.dropout_keeprate_holder:1.0}))
test_loss /= int(ids_test.shape[0] / batch_size)
'''
y_pred=np.concatenate(y_preds, axis=0).reshape((-1))
predictions_bounded = np.maximum(y_pred, np.ones(len(y_pred)) * -1) # bound the lower values
predictions_bounded = np.minimum(predictions_bounded, np.ones(len(y_pred)) * 1) # bound the higher values
# override test_loss
test_loss = np.sqrt(np.mean(np.square(y_test.reshape(predictions_bounded.shape)- predictions_bounded)))
'''
#sklearn auc mode
if self.metric_type: # override test_loss
self.y_pred_train = np.concatenate(y_preds_train, axis=0)
self.y_pred = np.concatenate(y_preds, axis=0)
train_loss = roc_auc_score(y_train, self.y_pred_train)
test_loss = roc_auc_score(y_test, self.y_pred)
metrics_ = 'loss' if self.metric_type is None else 'auc'
print("epoch:%s train_%s:%s test_%s:%s" %
(epoch + 1, metrics_, train_loss, metrics_, test_loss))
#print("self.pred=",self.sess.run(self.pred,feed_dict={self.ids:ids_test,self.y:y_test}))
#print("self.y=",y_test)
if isBetter(test_loss, cur_min_loss, is_greater_better):
cur_min_loss = test_loss
cur_best_rounds = epoch + 1
best_weights = {
v.name: v.eval(self.sess)
for v in tf.trainable_variables()
}
if epoch + 1 - cur_best_rounds >= early_stopping_rounds:
print(
"[Early Stop]Early Stopping because not improved for %s rounds"
% early_stopping_rounds)
self.sess.run(
tf.tuple([
tf.assign(var, best_weights[var.name])
for var in tf.trainable_variables()
]))
best_score = cur_min_loss #self.sess.run(self.loss, feed_dict={self.ids: ids_test, self.y: y_test, })
print("[Early Stop]Best Score:", best_score, ' at round ',
cur_best_rounds)
print(
"Train finish. Fit time:%.2f seconds. Epoch time:%.2f seconds"
% (time.time() - start_time,
(time.time() - start_time) / (epoch + 1)))
return best_score
#auc reset op
self.sess.run(tf.local_variables_initializer())
self.sess.run(
tf.tuple([
tf.assign(var, best_weights[var.name])
for var in tf.trainable_variables()
]))
best_score = cur_min_loss #self.sess.run(self.loss, feed_dict={self.ids: ids_test, self.y: y_test,})
print("[Epoch Maxi]Best Score:", best_score, ' at round ',
cur_best_rounds)
print("Train finish. Fit time:%.2f seconds. Epoch time:%.2f seconds" %
(time.time() - start_time, (time.time() - start_time) / N_EPOCH))
return best_score
# redirect stdout and stderr to log files
if prm.log_file:
sys.stdout = open(exp_dir + '/train.log', 'a')
sys.stderr = open(exp_dir + '/info.log', 'a')
# Check for saved weights & find iter
vsave = ut.ckpter(exp_dir + '/iter_*.vmodel.npz')
osave = ut.ckpter(exp_dir + '/iter_*.opt.npz')
vpath = lambda itr: '%s/iter_%07d.vmodel.npz' % (exp_dir, itr)
opath = lambda itr: '%s/iter_%07d.opt.npz' % (exp_dir, itr)
niter = vsave.iter
# Load annotations
ut.mprint("Loading annotations")
tbchr = ut.batcher(prm.trn_anns, prm.batch_size, niter)
vbchr = ut.batcher(prm.val_anns, prm.batch_size, niter)
ut.mprint("Done!")
#########################################################################
# Set up data fetch
camera_fps = [tf.placeholder(tf.string) for i in range(prm.batch_size)]
pts_xyz_fps = [tf.placeholder(tf.string) for i in range(prm.batch_size)]
pts_rgb_fps = [tf.placeholder(tf.string) for i in range(prm.batch_size)]
pts_sift_fps = [tf.placeholder(tf.string) for i in range(prm.batch_size)]
gt_depth_fps = [tf.placeholder(tf.string) for i in range(prm.batch_size)]
getfeed = lambda fps: \
dict([(ph,'data/'+fps[i,3]) for i,ph in enumerate(camera_fps)]+\
[(ph,'data/'+fps[i,0]) for i,ph in enumerate(pts_xyz_fps)]+\
def fit(self,
ids_train,
ids_test,
y_train,
y_test,
lr=0.001,
N_EPOCH=50,
batch_size=200,
early_stopping_rounds=20):
self.batch_size = batch_size
#data preprocess:对ids的每个features，label encoder都要从上一个的末尾开始。函数输入时则保证每个都从0起.
if self.hash_size is None:
for i, column in enumerate(ids_train.columns):
if i >= 1:
ids_train.loc[:, column] = ids_train[column] + sum(
self.features_sizes[:i])
ids_test.loc[:, column] = ids_test[column] + sum(
self.features_sizes[:i])
if True: #self.attention_FM:#储存为classs变量并用在get_attention里获取attention
self.ids_train, self.ids_test, self.y_train, self.y_test = ids_train, ids_test, y_train, y_test
self.ids = tf.placeholder(tf.int32, [None, self.fields])
self.y = tf.placeholder(tf.float32, [None, 1])
self.dropout_keeprate_holder = tf.placeholder(tf.float32)
if self.use_FM or self.use_MLP or self.use_AutoInt:
self.embedding = self.Embedding(
self.ids, self.embedding_weights) #(None,fields,k)
self.pred = 0
self.L2_reg = 0
if self.use_LR:
#bug detected. LR didn't keepdims
self.pred = self.LR(self.ids, self.w, self.b)
#only one FM will be used.
if self.use_NFM:
print("use NFM")
self.pred += self.NFM(self.embedding, self.NFM_weights)
elif self.use_FM and not self.attention_FM:
print("use FM")
if len(self.FM_ignore_interaction
) == 0: #if self.use_FM and self.FM_ignore_interaction==[]
self.pred += self.FM2(self.embedding)
if len(self.FM_ignore_interaction) > 0:
self.pred += self.FMDE(self.embedding)
elif self.use_FM and self.attention_FM:
print("use CFM")
afm_out, reg = self.CFM(self.embedding, self.AFM_weights)
self.pred += afm_out
self.L2_reg += reg
if self.use_AutoInt:
self.y_deep = self.embedding
for l in range(3):
self.y_deep = self.AutoInt(self.y_deep,
self.AutoInt_weights,
layer=l) #N,f,d
self.pred += tf.matmul(
tf.reshape(self.y_deep,
shape=[-1, self.fields * self.autoint_d]),
self.AutoInt_weights['W_out']) + self.AutoInt_weights['b_out']
if self.use_MLP:
MLP_in = tf.reshape(self.embedding, [-1, self.fields * self.k])
self.pred += self.MLP(MLP_in, self.weights, self.bias)
#self.pred=self.SqueezeEmbLR(self.embedding,self.SqueezeEmb_LRWeight)
assert self.pred is not None, "must have one predicion layer"
if self.loss_type == 'rmse':
self.loss = tf.sqrt(tf.reduce_mean(tf.square(self.y - self.pred)))
elif self.loss_type == 'mse':
self.loss = tf.reduce_mean(tf.square(self.y - self.pred))
elif self.loss_type in ['binary_crossentropy', 'binary', 'logloss']:
self.loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(labels=self.y,
logits=self.pred))
else:
raise Exception("Loss type %s not supported" % self.loss_type)
self.loss += self.lambda_l2 * self.L2_reg
self.optimizer = tf.train.AdamOptimizer(lr).minimize(self.loss)
if self.metric_type is not None:
assert self.metric_type == 'auc'
assert self.loss_type in [
'binary_crossentropy', 'binary', 'logloss'
]
#self.loss=tf.metrics.auc(labels=self.y,predictions=tf.nn.sigmoid(self.pred))
self.sess = self._init_session()
self.sess.run(tf.global_variables_initializer())
#self.sess.run(tf.local_variables_initializer())
cur_best_rounds = 0
is_greater_better = False if self.metric_type is None else True #默认Loss越小越好
cur_min_loss = 1e8 if not is_greater_better else -1e8
best_weights = {
v.name: v.eval(self.sess)
for v in tf.trainable_variables()
}
for epoch in range(N_EPOCH):
train_loss = 0.
y_preds_train = []
total_batches = int(ids_train.shape[0] / batch_size)
for bx, by in batcher(ids_train, y_train, batch_size,
self.hash_size):
_, l = self.sess.run(
[self.optimizer, self.loss],
feed_dict={
self.ids: bx,
self.y: by,
self.dropout_keeprate_holder: self.dropout_keeprate
})
train_loss += l #if not self.metric_type else l[1]
if self.metric_type:
y_preds_train.append(
self.sess.run(self.pred,
feed_dict={
self.ids: bx,
self.dropout_keeprate_holder: 1.0
}))
train_loss /= total_batches
#todo movielens afm rounded
test_loss = 0.
y_preds = []
for bx, by in batcher(ids_test, y_test, batch_size,
self.hash_size):
l = self.sess.run(self.loss,
feed_dict={
self.ids: bx,
self.y: by
})
test_loss += l #if not self.metric_type else l[1]
if self.metric_type:
y_preds.append(
self.sess.run(self.pred,
feed_dict={
self.ids: bx,
self.dropout_keeprate_holder: 1.0
}))
test_loss /= int(ids_test.shape[0] / batch_size)
'''
y_pred=np.concatenate(y_preds, axis=0).reshape((-1))
predictions_bounded = np.maximum(y_pred, np.ones(len(y_pred)) * -1) # bound the lower values
predictions_bounded = np.minimum(predictions_bounded, np.ones(len(y_pred)) * 1) # bound the higher values
# override test_loss
test_loss = np.sqrt(np.mean(np.square(y_test.reshape(predictions_bounded.shape)- predictions_bounded)))
'''
if self.metric_type: # override test_loss
self.y_pred_train = np.concatenate(y_preds_train, axis=0)
self.y_pred = np.concatenate(y_preds, axis=0)
train_loss = roc_auc_score(y_train, self.y_pred_train)
test_loss = roc_auc_score(y_test, self.y_pred)
metrics_ = 'loss' if self.metric_type is None else 'auc'
print("epoch:%s train_%s:%s test_%s:%s" %
(epoch + 1, metrics_, train_loss, metrics_, test_loss))
#print("self.pred=",self.sess.run(self.pred,feed_dict={self.ids:ids_test,self.y:y_test}))
#print("self.y=",y_test)
if isBetter(test_loss, cur_min_loss, is_greater_better):
print("better")
cur_min_loss = test_loss
cur_best_rounds = epoch + 1
best_weights = {
v.name: v.eval(self.sess)
for v in tf.trainable_variables()
}
if epoch + 1 - cur_best_rounds >= early_stopping_rounds:
print(
"[Early Stop]Early Stopping because not improved for %s rounds"
% early_stopping_rounds)
self.sess.run(
tf.tuple([
tf.assign(var, best_weights[var.name])
for var in tf.trainable_variables()
]))
best_score = cur_min_loss #self.sess.run(self.loss, feed_dict={self.ids: ids_test, self.y: y_test, })
print("[Early Stop]Best Score:", best_score, ' at round ',
cur_best_rounds)
return best_score
#auc reset op
#self.sess.run(tf.local_variables_initializer())
self.sess.run(
tf.tuple([
tf.assign(var, best_weights[var.name])
for var in tf.trainable_variables()
]))
best_score = cur_min_loss #self.sess.run(self.loss, feed_dict={self.ids: ids_test, self.y: y_test,})
print("Best Score:", best_score, ' at round ', cur_best_rounds)
return best_score