def get_attention_mask(self): if not self.attention_FM and not self.use_AutoInt: return self.attention_masks = [] for bx, bx_dense, by in batcher(X_=self.ids_test, y_=self.y_test, batch_size=500, hash_size=self.hash_size): #dense if self.attention_FM: self.attention_masks.append( self.sess.run(self.normalize_att_score, feed_dict={ self.ids: bx, self.y: by, self.dropout_keeprate_holder: 1.0 })) else: self.attention_masks.append( self.sess.run(self.normalize_autoint_att_score, feed_dict={ self.ids: bx, self.y: by, self.dropout_keeprate_holder: 1.0 })) #return np.array(self.attention_masks) return self.attention_masks
def get_attention_mask(self): if not self.attention_FM: return self.attention_masks = [] for bx, by in batcher(self.ids_test, self.y_test, 500, hash_size=self.hash_size): self.attention_masks.append( self.sess.run(self.normalize_att_score, feed_dict={ self.ids: bx, self.y: by, self.dropout_keeprate_holder: 1.0 })) return np.array(self.attention_masks)
def main(): FLAGS = parse_args() try: triton_client = tritonclient.grpc.InferenceServerClient(url=FLAGS.url, verbose=FLAGS.verbose) except Exception as e: print("channel creation failed: " + str(e)) sys.exit() model_name = FLAGS.model_name model_version = -1 print("Loading images") image_data = load_images(FLAGS.img_dir if FLAGS.img_dir is not None else FLAGS.img, max_images=FLAGS.batch_size * FLAGS.n_iter) image_data = array_from_list(image_data) print("Images loaded") for batch in tqdm(utils.batcher(image_data, FLAGS.batch_size, n_iterations=FLAGS.n_iter), desc="Inferring", total=FLAGS.n_iter): inputs = generate_inputs(FLAGS.input_name, batch.shape, "UINT8") outputs = generate_outputs(FLAGS.output_name) # Initialize the data inputs[0].set_data_from_numpy(batch) # Test with outputs results = triton_client.infer(model_name=model_name, inputs=inputs, outputs=outputs) # Get the output arrays from the results output0_data = results.as_numpy(FLAGS.output_name) maxs = np.argmax(output0_data, axis=1) if FLAGS.statistics: for i in range(len(maxs)): print("Sample ", i, " - label: ", maxs[i], " ~ ", output0_data[i, maxs[i]]) statistics = triton_client.get_inference_statistics(model_name="dali") if len(statistics.model_stats) != 1: print("FAILED: Inference Statistics") sys.exit(1) if FLAGS.statistics: print(statistics) print('PASS: infer')
def predict(self, ids_pred): if self.hash_size is None: for i, column in enumerate(ids_pred.columns): if i >= 1: ids_pred.loc[:, column] = ids_pred[column] + sum( self.features_sizes[:i]) outputs = [] self.ids_pred = ids_pred for bx in batcher(ids_pred, None, batch_size=self.batch_size, hash_size=self.hash_size): #y=None outputs.append( self.sess.run(self.pred, feed_dict={ self.ids: bx, self.dropout_keeprate_holder: 1.0 })) self.output = np.concatenate(outputs, axis=0) #.reshape((-1)) return self.output
def valRes(weightName, iter): net.load(weightName, sess) batcher = ut.batcher(LIST, sourcePath, BSZ, iter) val_loss = [] val_accuracy = [] imgs, nlbls = batcher.get_batch() _ = sess.run(data.fetchOp, feed_dict=data.getfeed(imgs)) for i in range(VAL_numBatches): _ = sess.run(data.swapOp) clbls = nlbls imgs, nlbls = batcher.get_batch() fdict = data.getfeed(imgs) fdict[labels] = clbls outs = sess.run([opt.accuracy, opt.loss, data.fetchOp], feed_dict=fdict) val_loss.append(outs[1]) val_accuracy.append(outs[0]) loss_val = np.mean(val_loss) accuracy_val = np.mean(val_accuracy) sys.stdout.write(" [%09d] Val loss = %.6f, Val accuracy = %.6f\n" % (iter, loss_val, accuracy_val))
def predict(self, ids_pred, dense_pred=None): # [bug fix]mutable prevention 19/06/27 start_time = time.time() ids_pred = ids_pred.copy() if self.hash_size is None: for i, column in enumerate(ids_pred.columns): if i >= 1: ids_pred.loc[:, column] = ids_pred[column] + sum( self.features_sizes[:i]) outputs = [] #self.ids_pred=ids_pred 0708 del for bx, bx_dense in batcher(ids_pred, y_=None, X_dense=dense_pred, batch_size=self.batch_size, hash_size=self.hash_size): #y=None if self.dense_features_size > 0: outputs.append( self.sess.run(self.pred, feed_dict={ self.ids: bx, self.dense_inputs: bx_dense, self.dropout_keeprate_holder: 1.0 })) else: #bx_dense==None outputs.append( self.sess.run(self.pred, feed_dict={ self.ids: bx, self.dropout_keeprate_holder: 1.0 })) self.output = np.concatenate(outputs, axis=0) #.reshape((-1)) time_cost = time.time() - start_time print("Time cost: %3f seconds, average epoch time:%2f ms" % (time_cost, (time_cost / len(outputs)) * 1000)) return self.output
WEIGHT_DECAY = 0. LR = 0.01 MOM = 0.9 MAX_ITER = int(1.2e7) # Update BSZ, lr BSZ = BSZ // AVG_IT LR_E = LR / float(AVG_IT) # Check for saved weights saved = ut.ckpter(WTS_DIR + 'iter_*.model.npz') iter = saved.iter # Set up batching batcher = ut.batcher(LIST, BSZ, iter * AVG_IT) # Set up data prep data = ldr.trainload(BSZ) labels = tf.placeholder(shape=(BSZ, ), dtype=tf.int32) # Load model-def net = md.model(data.batch, train=True) # Load trainer-def opt = tr.train(net, labels, LR_E, MOM, WEIGHT_DECAY) # Start session sess = tf.Session() sess.run(tf.initialize_all_variables())
import cifar100 as cf import trainer as tr import model as md import utils as ut from params import * # Load dataset tset, vset = cf.get_tval() # Restore any saved weights saved = ut.ckpter(WTS_DIR + 'iter_*.model.npz') iter = saved.iter # Set up batching tbatch = ut.batcher(tset[0], tset[1], BSZ, iter) vbatch = ut.batcher(vset[0], vset[1], BSZ, 0, False) # Create placeholders data = tf.placeholder(shape=(BSZ, ) + tset[0].shape[1:], dtype=tset[0].dtype) labels = tf.placeholder(shape=(BSZ, ) + tset[1].shape[1:], dtype=tset[1].dtype) lr = tf.placeholder(shape=(), dtype=tf.float32) phase = tf.placeholder(shape=(), dtype=tf.int32) # Load model-def net = md.model(data, phase) # Load trainer-def opt = tr.train(net, labels, lr, MOM, WD)
img = np.clip(img, 0., 1.) * 255 img = np.uint8(img) img = Image.fromarray(img) return img saved = ut.ckpter(args.WTS + 'iter_*.model.npz') iter = saved.iter std_num = len(args.STD.split(' ')) stds = [float(i) for i in args.STD.split(' ')] # Start session sess = tf.Session() batcher = ut.batcher(args.LIST, 1) data = tldr.testload(args.SIZE, args.BSIZE) std_img = tf.placeholder(shape=[args.BSIZE * args.BSIZE], dtype=tf.float32) std_denoise = tf.placeholder(shape=[args.BSIZE * args.BSIZE], dtype=tf.float32) rnd = tf.placeholder(shape=[], dtype=tf.float32) comp = tf.placeholder(shape=[], dtype=tf.float32) if args.MD == 1: net = md.model(data.batch, std_img, std_denoise) elif args.MD == 2: net = mdv2.model(data.batch, std_img, std_denoise) elif args.MD == 3: net = mdv3.model(data.batch, std_img, std_denoise) elif args.MD == 4: net = mdv4.model(data.batch, std_img, std_denoise) elif args.MD == 5: net = mdv5.model(data.batch, std_img, std_denoise)
# Ayan Chakrabarti <*****@*****.**> import sys import tensorflow as tf import numpy as np import cifar100 as cf import trainer as tr import model as md import utils as ut BSZ = 500 testset = cf.get_test() # Set up batching tbatch = ut.batcher(testset[0], testset[1], BSZ, 0, False) # Create placeholders data = tf.placeholder(shape=(BSZ, ) + testset[0].shape[1:], dtype=testset[0].dtype) # Load model-def net = md.model(data) # Start session sess = tf.Session() sys.stdout.write("Restoring from " + sys.argv[1] + "\n") sys.stdout.flush() net.load(sys.argv[1], sess) # Test nIter = testset[0].shape[0] // BSZ
def train(config): # Store results results = [] # Timestamp to name the model timestamp = datetime.datetime.now().strftime("%y%m%d%M%S") # Build the Folds folds = utils.fold_iter(dataset=config.train, k_folds=config.k_folds, test_split=config.test_split) # Iterate through the folds for fold, (train, test) in enumerate(folds): # Define the model if config.model == "cnn": model = CNN(config) else: model = RNN(config) # Build the model model.build() # Fold metrics fold_best, patience = 0, 0 result = [] print("\nFold ", fold + 1, "\n") for ep in range(config.n_epochs): # At the start of each epoch we shuffle the training set train = utils.shuffle(train) # Batch batcher = utils.batcher(dataset=train, batch_size=config.batch_size) # For each batch for batch in batcher: # Pad the data such that every word and sentence has the same length # for the sake of less code, tags in the classification task holds # the labels words, chars, labels = config.pad(dataset=batch) # Build the feed dictionary feed = config.feed(model, words, labels, chars=chars, dropout=config.dropout) # Train _, loss, acc = model.sess.run( [model.train_op, model.loss, model.accuracy], feed) # print(np.shape(model.features.eval(session=model.sess, feed_dict = feed))) # print(np.shape(model.max_pool.eval(session=model.sess, feed_dict = feed))) # Write the metrics summary = "Epoch " + str(ep + 1) + " :\n\tLoss: " + str(loss) # Score agasint test set f1_acc = [] epoch_results = [] metrics = eval.evaluate(model=model, test_set=train, config=config) epoch_results.append(metrics) f1 = (2 * metrics["tpr"] * metrics["tnr"]) / (metrics["tpr"] + metrics["tnr"]) # Write the metrics summary += "\n\tTPR D1: " + \ format(metrics["tpr"], ".3f") + "\tTNR D1: " + format(metrics["tnr"], ".3f") + "\tF1 D1: " + format(f1, ".3f") line = "\t" + str(metrics["tpr"]) + "\t" + \ str(metrics["tnr"]) + "\t" + str(f1) # IF there is only one fold and no test split %, then no testing occurs if not (config.k_folds == 1 and config.test_split == 0.0): # Fold test subset metrics = eval.evaluate(model=model, test_set=test, config=config) epoch_results.append(metrics) f1 = (2 * metrics["tpr"] * metrics["tnr"]) / (metrics["tpr"] + metrics["tnr"]) # Write the metrics summary += "\n\tTPR D1: " + \ format(metrics["tpr"], ".3f") + "\tTNR D1: " + format(metrics["tnr"], ".3f") + "\tF1 D1: " + format(f1, ".3f") # Test sets for i, _set in enumerate(config.test): metrics = eval.evaluate(model=model, test_set=_set, config=config) epoch_results.append(metrics) # Calculate F-measure F1 f1 = (2 * metrics["tpr"] * metrics["tnr"]) / (metrics["tpr"] + metrics["tnr"]) f1_acc.append(f1) line += "\t" + str(metrics["tpr"]) + "\t" + \ str(metrics["tnr"]) + "\t" + str(f1) # Write the metrics summary += "\n\tTPR D" + \ str(i + 2) + ": " + format(metrics["tpr"], ".3f") + \ "\tTNR D" + str(i + 2) + ": " + format(metrics["tnr"], ".3f") + "\tF1 D" + str( i + 2) + ": " + format(f1, ".3f") # Average the F1 scores f1_acc = f1_acc[0] # Output current metrics print(summary) # Check if the model has improved if f1_acc > fold_best: result = epoch_results fold_best = f1_acc # Reset patience patience = 0 # save the model if not config.save == None: model.class_save(fold, timestamp, config.save, ep) else: patience += 1 # After 3 epochs, if the model has not improved then we can stop if patience > 300: break # Store all metrics results.append(result) # Print some metrics for i, metrics in enumerate(result): print("\nTest set", i + 1, "\t--acc: ", format(metrics["accuracy"], '.3f'), " --tpr: ", format(metrics["tpr"], '.3f'), " --tnr: ", format(metrics["tnr"], '.3f')) model.close_session() line = eval.getstats(config, results) print(line) return line
parser.add_argument('--ITER', type=int, default=int(1e7), help='total iterations') args = parser.parse_args() saved = ut.ckpter(args.WTS + 'iter_*.model.npz') iter = saved.iter std_num = len(args.STD.split(' ')) stds = np.zeros(BSZ) for i in range(std_num): stds[i * 16:(i + 1) * 16] = float(args.STD.split(' ')[i]) batcher = ut.batcher(args.LIST, BSZ, iter) # Set up data prep data = ldr.trainload(BSZ, IMG_SZ) # Noise STD noise_std = tf.placeholder(shape=[BSZ], dtype=tf.float32) # Load model-def net = md.model(data.batch, noise_std) # Learning rate lr = tf.placeholder(shape=[], dtype=tf.float32) # Start session sess = tf.Session()
parser.add_argument('--LOSSWEIGHT', type=float, default=0., help='comb loss') args = parser.parse_args() # Check for saved weights saved = ut.ckpter(args.WTS + 'iter_*.model.npz') iter = saved.iter std_num = len(args.STD.split(' ')) stds = np.zeros(BSZ) s_loss = [] for i in range(std_num): s_loss.append([]) stds[i * 16:(i + 1) * 16] = float(args.STD.split(' ')[i]) # Set up batching batcher_train = ut.batcher(args.LISTTRAIN, BSZ, iter) # Set up data prep data = ldr.trainload(BSZ, args.SIZE) # Noise STD std_img = tf.placeholder(shape=[BSZ], dtype=tf.float32) std_denoise = tf.placeholder(shape=[BSZ], dtype=tf.float32) # Load model-def if args.MD == 1: net = md.model(data.batch, std_img, std_denoise) elif args.MD == 2: net = mdv2.model(data.batch, std_img, std_denoise) elif args.MD == 3: net = mdv3.model(data.batch, std_img, std_denoise)
def train(config): # Store results results = [] # Timestamp to name the model timestamp = datetime.datetime.now().strftime("%y%m%d%H%M%S") # Build the Folds folds = utils.fold_iter(dataset=config.train, k_folds=config.k_folds, test_split=config.test_split) # Iterate through the folds for fold, (train, test) in enumerate(folds): # Define the model if config.model == "cnn": model = CNN(config) else: model = RNN(config) # Build the model model.build() # Fold metrics fold_best, patience = 0, 0 result = [] print("\nFold ", fold + 1, "\n") for ep in range(config.n_epochs): # At the start of each epoch we shuffle the training set train = utils.shuffle(train) # Batch batcher = utils.batcher(dataset=train, batch_size=config.batch_size) # For each batch for batch in batcher: # Pad the data such that every word and sentence has the same length # for the sake of less code, tags in the classification task holds # the labels words, tags, chars = config.pad(dataset=batch, use_chars=config.use_chars) # Build the feed dictionary feed = config.feed(model, words=words, tags=tags, chars=chars, dropout=config.dropout, use_chars=config.use_chars) # Train _, loss = model.sess.run([model.train_op, model.loss], feed_dict=feed) # print(np.shape(model.word_vectors.eval(session=model.sess,feed_dict=feed))) # print(np.shape(model.conv.eval(session=model.sess,feed_dict=feed))) # print(np.shape(model.h_flat.eval(session=model.sess,feed_dict=feed))) # print((model.h_nodes)) # Write the metrics summary = "Epoch " + str(ep + 1) + " :\n\tLoss: " + str(loss) # Score agasint test set f1 = [] epoch_results = [] # IF there is only one fold and no test split %, then no testing occurs if not (config.k_folds == 1 and config.test_split == 0.0): # Fold test subset metrics = eval.evaluate(model=model, test_set=test, config=config) epoch_results.append(metrics) f1.append(metrics["f1"]) # Write the metrics summary += "\n\tPrecision D1: " + \ format(metrics["precision"], ".3f") + "\tRecall D1: " + format(metrics["recall"], ".3f") + "\tF1 D1: " + format(metrics["f1"], ".3f") # Test sets for i, _set in enumerate(config.test): metrics = eval.evaluate(model=model, test_set=_set, config=config) epoch_results.append(metrics) # Sum the F1s for averaging f1.append(metrics["f1"]) # Write the metrics summary += "\n\tPrecision D" + \ str(i + 2) + ": " + format(metrics["precision"], ".3f") + \ "\tRecall D" + str(i + 2) + ": " + format(metrics["recall"], ".3f") + "\tF1 D" + str( i + 2) + ": " + format(metrics["f1"], ".3f") # Output current metrics print(summary) # Average the F1 scores #f1 = np.average(np.array(f1)) if len(f1) > 0 else 0 f1 = f1[0] # Check if the model has improved if f1 > fold_best: result = epoch_results fold_best = f1 # Reset patience patience = 0 # save the model if not config.save == None: model.ner_save(fold, timestamp, config.save, ep) else: patience += 1 # After 3 epochs, if the model has not improved then we can stop if patience > config.patience: break # Store all metrics results.append(result) # Print some metrics for i, metrics in enumerate(result): print("\nTest set", i + 1, "\t--f1: ", format(metrics["f1"], '.3f'), " --acc: ", format(metrics["accuracy"], '.3f'), " --precis: ", format(metrics["precision"], '.3f'), " --recall: ", format(metrics["recall"], '.3f')) model.close_session() line = eval.getstats(config=config, results=results) return line
DISP_FREQ = 10 BSZ = 32 WEIGHT_DECAY = 0. LR = 0.001 MOM = 0.9 MAX_ITER = int(1.2e7) # Check for saved weights saved = ut.ckpter(WTS_DIR + 'iter_*.model.npz') iter = saved.iter # Set up batching batcher = ut.batcher(LIST, BSZ, iter) # Set up data prep data = ldr.trainload(BSZ) labels = tf.placeholder(shape=(BSZ, ), dtype=tf.int32) # Load model-def net = md.model(data.batch, train=True) # Load trainer-def opt = tr.train(net, labels, LR, MOM, WEIGHT_DECAY) # Start session sess = tf.Session() sess.run(tf.initialize_all_variables())
SAVE_FREQ = 1000 DISP_FREQ = 100 BSZ = 64 WEIGHT_DECAY = 0. LR = 0.0001 MOM = 0.9 MAX_ITER = int(1.2e7) # val VAL_FREQ = 100 VAL_numBatches = 20 # Check for saved weights saved = ut.ckpter(WTS_DIR + 'iter_*.model.npz') iter = saved.iter # Set up batching batcher = ut.batcher(LIST, sourcePath, BSZ, iter) batcher_val = ut.batcher(VAL, sourcePath, BSZ, iter) # Set up data prep data = ldr.trainload(BSZ) labels = tf.placeholder(shape=(BSZ, ), dtype=tf.int32) labels_val = tf.placeholder(shape=(BSZ, ), dtype=tf.int32) # Load model-def iftrain = tf.placeholder(tf.bool, shape=[]) net = md.model(data.batch, iftrain) # Load trainer-def opt = tr.train(net, labels, LR, MOM, WEIGHT_DECAY) # Start session
def fit( self, ids_train, ids_test, y_train, y_test, dense_train=None, dense_test=None, lr=0.001, N_EPOCH=50, batch_size=200, early_stopping_rounds=20, ): start_time = time.time() #[bug fix]mutable prevention 19/06/27 ids_train = ids_train.copy() ids_test = ids_test.copy() self.batch_size = batch_size #data preprocess:对ids的每个features,label encoder都要从上一个的末尾开始。函数输入时则保证每个都从0起. if self.hash_size is None: for i, column in enumerate(ids_train.columns): if i >= 1: ids_train.loc[:, column] = ids_train[column] + sum( self.features_sizes[:i]) ids_test.loc[:, column] = ids_test[column] + sum( self.features_sizes[:i]) if self.attention_FM or self.use_AutoInt: #储存为classs变量并用在get_attention里获取attention self.ids_train, self.ids_test, self.y_train, self.y_test = ids_train, ids_test, y_train, y_test self.ids = tf.placeholder(tf.int32, [None, self.fields]) self.dense_inputs = tf.placeholder(tf.float32, [None, self.dense_features_size]) self.y = tf.placeholder(tf.float32, [None, 1]) self.L2_reg = 0 self.dropout_keeprate_holder = tf.placeholder(tf.float32) embed_L2 = 0 if self.use_FM or self.use_MLP or self.use_AutoInt: self.embedding, embed_L2 = self.Embedding( self.ids, self.embedding_weights) #(None,fields,k) if self.use_SE: self.embeddingSE = self.SELayer(self.embedding, self.SE_weights) self.pred = 0 if self.use_LR: #bug detected. LR didn't keepdims self.pred = self.LR(self.ids, self.w, self.b) if self.use_MLR: print("use Mix of LR.") self.pred += self.MLR(self.ids, self.MLR_u, self.MLR_w) #only one FM will be used. if self.use_NFM: print("use NFM") self.pred += self.NFM(self.embedding, self.NFM_weights) elif self.use_BiFM: if self.use_SE: cross_term = tf.concat([ self.Bilinear_FM( self.embedding, self.bilinear_weights, se_emb=False), self.Bilinear_FM( self.embeddingSE, self.bilinear_weights, se_emb=True), ], axis=-1) # N,c,2k if self.use_FiBiNet: print("use FiBiNet") #deep backend cross_term = tf.reshape( cross_term, [-1, self.c * self.k * 2]) #None,2ck cross_term = tf.nn.relu( tf.matmul(cross_term, self.FiBiNet_weights['W1']) + self.FiBiNet_weights['b1']) self.pred += ( tf.matmul(cross_term, self.FiBiNet_weights['W2']) + self.FiBiNet_weights['b2']) else: print("use Fibifm") self.pred += tf.expand_dims(tf.reduce_sum(cross_term, axis=[1, 2]), axis=1) # N,1 else: print("use bifm") cross_term = self.Bilinear_FM(self.embedding, self.bilinear_weights, se_emb=False) # N,c,k self.pred += tf.expand_dims(tf.reduce_sum(cross_term, axis=[1, 2]), axis=1) # N,1 elif self.use_FM and not self.attention_FM and not self.use_CFM: print("use FM") if len(self.FM_ignore_interaction ) == 0: #if self.use_FM and self.FM_ignore_interaction==[] self.pred += self.FM2(self.embedding) if len(self.FM_ignore_interaction) > 0: self.pred += self.FMDE(self.embedding) elif self.use_FM and self.attention_FM: print("use AFM") afm_out, reg = self.AFM(self.embedding, self.AFM_weights) self.pred += afm_out self.L2_reg += reg elif self.use_FM and self.use_CFM: print("use CFM") cfm_out, reg = self.CFM(self.embedding, self.CFM_weights) self.pred += cfm_out self.L2_reg += reg if self.use_AutoInt: self.y_deep = self.embedding for _l in range(self.autoint_params['autoint_layers']): self.y_deep = self.AutoInt(self.y_deep, self.AutoInt_weights, layer=_l) #N,f,d self.pred += tf.matmul( tf.reshape(self.y_deep, shape=[ -1, self.fields * self.autoint_d * self.autoint_head ]), self.AutoInt_weights['W_out']) + self.AutoInt_weights['b_out'] if self.use_CrossNet_layers > 0: #combine crossnet with DNN MLP_in = tf.reshape(self.embedding, [-1, self.fields * self.k]) #(N,f*k) if self.dense_features_size > 0: MLP_in = tf.concat([MLP_in, self.dense_inputs], axis=1) #(N,f*k+dense) self.MLP_out = self.MLP(MLP_in, self.weights, self.bias, return_pred=False) #(None,last_layers) self.CrossNet_out = self.CrossNet(tf.expand_dims( MLP_in, axis=-1), self.CrossNet_weights) #(None,f*k+d) self.pred += tf.keras.layers.Dense( 1, use_bias=False, activation=None)(tf.concat([self.MLP_out, self.CrossNet_out], axis=1)) elif self.use_MLP: #并联dnn pred MLP_in = tf.reshape(self.embedding, [-1, self.fields * self.k]) #(N,f*k) if self.dense_features_size > 0: MLP_in = tf.concat([MLP_in, self.dense_inputs], axis=1) #(N,f*k+dense) self.pred += self.MLP(MLP_in, self.weights, self.bias) assert self.pred is not None, "must have one predicion layer" if self.loss_type == 'rmse': self.loss = tf.sqrt(tf.reduce_mean(tf.square(self.y - self.pred))) elif self.loss_type == 'mse': self.loss = tf.reduce_mean(tf.square(self.y - self.pred)) elif self.loss_type in ['binary_crossentropy', 'binary', 'logloss']: self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(labels=self.y, logits=self.pred)) else: raise Exception("Loss type %s not supported" % self.loss_type) #todo EMBEDL2 coef self.loss += self.lambda_l2 * self.L2_reg #+ embed_L2*1e-5 self.optimizer = tf.train.AdamOptimizer(lr).minimize(self.loss) if self.metric_type is not None: assert self.metric_type == 'auc' assert self.loss_type in [ 'binary_crossentropy', 'binary', 'logloss' ] #tf.auc mode: remove sklearn auc part #self.loss=tf.metrics.auc(labels=self.y,predictions=tf.nn.sigmoid(self.pred)) self.sess = self._init_session() self.sess.run(tf.global_variables_initializer()) self.sess.run(tf.local_variables_initializer()) cur_best_rounds = 0 is_greater_better = False if self.metric_type is None else True #默认Loss越小越好 cur_min_loss = 1e8 if not is_greater_better else -1e8 best_weights = { v.name: v.eval(self.sess) for v in tf.trainable_variables() } for epoch in range(N_EPOCH): train_loss = 0. y_preds_train = [] total_batches = int(ids_train.shape[0] / batch_size) # id input + dense input for bx, bx_dense, by in batcher(ids_train, y_train, X_dense=dense_train, batch_size=batch_size, hash_size=self.hash_size): if self.dense_features_size > 0: _, l = self.sess.run( [self.optimizer, self.loss], feed_dict={ self.ids: bx, self.y: by, self.dense_inputs: bx_dense, self.dropout_keeprate_holder: self.dropout_keeprate }) else: _, l = self.sess.run( [self.optimizer, self.loss], feed_dict={ self.ids: bx, self.y: by, self.dropout_keeprate_holder: self.dropout_keeprate }) train_loss += l #if not self.metric_type else l[1] if self.metric_type: y_preds_train.append(self.sess.run(self.pred,feed_dict={self.ids:bx,self.dense_inputs:bx_dense,self.dropout_keeprate_holder:1.0})) \ if self.dense_features_size>0 \ else y_preds_train.append(self.sess.run(self.pred,feed_dict={self.ids:bx,self.dropout_keeprate_holder:1.0})) train_loss /= total_batches if self.coldStartAvg: print("Cold Start Averaging start") if epoch == 0 else None self.coldStartAvgTool() #todo movielens afm rounded test_loss = 0. y_preds = [] for bx, bx_dense, by in batcher(ids_test, y_test, X_dense=dense_test, batch_size=batch_size, hash_size=self.hash_size): if self.dense_features_size > 0: l = self.sess.run(self.loss, feed_dict={ self.ids: bx, self.y: by, self.dense_inputs: bx_dense, self.dropout_keeprate_holder: 1.0 }) else: l = self.sess.run(self.loss, feed_dict={ self.ids: bx, self.y: by, self.dropout_keeprate_holder: 1.0 }) test_loss += l #if not self.metric_type else l[1] if self.metric_type: y_preds.append(self.sess.run(self.pred,feed_dict={self.ids:bx,self.dense_inputs:bx_dense,self.dropout_keeprate_holder:1.0})) \ if self.dense_features_size>0 \ else y_preds.append(self.sess.run(self.pred,feed_dict={self.ids:bx,self.dropout_keeprate_holder:1.0})) test_loss /= int(ids_test.shape[0] / batch_size) ''' y_pred=np.concatenate(y_preds, axis=0).reshape((-1)) predictions_bounded = np.maximum(y_pred, np.ones(len(y_pred)) * -1) # bound the lower values predictions_bounded = np.minimum(predictions_bounded, np.ones(len(y_pred)) * 1) # bound the higher values # override test_loss test_loss = np.sqrt(np.mean(np.square(y_test.reshape(predictions_bounded.shape)- predictions_bounded))) ''' #sklearn auc mode if self.metric_type: # override test_loss self.y_pred_train = np.concatenate(y_preds_train, axis=0) self.y_pred = np.concatenate(y_preds, axis=0) train_loss = roc_auc_score(y_train, self.y_pred_train) test_loss = roc_auc_score(y_test, self.y_pred) metrics_ = 'loss' if self.metric_type is None else 'auc' print("epoch:%s train_%s:%s test_%s:%s" % (epoch + 1, metrics_, train_loss, metrics_, test_loss)) #print("self.pred=",self.sess.run(self.pred,feed_dict={self.ids:ids_test,self.y:y_test})) #print("self.y=",y_test) if isBetter(test_loss, cur_min_loss, is_greater_better): cur_min_loss = test_loss cur_best_rounds = epoch + 1 best_weights = { v.name: v.eval(self.sess) for v in tf.trainable_variables() } if epoch + 1 - cur_best_rounds >= early_stopping_rounds: print( "[Early Stop]Early Stopping because not improved for %s rounds" % early_stopping_rounds) self.sess.run( tf.tuple([ tf.assign(var, best_weights[var.name]) for var in tf.trainable_variables() ])) best_score = cur_min_loss #self.sess.run(self.loss, feed_dict={self.ids: ids_test, self.y: y_test, }) print("[Early Stop]Best Score:", best_score, ' at round ', cur_best_rounds) print( "Train finish. Fit time:%.2f seconds. Epoch time:%.2f seconds" % (time.time() - start_time, (time.time() - start_time) / (epoch + 1))) return best_score #auc reset op self.sess.run(tf.local_variables_initializer()) self.sess.run( tf.tuple([ tf.assign(var, best_weights[var.name]) for var in tf.trainable_variables() ])) best_score = cur_min_loss #self.sess.run(self.loss, feed_dict={self.ids: ids_test, self.y: y_test,}) print("[Epoch Maxi]Best Score:", best_score, ' at round ', cur_best_rounds) print("Train finish. Fit time:%.2f seconds. Epoch time:%.2f seconds" % (time.time() - start_time, (time.time() - start_time) / N_EPOCH)) return best_score
# redirect stdout and stderr to log files if prm.log_file: sys.stdout = open(exp_dir + '/train.log', 'a') sys.stderr = open(exp_dir + '/info.log', 'a') # Check for saved weights & find iter vsave = ut.ckpter(exp_dir + '/iter_*.vmodel.npz') osave = ut.ckpter(exp_dir + '/iter_*.opt.npz') vpath = lambda itr: '%s/iter_%07d.vmodel.npz' % (exp_dir, itr) opath = lambda itr: '%s/iter_%07d.opt.npz' % (exp_dir, itr) niter = vsave.iter # Load annotations ut.mprint("Loading annotations") tbchr = ut.batcher(prm.trn_anns, prm.batch_size, niter) vbchr = ut.batcher(prm.val_anns, prm.batch_size, niter) ut.mprint("Done!") ######################################################################### # Set up data fetch camera_fps = [tf.placeholder(tf.string) for i in range(prm.batch_size)] pts_xyz_fps = [tf.placeholder(tf.string) for i in range(prm.batch_size)] pts_rgb_fps = [tf.placeholder(tf.string) for i in range(prm.batch_size)] pts_sift_fps = [tf.placeholder(tf.string) for i in range(prm.batch_size)] gt_depth_fps = [tf.placeholder(tf.string) for i in range(prm.batch_size)] getfeed = lambda fps: \ dict([(ph,'data/'+fps[i,3]) for i,ph in enumerate(camera_fps)]+\ [(ph,'data/'+fps[i,0]) for i,ph in enumerate(pts_xyz_fps)]+\
def fit(self, ids_train, ids_test, y_train, y_test, lr=0.001, N_EPOCH=50, batch_size=200, early_stopping_rounds=20): self.batch_size = batch_size #data preprocess:对ids的每个features,label encoder都要从上一个的末尾开始。函数输入时则保证每个都从0起. if self.hash_size is None: for i, column in enumerate(ids_train.columns): if i >= 1: ids_train.loc[:, column] = ids_train[column] + sum( self.features_sizes[:i]) ids_test.loc[:, column] = ids_test[column] + sum( self.features_sizes[:i]) if True: #self.attention_FM:#储存为classs变量并用在get_attention里获取attention self.ids_train, self.ids_test, self.y_train, self.y_test = ids_train, ids_test, y_train, y_test self.ids = tf.placeholder(tf.int32, [None, self.fields]) self.y = tf.placeholder(tf.float32, [None, 1]) self.dropout_keeprate_holder = tf.placeholder(tf.float32) if self.use_FM or self.use_MLP or self.use_AutoInt: self.embedding = self.Embedding( self.ids, self.embedding_weights) #(None,fields,k) self.pred = 0 self.L2_reg = 0 if self.use_LR: #bug detected. LR didn't keepdims self.pred = self.LR(self.ids, self.w, self.b) #only one FM will be used. if self.use_NFM: print("use NFM") self.pred += self.NFM(self.embedding, self.NFM_weights) elif self.use_FM and not self.attention_FM: print("use FM") if len(self.FM_ignore_interaction ) == 0: #if self.use_FM and self.FM_ignore_interaction==[] self.pred += self.FM2(self.embedding) if len(self.FM_ignore_interaction) > 0: self.pred += self.FMDE(self.embedding) elif self.use_FM and self.attention_FM: print("use CFM") afm_out, reg = self.CFM(self.embedding, self.AFM_weights) self.pred += afm_out self.L2_reg += reg if self.use_AutoInt: self.y_deep = self.embedding for l in range(3): self.y_deep = self.AutoInt(self.y_deep, self.AutoInt_weights, layer=l) #N,f,d self.pred += tf.matmul( tf.reshape(self.y_deep, shape=[-1, self.fields * self.autoint_d]), self.AutoInt_weights['W_out']) + self.AutoInt_weights['b_out'] if self.use_MLP: MLP_in = tf.reshape(self.embedding, [-1, self.fields * self.k]) self.pred += self.MLP(MLP_in, self.weights, self.bias) #self.pred=self.SqueezeEmbLR(self.embedding,self.SqueezeEmb_LRWeight) assert self.pred is not None, "must have one predicion layer" if self.loss_type == 'rmse': self.loss = tf.sqrt(tf.reduce_mean(tf.square(self.y - self.pred))) elif self.loss_type == 'mse': self.loss = tf.reduce_mean(tf.square(self.y - self.pred)) elif self.loss_type in ['binary_crossentropy', 'binary', 'logloss']: self.loss = tf.reduce_mean( tf.nn.sigmoid_cross_entropy_with_logits(labels=self.y, logits=self.pred)) else: raise Exception("Loss type %s not supported" % self.loss_type) self.loss += self.lambda_l2 * self.L2_reg self.optimizer = tf.train.AdamOptimizer(lr).minimize(self.loss) if self.metric_type is not None: assert self.metric_type == 'auc' assert self.loss_type in [ 'binary_crossentropy', 'binary', 'logloss' ] #self.loss=tf.metrics.auc(labels=self.y,predictions=tf.nn.sigmoid(self.pred)) self.sess = self._init_session() self.sess.run(tf.global_variables_initializer()) #self.sess.run(tf.local_variables_initializer()) cur_best_rounds = 0 is_greater_better = False if self.metric_type is None else True #默认Loss越小越好 cur_min_loss = 1e8 if not is_greater_better else -1e8 best_weights = { v.name: v.eval(self.sess) for v in tf.trainable_variables() } for epoch in range(N_EPOCH): train_loss = 0. y_preds_train = [] total_batches = int(ids_train.shape[0] / batch_size) for bx, by in batcher(ids_train, y_train, batch_size, self.hash_size): _, l = self.sess.run( [self.optimizer, self.loss], feed_dict={ self.ids: bx, self.y: by, self.dropout_keeprate_holder: self.dropout_keeprate }) train_loss += l #if not self.metric_type else l[1] if self.metric_type: y_preds_train.append( self.sess.run(self.pred, feed_dict={ self.ids: bx, self.dropout_keeprate_holder: 1.0 })) train_loss /= total_batches #todo movielens afm rounded test_loss = 0. y_preds = [] for bx, by in batcher(ids_test, y_test, batch_size, self.hash_size): l = self.sess.run(self.loss, feed_dict={ self.ids: bx, self.y: by }) test_loss += l #if not self.metric_type else l[1] if self.metric_type: y_preds.append( self.sess.run(self.pred, feed_dict={ self.ids: bx, self.dropout_keeprate_holder: 1.0 })) test_loss /= int(ids_test.shape[0] / batch_size) ''' y_pred=np.concatenate(y_preds, axis=0).reshape((-1)) predictions_bounded = np.maximum(y_pred, np.ones(len(y_pred)) * -1) # bound the lower values predictions_bounded = np.minimum(predictions_bounded, np.ones(len(y_pred)) * 1) # bound the higher values # override test_loss test_loss = np.sqrt(np.mean(np.square(y_test.reshape(predictions_bounded.shape)- predictions_bounded))) ''' if self.metric_type: # override test_loss self.y_pred_train = np.concatenate(y_preds_train, axis=0) self.y_pred = np.concatenate(y_preds, axis=0) train_loss = roc_auc_score(y_train, self.y_pred_train) test_loss = roc_auc_score(y_test, self.y_pred) metrics_ = 'loss' if self.metric_type is None else 'auc' print("epoch:%s train_%s:%s test_%s:%s" % (epoch + 1, metrics_, train_loss, metrics_, test_loss)) #print("self.pred=",self.sess.run(self.pred,feed_dict={self.ids:ids_test,self.y:y_test})) #print("self.y=",y_test) if isBetter(test_loss, cur_min_loss, is_greater_better): print("better") cur_min_loss = test_loss cur_best_rounds = epoch + 1 best_weights = { v.name: v.eval(self.sess) for v in tf.trainable_variables() } if epoch + 1 - cur_best_rounds >= early_stopping_rounds: print( "[Early Stop]Early Stopping because not improved for %s rounds" % early_stopping_rounds) self.sess.run( tf.tuple([ tf.assign(var, best_weights[var.name]) for var in tf.trainable_variables() ])) best_score = cur_min_loss #self.sess.run(self.loss, feed_dict={self.ids: ids_test, self.y: y_test, }) print("[Early Stop]Best Score:", best_score, ' at round ', cur_best_rounds) return best_score #auc reset op #self.sess.run(tf.local_variables_initializer()) self.sess.run( tf.tuple([ tf.assign(var, best_weights[var.name]) for var in tf.trainable_variables() ])) best_score = cur_min_loss #self.sess.run(self.loss, feed_dict={self.ids: ids_test, self.y: y_test,}) print("Best Score:", best_score, ' at round ', cur_best_rounds) return best_score