def eval(self, T, dev_data, hparams, sess): preds = self.infer(dev_data) if hparams.metric == 'logloss': log_loss = metrics.log_loss(dev_data[1], preds) if self.best_score > log_loss: self.best_score = log_loss try: os.makedirs('model_tmp/') except: pass self.saver.save(sess, 'model_tmp/model') utils.print_out("# Epcho-time %.2fs Eval logloss %.6f. Best logloss %.6f." \ %(T,log_loss,self.best_score)) elif hparams.metric == 'auc': fpr, tpr, thresholds = metrics.roc_curve(dev_data[1] + 1, preds, pos_label=2) auc = metrics.auc(fpr, tpr) if self.best_score < auc: self.best_score = auc try: os.makedirs('model_tmp/') except: pass self.saver.save(sess, 'model_tmp/model') utils.print_out("# Epcho-time %.2fs Eval AUC %.6f. Best AUC %.6f." \ %(T,auc,self.best_score))
def eval(self, T, dev_data, hparams, sess, test_data=None): preds1 = self.infer(test_data) preds = self.infer(dev_data) import pickle if test_data: pk = [test_data[1], preds1] with open(str(T) + '.pk', 'wb') as f: pickle.dump(pk, f) print('save test pikel file') if hparams.metric == 'logloss': log_loss = metrics.log_loss(dev_data[1], preds) if self.best_score > log_loss: self.best_score = log_loss try: os.makedirs('model_tmp/') except: pass self.saver.save(sess, 'model_tmp/model') utils.print_out("# Epcho-time %.2fs Eval logloss %.6f. Best logloss %.6f." \ %(T,log_loss,self.best_score)) elif hparams.metric == 'auc': fpr, tpr, thresholds = metrics.roc_curve(dev_data[1], preds, pos_label=1) auc = metrics.auc(fpr, tpr) if self.best_score < auc: self.best_score = auc try: os.makedirs('model_tmp/') except: pass self.saver.save(sess, 'model_tmp/model') utils.print_out("# Epcho-time %.2fs Eval AUC %.6f. Best AUC %.6f." \ %(T,auc,self.best_score))
def eval(self,T,dev,hparams,sess): preds=self.infer(dev) if True: fpr, tpr, thresholds = metrics.roc_curve(dev[hparams.label]+1, preds, pos_label=2) logloss=metrics.log_loss(dev[hparams.label],preds) auc=metrics.auc(fpr, tpr) if self.best_score<auc: self.best_score=auc utils.print_out(("# Epcho-time %.2fs Eval AUC %.6f. Eval logloss %.6f. Best AUC %.6f."+hparams.label)%(T,auc,logloss,self.best_score))
def eval(self, T, dev, hparams, sess): preds = self.infer(dev) dev['predict_imp'] = preds dev['rank'] = dev[['aid', 'bid']].groupby('aid')['bid'].apply( lambda row: pd.Series(dict(zip(row.index, row.rank())))) - 1 dev['predict_imp'] = dev['predict_imp'].apply(lambda x: np.exp(x) - 1) dev['predict_imp'] = dev['predict_imp'].apply(round) dev['predict_imp'] = dev['predict_imp'].apply(lambda x: 0 if x < 0 else x) dev['predict_imp'] = dev['predict_imp'] + dev['rank'] * 0.0001 dev['predict_imp'] = dev['predict_imp'].apply(lambda x: round(x, 4)) gold_dev = dev[dev['gold'] == True] score = abs(gold_dev['gold_imp'] - gold_dev['predict_imp']) / ( (gold_dev['gold_imp'] + gold_dev['predict_imp']) / 2 + 1e-15) SMAPE = score.mean() try: last_aid = None gold_imp = None gold_bid = None s = None score = [] for item in dev[['aid', 'bid', 'predict_imp']].values: item = list(item) if item[0] != last_aid: last_aid = item[0] gold_bid = item[1] gold_imp = item[2] if s is not None: score.append(s / cont) s = 0 cont = 0 else: if (gold_imp - item[2]) * (gold_bid - item[1]) == 0: s += -1 else: s += ((gold_imp - item[2]) * (gold_bid - item[1])) / (abs( ((gold_imp - item[2]) * (gold_bid - item[1])))) cont += 1 MonoScore = np.mean(score) score = 0.4 * (1 - SMAPE / 2) + 0.6 * (MonoScore + 1) / 2 except: MonoScore = 0 if SMAPE < self.best_score: self.best_score = SMAPE utils.print_out(( "# Epcho-time %.2fs AVG %.4f. Eval SMAPE %.4f. #Eval MonoScore %.4f. Best Score %.4f" ) % (T, dev['predict_imp'].mean(), SMAPE, MonoScore, self.best_score)) return SMAPE
def __init__(self,hparams): self.hparams=hparams if hparams.metric in ['logloss']: self.best_score=100000 else: self.best_score=0 self.build_graph(hparams) self.optimizer(hparams) params = tf.trainable_variables() utils.print_out("# Trainable variables") for param in params: utils.print_out(" %s, %s, %s" % (param.name, str(param.get_shape()),param.op.device))
def train(hparams): tf.set_random_seed(2018) random.seed(2018) train_iterator = TextIterator(hparams, mode="train") dev_iterator = TextIterator(hparams, mode="dev", batch_size=hparams.evl_batch_size) test_iterator = TextIterator(hparams, mode="test", batch_size=hparams.evl_batch_size) model = Model(hparams) config_proto = tf.ConfigProto(log_device_placement=0, allow_soft_placement=0) config_proto.gpu_options.allow_growth = True sess = tf.Session(config=config_proto) sess.run(tf.global_variables_initializer()) global_step = 0 train_loss = 0 train_norm = 0 best_loss = 0 dey_cont = 0 pay_cont = 0 epoch = False epoch_cont = 0 start_time = time.time() if True: while True: try: cost, _, norm = model.train(sess, train_iterator) except StopIteration: epoch = True epoch_cont += 1 continue global_step += 1 train_loss += cost train_norm += norm if global_step % hparams.num_display_steps == 0 or epoch: info = {} info['learning_rate'] = hparams.learning_rate info["train_ppl"] = train_loss / hparams.num_display_steps info["avg_grad_norm"] = train_norm / hparams.num_display_steps train_loss = 0 train_norm = 0 print_step_info(" ", global_step, info) if global_step % hparams.num_eval_steps == 0 or epoch: epoch = False preds = [] label = [] aid = [] while True: try: pred = model.infer(sess, dev_iterator, label, aid) preds += list(pred) except StopIteration: break res = {} for i in range(len(aid)): if aid[i] not in res: res[aid[i]] = {} res[aid[i]]['label'] = [] res[aid[i]]['pred'] = [] res[aid[i]]['label'].append(label[i] + 1) res[aid[i]]['pred'].append(preds[i]) auc = [] for u in res: fpr, tpr, thresholds = metrics.roc_curve( res[u]['label'], res[u]['pred'], pos_label=2) loss_ = metrics.auc(fpr, tpr) if np.isnan(loss_): continue gloab_auc[u] = loss_ auc.append(loss_) loss_ = np.mean(auc) if best_loss <= loss_: model.saver_ffm.save( sess, os.path.join(hparams.path, 'model_' + str(hparams.sub_name))) best_loss = loss_ T = (time.time() - start_time) start_time = time.time() utils.print_out( "# Epcho-time %.2fs Eval AUC %.6f. Best AUC %.6f." % (T, loss_, best_loss)) else: utils.print_out( "# Epcho-time %.2fs Eval AUC %.6f. Best AUC %.6f." % (T, loss_, best_loss)) model.saver_ffm.restore( sess, os.path.join(hparams.path, 'model_' + str(hparams.sub_name))) dey_cont += 1 if epoch_cont == hparams.epoch or dey_cont == hparams.dey_cont: model.saver_ffm.restore( sess, os.path.join(hparams.path, 'model_' + str(hparams.sub_name))) break print("Dev inference ...") preds = [] label = [] aid = [] while True: try: pred = model.infer(sess, dev_iterator, label, aid) preds += list(pred) except StopIteration: break data = [] for i in range(len(preds)): data.append([aid[i], label[i], preds[i]]) df = pd.DataFrame(data) df.columns = ['aid', 'label', 'score'] df.to_csv('result_sub/submission_dev_' + str(hparams.sub_name) + '.csv', index=False) print('Dev inference done!') res = {} for i in range(len(aid)): if aid[i] not in res: res[aid[i]] = {} res[aid[i]]['label'] = [] res[aid[i]]['pred'] = [] res[aid[i]]['label'].append(label[i] + 1) res[aid[i]]['pred'].append(preds[i]) auc = [] for u in res: fpr, tpr, thresholds = metrics.roc_curve(res[u]['label'], res[u]['pred'], pos_label=2) loss_ = metrics.auc(fpr, tpr) if np.isnan(loss_): continue auc.append(loss_) loss_ = np.mean(auc) print("Dev auc:", loss_) print("Test inference ...") preds = [] label = [] aid = [] while True: try: pred = model.infer(sess, test_iterator, label, aid) preds += list(pred) except StopIteration: break print('Test inference done!') return preds
def print_step_info(prefix, global_step, info): utils.print_out("%sstep %d lr %g logloss %.6f gN %.2f, %s" % (prefix, global_step, info["learning_rate"], info["train_ppl"], info["avg_grad_norm"], time.ctime()))
def __init__(self, hparams): self.f1 = hparams.aid.copy() self.f2 = hparams.user.copy() self.batch_norm_decay = 0.9 self.single_ids = {} self.num_ids = {} self.mulit_ids = {} self.mulit_mask = {} self.emb_v1 = {} self.emb_v2 = {} self.emb_combine_aid_v2 = {} self.norm_num = {} self.cross_params = [] self.layer_params = [] self.embed_params = [] self.length = {} self.bias = tf.Variable(tf.truncated_normal(shape=[1], mean=0.0, stddev=0.0001), name='bias') self.use_dropout = tf.placeholder(tf.bool) initializer = tf.random_uniform_initializer(-0.1, 0.1) self.feature_all_length = len(hparams.single_features) + len( hparams.mutil_features) feature_all_length = self.feature_all_length self.label = tf.placeholder(shape=(None), dtype=tf.float32) norm = [ 'uid_count', 'interest1_len', 'interest2_len', 'interest3_len', 'interest4_len', 'interest5_len', 'kw1_len', 'kw2_len', 'kw3_len', 'topic1_len', 'topic2_len', 'topic3_len' ] for s in hparams.num_features: self.num_ids[s] = tf.placeholder(shape=(None, ), dtype=tf.float32) if s in norm: self.norm_num[s] = self.batch_norm_layer( tf.reshape(self.num_ids[s], [-1, 1]), self.use_dropout, s) else: self.norm_num[s] = self.num_ids[s][:, None] for s in hparams.single_features: self.single_ids[s] = tf.placeholder(shape=(None, ), dtype=tf.int32) self.emb_v1[s] = tf.Variable(tf.truncated_normal( shape=[len(hparams.dict[s]) + 2, 1], mean=0.0, stddev=0.0001), name='emb_v1_' + s) self.embed_params.append(self.emb_v1[s]) self.emb_v2[s] = tf.Variable(tf.truncated_normal( shape=[len(hparams.dict[s]) + 2, hparams.k], mean=0.0, stddev=0.0001), name='emb_v2_' + s) self.embed_params.append(self.emb_v2[s]) for s in hparams.mutil_features: self.mulit_ids[s] = tf.placeholder(shape=(None, None), dtype=tf.int32) self.length[s] = tf.placeholder(shape=(None, ), dtype=tf.int32) self.mulit_mask[s] = tf.sequence_mask(self.length[s], 100, dtype=tf.float32) self.emb_v1[s] = tf.Variable(tf.truncated_normal( shape=[len(hparams.dict[s]) + 2, 1], mean=0.0, stddev=0.0001), name='emb_v1_' + s) self.embed_params.append(self.emb_v1[s]) self.emb_v2[s] = tf.Variable(tf.truncated_normal( shape=[len(hparams.dict[s]) + 2, hparams.k], mean=0.0, stddev=0.0001), name='emb_v2_' + s) self.embed_params.append(self.emb_v2[s]) self.build_graph(hparams) params = tf.trainable_variables() utils.print_out("# Trainable variables") for param in params: if 'W_' in param.name or 'b_' in param.name or 'emb' in param.name: utils.print_out( " %s, %s, %s" % (param.name, str(param.get_shape()), param.op.device)) self.optimizer(hparams) params = tf.trainable_variables()