def keras_pred(valide_data, valide_label): model = load_model(FLAGS.input_previous_model_path + '/model_0986303.h5') print (model.summary()) y_pred = model.predict(DNN_Model.DNN_DataSet(None, valide_data), verbose=0, batch_size=10240) score = metrics.roc_auc_score(valide_label, y_pred) print ("Keras AUC: {0}".format(score)) return y_pred
def nfold_train(train_data, train_label, model_types=None, stacking=False, valide_data=None, valide_label=None, test_data=None, train_weight=None, valide_weight=None, flags=None, tokenizer=None, scores=None, emb_weight=None, cat_max=None, leak_target=None): """ nfold Training """ print("Over all training size:") print(train_data.shape) print("Over all label size:") print(train_label.shape) fold = flags.nfold kf = KFold(n_splits=fold, shuffle=True) # wv_model = gensim.models.Word2Vec.load("wv_model_norm.gensim") stacking = flags.stacking stacking_data = None stacking_label = None test_preds = None num_fold = 0 models = [] losses = [] for train_index, test_index in kf.split(train_data): # print(test_index[:100]) # exit(0) if valide_label is None: train_part = train_data[train_index] train_part_label = train_label[train_index] valide_part = train_data[test_index] valide_part_label = train_label[test_index] else: train_part = train_data train_part_label = train_label valide_part = valide_data valide_part_label = valide_label if train_weight is not None: train_part_weight, valide_part_weight = train_weight, valide_weight print('fold: %d th train :-)' % (num_fold)) print('Train size: {} Valide size: {}'.format(train_part.shape[0], valide_part.shape[0])) print('Train target nunique: ', np.unique(np.argwhere(train_part_label == 1)[:, 1]).shape[0], 'Validate target nuique: ', np.unique(np.argwhere(valide_part_label == 1)[:, 1]).shape[0]) onefold_models = [] for model_type in model_types: if model_type == 'k' or model_type == 'r': # with tf.device('/cpu:0'): model = DNN_Model(scores=scores, cat_max=train_part_label.shape[1], flags=flags, emb_weight=emb_weight, model_type=model_type) if num_fold == 0: print(model.model.summary()) model.train(train_part, train_part_label, valide_part, valide_part_label) model = Model( inputs=model.model.inputs, outputs=model.model.get_layer(name='avg_pool').output) onefold_models.append((model, model_type)) elif model_type == 'v': # with tf.device('/cpu:0'): model = VAE_Model(flags=flags) if num_fold == 0: print(model.model.summary()) model.train(train_part, train_part_label, valide_part, valide_part_label) model = Model(inputs=model.model.inputs, outputs=model.model.get_layer(name='z').output) # if stacking: # model = Model(inputs = model.model.inputs, outputs = model.model.get_layer(name = 'merge_sparse_emb').output) onefold_models.append((model, 'v')) stacking_data = model_eval( model, 'v', train_data) # for model in onefold_models] # stacking_data = reduce((lambda x, y: np.c_[x, y]), stacking_data) print('stacking_data shape: {0}'.format(stacking_data.shape)) elif model_type == 'x': model = xgb_train(train_part, train_part_label, valide_part, valide_part_label, num_fold) onefold_models.append((model, 'x')) elif model_type == 'l': model = lgbm_train(train_part, train_part_label, valide_part, valide_part_label, num_fold, fold, flags=flags) onefold_models.append((model, 'l')) # print (leak_train.head) # if stacking: # valide_pred = [model_eval(model[0], model[1], valide_part) for model in onefold_models] # valide_pred = reduce((lambda x, y: np.c_[x, y]), valide_pred) # test_pred = [model_eval(model[0], model[1], test_data) for model in onefold_models] # test_pred = reduce((lambda x, y: np.c_[x, y]), test_pred) # if stacking_data is None: # stacking_data = valide_pred #np.c_[valide_part, valide_pred] # stacking_label = valide_part_label # test_preds = test_pred # else: # stacking_data = np.append(stacking_data, valide_pred, axis = 0) #np.append(stacking_data, np.c_[valide_part, valide_pred], axis = 0) # stacking_label = np.append(stacking_label, valide_part_label, axis = 0) # test_preds += test_pred # print('stacking_data shape: {0}'.format(stacking_data.shape)) # print('stacking_label shape: {0}'.format(stacking_label.shape)) # print('stacking test data shape: {0}'.format(test_preds.shape)) models.append(onefold_models[0]) num_fold += 1 if num_fold == flags.ensemble_nfold: break # if stacking: # test_preds /= flags.ensemble_nfold # test_data = np.c_[test_data, test_preds] return models, stacking_data, stacking_label, test_preds
def nfold_train(train_data, train_label, model_types=None, stacking=False, valide_data=None, valide_label=None, test_data=None, train_weight=None, valide_weight=None, flags=None, tokenizer=None, scores=None, emb_weight=None): """ nfold Training """ print("Over all training size:") print(train_data.shape) print("Over all label size:") print(train_label.shape) fold = flags.nfold kf = KFold(n_splits=fold, shuffle=False) # wv_model = gensim.models.Word2Vec.load("wv_model_norm.gensim") stacking = flags.stacking stacking_data = None stacking_label = None test_preds = None num_fold = 0 models = [] for train_index, test_index in kf.split(train_data): # print(test_index[:100]) # exit(0) if valide_label is None: train_part = train_data[train_index] train_part_label = train_label[train_index] valide_part = train_data[test_index] valide_part_label = train_label[test_index] if train_weight is not None: train_part_weight = train_weight[train_index] valide_part_weight = train_weight[test_index] else: train_part = train_data train_part_label = train_label valide_part = valide_data valide_part_label = valide_label if train_weight is not None: train_part_weight, valide_part_weight = train_weight, valide_weight print('fold: %d th train :-)' % (num_fold)) print('Train size: {} Valide size: {}'.format( train_part_label.shape[0], valide_part_label.shape[0])) onefold_models = [] for model_type in model_types: if model_type == 'k': # with tf.device('/cpu:0'): if flags.load_only_singleCnt: input_len = train_part.shape[1] model = DNN_Model(hidden_dim = [int(hn.strip()) for hn in flags.full_connect_hn.strip().split(',')], \ batch_size = flags.batch_size, epochs = flags.epochs, \ batch_interval = flags.batch_interval, emb_dropout = flags.emb_dropout, \ full_connect_dropout = flags.full_connect_dropout, scores = scores, \ emb_dim = [int(e.strip()) for e in flags.emb_dim.strip().split(',')], \ load_only_singleCnt = flags.load_only_singleCnt, input_len = input_len) if num_fold == 0: print(model.model.summary()) # if flags.load_only_singleCnt: # model.train(train_part, train_part_label, valide_part, valide_part_label) # else: model.train(train_part, train_part_label, valide_part, valide_part_label) # if stacking: # model = Model(inputs = model.model.inputs, outputs = model.model.get_layer(name = 'merge_sparse_emb').output) onefold_models.append((model, 'k')) elif model_type == 'x': model = xgb_train(train_part, train_part_label, valide_part, valide_part_label, num_fold) onefold_models.append((model, 'x')) elif model_type == 'l': model = lgbm_train(train_part, train_part_label, valide_part, valide_part_label, num_fold, fold, flags=flags) onefold_models.append((model, 'l')) # if stacking: # valide_pred = [model_eval(model[0], model[1], valide_part) for model in onefold_models] # valide_pred = reduce((lambda x, y: np.c_[x, y]), valide_pred) # test_pred = [model_eval(model[0], model[1], test_data) for model in onefold_models] # test_pred = reduce((lambda x, y: np.c_[x, y]), test_pred) # if stacking_data is None: # stacking_data = valide_pred #np.c_[valide_part, valide_pred] # stacking_label = valide_part_label # test_preds = test_pred # else: # stacking_data = np.append(stacking_data, valide_pred, axis = 0) #np.append(stacking_data, np.c_[valide_part, valide_pred], axis = 0) # stacking_label = np.append(stacking_label, valide_part_label, axis = 0) # test_preds += test_pred # print('stacking_data shape: {0}'.format(stacking_data.shape)) # print('stacking_label shape: {0}'.format(stacking_label.shape)) # print('stacking test data shape: {0}'.format(test_preds.shape)) models.append(onefold_models[0]) num_fold += 1 if num_fold == flags.ensemble_nfold: break # if stacking: # test_preds /= flags.ensemble_nfold # test_data = np.c_[test_data, test_preds] return models, stacking_data, stacking_label, test_preds
def nfold_train(train_data, train_label, model_types=None, stacking=False, valide_data=None, valide_label=None, test_data=None, train_weight=None, valide_weight=None, flags=None, tokenizer=None, scores=None, emb_weight=None, cat_max=None, leak_target=None): """ nfold Training """ print("Over all training size:") print(train_data.shape) print("Over all label size:") print(train_label.shape) fold = flags.nfold kf = KFold(n_splits=fold, shuffle=True, random_state=100) # wv_model = gensim.models.Word2Vec.load("wv_model_norm.gensim") stacking = flags.stacking stacking_data = pd.Series([np.zeros(1)] * train_data.shape[0]) stacking_label = pd.Series([np.zeros(1)] * train_data.shape[0]) test_preds = None num_fold = 0 models = [] losses = [] train_part_img_id = [] validate_part_img_id = [] for train_index, test_index in kf.split(train_data): # print(test_index[:100]) # exit(0) if valide_label is None: train_img = preprocess_img(train_data['img']) train_part = train_img[train_index] train_part_label = train_label[train_index] validate_part = train_img[test_index] validate_part_label = train_label[test_index] train_part_img_id.append(train_data.iloc[train_index].img_id) validate_part_img_id.append(train_data.iloc[test_index].img_id) print('\nfold: %d th train :-)' % (num_fold)) print('Train size: {} Valide size: {}'.format( train_part_label.shape[0], validate_part_label.shape[0])) print('Train target nunique: ', np.unique(np.argwhere(train_part_label == 1)[:, 1]).shape[0], 'Validate target nuique: ', np.unique(np.argwhere(validate_part_label == 1)[:, 1]).shape[0]) onefold_models = [] for model_type in model_types: if model_type == 'k' or model_type == 'r': # with tf.device('/cpu:0'): model = DNN_Model(scores=scores, cat_max=train_part_label.shape[1], flags=flags, emb_weight=emb_weight, model_type=model_type) if num_fold == 0: print(model.model.summary()) model.train(train_part, train_part_label, validate_part, validate_part_label) onefold_models.append((model, model_type)) if stacking: flat_model = Model( inputs=model.model.inputs, outputs=model.model.get_layer(name='avg_pool').output) stacking_data[test_index] = list( flat_model.predict(validate_part)) stacking_label[test_index] = list(model.predict(validate_part)) models.append(onefold_models[0]) num_fold += 1 if num_fold == flags.ensemble_nfold: break # if stacking: # test_preds /= flags.ensemble_nfold # test_data = np.c_[test_data, test_preds] return models, stacking_data, stacking_label, test_preds, train_part_img_id, validate_part_img_id