def categoryEncode(df, cols=None, mode="binary"): if(mode == "ordinal"): encoder = OrdinalEncoder(cols=cols, handle_missing="return_nan", handle_unknown="return_nan") elif(mode == "binary"): encoder = BinaryEncoder(cols=cols) df_new = encoder.fit_transform(df) return df_new
def encode_result(df_orig): df = df_orig.copy(deep=True) mapping_dict = { 'col': 'result', 'mapping': { 'hwin': 1, 'draw': 2, 'awin': 3 } } ord_enc = OrdinalEncoder(mapping=[mapping_dict]) df['ordinal_result'] = ord_enc.fit_transform(df[['result']]) return df
def _encode_categories(self): """ This private method stands for encoding categorical variables. Label encoding used for ordinal categories and one-hot encoding used for nominal categories. """ logging.info(f'#{self._index()} - Encoding categorical columns...') # get column names for categorical and numerical features categorical_vars = self.X.select_dtypes(include='object').columns numerical_vars = self.X.columns.difference(categorical_vars) ordinal = pd.Index([ 'ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond', 'HeatingQC', 'KitchenQual', 'FireplaceQu', 'GarageQual', 'GarageCond', 'PoolQC' ]) nominal = categorical_vars.difference(ordinal) standard_mapping = { 'NA': 0, 'Po': 1, 'Fa': 2, 'TA': 3, 'Gd': 4, 'Ex': 5 } mapping_for_ordinals = [{ 'col': column, 'mapping': standard_mapping } for column in ordinal] x_num = self.X[numerical_vars] x_test_num = self.X_test[numerical_vars] # one hot encode categorical columns one_hot_encoder = OneHotEncoder(use_cat_names=True) label_encoder = OrdinalEncoder(drop_invariant=True, mapping=mapping_for_ordinals, handle_unknown='error') x_cat_nom = one_hot_encoder.fit_transform(self.X[nominal]) x_cat_ord = label_encoder.fit_transform(self.X[ordinal]) x_test_cat_nom = one_hot_encoder.transform(self.X_test[nominal]) x_test_cat_ord = label_encoder.transform(self.X_test[ordinal]) self.X = x_num.join(x_cat_ord).join(x_cat_nom) self.X_test = x_test_num.join(x_test_cat_ord).join(x_test_cat_nom) logging.info(f'#{self._step_index} - DONE!')
def out_of_folds_predict(X, y): callbacks = [ EarlyStopping( # Stop training when loss is no longer improving monitor="loss", # "no longer improving" being defined as "no better than 1e-2 less" min_delta=1e-5, # "no longer improving" being further defined as "for at least 2 epochs" patience=2, verbose=0, ) ] preds = np.zeros(X.shape[0]) n_splits = 4 if y.sum() < 2: kfold = KFold(n_splits=n_splits) else: kfold = StratifiedKFold(n_splits=n_splits) for i, (train_index, test_index) in enumerate(kfold.split(X, y)): print(f'Split {i+1} of {n_splits}...') pipe = build_pipe() X_train, X_test = X[train_index], X[test_index] y_train, y_test = y[train_index], y[test_index] encoder = OrdinalEncoder() X_train = encoder.fit_transform(X_train, y_train).astype(np.float) pipe.fit(X_train, y_train, epochs=20, callbacks=callbacks, verbose=0) X_test = encoder.transform(X_test).astype(np.float) pipe.evaluate(X_test, y_test, verbose=1) preds[test_index] = pipe.predict(X_test).flatten() pipe = build_pipe() return preds
def convert_meta_to_dict(self): meta = self.meta[['productid'] + self.META_COLS].copy() # Encode to int encoder = OrdinalEncoder(cols=self.META_COLS) meta = encoder.fit_transform(meta) save_model(encoder, '{}/encoder'.format(MODEL_PATH)) meta['values'] = meta.apply(get_dict_values, args=(self.META_COLS, ), axis=1) meta_dict = meta.set_index('productid')['values'].to_dict() meta_dict = {self.word2id[k]: v for k, v in meta_dict.items()} meta_counts_dict = ( meta[self.META_COLS].max() + 1).to_dict() # Need to +1 to account for index starting from zero # Without +1 the embedding size will be insufficient by 1 ordered_meta_counts_dict = OrderedDict() for col in ['product'] + self.META_COLS: ordered_meta_counts_dict[col] = meta_counts_dict.get(col, 0) return meta_dict, ordered_meta_counts_dict
preran=False, drop_original=True, ) LENGTH_ENCODE = False if LENGTH_ENCODE: len_encode = ["URL"] for col in len_encode: X[f"{col}_len"] = X[col].apply(len) X = X.drop(col, axis=1) CATEGORIZE = True if CATEGORIZE: X[obj_cols] = X[obj_cols].astype("category") enc = OrdinalEncoder() X = enc.fit_transform(X) DATE_ENCODE = False if DATE_ENCODE: X = encode_dates(X, "date") sns.displot(y) plt.title("Distribution") plt.show() SEED = 0 SAMPLE_SIZE = 10000 Xt, Xv, yt, yv = train_test_split( X, y, random_state=SEED) # split into train and validation set dt = xgb.DMatrix(Xt, yt)
# OrdinalEncoder(), # LGBMRegressor() # ) # pipe.fit(X_train, y_train) # print('훈련 R^2: ', pipe.score(X_train, y_train)) # print('검증 R^2: ', pipe.score(X_val, y_val)) # print('TEST R^2: ', pipe.score(X_test, y_test)) # print('\n훈련 MAE: ', mean_absolute_error(pipe.predict(X_train), y_train)) # print('검증 MAE: ', mean_absolute_error(pipe.predict(X_val), y_val)) # print('TEST MAE: ', mean_absolute_error(pipe.predict(X_test), y_test)) ## 인코딩 encoder = OrdinalEncoder() X_train_encoded = encoder.fit_transform(X_train) X_val_encoded = encoder.transform(X_val) X_test_encoded = encoder.transform(X_test) # ## 파라미터 튜닝 # lightGB = LGBMRegressor(learning_rate=0.01, max_depth=15, n_estimators=300000, num_leaves=250, # random_state=1, reg_alpha=1, reg_lambda=1, subsample=0.7) # eval_set = [(X_train_encoded, y_train), # (X_val_encoded,y_val), # (X_test_encoded, y_test)] # lightGB.fit(X_train_encoded, y_train, # eval_set=eval_set, # early_stopping_rounds=1000, # eval_metric='mae',
'CoapplicantIncome': CoapplicantIncome, 'LoanAmount': LoanAmount, 'Loan_Amount_Term': Loan_Amount_Term } features = pd.DataFrame(data, index=[0]) return features df0 = user_input_features() ################################################################################## st.write("Exibindo os dados de entrada", df0) # Codificando e normalizando os dados de entrada ordinal = OrdinalEncoder() df0 = ordinal.fit_transform(df0) stand = StandardScaler().fit(df0) #df0 = stand.transform(df0) ################################################################################## ################################################################################## # Carregando o dataset e separando as features e target loan = pd.read_csv('emprestimo_app.csv') if st.sidebar.checkbox("Mostrar todos os dados"): st.subheader("Exibindo todos os dados") st.write(loan) X = loan.drop('Loan_Status', axis=1)
application_train = pd.read_feather('../data/input/application_train.ftr') application_test = pd.read_feather('../data/input/application_test.ftr') train = application_train test = application_test categorical_columns = [ 'NAME_CONTRACT_TYPE', 'CODE_GENDER', 'FLAG_OWN_CAR', 'FLAG_OWN_REALTY', 'NAME_TYPE_SUITE', 'NAME_INCOME_TYPE', 'NAME_EDUCATION_TYPE', 'NAME_FAMILY_STATUS', 'NAME_HOUSING_TYPE', 'OCCUPATION_TYPE', 'WEEKDAY_APPR_PROCESS_START', 'ORGANIZATION_TYPE', 'FONDKAPREMONT_MODE', 'HOUSETYPE_MODE', 'WALLSMATERIAL_MODE', 'EMERGENCYSTATE_MODE' ] enc = OrdinalEncoder(cols=categorical_columns, verbose=1) train[categorical_columns] = enc.fit_transform(train[categorical_columns]) test[categorical_columns] = enc.transform(test[categorical_columns]) X_train = train.drop(['SK_ID_CURR', 'TARGET'], axis=1) y_train = train.TARGET.values X_test = test.drop(['SK_ID_CURR'], axis=1) params = { 'metric': ['auc'], 'learning_rate': [0.1], 'num_leaves': [i * 10 for i in range(2, 6)], 'min_data_in_leaf': [5, 10, 15, 20], 'random_state': [SEED], 'verbose': [1] }
class Encoder(): encode_methods = { 'OrdinalEncoder': OrdinalEncoder, 'OneHotEncoder': OneHotEncoder, 'CountEncoder': CountEncoder, 'TargetEncoder': TargetEncoder, } # spark_encode_methods = { # 'mean_encoder':, # 'target_encoder':, # 'label_encoder':, # 'onehot_encoder' # } # target_encoder,mean_encoder在编码时,不能够把训练集和验证机concat在一起进行编码 # label_encoder,onehot_encoder可以 def __init__(self, sparksess=None, logdir='/encoder', handle_unknown='-99999', save_encoder=False): self.spark = sparksess self.logdir = logdir self.save_encoder self.ordinal_encoder_features = [] self.onehot_encoder_features = [] self.count_encoder_features = [] self.target_encoder_features = [] self.ordinal_encoder = OrdinalEncoder( cols=self.ordinal_encoder_features, return_df=True, handle_unknown=handle_unknown) self.onehot_encoder = OneHotEncoder(cols=self.onehot_encoder_features, return_df=True, handle_unknown=handle_unknown) self.count_encoder = CountEncoder(cols=self.count_encoder_features, return_df=True, handle_unknown=handle_unknown) self.target_encoder = TargetEncoder(cols=self.target_encoder_features, return_df=True, handle_unknown=handle_unknown) def fit(self, x_train, x_val=None, y_train=None, y_val=None, method_mapper=None): """ Parameters ---------- x_train: pd.DataFrame x_val: pd.DataFrame y_train: pd.DataFrame y_val: pd.DataFrame method_mapper: dict a mapping of feature to EncodeMethod example mapping: { 'feature1': OrdinalEncoder, 'feature2': OneHotEncoder, 'feature3': CountEncoder, 'feature4': TargetEncoder, } """ for feat in method_mapper: if method_mapper[feat] == 'OrdinalEncoder': self.ordinal_encoder_features.append(feat) elif method_mapper[feat] == 'OneHotEncoder': self.onehot_encoder_features.append(feat) elif method_mapper[feat] == 'CountEncoder': self.count_encoder_features.append(feat) elif method_mapper[feat] == 'TargetEncoder': self.target_encoder_features.append(feat) else: raise ValueError( '编码方式只支持[OrdinalEncoder, OneHotEncoder, CountEncoder, TargetEncoder], 接收到%s' % feat) if self.spark is None: if len(self.ordinal_encoder_features) != 0 or len( self.onehot_encoder_features) != 0: x_whole = x_train.append(x_val) y_whole = None if not y_train is None and not y_val is None: y_whole = y_train.append(y_val) x_whole = self.ordinal_encoder.fit_transform(x_whole, y_whole) x_whole = self.onehot_encoder.fit_transform(x_whole, y_whole) x_train = x_whole[:len(x_train)] x_val = x_whole[len(x_train):] x_train = self.count_encoder.fit_transform(x_train, y_train) x_val = self.count_encoder.transform(x_val, y_val) x_train = self.target_encoder.fit_transform(x_train, y_train) x_val = self.target_encoder.transform(x_val, y_val) if self.save_encoder: self.save_encoder() return x_train, y_train, x_val, y_val def transform(self, x, y=None): x = self.ordinal_encoder.transform(x, y) x = self.onehot_encoder.transform(x, y) x = self.count_encoder.transform(x, y) x = self.target_encoder.transform(x, y) return x, y def fit_transform(self, x_train, x_val=None, y_train=None, y_val=None, method_mapper=None): """ Parameters ---------- x_train: pd.DataFrame x_val: pd.DataFrame y_train: pd.DataFrame y_val: pd.DataFrame method_mapper: dict a mapping of feature to EncodeMethod example mapping: { 'feature1': OrdinalEncoder, 'feature2': OneHotEncoder, 'feature3': CountEncoder, 'feature4': TargetEncoder, } """ self.fit(x_train, x_val, y_train, y_val, method_mapper) x_train, y_train = self.transform(x_train, y_train) if x_val is not None: x_val, y_val = self.transform(x_val, y_val) return x_train, y_train, x_val, y_val def save_encoder(self): now = time.strftime("%Y-%m-%d-%H_%M_%S", time.localtime(time.time())) os.makedirs(os.path.join(self.logdir, now)) with open(os.path.join(self.logdir, now, 'OrdinalEncoder.pkl'), 'wb') as f: pickle.dump(self.ordinal_encoder, f) with open(os.path.join(self.logdir, now, 'OneHotEncoder.pkl'), 'wb') as f: pickle.dump(self.onehot_encoder, f) with open(os.path.join(self.logdir, now, 'CountEncoder.pkl'), 'wb') as f: pickle.dump(self.count_encoder, f) with open(os.path.join(self.logdir, now, 'TargetEncoder.pkl'), 'wb') as f: pickle.dump(self.target_encoder, f) with open( os.path.join(self.logdir, now, 'OrdinalEncoderFeatures.json'), 'w') as f: json.dump(self.ordinal_encoder_features, f) with open(os.path.join(self.logdir, now, 'OneHotEncoderFeatures.json'), 'w') as f: json.dump(self.onehot_encoder_features, f) with open(os.path.join(self.logdir, now, 'CountEncoderFeatures.json'), 'w') as f: json.dump(self.count_encoder_features, f) with open(os.path.join(self.logdir, now, 'TargetEncoderFeatures.json'), 'w') as f: json.dump(self.target_encoder_features, f) def load_encoder(self, logdir=None): with open(os.path.join(self.logdir, 'OrdinalEncoder.pkl'), 'wb') as f: pickle.dump(self.ordinal_encoder, f) with open(os.path.join(self.logdir, 'OneHotEncoder.pkl'), 'wb') as f: pickle.dump(self.onehot_encoder, f) with open(os.path.join(self.logdir, 'CountEncoder.pkl'), 'wb') as f: pickle.dump(self.count_encoder, f) with open(os.path.join(self.logdir, 'TargetEncoder.pkl'), 'wb') as f: pickle.dump(self.target_encoder, f) with open(os.path.join(self.logdir, 'OrdinalEncoderFeatures.json'), 'w') as f: json.dump(self.ordinal_encoder_features, f) with open(os.path.join(self.logdir, 'OneHotEncoderFeatures.json'), 'w') as f: json.dump(self.onehot_encoder_features, f) with open(os.path.join(self.logdir, 'CountEncoderFeatures.json'), 'w') as f: json.dump(self.count_encoder_features, f) with open(os.path.join(self.logdir, 'TargetEncoderFeatures.json'), 'w') as f: json.dump(self.target_encoder_features, f)