def xgb_class(X_train, y_train, X_test, y_test):
"""
Baseline XGB Classifier that prints out ROC score for Train and Test
sets provided.
"""
class_index = 1
# processor = make_pipeline(
# ce.ordinal.OrdinalEncoder(),
# SimpleImputer(strategy='median')
# )
# X_train_processed = processor.fit_transform(X_train)
# X_test_processed = processor.transform(X_test)
encoder = OrdinalEncoder()
imputer = SimpleImputer()
X_train_encoded = encoder.fit(X_train)
X_train_encoded = encoder.transform(X_train)
X_train_imputed = imputer.fit_transform(X_train_encoded)
X_test_encoded = encoder.fit(X_test)
X_test_encoded = encoder.transform(X_test)
X_test_imputed = imputer.fit_transform(X_test_encoded)
model = XGBClassifier(n_estimators=100, n_jobs=-1, max_depth=10)
model.fit(X_train_imputed, y_train, eval_metric='auc')
# Getting the predicted probabilities
y_pred = model.predict(X_test_processed)
y_pred_proba_train = model.predict_proba(X_train_imputed)[:,
class_index]
y_pred_proba_test = model.predict_proba(X_test_imputed)[:, class_index]
train_roc = roc_auc_score(y_train, y_pred_proba_train)
test_roc = roc_auc_score(y_test, y_pred_proba_test)
# Making a new Series for mean baseline print
s1 = pd.Series(y_train)
s2 = pd.Series(y_test)
s3 = s1.append(s2)
print('Mean Baseline of Target')
print(s3.value_counts(normalize=True))
print()
print(f'Train ROC AUC for class: {train_roc} \n')
print(f'Test ROC AUC for class: {test_roc}')
return
def encode_cat_col(self):
enc = OrdinalEncoder(return_df=False).fit(self.categ_col)
self.categ_col = enc.transform(self.categ_col)
# DEBUG
print(self.DS)
print(self.categ_col)
def fit_label(input_df: pd.DataFrame, cols: List[str], na_value: Any = None):
"""
Creates the label encoder by fitting it through the given DataFrame
NaN values and Special value specified under `na_value` in the DataFrame will be encoded as unseen value.
Args:
input_df: DataFrame used to fit the encoder
cols: List of categorical columns to be encoded
na_value: Default null value for DataFrame
Returns:
result_df: encoded input_df DataFrame
model : encoder model to be passed to `transform_label` method
"""
df = input_df.copy()
if na_value is not None:
for col in cols:
df[col] = df[col].replace({na_value: np.nan})
encoder = OrdinalEncoder(cols=cols)
encoder = encoder.fit(df)
for idx in range(len(encoder.mapping)):
encoder.mapping[idx]["mapping"].loc[np.nan] = -2
result_df = encoder.transform(df)
for col in cols:
result_df[col] = result_df[col].replace({-1: 0, -2: 0})
result_df[col] = result_df[col].astype(int)
model = {"encoder": encoder, "cols": cols, "na_value": na_value}
return result_df, model
def test_display_dataset_analysis_3(self, mock_correlation_matrix):
"""
Test we don't have a problem when only categorical features
"""
df = self.df.copy()
df['x1'] = 'a'
df['x2'] = df['x2'].astype(str)
encoder = OrdinalEncoder(
cols=['x1', 'x2'],
handle_unknown='ignore',
return_df=True).fit(df)
df = encoder.transform(df)
clf = cb.CatBoostClassifier(n_estimators=1).fit(df[['x1', 'x2']], df['y'])
xpl = SmartExplainer()
xpl.compile(model=clf, x=df[['x1', 'x2']])
report = ProjectReport(
explainer=xpl,
project_info_file=os.path.join(current_path, '../../data/metadata.yaml'),
x_train=df[['x1', 'x2']],
)
report.display_dataset_analysis()
self.assertEqual(mock_correlation_matrix.call_count, 0)
def encode_cat_col(self): # TODO pandas to numpy
from category_encoders import OrdinalEncoder
enc = OrdinalEncoder(return_df=False).fit(self.categ_col)
self.categ_col = enc.transform(self.categ_col)
# DEBUG
print(self.DS)
print(self.categ_col)
def __init__(self, df_train: pd.DataFrame, df_valid: pd.DataFrame, df_test: pd.DataFrame, use_columns, label_column):
encoder = OrdinalEncoder(cols=use_columns, handle_unknown='impute').fit(df_train)
df_train_X = encoder.transform(df_train).astype('int64')
df_valid_X = encoder.transform(df_valid).astype('int64')
df_test_X = encoder.transform(df_test).astype('int64')
self.train_X = torch.from_numpy(df_train_X[use_columns].values).long()
self.train_y = df_train[label_column].values
self.valid_X = torch.from_numpy(df_valid_X[use_columns].values).long()
self.valid_y = df_valid[label_column].values
self.test_X = torch.from_numpy(df_test_X[use_columns].values).long()
self.test_y = df_test[label_column].values
field_dims = list(df_train_X[use_columns].max())
self.field_dims = list(map(add_one, field_dims))
self.data_num = self.train_X.size()[0]
def main():
print('started experimnent')
with neptune.create_experiment(
name='feature engineering',
tags=['feature-extraction', FEATURE_NAME],
upload_source_files=get_filepaths(),
properties={'feature_version': FEATURE_NAME}):
print('loading data')
train = load_and_merge(RAW_DATA_PATH, 'train',
NROWS)[ID_COLS + V1_COLS + ['isFraud']]
test = load_and_merge(RAW_DATA_PATH, 'test', NROWS)[ID_COLS + V1_COLS]
categorical_cols = set(V1_CAT_COLS)
print('cleaning data')
email_cols = ['P_emaildomain', 'R_emaildomain']
train, new_email_cols = clean_email(train, email_cols)
test, _ = clean_email(test, email_cols)
categorical_cols.update(new_email_cols)
for col in email_cols:
categorical_cols.remove(col)
categorical_cols = list(categorical_cols)
neptune.set_property('categorical_columns', str(categorical_cols))
print('encoding categoricals')
encoder = OrdinalEncoder(cols=categorical_cols).fit(
train[ID_COLS + categorical_cols])
train[ID_COLS + categorical_cols] = encoder.transform(
train[ID_COLS + categorical_cols])
test[ID_COLS + categorical_cols] = encoder.transform(
test[ID_COLS + categorical_cols])
train_features_path = os.path.join(
FEATURES_DATA_PATH, 'train_features_{}.csv'.format(FEATURE_NAME))
print('saving train to {}'.format(train_features_path))
train.to_csv(train_features_path, index=None)
log_data_version(train_features_path, prefix='train_features_')
test_features_path = os.path.join(
FEATURES_DATA_PATH, 'test_features_{}.csv'.format(FEATURE_NAME))
print('saving test to {}'.format(test_features_path))
test.to_csv(test_features_path, index=None)
log_data_version(test_features_path, prefix='test_features_')
class df_OrdinalEncoder(TransformerMixin):
def __init__(self, handle_unknown='ignore'):
self.handle_unknown = handle_unknown
def fit(self, X, y=None):
self.enc = OrdinalEncoder(handle_unknown=self.handle_unknown)
self.enc.fit(X)
return self
def transform(self, X):
X_encoded = self.enc.transform(X)
X_encoded_df = pd.DataFrame(data=X_encoded, index=X.index, columns=X.columns)
return X_encoded_df
def _encode_categories(self):
"""
This private method stands for encoding categorical variables. Label encoding used for ordinal categories and
one-hot encoding used for nominal categories.
"""
logging.info(f'#{self._index()} - Encoding categorical columns...')
# get column names for categorical and numerical features
categorical_vars = self.X.select_dtypes(include='object').columns
numerical_vars = self.X.columns.difference(categorical_vars)
ordinal = pd.Index([
'ExterQual', 'ExterCond', 'BsmtQual', 'BsmtCond', 'HeatingQC',
'KitchenQual', 'FireplaceQu', 'GarageQual', 'GarageCond', 'PoolQC'
])
nominal = categorical_vars.difference(ordinal)
standard_mapping = {
'NA': 0,
'Po': 1,
'Fa': 2,
'TA': 3,
'Gd': 4,
'Ex': 5
}
mapping_for_ordinals = [{
'col': column,
'mapping': standard_mapping
} for column in ordinal]
x_num = self.X[numerical_vars]
x_test_num = self.X_test[numerical_vars]
# one hot encode categorical columns
one_hot_encoder = OneHotEncoder(use_cat_names=True)
label_encoder = OrdinalEncoder(drop_invariant=True,
mapping=mapping_for_ordinals,
handle_unknown='error')
x_cat_nom = one_hot_encoder.fit_transform(self.X[nominal])
x_cat_ord = label_encoder.fit_transform(self.X[ordinal])
x_test_cat_nom = one_hot_encoder.transform(self.X_test[nominal])
x_test_cat_ord = label_encoder.transform(self.X_test[ordinal])
self.X = x_num.join(x_cat_ord).join(x_cat_nom)
self.X_test = x_test_num.join(x_test_cat_ord).join(x_test_cat_nom)
logging.info(f'#{self._step_index} - DONE!')
def out_of_folds_predict(X, y):
callbacks = [
EarlyStopping(
# Stop training when loss is no longer improving
monitor="loss",
# "no longer improving" being defined as "no better than 1e-2 less"
min_delta=1e-5,
# "no longer improving" being further defined as "for at least 2 epochs"
patience=2,
verbose=0,
)
]
preds = np.zeros(X.shape[0])
n_splits = 4
if y.sum() < 2:
kfold = KFold(n_splits=n_splits)
else:
kfold = StratifiedKFold(n_splits=n_splits)
for i, (train_index, test_index) in enumerate(kfold.split(X, y)):
print(f'Split {i+1} of {n_splits}...')
pipe = build_pipe()
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
encoder = OrdinalEncoder()
X_train = encoder.fit_transform(X_train, y_train).astype(np.float)
pipe.fit(X_train, y_train, epochs=20, callbacks=callbacks, verbose=0)
X_test = encoder.transform(X_test).astype(np.float)
pipe.evaluate(X_test, y_test, verbose=1)
preds[test_index] = pipe.predict(X_test).flatten()
pipe = build_pipe()
return preds
def create_label(df, test_df, topic):
result_dict = {}
feature_df = df[['title']].copy()
label_df = df.drop(columns=['itemid', 'title', 'image_path']).copy()
feature_df['title'] = feature_df['title'].apply(lambda x: text_process(x))
feature_array = feature_df['title'].values.tolist()
feature_encoder = TfidfVectorizer()
feature_encoder.fit(feature_array)
feature_attr = feature_encoder.transform(feature_array)
feature_decomposer = TruncatedSVD(500)
feature_decomposer.fit(feature_attr)
feature_attr = feature_decomposer.transform(feature_attr)
test_df['title'] = test_df['title'].apply(lambda x: text_process(x))
test_array = test_df['title'].values.tolist()
test_attr = feature_encoder.transform(test_array)
test_attr = feature_decomposer.transform(test_attr)
train_itemid = df['itemid']
test_itemid = test_df['itemid']
result_dict['itemid_train_{}'.format(topic)] = train_itemid
result_dict['itemid_test_{}'.format(topic)] = test_itemid
result_dict['X_train_{}'.format(topic)] = feature_attr
result_dict['X_encoder_{}'.format(topic)] = feature_encoder
result_dict['X_decomposer_{}'.format(topic)] = feature_decomposer
result_dict['X_test_{}'.format(topic)] = test_attr
for column in label_df.columns:
label_encoder = OrdinalEncoder(cols=[column], handle_unknown='impute')
label_encoder.fit(label_df[[column]])
label_attr = label_encoder.transform(label_df[[column]])
result_dict['Y_train_{}_{}'.format(topic, column)] = label_attr
result_dict['Y_encoder_{}_{}'.format(topic, column)] = label_encoder
result_dict['Y_colname_{}_{}'.format(topic,
column)] = label_attr.columns
return result_dict
def encode_ordinal_df(dataframe, fit=False):
"""
Encode ordinal features, preserving the notion of order and dropping invariant features
---
Arguments
dataframe: pd.DataFrame
Dataframe with pre-processed data (i.e. renamed features), ordinal features only
fit: boolean
Indicates if we should train or load an encoder
Returns
dataframe: pd.DataFrame
Dataframe with encoded data
"""
# Train or load an encoder
if fit:
encoder = OrdinalEncoder(cols=dataframe.columns.values, drop_invariant=True)
encoder.fit(dataframe)
pickle_obj(encoder, 'ordinal_encoder')
else:
encoder = unpickle_obj('ordinal_encoder')
# transform data
return encoder.transform(dataframe)
application_train = pd.read_feather('../data/input/application_train.ftr')
application_test = pd.read_feather('../data/input/application_test.ftr')
train = application_train
test = application_test
categorical_columns = [
'NAME_CONTRACT_TYPE', 'CODE_GENDER', 'FLAG_OWN_CAR', 'FLAG_OWN_REALTY',
'NAME_TYPE_SUITE', 'NAME_INCOME_TYPE', 'NAME_EDUCATION_TYPE',
'NAME_FAMILY_STATUS', 'NAME_HOUSING_TYPE', 'OCCUPATION_TYPE',
'WEEKDAY_APPR_PROCESS_START', 'ORGANIZATION_TYPE', 'FONDKAPREMONT_MODE',
'HOUSETYPE_MODE', 'WALLSMATERIAL_MODE', 'EMERGENCYSTATE_MODE'
]
enc = OrdinalEncoder(cols=categorical_columns, verbose=1)
train[categorical_columns] = enc.fit_transform(train[categorical_columns])
test[categorical_columns] = enc.transform(test[categorical_columns])
X_train = train.drop(['SK_ID_CURR', 'TARGET'], axis=1)
y_train = train.TARGET.values
X_test = test.drop(['SK_ID_CURR'], axis=1)
params = {
'metric': ['auc'],
'learning_rate': [0.1],
'num_leaves': [i * 10 for i in range(2, 6)],
'min_data_in_leaf': [5, 10, 15, 20],
'random_state': [SEED],
'verbose': [1]
}
cv = StratifiedKFold(3, shuffle=True, random_state=SEED)
X,
op_version=opv,
output_names=operator.outputs[:1])
cat.add_to(scope, container)
update_registered_converter(OrdinalEncoder, "CategoricalEncoderOrdinalEncoder",
ordinal_encoder_shape_calculator,
ordinal_encoder_converter)
###################################
# Let's compute the output one a short example.
enc = OrdinalEncoder(cols=[0, 1])
enc.fit(X)
print(enc.transform(X[:5]))
###################################
# Let's check the ONNX conversion produces the same results.
ord_onx = to_onnx(enc, X[:1], target_opset=14)
sess = InferenceSession(ord_onx.SerializeToString())
print(sess.run(None, {'X': X[:5]})[0])
######################################
# That works.
#
# Custom converter for WOEEncoder
# +++++++++++++++++++++++++++++++
#
# We start from example :ref:`l-plot-custom-converter`
from skimpute import MissForest
df = pd.read_csv("../exmaple/train_classification.csv")
start = time()
df.pop("Name")
df.pop("Ticket")
df.pop("PassengerId")
y = df.pop("Survived").values
cv = ShuffleSplit(n_splits=1, test_size=0.25, random_state=42)
train_ix, test_ix = next(cv.split(df, y))
train_X = (df.iloc[train_ix, :])
train_y = y[train_ix]
test_X = (df.iloc[test_ix, :])
test_y = y[test_ix]
imputer = MissForest()
imputer.fit(df)
train_X = imputer.transform(train_X)
test_X = imputer.transform(test_X)
print(train_X)
print(train_X.dtypes)
print(time() - start)
encoder = OrdinalEncoder()
encoder.fit(df)
train_X = encoder.transform(train_X)
test_X = encoder.transform(test_X)
rf = RandomForestClassifier(random_state=42)
rf.fit(train_X, train_y)
score = rf.score(test_X, test_y)
print(score) # 0.8295964125560538
# LGBMRegressor()
# )
# pipe.fit(X_train, y_train)
# print('훈련 R^2: ', pipe.score(X_train, y_train))
# print('검증 R^2: ', pipe.score(X_val, y_val))
# print('TEST R^2: ', pipe.score(X_test, y_test))
# print('\n훈련 MAE: ', mean_absolute_error(pipe.predict(X_train), y_train))
# print('검증 MAE: ', mean_absolute_error(pipe.predict(X_val), y_val))
# print('TEST MAE: ', mean_absolute_error(pipe.predict(X_test), y_test))
## 인코딩
encoder = OrdinalEncoder()
X_train_encoded = encoder.fit_transform(X_train)
X_val_encoded = encoder.transform(X_val)
X_test_encoded = encoder.transform(X_test)
# ## 파라미터 튜닝
# lightGB = LGBMRegressor(learning_rate=0.01, max_depth=15, n_estimators=300000, num_leaves=250,
# random_state=1, reg_alpha=1, reg_lambda=1, subsample=0.7)
# eval_set = [(X_train_encoded, y_train),
# (X_val_encoded,y_val),
# (X_test_encoded, y_test)]
# lightGB.fit(X_train_encoded, y_train,
# eval_set=eval_set,
# early_stopping_rounds=1000,
# eval_metric='mae',
# verbose=100
from shapash.data.data_loader import data_loading
from shapash.explainer.smart_explainer import SmartExplainer
house_df, house_dict = data_loading('house_prices')
y_df = house_df['SalePrice'].to_frame()
X_df = house_df[house_df.columns.difference(['SalePrice'])]
house_df.head()
categorical_features = [
col for col in X_df.columns if X_df[col].dtype == 'object'
]
encoder = OrdinalEncoder(cols=categorical_features,
handle_unknown='ignore',
return_df=True).fit(X_df)
X_df = encoder.transform(X_df)
Xtrain, Xtest, ytrain, ytest = train_test_split(X_df,
y_df,
train_size=0.75,
random_state=1)
regressor = LGBMRegressor(n_estimators=200).fit(Xtrain, ytrain)
y_pred = pd.DataFrame(regressor.predict(Xtest),
columns=['pred'],
index=Xtest.index)
xpl = SmartExplainer(features_dict=house_dict)
xpl.compile(x=Xtest, model=regressor, preprocessing=encoder, y_pred=y_pred)
evals=[(dt, "training"), (dv, "valid")],
num_boost_round=MAX_ROUNDS,
early_stopping_rounds=EARLY_STOPPING_ROUNDS,
verbose_eval=REPORT_ROUNDS,
)
score = model.best_score
print(f"{METRIC}: {score:.4f}")
xgb.plot_importance(
model,
grid=False,
max_num_features=20,
importance_type="gain",
)
figure_path = Path("figures")
figure_path.mkdir(exist_ok=True)
plt.savefig(figure_path / "xgboost_importance.png")
# test score
TEST = True
if TEST:
df_test = pd.read_csv("data/test.csv")
df_test[obj_cols] = df_test[obj_cols].astype("category")
df_test = enc.transform(df_test)
X_test = df_test.drop(DROP_FEATURES, axis=1)
dtest = xgb.DMatrix(X_test)
y_pred = model.predict(dtest)
submission = pd.Series(y_pred, index=df_test["id"])
submission.name = "target"
submission.to_csv("submission/xgboost.csv")
class OrdinalEncoder():
"""Maps each categorical value to one column using ordinal encoding.
Parameters:
cols: [str]
list of column names to encode.
"""
name = 'ordinal'
def __init__(self, cols=None):
self.encoder = Ordinal(cols=cols)
def fit(self, X, features, y=None):
"""Fits encoder to data table.
returns self
"""
self.encoder.fit(X, y=None)
self.features = self.encode_features_list(X, features)
return self
def transform(self, X):
"""Encodes matrix and updates features accordingly.
returns encoded matrix (dataframe)
"""
X_new = self.encoder.transform(X)
feature_names = []
for feature in self.features:
for fname in feature.get_feature_names():
feature_names.append(fname)
X_new.columns = feature_names
return X_new
def fit_transform(self, X, features, y=None):
"""First fits, then transforms matrix.
returns encoded matrix (dataframe)
"""
return self.fit(X, features, y).transform(X)
def get_mapping(self, category):
"""Gets the mapping the ordinal encoder.
returns mapping (dict)
"""
if isinstance(category, str):
for map in self.encoder.mapping:
if map['col'] == category:
return map['mapping']
return self.encoder.mapping[category]['mapping']
def encode_features_list(self, X, features):
feature_list = []
index = 0
for f in features:
if f.get_name() in self.encoder.cols:
f = ft.Feature([f], primitive=OrdinalEnc(self, index))
index += 1
feature_list.append(f)
return feature_list
def get_features(self):
return self.features
def get_name(self):
return self.name
def load_dataset(refresh_days=1,
dataset='general',
thresh=0.7,
simfin_api_key='free',
simfin_directory='simfin_data/',
data_directory=DATA_DIR,
shareprices_df=''):
# Set Simfin Settings
sf.set_api_key(simfin_api_key)
sf.set_data_dir(simfin_directory)
derived_shareprice_df = sf.load_derived_shareprices(variant='latest',
market='us')
derived_shareprice_df.to_csv(data_directory / 'stock_derived.csv')
company_df = sf.load_companies(market='us', refresh_days=1)
company_df.to_csv(data_directory / 'company.csv')
industry_df = sf.load_industries(refresh_days=1)
industry_df.to_csv(data_directory / 'industry.csv')
if dataset == 'general':
# Load Data from Simfin
income_df = sf.load_income(variant='ttm',
market='us',
refresh_days=refresh_days)
income_df = income_df.sort_index(level=['Ticker', 'Report Date'],
ascending=[1, 1])
income_quarterly_df = sf.load_income(variant='quarterly',
market='us',
refresh_days=refresh_days)
income_quarterly_df = income_quarterly_df.sort_index(
level=['Ticker', 'Report Date'], ascending=[1, 1])
income_df.groupby('Ticker').last().to_csv(data_directory /
'general_income.csv')
balance_df = sf.load_balance(variant='ttm',
market='us',
refresh_days=refresh_days)
balance_df = balance_df.sort_index(level=['Ticker', 'Report Date'],
ascending=[1, 1])
balance_quarterly_df = sf.load_balance(variant='quarterly',
market='us',
refresh_days=refresh_days)
balance_quarterly_df = balance_quarterly_df.sort_index(
level=['Ticker', 'Report Date'], ascending=[1, 1])
balance_df.groupby('Ticker').last().to_csv(data_directory /
'general_balance.csv')
cashflow_df = sf.load_cashflow(variant='ttm',
market='us',
refresh_days=refresh_days)
cashflow_df = cashflow_df.sort_index(level=['Ticker', 'Report Date'],
ascending=[1, 1])
cashflow_quarterlay_df = sf.load_cashflow(variant='quarterly',
market='us',
refresh_days=refresh_days)
cashflow_quarterlay_df = cashflow_quarterlay_df.sort_index(
level=['Ticker', 'Report Date'], ascending=[1, 1])
cashflow_df.groupby('Ticker').last().to_csv(data_directory /
'general_cashflow.csv')
derived_df = sf.load_derived(variant='ttm',
market='us',
refresh_days=refresh_days)
derived_df = derived_df.sort_index(level=['Ticker', 'Report Date'],
ascending=[1, 1])
derived_df.groupby('Ticker').last().to_csv(
data_directory / 'general_fundamental_derived.csv')
cache_args = {
'cache_name': 'financial_signals',
'cache_refresh': refresh_days
}
fin_signal_df = sf.fin_signals(df_income_ttm=income_df,
df_balance_ttm=balance_df,
df_cashflow_ttm=cashflow_df,
**cache_args)
growth_signal_df = sf.growth_signals(
df_income_ttm=income_df,
df_income_qrt=income_quarterly_df,
df_balance_ttm=balance_df,
df_balance_qrt=balance_quarterly_df,
df_cashflow_ttm=cashflow_df,
df_cashflow_qrt=cashflow_quarterlay_df,
**cache_args)
# Remove Columns that exist in other Fundamental DataFrames
balance_columns = balance_df.columns[~balance_df.columns.isin(set(
).union(income_df.columns))]
cashflow_columns = cashflow_df.columns[~cashflow_df.columns.isin(set(
).union(income_df.columns))]
derived_df_columns = derived_df.columns[~derived_df.columns.isin(set(
).union(income_df.columns, growth_signal_df.columns, fin_signal_df.
columns))]
# Merge the fundamental data into a single dataframe
fundamental_df = income_df.join(balance_df[balance_columns]).join(
cashflow_df[cashflow_columns]).join(fin_signal_df).join(
growth_signal_df).join(derived_df[derived_df_columns])
fundamental_df['Dataset'] = 'general'
elif dataset == 'banks':
# Load Data from Simfin
income_df = sf.load_income_banks(variant='ttm',
market='us',
refresh_days=refresh_days)
income_df = income_df.sort_index(level=['Ticker', 'Report Date'],
ascending=[1, 1])
income_df.groupby('Ticker').last().to_csv(data_directory /
'banks_income.csv')
balance_df = sf.load_balance_banks(variant='ttm',
market='us',
refresh_days=refresh_days)
balance_df = balance_df.sort_index(level=['Ticker', 'Report Date'],
ascending=[1, 1])
balance_df.groupby('Ticker').last().to_csv(data_directory /
'banks_balance.csv')
cashflow_df = sf.load_cashflow_banks(variant='ttm',
market='us',
refresh_days=refresh_days)
cashflow_df = cashflow_df.sort_index(level=['Ticker', 'Report Date'],
ascending=[1, 1])
cashflow_df.groupby('Ticker').last().to_csv(data_directory /
'banks_cashflow.csv')
derived_df = sf.load_derived_banks(variant='ttm',
market='us',
refresh_days=refresh_days)
derived_df = derived_df.sort_index(level=['Ticker', 'Report Date'],
ascending=[1, 1])
derived_df.groupby('Ticker').last().to_csv(
data_directory / 'banks_fundamental_derived.csv')
derived_df.groupby('Ticker').last().to_csv(
data_directory / 'banks_fundamental_derived.csv')
# Remove Columns that exist in other Fundamental DataFrames
balance_columns = balance_df.columns[~balance_df.columns.isin(set(
).union(income_df.columns))]
cashflow_columns = cashflow_df.columns[~cashflow_df.columns.isin(set(
).union(income_df.columns))]
derived_df_columns = derived_df.columns[~derived_df.columns.isin(set(
).union(income_df.columns))]
# Merge the fundamental data into a single dataframe
fundamental_df = income_df.join(balance_df[balance_columns]).join(
cashflow_df[cashflow_columns]).join(derived_df[derived_df_columns])
fundamental_df['Dataset'] = 'banks'
elif dataset == 'insurance':
# Load Data from Simfin
income_df = sf.load_income_insurance(variant='ttm',
market='us',
refresh_days=refresh_days)
income_df = income_df.sort_index(level=['Ticker', 'Report Date'],
ascending=[1, 1])
income_df.groupby('Ticker').last().to_csv(data_directory /
'insurance_income.csv')
balance_df = sf.load_balance_insurance(variant='ttm',
market='us',
refresh_days=refresh_days)
balance_df = balance_df.sort_index(level=['Ticker', 'Report Date'],
ascending=[1, 1])
balance_df.groupby('Ticker').last().to_csv(data_directory /
'insurance_balance.csv')
cashflow_df = sf.load_cashflow_insurance(variant='ttm',
market='us',
refresh_days=refresh_days)
cashflow_df = cashflow_df.sort_index(level=['Ticker', 'Report Date'],
ascending=[1, 1])
cashflow_df.groupby('Ticker').last().to_csv(data_directory /
'insurance_cashflow.csv')
derived_df = sf.load_derived_insurance(variant='ttm',
market='us',
refresh_days=refresh_days)
derived_df = derived_df.sort_index(level=['Ticker', 'Report Date'],
ascending=[1, 1])
derived_df.groupby('Ticker').last().to_csv(
data_directory / 'insurance_fundamental_derived.csv')
# Remove Columns that exist in other Fundamental DataFrames
balance_columns = balance_df.columns[~balance_df.columns.isin(set(
).union(income_df.columns))]
cashflow_columns = cashflow_df.columns[~cashflow_df.columns.isin(set(
).union(income_df.columns))]
derived_df_columns = derived_df.columns[~derived_df.columns.isin(set(
).union(income_df.columns))]
# Merge the fundamental data into a single dataframe
fundamental_df = income_df.join(balance_df[balance_columns]).join(
cashflow_df[cashflow_columns]).join(derived_df[derived_df_columns])
fundamental_df['Dataset'] = 'insurance'
# Drop Columns with more then 1-thresh nan values
fundamental_df = fundamental_df.dropna(thresh=int(thresh *
len(fundamental_df)),
axis=1)
# Drop Duplicate Index
fundamental_df = fundamental_df[~fundamental_df.index.duplicated(
keep='first')]
# Replace Report Date with the Publish Date because the Publish Date is when the Fundamentals are known to the Public
fundamental_df['Published Date'] = fundamental_df['Publish Date']
fundamental_df = fundamental_df.reset_index().set_index(
['Ticker', 'Publish Date'])
df = sf.reindex(df_src=fundamental_df,
df_target=shareprices_df,
group_index=TICKER,
method='ffill').dropna(how='all').join(shareprices_df)
# General
# Clean Up
df = df.drop([
'SimFinId', 'Currency', 'Fiscal Year', 'Report Date', 'Restated Date',
'Fiscal Period', 'Published Date'
],
axis=1)
if dataset == 'general':
# Remove Share Prices Over Amazon Share Price
df = df[df['Close'] <= df.loc['AMZN']['Close'].max()]
df = df.dropna(subset=[
'Shares (Basic)', 'Shares (Diluted)', 'Revenue', 'Earnings Growth'
])
non_per_share_cols = [
'Currency', 'Fiscal Year', 'Fiscal Period', 'Published Date',
'Restated Date', 'Shares (Basic)', 'Shares (Diluted)', 'Close',
'Dataset'
] + fin_signal_df.columns.tolist() + growth_signal_df.columns.tolist(
) + derived_df_columns.difference(
['EBITDA', 'Total Debt', 'Free Cash Flow']).tolist()
else:
df = df.dropna(
subset=['Shares (Basic)', 'Shares (Diluted)', 'Revenue'])
non_per_share_cols = [
'Currency', 'Fiscal Year', 'Fiscal Period', 'Published Date',
'Restated Date', 'Shares (Basic)', 'Shares (Diluted)', 'Close',
'Dataset'
] + derived_df_columns.difference(
['EBITDA', 'Total Debt', 'Free Cash Flow']).tolist()
df = df.replace([np.inf, -np.inf], 0)
df = df.fillna(0)
per_share_cols = df.columns[~df.columns.isin(non_per_share_cols)]
df[per_share_cols] = df[per_share_cols].div(df['Shares (Diluted)'], axis=0)
# Add Company and Industry Information and Categorize
df = df.join(company_df).merge(
industry_df, left_on='IndustryId', right_index=True).drop(
columns=['IndustryId', 'Company Name', 'SimFinId'])
categorical_features = [
col for col in df.columns if df[col].dtype == 'object'
]
encoder = OrdinalEncoder(cols=categorical_features,
handle_unknown='ignore',
return_df=True).fit(df)
df = encoder.transform(df)
# Sort
df = df.sort_index(level=['Ticker', 'Date'], ascending=[1, 1])
return df
class Encoder():
encode_methods = {
'OrdinalEncoder': OrdinalEncoder,
'OneHotEncoder': OneHotEncoder,
'CountEncoder': CountEncoder,
'TargetEncoder': TargetEncoder,
}
# spark_encode_methods = {
# 'mean_encoder':,
# 'target_encoder':,
# 'label_encoder':,
# 'onehot_encoder'
# }
# target_encoder，mean_encoder在编码时，不能够把训练集和验证机concat在一起进行编码
# label_encoder,onehot_encoder可以
def __init__(self,
sparksess=None,
logdir='/encoder',
handle_unknown='-99999',
save_encoder=False):
self.spark = sparksess
self.logdir = logdir
self.save_encoder
self.ordinal_encoder_features = []
self.onehot_encoder_features = []
self.count_encoder_features = []
self.target_encoder_features = []
self.ordinal_encoder = OrdinalEncoder(
cols=self.ordinal_encoder_features,
return_df=True,
handle_unknown=handle_unknown)
self.onehot_encoder = OneHotEncoder(cols=self.onehot_encoder_features,
return_df=True,
handle_unknown=handle_unknown)
self.count_encoder = CountEncoder(cols=self.count_encoder_features,
return_df=True,
handle_unknown=handle_unknown)
self.target_encoder = TargetEncoder(cols=self.target_encoder_features,
return_df=True,
handle_unknown=handle_unknown)
def fit(self,
x_train,
x_val=None,
y_train=None,
y_val=None,
method_mapper=None):
"""
Parameters
----------
x_train: pd.DataFrame
x_val: pd.DataFrame
y_train: pd.DataFrame
y_val: pd.DataFrame
method_mapper: dict
a mapping of feature to EncodeMethod
example mapping:
{
'feature1': OrdinalEncoder,
'feature2': OneHotEncoder,
'feature3': CountEncoder,
'feature4': TargetEncoder,
}
"""
for feat in method_mapper:
if method_mapper[feat] == 'OrdinalEncoder':
self.ordinal_encoder_features.append(feat)
elif method_mapper[feat] == 'OneHotEncoder':
self.onehot_encoder_features.append(feat)
elif method_mapper[feat] == 'CountEncoder':
self.count_encoder_features.append(feat)
elif method_mapper[feat] == 'TargetEncoder':
self.target_encoder_features.append(feat)
else:
raise ValueError(
'编码方式只支持[OrdinalEncoder, OneHotEncoder, CountEncoder, TargetEncoder], 接收到%s'
% feat)
if self.spark is None:
if len(self.ordinal_encoder_features) != 0 or len(
self.onehot_encoder_features) != 0:
x_whole = x_train.append(x_val)
y_whole = None
if not y_train is None and not y_val is None:
y_whole = y_train.append(y_val)
x_whole = self.ordinal_encoder.fit_transform(x_whole, y_whole)
x_whole = self.onehot_encoder.fit_transform(x_whole, y_whole)
x_train = x_whole[:len(x_train)]
x_val = x_whole[len(x_train):]
x_train = self.count_encoder.fit_transform(x_train, y_train)
x_val = self.count_encoder.transform(x_val, y_val)
x_train = self.target_encoder.fit_transform(x_train, y_train)
x_val = self.target_encoder.transform(x_val, y_val)
if self.save_encoder:
self.save_encoder()
return x_train, y_train, x_val, y_val
def transform(self, x, y=None):
x = self.ordinal_encoder.transform(x, y)
x = self.onehot_encoder.transform(x, y)
x = self.count_encoder.transform(x, y)
x = self.target_encoder.transform(x, y)
return x, y
def fit_transform(self,
x_train,
x_val=None,
y_train=None,
y_val=None,
method_mapper=None):
"""
Parameters
----------
x_train: pd.DataFrame
x_val: pd.DataFrame
y_train: pd.DataFrame
y_val: pd.DataFrame
method_mapper: dict
a mapping of feature to EncodeMethod
example mapping:
{
'feature1': OrdinalEncoder,
'feature2': OneHotEncoder,
'feature3': CountEncoder,
'feature4': TargetEncoder,
}
"""
self.fit(x_train, x_val, y_train, y_val, method_mapper)
x_train, y_train = self.transform(x_train, y_train)
if x_val is not None:
x_val, y_val = self.transform(x_val, y_val)
return x_train, y_train, x_val, y_val
def save_encoder(self):
now = time.strftime("%Y-%m-%d-%H_%M_%S", time.localtime(time.time()))
os.makedirs(os.path.join(self.logdir, now))
with open(os.path.join(self.logdir, now, 'OrdinalEncoder.pkl'),
'wb') as f:
pickle.dump(self.ordinal_encoder, f)
with open(os.path.join(self.logdir, now, 'OneHotEncoder.pkl'),
'wb') as f:
pickle.dump(self.onehot_encoder, f)
with open(os.path.join(self.logdir, now, 'CountEncoder.pkl'),
'wb') as f:
pickle.dump(self.count_encoder, f)
with open(os.path.join(self.logdir, now, 'TargetEncoder.pkl'),
'wb') as f:
pickle.dump(self.target_encoder, f)
with open(
os.path.join(self.logdir, now, 'OrdinalEncoderFeatures.json'),
'w') as f:
json.dump(self.ordinal_encoder_features, f)
with open(os.path.join(self.logdir, now, 'OneHotEncoderFeatures.json'),
'w') as f:
json.dump(self.onehot_encoder_features, f)
with open(os.path.join(self.logdir, now, 'CountEncoderFeatures.json'),
'w') as f:
json.dump(self.count_encoder_features, f)
with open(os.path.join(self.logdir, now, 'TargetEncoderFeatures.json'),
'w') as f:
json.dump(self.target_encoder_features, f)
def load_encoder(self, logdir=None):
with open(os.path.join(self.logdir, 'OrdinalEncoder.pkl'), 'wb') as f:
pickle.dump(self.ordinal_encoder, f)
with open(os.path.join(self.logdir, 'OneHotEncoder.pkl'), 'wb') as f:
pickle.dump(self.onehot_encoder, f)
with open(os.path.join(self.logdir, 'CountEncoder.pkl'), 'wb') as f:
pickle.dump(self.count_encoder, f)
with open(os.path.join(self.logdir, 'TargetEncoder.pkl'), 'wb') as f:
pickle.dump(self.target_encoder, f)
with open(os.path.join(self.logdir, 'OrdinalEncoderFeatures.json'),
'w') as f:
json.dump(self.ordinal_encoder_features, f)
with open(os.path.join(self.logdir, 'OneHotEncoderFeatures.json'),
'w') as f:
json.dump(self.onehot_encoder_features, f)
with open(os.path.join(self.logdir, 'CountEncoderFeatures.json'),
'w') as f:
json.dump(self.count_encoder_features, f)
with open(os.path.join(self.logdir, 'TargetEncoderFeatures.json'),
'w') as f:
json.dump(self.target_encoder_features, f)