def to_categorical(
training_data: pd.DataFrame, test_data: pd.DataFrame
) -> (pd.DataFrame, pd.DataFrame):
categorical_columns_list = list(training_data.columns[training_data.dtypes==object])
ce_be = BinaryEncoder(cols=categorical_columns_list, handle_unknown="inpute")
training_data_ce_binary = ce_be.fit_transform(training_data)
test_data_ce_binary = ce_be.transform(test_data)
return dict(train_data_categorical=training_data_ce_binary,
test_data_categorical=test_data_ce_binary)
#ordinal encoding
from sklearn.preprocessing import OrdinalEncoder
ord1 = OrdinalEncoder()
ord1.fit([df['ord_2']])
df["ord_2"] = ord1.fit_transform(df[["ord_2"]])
df.head(10)
dnew = df.copy()
#ordinal encoding through mapping
temp_dict = {'Cold': 1, 'Warm': 2, 'Hot': 3}
dnew['Ord_2_encod'] = dnew.ord_2.map(temp_dict)
dnew = dnew.drop(['ord_2'], axis=1)
#Binary encoding
from category_encoders import BinaryEncoder
encoder = BinaryEncoder(cols=['ord_2'])
newdata = encoder.fit_transform(df['ord_2'])
df = pd.concat([df, newdata], axis=1)
df = df.drop(['ord_2'], axis=1)
df.head(10)
#Hash encoding
from sklearn.feature_extraction import FeatureHasher
h = FeatureHasher(n_features=3, input_type='string')
hashed_Feature = h.fit_transform(df['nom_0'])
hashed_Feature = hashed_Feature.toarray()
df = pd.concat([df, pd.DataFrame(hashed_Feature)], axis=1)
df.head(10)
df.insert(6, "Target", [0, 1, 1, 0, 0, 1, 0, 0, 0, 1], True)
#mean Ecoding /Target encoding
def predict():
'''
For rendering results on HTML GUI
'''
features = [x for x in request.form.values()]
#final_features = [np.array(int_features)]
#prediction = model.predict(final_features)
#output = round(prediction[0], 2)
features = np.array(features)
features = features.reshape(1, 6)
features = pd.DataFrame(data=features,
columns=[
'Name', 'Genre', 'Comments', 'Likes',
'Popularity', 'Followers'
])
df = pd.read_csv('data.csv')
cv = {'Comments': int, 'Likes': int, 'Popularity': int, 'Followers': int}
df = df.astype(cv)
features = features.astype(cv)
#x=df[df['Views']==0].index
df.drop(index=df[df['Views'] < df['Likes']].index, axis=1, inplace=True)
df.drop(index=df[df['Views'] < df['Comments']].index, axis=1, inplace=True)
df.drop(index=df[df['Views'] < df['Popularity']].index,
axis=1,
inplace=True)
Q1 = df.quantile(0.25)
Q3 = df.quantile(0.75)
IQR = Q3 - Q1
(df < (Q1 - 1.5 * IQR)) | (df > (Q3 + 1.5 * IQR))
df = df[~((df < (Q1 - 3 * IQR)) | (df > (Q3 + 3 * IQR))).any(axis=1)]
df = df.drop(
columns=['Unique_ID', 'Country', 'Song_Name', 'Timestamp', 'index'])
y = df['Views']
df = df.drop(columns=['Views'])
be = BinaryEncoder()
df = be.fit_transform(df)
f = be.transform(features)
X = df.iloc[:, :]
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.3,
random_state=0)
rg1 = AdaBoostRegressor()
rg1.fit(X_train, y_train)
#ypred=rg1.predict(X_test)
#sqrt(mean_squared_error(y_test,ypred))
rg2 = GradientBoostingRegressor(n_estimators=300, learning_rate=0.1)
# para={'n_estimators':[250,300],'learning_rate':[1,0.1,0.01]}
# grid=GridSearchCV(estimator=rg8,param_grid=para,verbose=1,cv=10,n_jobs=-1)
rg2.fit(X_train, y_train)
#ypred=rg2.predict(X_test)
#sqrt(mean_squared_error(y_test,ypred))
rg3 = RandomForestRegressor(random_state=0, n_estimators=20, max_depth=15)
# para={'n_estimators':[5,10,30,20],'max_depth':[5,8,20,17]}
# grid=GridSearchCV(estimator=rg9,param_grid=para,cv=10,verbose=1,n_jobs=-1)
rg3.fit(X_train, y_train)
#ypred=rg3.predict(X_test)
#sqrt(mean_squared_error(y_test,ypred))
rg6 = StackingRegressor([rg1, rg2], meta_regressor=rg3)
rg6.fit(X_train, y_train)
#ypred=rg6.predict(X_test)
#sqrt(mean_squared_error(y_test,ypred))
f = f.iloc[:, :]
y_pred = rg6.predict(f)
y_pred = y_pred.astype(int)
return render_template(
'index.html', prediction_text='Numberb of Views is {}'.format(y_pred))
def binaryEncoding(df,column): from category_encoders import BinaryEncoder encoder=BinaryEncoder(cols=[column]) df=encoder.fit_transform(df) return df
DataFrame = pd.core.frame.DataFrame
Series = pd.core.series.Series
Array = np.ndarray
Imputer = Callable[[DataFrame], DataFrame]
nan = np.nan
df = pd.read_csv('data.csv').drop(['name'], axis=1)
X_ = df.drop('status_group', axis=1)
y = df.status_group
be = BinaryEncoder()
FEATS = 70
pca = PCA(n_components=FEATS)
vals = pca.fit_transform(StandardScaler().fit_transform(be.fit_transform(X_)))
X = pd.DataFrame(vals,
columns=[f"pc{k+1}" for k in range(FEATS)],
index=y.index).assign(y=y)
def mcar_goblin(dat: DataFrame, ratio: float) -> DataFrame:
''' Simulate MCAR with bernoulli '''
def ident_or_nan(x: float) -> float:
''' if heads, replace value with nan. if tails, identity '''
coin = bernoulli(ratio)
if coin.rvs() == 1:
return nan
else:
return x
nrows=500)
test = pd.read_csv(os.path.join(config["input_path"], "test.csv"),
na_values=-1,
nrows=500)
train_feature, train_label = train.iloc[:,
2:].copy(), train.iloc[:,
1].copy()
test_feature = test.iloc[:, 1:].copy()
del train, test
train_feature = train_feature[[
col for col in train_feature.columns if not col.startswith("ps_calc_")
]]
test_feature = test_feature[train_feature.columns]
ncs = [
col for col in train_feature.columns
if not col.endswith(("_bin", "_cat"))
]
ccs = [
col for col in train_feature.columns if col.endswith(("_bin", "_cat"))
]
eet = EntityEmbeddingTree(numeric_columns=ncs, categorical_columns=ccs)
eet.fit(X=train_feature, y=train_label)
encoder = BinaryEncoder()
print(encoder.fit_transform(eet.transform(X=train_feature)).shape)
print(encoder.transform(eet.transform(X=test_feature)).shape)