def initialize_regressors(sgd_grid):
l = SGDRegressor()
regressions_to_fit = []
for params in ParameterGrid(sgd_grid):
l.set_params(**params)
regressions_to_fit.append(clone(l))
return regressions_to_fit
def meshes_loop(X, y, params):
alpha = params['alpha']
l1_ratio = params['l1_ratio']
results = pd.DataFrame(
columns=['alpha', 'l1_ratio', 'y', 'r2_test', 'r2_train'])
row = 0
for a in alpha:
for l in l1_ratio:
sgdr = SGDRegressor(penalty='elasticnet',
tol=1e-6,
max_iter=params['max_iter'])
sgdr.set_params(**{'alpha': a, 'l1_ratio': l})
print('PARAMS alpha: {}, l1_ratio: {}'.format(a, l))
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.2,
random_state=0)
sgdr.fit(X_train, y_train)
r2_test = r2_score(y_test, sgdr.predict(X_test))
r2_train = r2_score(y_train, sgdr.predict(X_train))
results.loc[row] = [a, l, params['y'], r2_test, r2_train]
row += 1
results.to_csv(params['name'])
class LinearAprxAgent:
def create_policy(self, func_approximator, epsilon=0):
# from lab 8
def policy_fn(state):
"""
Input:
state: a 2D array with the position and velocity
Output:
A,q_values:
"""
action_index = np.ones(self.num_of_actions,
dtype=float) * epsilon / self.num_of_actions
#transform to the same shape as the model was trained on
state_transformed = self.feature_transformer.transform([state])
q_values = self.func_approximator.predict(state_transformed)
best_action = np.argmax(q_values)
action_index[best_action] += (1.0 - epsilon)
return action_index, q_values # return the potentially stochastic policy (which is due to the exploration)
return policy_fn # return a handle to the function so we can call it in the future
def __init__(self, env):
#RBF Hyper parameters
self.SGD_learning_rate = "optimal" #‘constant’, ‘optimal’, ‘invscaling’, ‘adaptive’
self.tol = 1e-5 #The stopping criterion
self.SGD_max_iter = 1e4
self.func_approximator = SGDRegressor(
learning_rate=self.SGD_learning_rate,
tol=self.tol,
max_iter=self.SGD_max_iter,
loss='huber')
self.feature_transformer = sklearn.pipeline.FeatureUnion([
("rbf1", RBFSampler(gamma=12.8, n_components=50)),
("rbf2", RBFSampler(gamma=6.4, n_components=50)),
("rbf3", RBFSampler(gamma=3.2, n_components=50)),
("rbf4", RBFSampler(gamma=1.6, n_components=50)),
("rbf5", RBFSampler(gamma=0.8, n_components=50)),
("rbf6", RBFSampler(gamma=0.4, n_components=50)),
("rbf7", RBFSampler(gamma=0.2, n_components=50)),
("rbf8", RBFSampler(gamma=0.1, n_components=50))
])
self.num_of_actions = env.action_space.n
self.env = env
#function which is the learned function
self.policy = self.create_policy(self.func_approximator, 1)
self.episodes = 200
# self.print_out_every_x_episodes = int(self.episodes/50)
self.times_exploited = 0
# hyper parameters for epsilon explore
self.initial_epsilon = 1 # initial
self.decrease_factor = (1 / self.episodes) / 1.25 # epsilon
# print("Decrease Factor: " + str(self.decrease_factor))
def train(self):
states, all_rewards, all_total_rewards = self.run_all_episodes(
"Training")
state_transformed = self.feature_transformer.transform(states)
q_values = self.func_approximator.predict(state_transformed)
return states, all_rewards, all_total_rewards, self.func_approximator, state_transformed, q_values
def evaluate(self,
intercept,
coeff,
states_transformed,
q_values,
last_reward,
episodes=100):
self.func_approximator = SGDRegressor(
learning_rate=self.SGD_learning_rate,
tol=self.tol,
max_iter=self.SGD_max_iter,
loss='huber')
self.func_approximator.fit(states_transformed,
last_reward,
coef_init=coeff,
intercept_init=intercept)
self.episodes = episodes
self.epsilon = -1000
states, all_rewards, all_total_rewards = self.run_all_episodes(
"Evaluation", evaluate=True)
return states, all_rewards, all_total_rewards
def run_all_episodes(self, title, evaluate=False):
all_total_rewards = []
all_rewards = []
epsilon = self.initial_epsilon # at the start only explore
power = 1
for episode in range(1, self.episodes + 1):
states, rewards = self.run_episode(epsilon, evaluate)
total_reward = np.sum(rewards)
# if episode % self.print_out_every_x_episodes == 0:
# print("Episode number: " + str(episode) + ". Total reward in episode: " + str(total_reward) + ". Episode executed with epsilon = " + str(epsilon))
# print("Average total reward in last " + str(self.print_out_every_x_episodes) + " episodes: " + str(np.mean(all_rewards[-self.print_out_every_x_episodes:])))
# print("Times exploited the last episode " + str(self.times_exploited))
# print("-----")
self.times_exploited = 0
all_rewards.append(rewards)
all_total_rewards.append(total_reward)
if not evaluate:
epsilon = self.decrease_epsilon(epsilon, power)
power += 0.10
#graph with orange smoothed reward
if not evaluate:
window_size = int(self.episodes / 10)
smoothed_rewards = pd.Series(all_total_rewards).rolling(
window_size, min_periods=window_size).mean()
this_smoothed_reward = smoothed_rewards.values[-1]
smooth_plot(all_total_rewards, smoothed_rewards, title)
return states, all_rewards, all_total_rewards
#exponential decrease in epsilon
def decrease_epsilon(self, epsilon, power):
decrease = 0.005
return epsilon * ((1 - decrease)**power)
def run_episode(self, epsilon, evaluate=False):
rewards = []
states = []
actions = []
done = False
state = self.env.reset()
states.append(state)
while not done:
random_number = np.random.random()
if random_number < epsilon:
#explore
action = np.random.choice(self.num_of_actions)
else:
#exploit
action = self.get_action(state)
self.times_exploited += 1
new_state, reward, done, i = self.env.step(action=action)
states.append(new_state)
actions.append(action)
rewards.append(reward)
if not evaluate:
#update policy function
self.update(states[1:], rewards, epsilon)
state = new_state
return states, rewards
def update(self, states, rewards, epsilon):
#update the linear function
self.feature_transformer.fit(states)
states_transformed = self.feature_transformer.transform(states)
self.func_approximator.fit(states_transformed, rewards)
self.policy = self.create_policy(self.func_approximator, 0)
def get_action(self, state):
#linear function to get best action
actions, q_values = self.policy(state)
return np.argmax(actions)
def get_action_text(self):
return action_text
def get_env(self):
return env
def get_chart_title(self):
return "Action = " + action_text
def get_weights(self):
return self.func_approximator.get_params()
def set_params(self, coef, intercept):
self.func_approximator.set_params(weights.get_items())
alpha = [0.1, 0.5, 1.0]
l1_ratio = [1e-3, 0.1, 0.4, 1]
sgdr = SGDRegressor(penalty='elasticnet', tol = 1e-3)
rs = ShuffleSplit(n_splits=3, test_size=0.2, random_state=0)
results = pd.DataFrame(columns=['alpha', 'l1_ratio', 'edge', 'mean_test', 'mean_train'])
row = 0
for a in alpha:
for l in l1_ratio:
sgdr = SGDRegressor(penalty='elasticnet', tol = 1e-6)
sgdr.set_params(**{'alpha':a, 'l1_ratio': l})
print('PARAMS alpha: {}, l1_ratio: {}'.format(a, l))
for i in sorted(c):
print('FOR EDGE NUMBER ' + str(i))
r2_test, r2_train = [], []
for train_index, test_index in rs.split(X):
X_train, y_train = X[train_index], Y[train_index, i]
X_test, y_test = X[test_index], Y[test_index, i]
sgdr.fit(X_train, y_train)
r2_train += [sgdr.score(X_train, y_train)]
r2_test += [sgdr.score(X_test, y_test)]
print('R2 train: {}, test: {}'.format(r2_train[-1], r2_test[-1]))
print('MEAN R2 TRAIN: {}, TEST: {}'.format(np.mean(r2_train), np.mean(r2_test)))
results.loc[row] = [a, l, i, np.mean(r2_test), np.mean(r2_train)]
row += 1
class ModelLookup():
def __init__(self, model_type, target_col):
self.model_type = model_type
self.target_col = target_col
self.get_model()
def get_model(self):
'''each lookup entry MUST set:
- model_name
- tuning_metric
- best_params
Optional, if you want to do param tuning:
- param space
- objective_function'''
lookup_dict = {
'xgboost': self.get_xgboost,
'reg_linear': self.get_reg_linear,
'lightgbm': self.get_lightgbm
}
if self.model_type not in lookup_dict.keys():
raise ValueError('No ModelLookup entry defined for ' +
self.model_type)
lookup_dict[self.model_type]()
def get_xgboost(self):
# ----------------
# base model attrs
# ----------------
self.model_name = 'xgboost_model_' + pd.to_datetime('today').strftime(
'%Y-%m-%d')
# self.cores = hf.detect_cores() - 2
if is_bool_dtype(self.target_col):
objective = 'binary:logistic'
self.tuning_metric = 'roc_auc'
self.model = xgb.XGBClassifier()
# self.tuning_metric = 'f1'
elif is_numeric_dtype(self.target_col):
objective = 'reg:linear'
self.tuning_metric = 'neg_mean_squared_error'
self.model = xgb.XGBRegressor()
elif is_string_dtype(self.target_col):
objective = 'multi:softprob'
num_class = len(pd.unique(self.target_col))
self.tuning_metric = 'accuracy'
self.target_lookup = pd.Categorical(self.target_col)
self.target_col = self.target_lookup.codes
self.model = xgb.XGBClassifier()
else:
print('some sort of error')
self.best_params = {
'base_score': 0.5,
'booster': 'gbtree',
'colsample_bylevel': 1,
'colsample_bytree': 0.61993666911880152,
'gamma': 3.5007109366333236,
'learning_rate': 0.042247990716033385,
'max_delta_step': 0,
'max_depth': 9,
'min_child_weight': 5,
'missing': None,
'n_estimators': 124,
'n_jobs': -1,
'nthread': -1,
'objective': objective,
'random_state': 0,
'reg_alpha': 0,
'reg_lambda': 9,
'scale_pos_weight': 1,
'seed': 0,
'silent': True,
'subsample': 1
}
try:
self.model.set_params(num_class=num_class)
except:
pass
# ------------------
# model tuning attrs
# ------------------
self.param_space = {
'base_score': [0.5],
'max_depth': (5, 10),
'n_estimators': (50, 125),
'learning_rate': (0.01, .3),
'min_child_weight': (1, 50),
'gamma': (0, 5.0),
'colsample_bytree': (0.50, .999),
'reg_alpha': (0, 5),
'reg_lambda': (0, 10),
'subsample': (.2, 1)
}
self.objective_function = hf.xgb_objective
def get_reg_linear(self):
# ----------------
# base model attrs
# ----------------
self.model_name = 'linear_model_' + pd.to_datetime('today').strftime(
'%Y-%m-%d')
# self.cores = hf.detect_cores() - 2
if is_bool_dtype(self.target_col):
loss = 'log'
self.tuning_metric = 'roc_auc'
self.model = SGDClassifier()
# self.tuning_metric = 'f1'
elif is_numeric_dtype(self.target_col):
loss = 'squared_loss'
self.tuning_metric = 'neg_mean_squared_error'
self.model = SGDRegressor()
# self.tuning_metric = 'neg_mean_absolute_error'
self.best_params = {
'alpha': 0.0001,
'average': False,
'class_weight': None,
'epsilon': 0.1,
'eta0': 0.0,
'fit_intercept': True,
'l1_ratio': 0.15,
'learning_rate': 'optimal',
'loss': loss,
'max_iter': 1000,
'n_iter': None,
'n_jobs': -1,
'penalty': 'elasticnet',
'power_t': 0.5,
'random_state': None,
'shuffle': True,
'tol': 0.001,
'verbose': 0,
'warm_start': False
}
# ------------------
# model tuning attrs
# ------------------
self.param_space = {'alpha': (0.0001, 3), 'l1_ratio': (0, 1.0)}
self.objective_function = hf.reg_linear_objective
def get_lightgbm(self):
# ----------------
# base model attrs
# ----------------
self.model_name = 'lightgbm_model_' + pd.to_datetime('today').strftime(
'%Y-%m-%d')
# self.cores = hf.detect_cores() - 2
if is_bool_dtype(self.target_col):
objective = 'binary'
self.tuning_metric = 'roc_auc'
self.model = lgbm.LGBMClassifier()
# self.tuning_metric = 'f1'
elif is_numeric_dtype(self.target_col):
objective = 'regression'
self.tuning_metric = 'neg_mean_squared_error'
self.model = lgbm.LGBMRegressor()
elif is_string_dtype(self.target_col):
objective = 'multiclass'
num_class = len(pd.unique(self.target_col))
self.tuning_metric = 'accuracy'
self.target_lookup = pd.Categorical(self.target_col)
self.target_col = pd.Series(self.target_lookup.codes)
self.model = lgbm.LGBMRegressor()
# self.tuning_metric = 'neg_mean_absolute_error'
self.best_params = {
'boosting_type': 'gbdt',
'class_weight': None,
'colsample_bytree': 1.0,
'learning_rate': 0.1,
'max_depth': -1,
'min_child_samples': 20,
'min_child_weight': 0.001,
'min_split_gain': 0.0,
'n_estimators': 100,
'n_jobs': -1,
'num_leaves': 31,
'objective': objective,
'random_state': None,
'reg_alpha': 0.0,
'reg_lambda': 0.0,
'silent': True,
'subsample': 1.0,
'subsample_for_bin': 200000,
'subsample_freq': 1
}
try:
self.model.set_params(num_class=num_class)
except:
pass
# ------------------
# model tuning attrs
# ------------------
self.param_space = {
'max_depth': (5, 10),
'n_estimators': (50, 125),
'learning_rate': (0.01, .3),
'min_child_weight': (0.01, 20),
'min_split_gain': (0.0, 0.1),
'colsample_bytree': (0.50, .999),
'reg_alpha': (0, 5),
'reg_lambda': (0, 10),
'subsample': (.2, 1)
}
self.objective_function = hf.lgbm_objective