def train_simple(data: DataFrame, esti: Estimator, eid: str) -> PipResult:
"""
Train without cross validation
"""
try:
print(f"--- train_simple {eid}")
# Prepare training and test data.
df_train, df_test = data.randomSplit([0.9, 0.1], seed=12345)
# Run TrainValidationSplit, and choose the best set of parameters.
trained_model: Transformer = esti.fit(df_train)
# Make predictions on test data. model is the model with combination of parameters
# that performed best.
predictions = trained_model.transform(df_test) \
.select("features", "label", "prediction")
# Select (prediction, true label) and compute test error
evaluator = RegressionEvaluator(labelCol="label",
predictionCol="prediction",
metricName="rmse")
rmse = evaluator.evaluate(predictions)
print(f"-- Root Mean Squared Error (RMSE) on test data = {rmse}")
fnam = cm.fnam(eid)
hlp.save_model(trained_model, hlp.get_datadir(), fnam)
print(f"-- saved model to {fnam}")
return PipResult(rmse, trained_model, "OK")
except Exception:
print(tb.format_exc())
return PipResult(0.0, None, "ERROR")
def fit_classifier(self) -> None:
"""
Fit the classifier and save the model using Pickle.
:return:
"""
self.clf.fit(self.X, self.y)
save_model(self.clf, config.dataset, config.model)
def linear_lasso_regression(X, y):
"""
Fit lasso linear regression model
:param X: train input
:param y: train output
:return: None
"""
# Train a linear regression model with Lasso regression and assess its performance.
lasso_regression = Lasso( # Optimal parameters from by grid search run.
alpha=0,
tol=0.01,
selection="random",
positive=False,
max_iter=1000,
normalize=False)
lasso_regression.fit(X, y)
quick_prediction_test(lasso_regression, X, y)
y_prediction = lasso_regression.predict(X)
evaluate_model_error(lasso_regression, y, y_prediction)
plot_best_fit(y, y_prediction, "lasso_reg")
save_model(lasso_regression, "lasso_reg")
# Using the same model with 10-fold cross validation.
k_fold_cross_validation(lasso_regression, X, y, folds=5)
if config.is_grid_search:
print("\nPerforming grid search to find optimal hyperparameters...")
# Applying grid search algorithm to the Lasso Regression model
parameters = {
"alpha": [0.0001, 0.001, 0.01, 0.1, 0, 1, 10],
"tol": [0.01, 0.1, 1],
"positive": [True, False],
"selection": ["cyclic", "random"]
}
lasso_reg_gs_results = grid_search_algorithm(Lasso(),
X,
y,
parameters,
folds=5)
lasso_reg_gs_results_df = pd.DataFrame(
lasso_reg_gs_results.cv_results_)
lasso_reg_gs_results_df.to_csv(
"grid_search_results/lasso_reg_grid_search_results.csv")
# Top 5 hyperparameters found for Lasso Regression.
print("Top 5 hyperparameters combinations found for Lasso Regression:")
lasso_reg_gs_results_df.sort_values(by=['rank_test_score']).head(5)
# Best model found by grid search algorithm for Lasso Regression.
final_lasso_reg_model = lasso_reg_gs_results.best_estimator_
print(
"\nBest model hyperparameters found by grid search algorithm for Lasso Regression:"
)
print(final_lasso_reg_model)
def linear_ridge_regression(X, y) -> None:
"""
Fit ridge linear regression model
:param X: train input
:param y: train output
:return: None
"""
# Train a linear regression model with Ridge regression and assess its performance.
ridge_regression = Ridge(
alpha=0.1, solver='svd',
normalize=False) # Optimal parameters from by grid search run.
ridge_regression.fit(X, y)
quick_prediction_test(ridge_regression, X, y)
y_prediction = ridge_regression.predict(X)
evaluate_model_error(ridge_regression, y, y_prediction)
plot_best_fit(y, y_prediction, "ridge_reg")
save_model(ridge_regression, "ridge_reg")
# Using the same model with 10-fold cross validation.
k_fold_cross_validation(ridge_regression, X, y)
if config.is_grid_search:
print("\nPerforming grid search to find optimal hyperparameters...")
# Applying grid search algorithm to the Ridge Regression model to explore combinations of hyperparameters to
# find the best hyperparameters for a model, since doing so manually is a very time consuming task.
parameters = {
"alpha": [0.1, 1, 10],
"normalize": [True, False],
"solver":
["auto", "svd", "cholesky", "lsqr", "sparse_cg", "sag", "saga"]
}
ridge_reg_gs_results = grid_search_algorithm(Ridge(),
X,
y,
parameters,
folds=5)
ridge_reg_gs_results_df = pd.DataFrame(
ridge_reg_gs_results.cv_results_)
ridge_reg_gs_results_df.to_csv(
"grid_search_results/ridge_reg_grid_search_results.csv")
# Top 5 hyperparameters found for Ridge Regression.
print("Top 5 hyperparameters combinations found for Ridge Regression:")
ridge_reg_gs_results_df.sort_values(by=['rank_test_score']).head(5)
# Best model found by grid search algorithm for Ridge Regression.
final_ridge_reg_model = ridge_reg_gs_results.best_estimator_
print(
"\nBest model hyperparameters found by grid search algorithm for Ridge Regression:"
)
print(final_ridge_reg_model)
def decision_tree_regression(X, y):
"""
Briefly explore decision tree regression model.
:param X:
:param y:
:return:
"""
tree_regressor = DecisionTreeRegressor()
tree_regressor.fit(X, y)
quick_prediction_test(tree_regressor, X, y)
evaluate_model_error(tree_regressor, y, tree_regressor.predict(X))
save_model(tree_regressor, "tree_reg")
k_fold_cross_validation(tree_regressor, X, y)
def svm_regression(X, y):
"""
Briefly explore SVM regression model.
:param X:
:param y:
:return:
"""
svr = SVR()
svr.fit(X, y)
quick_prediction_test(svr, X, y)
evaluate_model_error(svr, y, svr.predict(X))
save_model(svr, "svr")
k_fold_cross_validation(svr, X, y)
def mlp_regression(X, y):
"""
Briefly explore MLP regression model.
:param X:
:param y:
:return:
"""
mlp_regressor = MLPRegressor()
mlp_regressor.fit(X, y)
quick_prediction_test(mlp_regressor, X, y)
evaluate_model_error(mlp_regressor, y, mlp_regressor.predict(X))
save_model(mlp_regressor, "mlp_reg")
k_fold_cross_validation(mlp_regressor, X, y)
def save_load_hlp():
SparkSession.builder \
.appName("tryout") \
.getOrCreate()
m = LinearRegression(regParam=0.5, maxIter=10)
pm = m.extractParamMap()
pprint(pm)
print()
pprint(str(path))
hlp.save_model(m, path, nam)
m2 = hlp.load_model(path, nam)
pm2 = m2.extractParamMap()
pprint(pm2)
def elastic_net_regression(X, y):
"""
Briefly explore elastic net linear regression model.
:param X:
:param y:
:return:
"""
elastic_net_regression = ElasticNet()
elastic_net_regression.fit(X, y)
quick_prediction_test(elastic_net_regression, X, y)
evaluate_model_error(elastic_net_regression, y,
elastic_net_regression.predict(X))
save_model(elastic_net_regression, "elasticnet_reg")
k_fold_cross_validation(elastic_net_regression, X, y)
def random_forest_generator_regression(X, y):
"""
Briefly explore random forest generator regression model.
:param X:
:param y:
:return:
"""
random_forest_generator_reg = RandomForestRegressor()
random_forest_generator_reg.fit(X, y)
quick_prediction_test(random_forest_generator_reg, X, y)
evaluate_model_error(random_forest_generator_reg, y,
random_forest_generator_reg.predict(X))
save_model(random_forest_generator_reg, "random_forest_generator_reg")
k_fold_cross_validation(random_forest_generator_reg, X, y)
def train(data: DataFrame, esti: Estimator, eid: str,
param_grid_builder: Callable[[Estimator], list]) -> PipResult:
try:
print(f"--- train {eid}")
# Prepare training and test data.
df_train, df_test = data.randomSplit([0.9, 0.1], seed=12345)
# We use a ParamGridBuilder to construct a grid of parameters to search over.
# TrainValidationSplit will try all combinations of values and determine best model using
# the evaluator.
params = param_grid_builder(esti)
print(f"--- params")
pprint(params)
# In this case the estimator is simply the linear regression.
# A TrainValidationSplit requires an Estimator, a set of Estimator ParamMaps, and an Evaluator.
tvs = TrainValidationSplit(
estimator=esti,
estimatorParamMaps=params,
evaluator=RegressionEvaluator(),
# 80% of the data will be used for training, 20% for validation.
trainRatio=0.8)
# Run TrainValidationSplit, and choose the best set of parameters.
trained_models: TrainValidationSplitModel = tvs.fit(df_train)
# Make predictions on test data. model is the model with combination of parameters
# that performed best.
predictions = trained_models.transform(df_test) \
.select("features", "label", "prediction")
# Select (prediction, true label) and compute test error
evaluator = RegressionEvaluator(labelCol="label",
predictionCol="prediction",
metricName="rmse")
rmse = evaluator.evaluate(predictions)
print(f"-- Root Mean Squared Error (RMSE) on test data = {rmse}")
fnam = cm.fnam(eid)
hlp.save_model(trained_models.bestModel, hlp.get_datadir(), fnam)
print(f"-- saved model to {fnam}")
return PipResult(rmse, trained_models.bestModel, "OK")
except Exception:
print(tb.format_exc())
return PipResult(0.0, None, "ERROR")
def general_linear_regression(X, y) -> None:
"""
Fit general linear regression model
:param X: train input
:param y: train output
:return: None
"""
# Train a linear regression model and assess its performance.
linear_regression = LinearRegression(fit_intercept=True, normalize=True)
linear_regression.fit(X, y)
quick_prediction_test(linear_regression, X, y)
y_prediction = linear_regression.predict(X)
evaluate_model_error(linear_regression, y, y_prediction)
plot_best_fit(y, y_prediction, "linear_reg")
save_model(linear_regression, "linear_reg")
# Using the same model with 10-fold cross validation.
k_fold_cross_validation(linear_regression, X, y)
from torch.nn import MSELoss
from torch.optim import SGD
import numpy as np
import sys
sys.path.append('..')
from helpers import load_data_in_chunks, save_model
from model import Net
from CustomLoss import CustomLoss
(Xs, Ys) = load_data_in_chunks('basic', 'train', chunk_size=5)
Xs = Xs.astype(np.float32)
Ys = Ys.astype(np.float32)
regr = NeuralNet(Net,
max_epochs=10000000000,
batch_size=100,
iterator_train__shuffle=True,
criterion=CustomLoss,
optimizer=SGD,
optimizer__lr=1e-5,
optimizer__momentum=0.9,
optimizer__nesterov=True,
optimizer__dampening=0,
verbose=5,
callbacks=[('early_stop', EarlyStopping())])
regr.fit(Xs, Ys / 5000)
save_model(regr, 'conv-mse')
def run_nlm(model,
corpus,
optimizer=None,
epochs=1,
eval_corpus=None,
status_interval=25,
str_pattern='{}_{}_epoch_{}.pkl',
rz_amplifier=5):
entity_offset = 1
for epoch in range(1, epochs + 1):
X_epoch_loss, E_epoch_loss, R_epoch_loss, L_epoch_loss = 0, 0, 0, 0
epoch_tokens, epoch_r_div, epoch_l_div, epoch_e_div = 0, 0, 0, 0
epoch_start = time.time()
count_E = 0
count_E_correct = 0
count_R = 0
r_true_positive = 0
r_false_positive = 0
for i_doc, doc in enumerate(corpus.gen()):
model.reset_state()
# initialize e_current
e_current = model.get_new_entity()
# forward first token through Embedding and RNN
# initialize states
# lstm initializes states with zeros when given None
h_t, states = model.forward_rnn(doc.X[0], states=None)
h_t = h_t.squeeze(0)
# initialize loss tensors
X_loss = torch.tensor(0, dtype=torch.float, device=device)
E_loss = torch.tensor(0, dtype=torch.float, device=device)
R_loss = torch.tensor(0, dtype=torch.float, device=device)
L_loss = torch.tensor(0, dtype=torch.float, device=device)
# counters to properly devide losses
r_div = 0
l_div = 0
e_div = 0
# counter for stats
doc_r_true_positive = 0
doc_r_false_positive = 0
doc_count_R = 0
# iterate over document
for t in range(doc.X.size(0) - 1):
# define target values
next_X = doc.X[t + 1] # next Token
next_E = doc.E[
t +
1] - entity_offset # next Entity, offset to match indices with self.entities
next_R = doc.R[t + 1] # next R type
next_L = doc.L[t + 1] # next Length
# ***START PAPER ALGORITHM***
# Define current value for L
current_L = doc.L[t]
if current_L == 1:
# 1.
# last L equals 1: not continuing entity mention
# predict next R
R_dist = model.get_next_R(h_t)
# create loss for R
r_current_loss = torch.nn.functional.cross_entropy(
R_dist, next_R.view(-1)) * rz_amplifier
# r_current_loss is used to make amplification of loss possible
R_loss += r_current_loss
# add division counter for R loss
r_div += 1
if next_R == 1:
# next token is within an entity mention
doc_count_R += 1
if R_dist.argmax():
# both True - correct pred
doc_r_true_positive += 1
#R_loss += r_current_loss
else:
# false negative prediction
# extra loss to increase recall
R_loss += r_current_loss * rz_amplifier
pass
# select the entity
E_dist = model.get_next_E(h_t, t)
# count for stats
count_E += 1
count_E_correct += int(E_dist.argmax() == next_E)
# calculate entity loss
E_loss += torch.nn.functional.cross_entropy(
E_dist, next_E.view(-1))
e_div += 1
# register entity
model.register_predicted_entity(next_E)
# set e_current to entity embedding e_t-1
e_current = model.get_entity_embedding(next_E)
# predict length of entity and calculate loss
L_dist = model.get_next_L(h_t, e_current)
L_loss += torch.nn.functional.cross_entropy(
L_dist, next_L.view(-1))
l_div += 1
else:
# only for stats and possibility to amplify loss
if R_dist.argmax():
# wrong True pred
doc_r_false_positive += 1
# extra loss
# R_loss += r_current_loss * rz_amplifier
else:
# correct False pred
# R_loss += r_current_loss
pass
else:
# 2. Otherwise
# last L unequal 1, continuing entity mention
# set last new_L = last_L - 1
# new_R = last_R
# new_E = last_E
# additional prediction for E to get more training cases
# (it also makes stats more comparable to deep-mind paper)
E_dist = model.get_next_E(h_t, t)
count_E += 1
count_E_correct += int(E_dist.argmax() == next_E)
E_loss += torch.nn.functional.cross_entropy(
E_dist, next_E.view(-1))
e_div += 1
pass
# 3. Sample X, get distribution for next Token
X_dist = model.get_next_X(h_t, e_current)
X_loss += torch.nn.functional.cross_entropy(
X_dist, next_X.view(-1))
# 4. Advance the RNN on predicted token, here in training next token
h_t, states = model.forward_rnn(doc.X[t + 1], states)
h_t = h_t.squeeze(0)
# new hidden state of next token from here (h_t, previous was h_t-1)
# 5. Update entity state
if next_R == 1:
model.update_entity_embedding(next_E, h_t, t)
# set e_current to embedding e_t
e_current = model.get_entity_embedding(next_E)
# 6. Nothing toDo?
# ***END PAPER ALGORITHM***
# calculate stats and divide loss values
r_true_positive += doc_r_true_positive
r_false_positive += doc_r_false_positive
count_R += doc_count_R
doc_r_prec = doc_r_true_positive / max(
(doc_r_true_positive + doc_r_false_positive), 1)
doc_r_recall = doc_r_true_positive / max(doc_count_R, 1)
doc_rf_score = 2 * ((doc_r_prec * doc_r_recall) /
max(doc_r_prec + doc_r_recall, 1))
R_loss = R_loss / max(doc_rf_score, 0.35)
X_epoch_loss += X_loss.item()
R_epoch_loss += R_loss.item()
E_epoch_loss += E_loss.item()
L_epoch_loss += L_loss.item()
X_loss /= len(doc)
R_loss /= max(r_div, 1)
E_loss /= max(e_div, 1)
L_loss /= max(l_div, 1)
epoch_tokens += len(doc)
epoch_r_div += r_div
epoch_l_div += l_div
epoch_e_div += e_div
if optimizer:
# optimization step
optimizer.zero_grad()
loss = X_loss + R_loss + E_loss + L_loss
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), 0.5)
optimizer.step()
if status_interval and i_doc % status_interval == 0:
# status output
r_prec = r_true_positive / max(
(r_true_positive + r_false_positive), 1)
r_recall = r_true_positive / max(count_R, 1)
rf_score = 2 * (
(r_prec * r_recall) / max(r_prec + r_recall, 1))
print(
f'Doc {i_doc}/{len(corpus)-1}: X_loss {X_epoch_loss / epoch_tokens:0.3}, R_loss {R_epoch_loss / epoch_r_div:0.3}, E_loss {E_epoch_loss / epoch_e_div:0.3}, L_loss {L_epoch_loss / epoch_l_div:0.3}, E_acc {count_E_correct/count_E:0.3}, R_prec {r_prec:0.3}, R_recall {r_recall:0.3}'
)
sys.stdout.flush()
# calulate readable time format
seconds = round(time.time() - epoch_start)
m, s = divmod(seconds, 60)
h, m = divmod(m, 60)
x_hour_and_ = f'{h} hours and ' * bool(h)
if optimizer:
print(f'Epoch {epoch} finished after {x_hour_and_}{m} minutes.')
else:
print(
f'Evaluation on "{corpus.partition}" partition finished after {x_hour_and_}{m} minutes.'
)
# calculate epoch stats: precision, recall and F-Score
r_prec = r_true_positive / max((r_true_positive + r_false_positive), 1)
r_recall = r_true_positive / max(count_R, 1)
rf_score = 2 * ((r_prec * r_recall) / max(r_prec + r_recall, 1))
print(
f'Loss: X_loss {X_epoch_loss / epoch_tokens:0.3}, R_loss {R_epoch_loss / epoch_r_div:0.3}, E_loss {E_epoch_loss / epoch_e_div:0.3}, L_loss {L_epoch_loss / epoch_l_div:0.3}, E_acc {count_E_correct/count_E:0.3}, R_prec {r_prec:0.3}, R_recall {r_recall:0.3}, R_Fscore {rf_score:0.3}'
)
print()
# if in train mode
if optimizer:
# save model
file_name = str_pattern.format(model.__class__.__name__,
model.lstm.hidden_size, epoch)
save_model(model, file_name)
if eval_corpus:
# evaluate on evaluation corpus
with torch.no_grad():
model.eval()
run_nlm(model,
eval_corpus,
status_interval=None,
rz_amplifier=rz_amplifier)
model.train()
def save_model(self, save_location, name):
save_model(save_location, name, self.model)
from sklearn.dummy import DummyRegressor
import sys
sys.path.append('..')
from helpers import load_data, save_model
(Xs, Ys) = load_data('train')
regr = DummyRegressor(strategy='median')
regr.fit(Xs, Ys)
save_model(regr, 'dummy')
from torch.nn import MSELoss
from torch.optim import SGD
import numpy as np
import sys
sys.path.append('..')
from helpers import load_data_in_chunks, save_model
from model import Net
from RelativeEntropyLoss import RelativeEntropyLoss
(Xs, Ys) = load_data_in_chunks('survival', 'train', chunk_size=5)
Xs = Xs.astype(np.float32)
Ys = Ys.astype(np.float32)
regr = NeuralNet(Net,
max_epochs=10000000000,
batch_size=100,
iterator_train__shuffle=True,
criterion=RelativeEntropyLoss,
optimizer=SGD,
optimizer__lr=1e-5,
optimizer__momentum=0.9,
optimizer__nesterov=True,
optimizer__dampening=0,
verbose=5,
callbacks=[('early_stop', EarlyStopping())])
regr.fit(Xs, Ys)
save_model(regr, 'conv-survival')
from sklearn.linear_model import LinearRegression
import sys
sys.path.append('..')
from helpers import load_data_in_chunks, save_model
(Xs, Ys) = load_data_in_chunks('train', chunk_size=5)
regr = LinearRegression()
regr.fit(Xs.reshape(Xs.shape[0], -1), Ys)
save_model(regr, 'linear-multiple')
def pipeline(directory: str,
frames: List[Frame],
parser_name: str,
extract_features: set,
filter_: dict,
prune_test_data=True,
log_data=False):
filter = copy.deepcopy(filter_)
# Prune sentences without an LU from the frames
# TODO: log no. sentences pruned
filter_faulty_sentences(frames)
# Filter frames
(frames, no_filtered_frames_sentences) = filter_data(frames, filter)
print(f"Filtered data")
# Add feature representation of each word to each word node
(no_data_points_features) = create_feature_representation(
frames, extract_features)
print(f"Feature representation created")
no_frames = f"Number of frames: {len(frames)}"
print(no_frames)
# Split data into training and test sets
(train_sentences, test_sentences) = split_data_train_test(frames)
# # For testing purpose adding all sentences to training and testing
# sentences = []
# for frame in frames:
# sentences.extend(frame.getSentences())
# (train_sentences, test_sentences) = (sentences, sentences)
no_sentences = f"Number of sentences: {len(train_sentences) + len(test_sentences)}"
print(no_sentences)
print(no_data_points_features)
filter = copy.deepcopy(filter_)
# Train models
id_clf, label_clf, report_training = train_models_2(
train_sentences, filter)
print(f"{report_training}")
if log_data:
model_path = f"{directory}/models"
if not os.path.isdir(model_path):
# Create model folder
try:
os.mkdir(f"{model_path}")
except:
raise OSError(f"Unable to create directory {model_path}")
# Save models
save_model(id_clf, f"{parser_name}_identification_model",
f"{model_path}")
save_model(label_clf, f"{parser_name}_labeling_model", f"{model_path}")
filter = copy.deepcopy(filter_)
# Test models
(id_evaluation, label_evaluation,
evaluation) = test_models_2(id_clf,
label_clf,
test_sentences,
filter,
prune_test_data=prune_test_data)
print(f"Models tested")
if log_data:
# Save evaluation
save_to_file(id_evaluation,
f"{directory}/{parser_name}_id_evaluation.txt")
save_to_file(label_evaluation,
f"{directory}/{parser_name}_label_evaluation.txt")
save_to_file(f"{evaluation}",
f"{directory}/{parser_name}_evaluation.txt")
save_to_file(
f"{no_frames}\n{no_sentences}\n{no_data_points_features}\n{report_training}",
f"{directory}/run_description.txt")
# Release memory using garbage collection
del id_clf
del label_clf
gc.collect()
return evaluation
# Backprop and update if any instances were actually provided
if len(cls_targets) > 0:
loss.backward()
nn.utils.clip_grad_norm(rfcn.parameters(), args.grad_clip)
opt.step()
return loss
# Training loop
for epoch in range(args.epochs):
# Get next batch of training data
examples, classes, boxes = helpers.next_batch()
# Run the train step
total_loss = update(examples, classes, boxes)
# Prevent premature logging
if epoch == 0:
continue
# Print training status
if epoch % args.print_every == 0:
print("")
# Save models
if epoch % args.save_every == 0:
helpers.save_model(rfcn, 'rfcn')
helpers.save_model(rfcn, 'rfcn')
from sklearn.linear_model import LinearRegression
import sys
sys.path.append('..')
from helpers import load_data, save_model
(Xs, Ys) = load_data('train')
regr = LinearRegression()
regr.fit(Xs, Ys)
save_model(regr, 'linear-basic')
'losses': losses,
'scores': scores,
'ave_scores': ave_scores,
'actor_losses': actor_losses,
'critic_losses': critic_losses
}
model = ActorCritic(model_params)
optimizer = torch.optim.Adam(lr=params['lr'], params=model.parameters())
start = perf_counter()
if __name__ == '__main__':
try:
worker(model, optimizer, params)
save_model(model, optimizer, 'actor_critic.pt')
plot_losses(params['losses'], 'loss.png')
plot_losses(params['actor_losses'], filename='actor_loss.png', plotName="Actor Losses")
plot_losses(params['critic_losses'], filename='critic_loss.png', plotName="Critic Losses")
plot_scores(params['scores'], params['ave_scores'], filename='scores.png')
end = perf_counter()
print((end - start))
except KeyboardInterrupt:
pass
finally:
save_model(model, optimizer, 'actor_critic.pt')
plot_losses(params['losses'], 'loss.png')
plot_losses(params['actor_losses'], filename='actor_loss.png', plotName="Actor Losses")
plot_losses(params['critic_losses'], filename='critic_loss.png', plotName="Critic Losses")
plot_scores(params['scores'], params['ave_scores'], filename='scores.png')
end = perf_counter()
((clouds, locations, _), (alt_clouds, alt_locs, _), labels) = data
clouds = clouds.cuda()
alt_clouds = alt_clouds.cuda()
labels = labels.cuda()
conv = scan_conv(clouds, alt_clouds)
#Compute match prediction
scores = scan_match(conv)
predictions = torch.argmax(F.softmax(scores, dim=1), dim=1)
metrics[0] += torch.sum((predictions + labels == 2)) # both prediction and lable are 1
metrics[1] += torch.sum((predictions - labels == 1)) # prediction is 1 but label is 0
metrics[2] += torch.sum((predictions + labels == 0)) # both prediction and label are 0
metrics[3] += torch.sum((predictions - labels == -1)) # prediction is 0, label is 1
acc = (metrics[0] + metrics[2]) / sum(metrics)
prec = (metrics[0]) / (metrics[0] + metrics[1])
rec = (metrics[0]) / (metrics[0] + metrics[3])
f1 = 2 * prec * rec / (prec + rec)
print_output('Metrics: (TP %d, FP %d, TN %d, FN %d)' % (metrics[0], metrics[1], metrics[2], metrics[3]))
print_output('(Acc: %f, Precision: %f, Recall: %f, F1: %f)' % (acc, prec, rec, f1))
if not config.lock_conv:
helpers.save_model(scan_conv, out_dir, epoch, 'conv')
if config.train_match:
helpers.save_model(scan_match, out_dir, epoch, 'match')
if config.train_transform:
helpers.save_model(scan_transform, out_dir, epoch, 'transform')
if (len(select.select([sys.stdin], [], [], 0)[0])):
break
print_output("Completed training for {0} epochs".format(epoch + 1))
from skorch import NeuralNet
from skorch.callbacks import EarlyStopping
from torch.nn import MSELoss
from torch.optim import SGD
import numpy as np
import sys
sys.path.append('..')
from helpers import load_data_in_chunks, save_model
from model import Net
from CustomLoss import CustomLoss
(Xs, Ys) = load_data_in_chunks('train', chunk_size=5)
Xs = Xs.astype(np.float32)
Ys = Ys.astype(np.float32)
regr = NeuralNet(Net,
max_epochs=10000000000,
batch_size=100,
iterator_train__shuffle=True,
criterion=MSELoss,
optimizer=SGD,
optimizer__lr=1e-5,
optimizer__momentum=0.95,
verbose=5,
callbacks=[('early_stop', EarlyStopping())])
regr.fit(Xs, Ys / 5000)
save_model(regr, 'lstm-mse')
def main():
start = time.time()
##### Run variables #####
# If the data should be pruned as a part of the evaluation
pruning_test_data = True
# Filter the data used in both training and testing
filter = {"min_sentences": 0, "min_role_occurrence": 6, "prune": 1}
# Features of data to use
features_ = {
"",
# "frame",
# "core_elements",
"word",
"lemma",
"pos",
# "deprel",
"ref",
# "lu_words",
# "lu_lemmas",
# "lu_deprels",
# "lu_pos",
# "head_word",
# "head_lemma",
# "head_deprel",
# "head_pos",
# "child_words",
# "child_lemmas",
# "child_deprels",
# "child_pos",
}
if not os.path.isfile('spacy_parse.pkl'):
try:
frames = parse_spacy()
save_model(frames, "spacy_parse", ".")
send_email("Parsing spaCy",
f"Finished parsing spaCy and saved to model",
email_address, send_mail)
except Exception as err:
send_email("Parsing spaCy",
f"Error when parsing spaCy\n{str(err)}", email_address,
send_mail)
quit()
######## RUNS ########
# for feature in features_:
features = features_.copy()
# features.remove(feature)
# Change this string to represent the data manipulation made
now = datetime.now()
dt_string = now.strftime("_%Y-%m-%d_%H-%M-%S")
directory = f"runs/run{dt_string}"
readable_time = now.strftime("%H:%M:%S %Y-%m-%d")
# Description of run
data_description = (
f"Testing all features not generated by parser. Features: {feature}. \nlinearSVC. \n{features=}. \n{filter=}. \n{pruning_test_data=}. \nTime: {readable_time}\n"
)
if log_data:
# Create new run folder
try:
os.mkdir(directory)
f = open(directory + "/run_description.txt", "a")
f.write(data_description)
f.close()
except:
raise OSError(f"Unable to create directory {directory}")
send_email(
directory,
f"New run started: \n{data_description}\n",
email_address,
send_mail,
)
run_malt(data_description,
directory,
features,
filter,
prune_test_data=pruning_test_data)
run_spacy(data_description,
directory,
features,
filter,
prune_test_data=pruning_test_data)
send_email("Finished runs", "Tests compleate :)", email_address, send_mail)
timestamp(start, "Total time: ")
quit()
'brain_name': brain_name,
'start_epsilon': start_epsilon,
'end_epsilon': end_epsilon,
'epochs': epochs,
'lr': lr,
'gamma': gamma,
'clc': clc,
'reward_leadup': reward_leadup,
'entropy_bonus': entropy_bonus,
'batch_size': batch_size,
'losses': losses,
'scores': scores,
'ave_scores': ave_scores,
'actor_losses': actor_losses,
'critic_losses': critic_losses
}
start = perf_counter()
worker(model, params)
save_model(model, 'actor_critic.pt')
end = perf_counter()
print((end - start))
plot_losses(params['losses'], 'loss.png')
plot_losses(params['actor_losses'],
filename='actor_loss.png',
plotName="Actor Losses")
plot_losses(params['critic_losses'],
filename='critic_loss.png',
plotName="Critic Losses")
plot_scores(params['scores'], params['ave_scores'], filename='scores.png')
def main_trained_models():
# If the data should be pruned as a part of the evaluation
prune_test_data = True
# Filter the data used in both training and testing
filter = {"min_sentences": 0, "min_role_occurrence": 6, "prune": 1}
if not os.path.isfile('spacy_parse.pkl'):
try:
print(f"Parsing spaCy frames")
frames = parse_spacy()
save_model(frames, "spacy_parse", ".")
send_email("Parsing spaCy",
f"Finished parsing spaCy and saved to model",
email_address, send_mail)
except Exception as err:
send_email("Parsing spaCy",
f"Error when parsing spaCy\n{str(err)}", email_address,
send_mail)
quit()
# Parse data
spacy_frames: List[Frame] = open_model("spacy_parse", ".")
malt_frames: List[Frame] = parse_malt()
#### runs ####
for x in os.walk("runs/tmp/"):
for folder in x[1]:
directory = x[0] + folder
assert os.path.isdir(directory)
features = {folder[4:]}
now = datetime.now()
readable_time = now.strftime("%H:%M:%S %Y-%m-%d")
# Description of run
data_description = (
f"Testing good guess, one feature at a time. \nlinearSVC. \n{features=}. \n{filter=}. \n{prune_test_data=}. \nTime: {readable_time}\n"
)
f = open(directory + "/run_description.txt", "w")
f.write(data_description)
f.close()
send_email(
directory,
f"New run started: \n{data_description}\n",
email_address,
send_mail,
)
pipeline_2(directory, malt_frames, "malt", features, filter,
prune_test_data, log_data)
send_email(
directory,
f"Pipeline for data parsed with Maltparser compleate.",
email_address,
send_mail,
)
pipeline_2(directory, spacy_frames, "spacy", features, filter,
prune_test_data, log_data)
send_email(
directory,
f"Pipeline for data parsed with spaCy compleate.",
email_address,
send_mail,
)
from skorch.callbacks import EarlyStopping
from torch.nn import MSELoss
from torch.optim import SGD
import numpy as np
import sys
sys.path.append('..')
from helpers import load_data_in_chunks, save_model
from model import Net
from CustomLoss import CustomLoss
(Xs, Ys) = load_data_in_chunks('basic', 'train', chunk_size=5)
Xs = Xs.astype(np.float32)
Ys = Ys.astype(np.float32)
regr = NeuralNet(Net,
max_epochs=10000000000,
batch_size=100,
iterator_train__shuffle=True,
criterion=CustomLoss,
optimizer=SGD,
optimizer__lr=1e-5,
optimizer__momentum=0.9,
optimizer__nesterov=True,
optimizer__dampening=0,
verbose=5,
callbacks=[('early_stop', EarlyStopping())])
regr.fit(Xs, Ys / 5000)
save_model(regr, 'conv-custom')
from sklearn.neural_network import MLPRegressor
import numpy as np
import sys
sys.path.append('..')
from helpers import load_data, save_model
(Xs, Ys) = load_data('train')
regr = MLPRegressor(hidden_layer_sizes=(60, 10), verbose=True, max_iter=1000)
regr.fit(Xs, Ys)
save_model(regr, 'mlp')
def hyperparameter_tuning(self) -> None:
"""
Performs a hyperparameter tuning search (either grid search or randomised search) on the defined parameters and
saves the results in a CSV file for further analysis.
Note: only designed to work with MLP (determined based on initial evaluations).
:return: None.
"""
# Determine scoring metric to use based on dataset.
scoring = str()
if config.dataset == "binary":
scoring = "f1"
elif config.dataset == "multi":
scoring = "f1_weighted"
parameters = dict()
search_alg_str = str()
# Initialise Grid Search.
if config.is_grid_search:
print("Hyperparameter tuning technique chosen: GRID SEARCH")
if config.dataset == "binary":
parameters = {
"hidden_layer_sizes": [(98,), (98, 98), (114,), (114, 114)],
"learning_rate_init": [0.001, 0.03, 0.04, 0.1],
"alpha": [0.0001, 0.26, 0.96]
}
print(parameters)
elif config.dataset == "multi":
parameters = {
"hidden_layer_sizes": [(68,), (68, 68), (100,), (100, 100)],
"learning_rate_init": [0.001, 0.01, 0.1],
"momentum": [0.1, 0.9],
"alpha": [0.0001, 0.1, 0.9]
}
searchCV = GridSearchCV(
param_grid=parameters,
estimator=self.clf,
cv=self.folds,
scoring=scoring
)
search_alg_str = "gs"
# Initialise Randomised Search.
elif config.is_randomised_search:
print("Hyperparameter tuning technique chosen: RANDOMISED SEARCH")
parameters = {
'hidden_layer_sizes': (sp_randint(1, 150)),
'learning_rate_init': sp_uniform(0.001, 1),
'momentum': sp_uniform(0.1, 0.9),
'alpha': sp_uniform(0.0001, 1)
}
searchCV = RandomizedSearchCV(
param_distributions=parameters,
estimator=self.clf,
n_iter=100,
cv=self.folds,
scoring=scoring
)
search_alg_str = "rs"
# Run the search and save results in a CSV file.
gs_results = searchCV.fit(self.X, self.y)
gs_results_df = pd.DataFrame(gs_results.cv_results_)
gs_results_df.to_csv("../results/grid_search/{}_{}_{}.csv".format(config.dataset, config.model, search_alg_str))
# Print the best model found by hyperparameter tuning algorithm for the MLP and save the model in a Pickle file.
final_model = gs_results.best_estimator_
print("\nBest model hyperparameters found by randomised search algorithm:")
print(final_model)
print("Score: {}".format(gs_results.best_score_))
save_model(final_model, config.dataset,
"{}_{}_{}_best_estimator".format(config.dataset, config.model, search_alg_str))
