def fit(self):
models_to_fit = list()
self.get_best_parameters()
self.instantiate_parameters()
for model_name, model in models.items():
self.model = model(self)
model_saved = self.model.load()
if model_saved:
continue
else:
models_to_fit.append([model_name, model])
if len(models_to_fit) == 0:
return
self.fit_dataset()
self.save_dataset()
for model_name, model in models_to_fit:
self.model = model(self)
model_saved = self.model.load()
if model_saved:
continue
run = wandb.init(project=self.gs.target,
entity='remissia',
reinit=True,
group=self.gs.tumor_type,
save_code=True)
self.parameters['Model'] = self.model.name
run.config.update(self.parameters)
self.model.fit_test()
scores = self.predict()
run.log(self.insert_section(scores))
self.model.save()
run.finish()
return
def get_models_results(text, model_type, **options):
if model_type == Config.ALL_MODELS_KEY:
current_models = {
key: val
for key, val in models.items() if val is not None
}
else:
current_models = {model_type: models[model_type]}
results = []
for model_name, model in current_models.items():
start = time.time()
audio = model.synthesize(text, **options)
filename = model.save(audio, Config.WAVES_FOLDER)
with open(filename, "rb") as f:
audio_bytes = f.read()
end = time.time()
results.append({
'name': model_name,
'time': round(end - start, 3),
'filename': filename,
'response_audio': audio_bytes
})
return results
def calculate_small_step_sensitivities(trace, models):
def analyses(v):
return flatten_lists(
[calculate_sensitivities(trace, spec) for spec in small_step_chart_specs(v)]
)
return {k: analyses(v) for k, v in models.items()}
def test():
results = {
model_name: {
result_name: try_eval(result)
for result_name, result in eval_model(model, givewell).items()
}
for model_name, model in models.items()
}
for k, v in results.items():
print(k, list(v.values())[0])
def kfold_eval_all_models(X, targets):
kf = KFold(n_splits=3, random_state=42, shuffle=True)
for target in ['DiffMeanHourlyPercent', 'DiffMedianHourlyPercent']:
y = targets
fold_idx = 0
for train_index, test_index in kf.split(X):
X_train, X_test = X.iloc[train_index], X.iloc[test_index]
y_train, y_test = y.iloc[train_index][target].values, y.iloc[
test_index][target].values
for name, model in models.items():
print(name)
model.fit(X_train, y_train)
y_pred = model.predict(X_test)
r2 = r2_score(y_test, y_pred)
mae = mean_absolute_error(y_test, y_pred)
mse = mean_squared_error(y_test, y_pred)
yield target, fold_idx, name, r2, mae, sqrt(mse)
fold_idx += 1
def add_Post(current_user):
if not current_user:
return jsonify(
{'message': 'please login first to perform this operation'})
# data = request.get_json()
data = json.loads(request.form['data'])
new_item = items(name=data['name'],
description=data['description'],
category=data['category'],
location=data['location'],
date=data['date'])
new_item.user_id = current_user.id
if 'category' in data:
if data['category'] == 'found':
imgaeUpload(new_item, current_user)
# return 'hello'
db.session.add(new_item)
db.session.commit()
return jsonify({'message': 'New item added'})
def select_models_(data, target, features_names, target_name):
"""select models based roc metrics using k-fold cross-validation
"""
X = np.array(data)
y = np.array(target)
kf = KFold(n_splits=10, shuffle=True, random_state=0)
#kf = StratifiedKFold(n_splits=5,shuffle=True,random_state=0)
metrics = []
for name, clf in models.items():
aucs = []
fprs = []
tprs = []
for train_index, test_index in kf.split(X, y):
X_train, X_test = X[train_index], X[test_index]
y_train, y_test = y[train_index], y[test_index]
clf.fit(X_train, y_train)
probas = clf.predict_proba(X_test)
fpr, tpr, thresholds = roc_curve(y_test, probas[:, 1])
clf_auc = auc(fpr, tpr)
aucs.append(clf_auc)
fprs.append(fpr)
tprs.append(tpr)
rocplots(fprs, tprs, name)
metrics.append([name, np.mean(aucs), np.std(aucs, ddof=1)])
metrics = sorted(metrics, key=lambda x: (-x[1], -x[2]))
best_model = models[metrics[0][0]]
fp = open("models.metrics.tsv", "w")
head = ["name", "mean_auc", "std_auc"]
line = "\t".join(head) + "\n"
fp.write(line)
for metric in metrics:
metric = [str(i) for i in metric]
line = "\t".join(metric) + "\n"
fp.write(line)
fp.close()
return best_model, models
def register_schemas(only: BaseModel = None) -> None:
for label, model in models.items():
app.schema(label)(model)
def main(params, num_samples, phases, save_plots=False):
"""`params` are inputs to GiveWell's models. `phases` is a set of keywords to control which analysis and plotting steps are actually performed."""
sns.set(style="darkgrid")
model_context = pm.Model()
models_with_params = {
k: prune_inputs_in_model(inline_params(v, bounds_to_log_normal_params(params)))
for k, v in models.items()
}
models_with_variables = {
k: register_model(model_context, v) for k, v in models_with_params.items()
}
with model_context:
trace = pm.sample(num_samples)
results = results_from_models(models_with_variables)
if "uncertainties" in phases:
plot_uncertainties_small_multiples(
trace, results, save_plots, show_distance=False
)
plot_uncertainties_overlaid(trace, results, save_plots)
if "regressions" in phases or "distances" in phases or "small steps" in phases:
angles, taus, footrules = compute_distances(models_with_variables, trace)
trace.add_values({"angle": angles, "tau": taus, "footrule": footrules})
df = pm.trace_to_dataframe(trace)
# Have to add these manually because pymc3 doesn't seem to do so in the line above
df["angle"] = angles
df["tau"] = taus
df["footrule"] = footrules
angle_sensitivities = calculate_distance_sensitivities(
trace, models_with_variables, "angle"
)
ordering = dict(
map(lambda x: (x[1], x[0]), enumerate(angle_sensitivities["names"]))
)
if "regressions" in phases:
plot_big_step_regressions(ordering, models_with_variables, df, save_plots)
if "sensitivities" in phases:
big_step_sensitivities = calculate_big_step_sensitivities(
trace, models_with_variables
)
for charity, sensitivities in big_step_sensitivities.items():
def namer(sensitivity_type):
return (
"plots/sensitivity-big-" + charity + "-" + sensitivity_type + ".png"
)
plot_sensitivities(namer, sensitivities, save_plots)
print(charity, "sum of deltas", sum(sensitivities["delta"]))
if "distances" in phases:
plot_uncertainties_small_multiples(
trace, results, save_plots, show_distance=True
)
plot_distance_regressions(
ordering, models_with_variables, df, "angle", save_plots
)
plot_distance_regressions(
ordering, models_with_variables, df, "tau", save_plots
)
plot_distance_regressions(
ordering, models_with_variables, df, "footrule", save_plots
)
def namer(outcome):
return (
lambda sensitivity_type: "plots/sensitivity-max-"
+ outcome
+ "-"
+ sensitivity_type
+ ".png"
)
plot_sensitivities(
namer("angle"), angle_sensitivities, save_plots, should_trim_prefix=False
)
tau_sensitivities = calculate_distance_sensitivities(
trace, models_with_variables, "tau"
)
plot_sensitivities(
namer("tau"), tau_sensitivities, save_plots, should_trim_prefix=False
)
footrule_sensitivities = calculate_distance_sensitivities(
trace, models_with_variables, "footrule"
)
plot_sensitivities(
namer("footrule"),
footrule_sensitivities,
save_plots,
should_trim_prefix=False,
)
tau_ranking = tau_sensitivities["names"]
footrule_ranking = footrule_sensitivities["names"]
angle_ranking = angle_sensitivities["names"]
print("tau-footrule by tau", kendall_tau(tau_ranking, footrule_ranking))
print(
"tau-footrule by footrule", spearman_footrule(tau_ranking, footrule_ranking)
)
print("tau-angle by tau", kendall_tau(tau_ranking, angle_ranking))
print("tau-angle by footrule", spearman_footrule(tau_ranking, angle_ranking))
print("footrule-angle by tau", kendall_tau(footrule_ranking, angle_ranking))
print(
"footrule-angle by footrule",
spearman_footrule(footrule_ranking, angle_ranking),
)
if "small steps" in phases:
small_step_sensitivities = calculate_small_step_sensitivities(
trace, models_with_variables
)
plot_small_step_regressions(ordering, models_with_variables, df, save_plots)
for charity, sensitivities in small_step_sensitivities.items():
for i, s in enumerate(sensitivities):
def namer(sensitivity_type):
return (
"plots/sensitivity-small-"
+ charity
+ "-"
+ str(i)
+ "-"
+ sensitivity_type
+ ".png"
)
plot_sensitivities(namer, s, save_plots)
def calculate_big_step_sensitivities(trace, models):
return {
k: calculate_sensitivities(trace, big_step_chart_spec(v))[0]
for k, v in models.items()
}
