def test_resampletomean():
# ******* setting up DINTModel
dm = SchemaMatcher(host="localhost", port=8080)
logging.info("Cleaning models from DINT server")
for m in dm.models:
dm.remove_model(m)
logging.info("Cleaning datasets from DINT server")
for ds in dm.datasets:
dm.remove_dataset(ds)
m1 = create_dint_model(dm, "full", "ResampleToMean")
m2 = create_dint_model(dm, "single", "ResampleToMean")
m3 = create_dint_model(dm, "full_chardist", "ResampleToMean")
m4 = create_dint_model(dm, "noheader", "ResampleToMean")
m5 = create_dint_model(dm, "chardistonly", "ResampleToMean")
models = [m1, m2, m3, m4, m5]
loo_experiment = Experiment(
models,
experiment_type="leave_one_out",
description="plain loo",
result_csv=os.path.join('results',
"performance_dint_resampletomean.csv"),
debug_csv=os.path.join("results", "debug_dint_resampletomean.csv"))
loo_experiment.run()
def test_models_holdout():
# ******* setting up DINTModel
dm = SchemaMatcher(host="localhost", port=8080)
logging.info("Cleaning models from DINT server")
for m in dm.models:
dm.remove_model(m)
logging.info("Cleaning datasets from DINT server")
for ds in dm.datasets:
dm.remove_dataset(ds)
m1 = create_dint_model(dm, "full", "NoResampling")
m2 = create_dint_model(dm, "single", "NoResampling")
m3 = create_dint_model(dm, "chardist", "NoResampling")
m4 = create_dint_model(dm, "noheader", "NoResampling")
m5 = create_dint_model(dm, "chardistonly", "NoResampling")
rf_model = NNetModel(['rf@charfreq'],
'rf@charfreq model: no headers',
add_headers=False,
p_header=0,
debug_csv=os.path.join("results",
"debug_nnet_rf_holdout.csv"))
models = [m1, m2, m3, m4, m5, rf_model]
rhold_experiment = Experiment(
models,
experiment_type="repeated_holdout",
description="repeated_holdout_0.5_10",
result_csv=os.path.join('results', "performance_models_holdout.csv"),
debug_csv=os.path.join("results", "debug_holdout.csv"),
holdout=0.5,
num=10)
rhold_experiment.run()
print(new_dataset)
print()
print("This should appear in the main set")
print(dm.datasets)
print()
print("We can look at the dataset properties")
print("filename:", new_dataset.set_filename)
print("id:", new_dataset.id)
print("sample: ", new_dataset.sample)
print("summary:", new_dataset.summary)
print()
print("We can also delete it")
dm.remove_dataset(new_dataset)
print()
print(dm.datasets)
#
# lists all the models...
#
print()
print("Here is a list of all the models on the server:")
print(dm.models)
#
# summary for all models
#
print()