def test_scenario5(self):
"""
Scenario: Successfully building multi-label evaluations from models retrieved by tag
Given I have previously executed "<scenario>" or reproduce it with arguments <kwargs>
And I create BigML multi-label resources using models tagged as "<tag>" to evaluate and log evaluation in "<output>"
And I check that the <number_of_labels> evaluations have been created
And I check that the evaluation is ready
Then the evaluation key "<key>" value for the model is greater than <value>
Examples:
|scenario | kwargs | tag | number_of_labels | output |key | value
| scenario_ml_e1| {"tag": "my_multilabel_e_1", "data": "../data/multilabel.csv", "label_separator": ":", "number_of_labels": 7, "training_separator": ",", "output": "./scenario_ml_e1/evaluation"} | my_multilabel_e_1 | 7 | ./scenario_ml_e5/evaluation | average_phi | 0.8180
"""
print self.test_scenario5.__doc__
examples = [[
'scenario_ml_e1',
'{"tag": "my_multilabel_e_1", "data": "data/multilabel.csv", "label_separator": ":", "number_of_labels": 7, "training_separator": ",", "output": "scenario_ml_e1/evaluation"}',
'my_multilabel_e_1', '7', 'scenario_ml_e5/evaluation',
'average_phi', '0.8180'
]]
for example in examples:
print "\nTesting with:\n", example
test_pred.i_have_previous_scenario_or_reproduce_it(
self, example[0], example[1])
ml_eval.i_create_ml_evaluations_from_tagged_models(
self, tag=example[2], output=example[4])
test_pred.i_check_create_evaluations(
self, number_of_evaluations=example[3])
ml_eval.i_check_evaluation_ready(self)
evaluation.i_check_evaluation_key(self,
key=example[5],
value=example[6])
def test_scenario7(self):
"""
Scenario: Successfully building ensemble evaluations from start and test-split:
Given I create BigML resources uploading train "<data>" file to evaluate an ensemble of <number_of_models> models with test-split <split> and log evaluation in "<output>"
And I check that the source has been created
And I check that the dataset has been created
And I check that the train dataset has been created
And I check that the test dataset has been created
And I check that the ensemble has been created
And I check that the evaluation has been created
Then the evaluation key "<key>" value for the model is greater than <value>
And I evaluate the ensemble in directory "<directory>" with the dataset in directory "<directory>" and log evaluation in "<output2>"
And I check that the evaluation has been created
Then the evaluation key "<key>" value for the model is greater than <value>
Examples:
| data | output | split | number_of_models | key | value | directory | output2
| ../data/iris.csv | ./scenario_e8/evaluation | 0.2 | 5 | average_phi | 0.94 | ./scenario_e8/ | ./scenario_e9/evaluation
"""
print self.test_scenario7.__doc__
examples = [
['data/iris.csv', 'scenario_e8/evaluation', '0.2', '5', 'average_phi', '0.94', 'scenario_e8', 'scenario_e9/evaluation']]
for example in examples:
print "\nTesting with:\n", example
evaluation.i_create_with_split_to_evaluate_ensemble(self, data=example[0], number_of_models=example[3], split=example[2], output=example[1])
test_pred.i_check_create_source(self)
test_pred.i_check_create_dataset(self, suffix=None)
test_pred.i_check_create_dataset(self, suffix='train ')
test_pred.i_check_create_dataset(self, suffix='test ')
test_pred.i_check_create_ensemble(self)
test_pred.i_check_create_evaluation(self)
evaluation.i_check_evaluation_key(self, key=example[4], value=example[5])
evaluation.i_evaluate_ensemble_with_dataset(self, ensemble_dir=example[6], dataset_dir=example[6], output=example[7])
test_pred.i_check_create_evaluation(self)
evaluation.i_check_evaluation_key(self, key=example[4], value=example[5])
def test_scenario6(self):
"""
Scenario: Successfully building evaluations from start and test-split:
Given I create BigML resources uploading train "<data>" file to evaluate with test-split <split> and log evaluation in "<output>"
And I check that the source has been created
And I check that the dataset has been created
And I check that the train dataset has been created
And I check that the test dataset has been created
And I check that the model has been created
And I check that the evaluation has been created
Then the evaluation key "<key>" value for the model is greater than <value>
Examples:
| data | output | split | key | value |
| ../data/iris.csv | ./scenario_e6/evaluation | 0.2 | average_phi | 0.85 |
"""
print self.test_scenario6.__doc__
examples = [
['data/iris.csv', 'scenario_e6/evaluation', '0.2', 'average_phi', '0.85']]
for example in examples:
print "\nTesting with:\n", example
evaluation.i_create_with_split_to_evaluate(self, data=example[0], split=example[2], output=example[1])
test_pred.i_check_create_source(self)
test_pred.i_check_create_dataset(self, suffix=None)
test_pred.i_check_create_dataset(self, suffix='train ')
test_pred.i_check_create_dataset(self, suffix='test ')
test_pred.i_check_create_model(self)
test_pred.i_check_create_evaluation(self)
evaluation.i_check_evaluation_key(self, key=example[3], value=example[4])
def test_scenario7(self):
"""
Scenario: Successfully building ensemble evaluations from start and test-split:
Given I create BigML resources uploading train "<data>" file to evaluate an ensemble of <number_of_models> models with test-split <split> and log evaluation in "<output>"
And I check that the source has been created
And I check that the dataset has been created
And I check that the train dataset has been created
And I check that the test dataset has been created
And I check that the ensemble has been created
And I check that the evaluation has been created
Then the evaluation key "<key>" value for the model is greater than <value>
And I evaluate the ensemble in directory "<directory>" with the dataset in directory "<directory>" and log evaluation in "<output2>"
And I check that the evaluation has been created
Then the evaluation key "<key>" value for the model is greater than <value>
Examples:
| data | output | split | number_of_models | key | value | directory | output2
| ../data/iris.csv | ./scenario_e8/evaluation | 0.2 | 5 | average_phi | 0.94 | ./scenario_e8/ | ./scenario_e9/evaluation
"""
print self.test_scenario7.__doc__
examples = [[
'data/iris.csv', 'scenario_e8/evaluation', '0.2', '5',
'average_phi', '0.94', 'scenario_e8', 'scenario_e9/evaluation'
]]
for example in examples:
print "\nTesting with:\n", example
evaluation.i_create_with_split_to_evaluate_ensemble(
self,
data=example[0],
number_of_models=example[3],
split=example[2],
output=example[1])
test_pred.i_check_create_source(self)
test_pred.i_check_create_dataset(self, suffix=None)
test_pred.i_check_create_dataset(self, suffix='train ')
test_pred.i_check_create_dataset(self, suffix='test ')
test_pred.i_check_create_ensemble(self)
test_pred.i_check_create_evaluation(self)
evaluation.i_check_evaluation_key(self,
key=example[4],
value=example[5])
evaluation.i_evaluate_ensemble_with_dataset(
self,
ensemble_dir=example[6],
dataset_dir=example[6],
output=example[7])
test_pred.i_check_create_evaluation(self)
evaluation.i_check_evaluation_key(self,
key=example[4],
value=example[5])
def test_scenario4(self):
"""
Scenario: Successfully building multi-label evaluations from models file
Given I have previously executed "<scenario>" or reproduce it with arguments <kwargs>
And I create BigML multi-label resources using models in file "<models_file>" to evaluate and log evaluation in "<output>"
And I check that the <number_of_labels> evaluations have been created
And I check that the evaluation is ready
Then the evaluation key "<key>" value for the model is greater than <value>
Examples:
|scenario | kwargs | models_file | number_of_labels | output |key | value
| scenario_ml_e1| {"tag": "my_multilabel_e_1", "data": "../data/multilabel.csv", "label_separator": ":", "number_of_labels": 7, "training_separator": ",", "output": "./scenario_ml_e1/evaluation"} | ./scenario_ml_e1/models | 7 | ./scenario_ml_e4/evaluation | average_phi | 0.8180
"""
print self.test_scenario4.__doc__
examples = [
['scenario_ml_e1', '{"tag": "my_multilabel_e_1", "data": "data/multilabel.csv", "label_separator": ":", "number_of_labels": 7, "training_separator": ",", "output": "scenario_ml_e1/evaluation"}', 'scenario_ml_e1/models', '7', 'scenario_ml_e4/evaluation', 'average_phi', '0.8180']]
for example in examples:
print "\nTesting with:\n", example
test_pred.i_have_previous_scenario_or_reproduce_it(self, example[0], example[1])
ml_eval.i_create_ml_evaluations_from_models(self, models_file=example[2], output=example[4])
test_pred.i_check_create_evaluations(self, number_of_evaluations=example[3])
ml_eval.i_check_evaluation_ready(self)
evaluation.i_check_evaluation_key(self, key=example[5], value=example[6])
def test_scenario6(self):
"""
Scenario: Successfully building evaluations from start and test-split:
Given I create BigML resources uploading train "<data>" file to evaluate with test-split <split> and log evaluation in "<output>"
And I check that the source has been created
And I check that the dataset has been created
And I check that the train dataset has been created
And I check that the test dataset has been created
And I check that the model has been created
And I check that the evaluation has been created
Then the evaluation key "<key>" value for the model is greater than <value>
Examples:
| data | output | split | key | value |
| ../data/iris.csv | ./scenario_e6/evaluation | 0.2 | average_phi | 0.85 |
"""
print self.test_scenario6.__doc__
examples = [[
'data/iris.csv', 'scenario_e6/evaluation', '0.2', 'average_phi',
'0.85'
]]
for example in examples:
print "\nTesting with:\n", example
evaluation.i_create_with_split_to_evaluate(self,
data=example[0],
split=example[2],
output=example[1])
test_pred.i_check_create_source(self)
test_pred.i_check_create_dataset(self, suffix=None)
test_pred.i_check_create_dataset(self, suffix='train ')
test_pred.i_check_create_dataset(self, suffix='test ')
test_pred.i_check_create_model(self)
test_pred.i_check_create_evaluation(self)
evaluation.i_check_evaluation_key(self,
key=example[3],
value=example[4])
