def test_scenario24(self):
"""
Scenario: Successfully test predictions remotely from boosted ensemble
Given I have previously executed "<scenario>" or reproduce it with arguments <kwargs>
And I create BigML resources using boosted ensemble in <iterations> iterations to remotely test "<test>" and log predictions in "<output>"
And I check that the ensemble has been created
And I check that the predictions are ready
And I check that the batch prediction is ready
And I check that the bath predictions datset is ready
Then the local prediction file is like "<predictions_file>"
Examples:
|scenario | kwargs | iterations | test | output |predictions_file |
"""
examples = [
['scenario1', '{"data": "data/iris.csv", "output": "scenario1/predictions.csv", "test": "data/test_iris.csv"}', '10', 'data/test_iris.csv', 'scenario24/predictions.csv', 'check_files/predictions_iris_boost.csv']]
show_doc(self.test_scenario24, examples)
for example in examples:
print "\nTesting with:\n", example
test_pred.i_have_previous_scenario_or_reproduce_it( \
self, example[0], example[1])
test_pred.i_create_resources_remotely_from_boosted_ensemble( \
self, iterations=example[2], test=example[3],
output=example[4])
test_pred.i_check_create_ensemble(self)
test_pred.i_check_create_batch_prediction(self)
test_pred.i_check_create_predictions(self)
test_pred.i_check_predictions(self, example[5])
def test_scenario21(self):
"""
Scenario 1: Successfully building test predictions from ensemble
And I create BigML resources from "<data>" using ensemble of <number_of_models> models to test "<test>" and log predictions in "<output>"
And I check that the source has been created
And I check that the dataset has been created
And I check that the ensemble has been created
And I check that the models in the ensembles have been created
And I check that the predictions are ready
Then the local prediction file is like "<predictions_file>"
Examples:
|data | number_of_models | test | output | predictions_file |
"""
examples = [
['data/grades.csv', '5', 'data/test_grades.csv', 'scenario21/predictions.csv', 'check_files/predictions_grades_e.csv']]
show_doc(self.test_scenario21, examples)
for example in examples:
print "\nTesting with:\n", example
test_pred.i_create_resources_in_prod_from_ensemble( \
self, data=example[0], number_of_models=example[1],
test=example[2], output=example[3])
test_pred.i_check_create_source(self)
test_pred.i_check_create_dataset(self, suffix=None)
test_pred.i_check_create_ensemble(self)
test_pred.i_check_create_models_in_ensembles(self,
in_ensemble=True)
test_pred.i_check_create_predictions(self)
test_pred.i_check_predictions(self, example[4])
def test_scenario22(self):
"""
Scenario: Successfully building threshold test predictions remotely from ensemble
Given I have previously executed "<scenario>" or reproduce it with arguments <kwargs>
And I create BigML resources using ensemble of <number_of_models> models with replacement to test "<test>" and log predictions in "<output>"
And I check that the ensemble has been created
And I check that the predictions are ready
And I create BigML resources using the previous ensemble with different thresholds "<threshold_class>" to remotely test "<test>" and log predictions in "<output2>" and "<output3>"
Then local predictions for different thresholds in "<output2>" and "<output3>" are different
Examples:
|scenario | kwargs | number_of_models | test | output | output2 | output3 | threshold_class
"""
examples = [
['scenario1', '{"data": "data/iris.csv", "output": "scenario1/predictions.csv", "test": "data/test_iris.csv"}', '10', 'data/test_iris.csv', 'scenario22/predictions.csv', 'scenario22/predictions2.csv', 'scenario22/predictions3.csv', 'Iris-virginica']]
show_doc(self.test_scenario22, examples)
for example in examples:
print "\nTesting with:\n", example
test_pred.i_have_previous_scenario_or_reproduce_it( \
self, example[0], example[1])
test_pred.i_create_resources_from_ensemble_with_replacement( \
self, number_of_models=example[2], test=example[3],
output=example[4])
test_pred.i_check_create_ensemble(self)
test_pred.i_check_create_predictions(self)
test_pred.i_create_resources_from_ensemble_with_threshold_rem( \
self, test=example[3], output2=example[5], output3=example[6],
threshold_class=example[7])
test_pred.i_check_predictions_with_different_thresholds( \
self, example[5], example[6])
def setup_scenario06(self):
"""
Scenario: Successfully building test predictions from ensemble
Given I have previously executed "<scenario>" or reproduce it with arguments <kwargs>
And I create BigML resources using ensemble of <number_of_models> models to test "<test>" and log predictions in "<output>"
And I check that the ensemble has been created
And I check that the predictions are ready
Then the local prediction file is like "<predictions_file>"
Examples:
|scenario | kwargs | number_of_models | test | output |predictions_file |
| scenario1| {"data": "../data/iris.csv", "output": "./scenario1/predictions.csv", "test": "../data/test_iris.csv"} | 10 | ../data/test_iris.csv | ./scenario5/predictions.csv | ./check_files/predictions_iris.csv |
"""
print self.setup_scenario06.__doc__
examples = [[
'scenario1',
'{"data": "data/iris.csv", "output": "scenario1/predictions.csv", "test": "data/test_iris.csv"}',
'10', 'data/test_iris.csv', 'scenario5/predictions.csv',
'check_files/predictions_iris.csv'
]]
for example in examples:
print "\nTesting with:\n", example
test_pred.i_have_previous_scenario_or_reproduce_it(
self, example[0], example[1])
test_pred.i_create_resources_from_ensemble(
self,
number_of_models=example[2],
test=example[3],
output=example[4])
test_pred.i_check_create_ensemble(self)
test_pred.i_check_create_predictions(self)
test_pred.i_check_predictions(self, example[5])
def test_scenario7(self):
"""
Scenario: Successfully building ensemble evaluations from start and test-split:
Given I create BigML resources uploading train "<data>" file to evaluate an ensemble of <number_of_models> models with test-split <split> and log evaluation in "<output>"
And I check that the source has been created
And I check that the dataset has been created
And I check that the train dataset has been created
And I check that the test dataset has been created
And I check that the ensemble has been created
And I check that the evaluation has been created
Then the evaluation key "<key>" value for the model is greater than <value>
And I evaluate the ensemble in directory "<directory>" with the dataset in directory "<directory>" and log evaluation in "<output2>"
And I check that the evaluation has been created
Then the evaluation key "<key>" value for the model is greater than <value>
Examples:
| data | output | split | number_of_models | key | value | directory | output2
| ../data/iris.csv | ./scenario_e8/evaluation | 0.2 | 5 | average_phi | 0.94 | ./scenario_e8/ | ./scenario_e9/evaluation
"""
print self.test_scenario7.__doc__
examples = [
['data/iris.csv', 'scenario_e8/evaluation', '0.2', '5', 'average_phi', '0.94', 'scenario_e8', 'scenario_e9/evaluation']]
for example in examples:
print "\nTesting with:\n", example
evaluation.i_create_with_split_to_evaluate_ensemble(self, data=example[0], number_of_models=example[3], split=example[2], output=example[1])
test_pred.i_check_create_source(self)
test_pred.i_check_create_dataset(self, suffix=None)
test_pred.i_check_create_dataset(self, suffix='train ')
test_pred.i_check_create_dataset(self, suffix='test ')
test_pred.i_check_create_ensemble(self)
test_pred.i_check_create_evaluation(self)
evaluation.i_check_evaluation_key(self, key=example[4], value=example[5])
evaluation.i_evaluate_ensemble_with_dataset(self, ensemble_dir=example[6], dataset_dir=example[6], output=example[7])
test_pred.i_check_create_evaluation(self)
evaluation.i_check_evaluation_key(self, key=example[4], value=example[5])
def test_scenario24(self):
"""
Scenario: Successfully test predictions remotely from boosted ensemble
Given I have previously executed "<scenario>" or reproduce it with arguments <kwargs>
And I create BigML resources using boosted ensemble in <iterations> iterations to remotely test "<test>" and log predictions in "<output>"
And I check that the ensemble has been created
And I check that the predictions are ready
And I check that the batch prediction is ready
And I check that the bath predictions datset is ready
Then the local prediction file is like "<predictions_file>"
Examples:
|scenario | kwargs | iterations | test | output |predictions_file |
"""
examples = [[
'scenario1',
'{"data": "data/iris.csv", "output": "scenario1/predictions.csv", "test": "data/test_iris.csv"}',
'10', 'data/test_iris.csv', 'scenario24/predictions.csv',
'check_files/predictions_iris_boost.csv'
]]
show_doc(self.test_scenario24, examples)
for example in examples:
print "\nTesting with:\n", example
test_pred.i_have_previous_scenario_or_reproduce_it( \
self, example[0], example[1])
test_pred.i_create_resources_remotely_from_boosted_ensemble( \
self, iterations=example[2], test=example[3],
output=example[4])
test_pred.i_check_create_ensemble(self)
test_pred.i_check_create_batch_prediction(self)
test_pred.i_check_create_predictions(self)
test_pred.i_check_predictions(self, example[5])
def test_scenario21(self):
"""
Scenario 1: Successfully building test predictions from ensemble
And I create BigML resources from "<data>" using ensemble of <number_of_models> models to test "<test>" and log predictions in "<output>"
And I check that the source has been created
And I check that the dataset has been created
And I check that the ensemble has been created
And I check that the models in the ensembles have been created
And I check that the predictions are ready
Then the local prediction file is like "<predictions_file>"
Examples:
|data | number_of_models | test | output | predictions_file |
"""
examples = [[
'data/grades.csv', '5', 'data/test_grades.csv',
'scenario21/predictions.csv',
'check_files/predictions_grades_e.csv'
]]
show_doc(self.test_scenario21, examples)
for example in examples:
print "\nTesting with:\n", example
test_pred.i_create_resources_in_prod_from_ensemble( \
self, data=example[0], number_of_models=example[1],
test=example[2], output=example[3])
test_pred.i_check_create_source(self)
test_pred.i_check_create_dataset(self, suffix=None)
test_pred.i_check_create_ensemble(self)
test_pred.i_check_create_models_in_ensembles(self,
in_ensemble=True)
test_pred.i_check_create_predictions(self)
test_pred.i_check_predictions(self, example[4])
def test_scenario16(self):
"""
Scenario: Successfully building threshold test predictions from ensemble
Given I have previously executed "<scenario>" or reproduce it with arguments <kwargs>
And I create BigML resources using ensemble of <number_of_models> models with replacement to test "<test>" and log predictions in "<output>"
And I check that the ensemble has been created
And I check that the predictions are ready
And I create BigML resources using the previous ensemble with different thresholds to test "<test>" and log predictions in "<output2>" and "<output3>"
Then local predictions for different thresholds in "<output2>" and "<output3>" are different
Examples:
|scenario | kwargs | number_of_models | test | output | output2 | output3
"""
examples = [[
'scenario1',
'{"data": "data/iris.csv", "output": "scenario1/predictions.csv", "test": "data/test_iris.csv"}',
'10', 'data/test_iris.csv', 'scenario16/predictions.csv',
'scenario16/predictions2.csv', 'scenario16/predictions3.csv'
]]
show_doc(self.test_scenario16, examples)
for example in examples:
print "\nTesting with:\n", example
test_pred.i_have_previous_scenario_or_reproduce_it(
self, example[0], example[1])
test_pred.i_create_resources_from_ensemble_with_replacement(
self,
number_of_models=example[2],
test=example[3],
output=example[4])
test_pred.i_check_create_ensemble(self)
test_pred.i_check_create_predictions(self)
test_pred.i_create_resources_from_ensemble_with_threshold(
self, test=example[3], output2=example[5], output3=example[6])
test_pred.i_check_predictions_with_different_thresholds(
self, example[5], example[6])
def test_scenario7(self):
"""
Scenario: Successfully building ensemble evaluations from start and test-split:
Given I create BigML resources uploading train "<data>" file to evaluate an ensemble of <number_of_models> models with test-split <split> and log evaluation in "<output>"
And I check that the source has been created
And I check that the dataset has been created
And I check that the train dataset has been created
And I check that the test dataset has been created
And I check that the ensemble has been created
And I check that the evaluation has been created
Then the evaluation key "<key>" value for the model is greater than <value>
And I evaluate the ensemble in directory "<directory>" with the dataset in directory "<directory>" and log evaluation in "<output2>"
And I check that the evaluation has been created
Then the evaluation key "<key>" value for the model is greater than <value>
Examples:
| data | output | split | number_of_models | key | value | directory | output2
| ../data/iris.csv | ./scenario_e8/evaluation | 0.2 | 5 | average_phi | 0.94 | ./scenario_e8/ | ./scenario_e9/evaluation
"""
print self.test_scenario7.__doc__
examples = [[
'data/iris.csv', 'scenario_e8/evaluation', '0.2', '5',
'average_phi', '0.94', 'scenario_e8', 'scenario_e9/evaluation'
]]
for example in examples:
print "\nTesting with:\n", example
evaluation.i_create_with_split_to_evaluate_ensemble(
self,
data=example[0],
number_of_models=example[3],
split=example[2],
output=example[1])
test_pred.i_check_create_source(self)
test_pred.i_check_create_dataset(self, suffix=None)
test_pred.i_check_create_dataset(self, suffix='train ')
test_pred.i_check_create_dataset(self, suffix='test ')
test_pred.i_check_create_ensemble(self)
test_pred.i_check_create_evaluation(self)
evaluation.i_check_evaluation_key(self,
key=example[4],
value=example[5])
evaluation.i_evaluate_ensemble_with_dataset(
self,
ensemble_dir=example[6],
dataset_dir=example[6],
output=example[7])
test_pred.i_check_create_evaluation(self)
evaluation.i_check_evaluation_key(self,
key=example[4],
value=example[5])
def setup_scenario06(self):
"""
Scenario: Successfully building test predictions from ensemble
Given I have previously executed "<scenario>" or reproduce it with arguments <kwargs>
And I create BigML resources using ensemble of <number_of_models> models to test "<test>" and log predictions in "<output>"
And I check that the ensemble has been created
And I check that the predictions are ready
Then the local prediction file is like "<predictions_file>"
Examples:
|scenario | kwargs | number_of_models | test | output |predictions_file |
"""
examples = [
['scenario1', '{"data": "data/iris.csv", "output": "scenario1/predictions.csv", "test": "data/test_iris.csv"}', '10', 'data/test_iris.csv', 'scenario5/predictions.csv', 'check_files/predictions_iris.csv']]
show_doc(self.setup_scenario06, examples)
for example in examples:
print "\nTesting with:\n", example
test_pred.i_have_previous_scenario_or_reproduce_it(self, example[0], example[1])
test_pred.i_create_resources_from_ensemble(self, number_of_models=example[2], test=example[3], output=example[4])
test_pred.i_check_create_ensemble(self)
test_pred.i_check_create_predictions(self)
test_pred.i_check_predictions(self, example[5])
def test_scenario1(self):
"""
Scenario 1: Successfully building test predictions from ensemble
Given I want to use api in DEV mode
And I create BigML resources in DEV from "<data>" using ensemble of <number_of_models> models to test "<test>" and log predictions in "<output>"
And I check that the source has been created
And I check that the dataset has been created
And I check that the ensemble has been created
And I check that the models in the ensembles have been created
And I check that the predictions are ready
Then the local prediction file is like "<predictions_file>"
Examples:
|data | number_of_models | test | output | predictions_file |
| ../data/grades.csv| 5 | ../data/test_grades.csv | ./scenario1_dev/predictions.csv | ./check_files/predictions_grades_e.csv |
"""
print self.test_scenario1.__doc__
examples = [
[
"data/grades.csv",
"5",
"data/test_grades.csv",
"scenario1_dev/predictions.csv",
"check_files/predictions_grades_e.csv",
]
]
for example in examples:
print "\nTesting with:\n", example
common.i_want_api_dev_mode(self)
test_pred.i_create_resources_in_dev_from_ensemble(
self, data=example[0], number_of_models=example[1], test=example[2], output=example[3]
)
test_pred.i_check_create_source(self)
test_pred.i_check_create_dataset(self, suffix=None)
test_pred.i_check_create_ensemble(self)
test_pred.i_check_create_models_in_ensembles(self, in_ensemble=True)
test_pred.i_check_create_predictions(self)
test_pred.i_check_predictions(self, example[4])
