def test_scenario3(self): """ Scenario: Successfully building evaluations from start: Given I create BigML resources uploading train "<data>" file to create model and log in "<output>" And I check that the source has been created And I check that the dataset has been created And I check that the model has been created And I evaluate "<test>" with proportional missing strategy And I check that the source has been created And I check that the dataset has been created And I check that the evaluation has been created Then the evaluation file is like "<json_evaluation_file>" Examples: | data | test | output | json_evaluation_file | | ../data/iris.csv | ../data/iris_nulls.csv | ./scenario_mis_3/evaluation | ./check_files/evaluation_iris_nulls.json | """ print self.test_scenario3.__doc__ examples = [ ['data/iris.csv', 'data/iris_nulls.csv', 'scenario_mis_3/evaluation', 'check_files/evaluation_iris_nulls.json']] for example in examples: print "\nTesting with:\n", example test_pred.i_create_all_resources_to_model(self, data=example[0], output=example[2]) test_pred.i_check_create_source(self) test_pred.i_check_create_dataset(self, suffix=None) test_pred.i_check_create_model(self) evaluation.i_create_proportional_to_evaluate(self, test=example[1]) test_pred.i_check_create_source(self) test_pred.i_check_create_dataset(self) test_pred.i_check_create_evaluation(self) evaluation.then_the_evaluation_file_is_like(self, example[3])
def setup_scenario1(self): """ Scenario: Successfully building evaluations from start: Given I create BigML resources uploading train "<data>" file to evaluate and log evaluation in "<output>" And I check that the source has been created And I check that the dataset has been created And I check that the model has been created And I check that the evaluation has been created Then the evaluation file is like "<json_evaluation_file>" Examples: | data | output | json_evaluation_file | | ../data/iris.csv | ./scenario_e1/evaluation | ./check_files/evaluation_iris.json | """ print self.setup_scenario1.__doc__ examples = [[ 'data/iris.csv', 'scenario_e1/evaluation', 'check_files/evaluation_iris.json' ]] for example in examples: print "\nTesting with:\n", example test_pred.i_create_all_resources_to_evaluate(self, data=example[0], output=example[1]) test_pred.i_check_create_source(self) test_pred.i_check_create_dataset(self, suffix=None) test_pred.i_check_create_model(self) test_pred.i_check_create_evaluation(self) evaluation.then_the_evaluation_file_is_like(self, example[2])
def test_scenario2(self): """ Scenario: Successfully building evaluations from source Given I have previously executed "<scenario>" or reproduce it with arguments <kwargs> And I create BigML resources using source to evaluate and log evaluation in "<output>" And I check that the dataset has been created And I check that the model has been created And I check that the evaluation has been created Then the evaluation file is like "<json_evaluation_file>" Examples: |scenario | kwargs | output | json_evaluation_file | | scenario_e1| {"data": "../data/iris.csv", "output": "./scenario_e1/predictions.csv"} |./scenario_e2/evaluation | ./check_files/evaluation_iris.json | """ print self.test_scenario2.__doc__ examples = [[ 'scenario_e1', '{"data": "data/iris.csv", "output": "scenario_e1/predictions.csv"}', 'scenario_e2/evaluation', 'check_files/evaluation_iris.json' ]] for example in examples: print "\nTesting with:\n", example test_pred.i_have_previous_scenario_or_reproduce_it( self, example[0], example[1]) evaluation.given_i_create_bigml_resources_using_source_to_evaluate( self, output=example[2]) test_pred.i_check_create_dataset(self, suffix=None) test_pred.i_check_create_model(self) test_pred.i_check_create_evaluation(self) evaluation.then_the_evaluation_file_is_like(self, example[3])
def test_scenario2(self): """ Scenario: Successfully building evaluations from source Given I have previously executed "<scenario>" or reproduce it with arguments <kwargs> And I create BigML resources using source to evaluate and log evaluation in "<output>" And I check that the dataset has been created And I check that the model has been created And I check that the evaluation has been created Then the evaluation file is like "<json_evaluation_file>" Examples: |scenario | kwargs | output | json_evaluation_file | | scenario_e1| {"data": "../data/iris.csv", "output": "./scenario_e1/predictions.csv"} |./scenario_e2/evaluation | ./check_files/evaluation_iris.json | """ print self.test_scenario2.__doc__ examples = [ ['scenario_e1', '{"data": "data/iris.csv", "output": "scenario_e1/predictions.csv"}', 'scenario_e2/evaluation', 'check_files/evaluation_iris.json']] for example in examples: print "\nTesting with:\n", example test_pred.i_have_previous_scenario_or_reproduce_it(self, example[0], example[1]) evaluation.given_i_create_bigml_resources_using_source_to_evaluate(self, output=example[2]) test_pred.i_check_create_dataset(self, suffix=None) test_pred.i_check_create_model(self) test_pred.i_check_create_evaluation(self) evaluation.then_the_evaluation_file_is_like(self, example[3])
def test_scenario5(self): """ Scenario: Successfully building evaluation from model and test file with data map Given I have previously executed "<scenario>" or reproduce it with arguments <kwargs> And I create BigML resources using test file "<test>" and a fields map "<fields_map>" to evaluate a model and log evaluation in "<output>" And I check that the source has been created And I check that the dataset has been created And I check that the evaluation has been created Then the evaluation file is like "<json_evaluation_file>" Examples: |scenario | kwargs | test | fields_map | output | json_evaluation_file | | scenario_e1| {"data": "../data/iris.csv", "output": "./scenario_e1/predictions.csv"} | ../data/iris_permuted.csv | ../data/fields_map.csv | ./scenario_e7/evaluation | ./check_files/evaluation_iris2.json | """ print self.test_scenario5.__doc__ examples = [[ 'scenario_e1', '{"data": "data/iris.csv", "output": "scenario_e1/predictions.csv"}', 'data/iris_permuted.csv', 'data/fields_map.csv', 'scenario_e7/evaluation', 'check_files/evaluation_iris2.json' ]] for example in examples: print "\nTesting with:\n", example test_pred.i_have_previous_scenario_or_reproduce_it( self, example[0], example[1]) evaluation.i_create_all_resources_to_evaluate_with_model_and_map( self, data=example[2], fields_map=example[3], output=example[4]) test_pred.i_check_create_source(self) test_pred.i_check_create_dataset(self, suffix=None) test_pred.i_check_create_evaluation(self) evaluation.then_the_evaluation_file_is_like(self, example[5])
def test_scenario7(self): """ Scenario: Successfully building ensemble evaluations from start and test-split: Given I create BigML resources uploading train "<data>" file to evaluate an ensemble of <number_of_models> models with test-split <split> and log evaluation in "<output>" And I check that the source has been created And I check that the dataset has been created And I check that the train dataset has been created And I check that the test dataset has been created And I check that the ensemble has been created And I check that the evaluation has been created Then the evaluation key "<key>" value for the model is greater than <value> And I evaluate the ensemble in directory "<directory>" with the dataset in directory "<directory>" and log evaluation in "<output2>" And I check that the evaluation has been created Then the evaluation key "<key>" value for the model is greater than <value> Examples: | data | output | split | number_of_models | key | value | directory | output2 | ../data/iris.csv | ./scenario_e8/evaluation | 0.2 | 5 | average_phi | 0.94 | ./scenario_e8/ | ./scenario_e9/evaluation """ print self.test_scenario7.__doc__ examples = [ ['data/iris.csv', 'scenario_e8/evaluation', '0.2', '5', 'average_phi', '0.94', 'scenario_e8', 'scenario_e9/evaluation']] for example in examples: print "\nTesting with:\n", example evaluation.i_create_with_split_to_evaluate_ensemble(self, data=example[0], number_of_models=example[3], split=example[2], output=example[1]) test_pred.i_check_create_source(self) test_pred.i_check_create_dataset(self, suffix=None) test_pred.i_check_create_dataset(self, suffix='train ') test_pred.i_check_create_dataset(self, suffix='test ') test_pred.i_check_create_ensemble(self) test_pred.i_check_create_evaluation(self) evaluation.i_check_evaluation_key(self, key=example[4], value=example[5]) evaluation.i_evaluate_ensemble_with_dataset(self, ensemble_dir=example[6], dataset_dir=example[6], output=example[7]) test_pred.i_check_create_evaluation(self) evaluation.i_check_evaluation_key(self, key=example[4], value=example[5])
def test_scenario6(self): """ Scenario: Successfully building evaluations from start and test-split: Given I create BigML resources uploading train "<data>" file to evaluate with test-split <split> and log evaluation in "<output>" And I check that the source has been created And I check that the dataset has been created And I check that the train dataset has been created And I check that the test dataset has been created And I check that the model has been created And I check that the evaluation has been created Then the evaluation key "<key>" value for the model is greater than <value> Examples: | data | output | split | key | value | | ../data/iris.csv | ./scenario_e6/evaluation | 0.2 | average_phi | 0.85 | """ print self.test_scenario6.__doc__ examples = [ ['data/iris.csv', 'scenario_e6/evaluation', '0.2', 'average_phi', '0.85']] for example in examples: print "\nTesting with:\n", example evaluation.i_create_with_split_to_evaluate(self, data=example[0], split=example[2], output=example[1]) test_pred.i_check_create_source(self) test_pred.i_check_create_dataset(self, suffix=None) test_pred.i_check_create_dataset(self, suffix='train ') test_pred.i_check_create_dataset(self, suffix='test ') test_pred.i_check_create_model(self) test_pred.i_check_create_evaluation(self) evaluation.i_check_evaluation_key(self, key=example[3], value=example[4])
def test_scenario5(self): """ Scenario: Successfully building evaluation from model and test file with data map Given I have previously executed "<scenario>" or reproduce it with arguments <kwargs> And I create BigML resources using test file "<test>" and a fields map "<fields_map>" to evaluate a model and log evaluation in "<output>" And I check that the source has been created And I check that the dataset has been created And I check that the evaluation has been created Then the evaluation file is like "<json_evaluation_file>" Examples: |scenario | kwargs | test | fields_map | output | json_evaluation_file | | scenario_e1| {"data": "../data/iris.csv", "output": "./scenario_e1/predictions.csv"} | ../data/iris_permuted.csv | ../data/fields_map.csv | ./scenario_e7/evaluation | ./check_files/evaluation_iris2.json | """ print self.test_scenario5.__doc__ examples = [ ['scenario_e1', '{"data": "data/iris.csv", "output": "scenario_e1/predictions.csv"}', 'data/iris_permuted.csv', 'data/fields_map.csv', 'scenario_e7/evaluation', 'check_files/evaluation_iris2.json']] for example in examples: print "\nTesting with:\n", example test_pred.i_have_previous_scenario_or_reproduce_it(self, example[0], example[1]) evaluation.i_create_all_resources_to_evaluate_with_model_and_map(self, data=example[2], fields_map=example[3], output=example[4]) test_pred.i_check_create_source(self) test_pred.i_check_create_dataset(self, suffix=None) test_pred.i_check_create_evaluation(self) evaluation.then_the_evaluation_file_is_like(self, example[5])
def test_scenario11(self): """ Scenario: Successfully building evaluations for deepnets from start: Given I create BigML deepnet resources uploading train "<data>" file to evaluate and log evaluation in "<output>" And I check that the source has been created And I check that the dataset has been created And I check that the deepnet has been created And I check that the evaluation has been created Then the evaluation file is like "<json_evaluation_file>" Examples: | data | output | json_evaluation_file | | ../data/iris.csv | ./scenario_e11/evaluation | ./check_files/evaluation_iris_dn.json | """ print self.test_scenario11.__doc__ examples = [ ['data/iris.csv', 'scenario_e11/evaluation', 'check_files/evaluation_iris_dn.json']] for example in examples: print "\nTesting with:\n", example dn_pred.i_create_all_dn_resources_to_evaluate(self, data=example[0], output=example[1]) test_pred.i_check_create_source(self) test_pred.i_check_create_dataset(self, suffix=None) dn_pred.i_check_create_dn_model(self) test_pred.i_check_create_evaluation(self) evaluation.then_the_evaluation_file_is_like(self, example[2])
def test_scenario7(self): """ Scenario: Successfully building ensemble evaluations from start and test-split: Given I create BigML resources uploading train "<data>" file to evaluate an ensemble of <number_of_models> models with test-split <split> and log evaluation in "<output>" And I check that the source has been created And I check that the dataset has been created And I check that the train dataset has been created And I check that the test dataset has been created And I check that the ensemble has been created And I check that the evaluation has been created Then the evaluation key "<key>" value for the model is greater than <value> And I evaluate the ensemble in directory "<directory>" with the dataset in directory "<directory>" and log evaluation in "<output2>" And I check that the evaluation has been created Then the evaluation key "<key>" value for the model is greater than <value> Examples: | data | output | split | number_of_models | key | value | directory | output2 | ../data/iris.csv | ./scenario_e8/evaluation | 0.2 | 5 | average_phi | 0.94 | ./scenario_e8/ | ./scenario_e9/evaluation """ print self.test_scenario7.__doc__ examples = [[ 'data/iris.csv', 'scenario_e8/evaluation', '0.2', '5', 'average_phi', '0.94', 'scenario_e8', 'scenario_e9/evaluation' ]] for example in examples: print "\nTesting with:\n", example evaluation.i_create_with_split_to_evaluate_ensemble( self, data=example[0], number_of_models=example[3], split=example[2], output=example[1]) test_pred.i_check_create_source(self) test_pred.i_check_create_dataset(self, suffix=None) test_pred.i_check_create_dataset(self, suffix='train ') test_pred.i_check_create_dataset(self, suffix='test ') test_pred.i_check_create_ensemble(self) test_pred.i_check_create_evaluation(self) evaluation.i_check_evaluation_key(self, key=example[4], value=example[5]) evaluation.i_evaluate_ensemble_with_dataset( self, ensemble_dir=example[6], dataset_dir=example[6], output=example[7]) test_pred.i_check_create_evaluation(self) evaluation.i_check_evaluation_key(self, key=example[4], value=example[5])
def test_scenario04(self): """ Scenario: Successfully building evaluation from fusion Given I have previously executed "<scenario>" or reproduce it with arguments <kwargs> And I create BigML fusion resources using model built from "<train>" to test "<test>" as an evaluation and log predictions in "<output>" And I check that the predictions are ready Then the local prediction file is like "<predictions_file>" Then the evaluation file is like "<json_evaluation_file>" Examples: | data | output | json_evaluation_file | | ../data/iris.csv | ./scenario_fs_e1/evaluation | ./check_files/evaluation_iris_dn.json | """ print self.test_scenario04.__doc__ examples = [ ['scenario_fs_e1/evaluation', 'check_files/evaluation_iris_fs.json']] for example in examples: print "\nTesting with:\n", example fs_pred.i_create_fs_resources_from_mode_and_evaluate(self, output=example[0]) fs_pred.i_check_create_fusion(self) test_pred.i_check_create_evaluation(self) evaluation.then_the_evaluation_file_is_like(self, example[1])
def test_scenario6(self): """ Scenario: Successfully building evaluations from start and test-split: Given I create BigML resources uploading train "<data>" file to evaluate with test-split <split> and log evaluation in "<output>" And I check that the source has been created And I check that the dataset has been created And I check that the train dataset has been created And I check that the test dataset has been created And I check that the model has been created And I check that the evaluation has been created Then the evaluation key "<key>" value for the model is greater than <value> Examples: | data | output | split | key | value | | ../data/iris.csv | ./scenario_e6/evaluation | 0.2 | average_phi | 0.85 | """ print self.test_scenario6.__doc__ examples = [[ 'data/iris.csv', 'scenario_e6/evaluation', '0.2', 'average_phi', '0.85' ]] for example in examples: print "\nTesting with:\n", example evaluation.i_create_with_split_to_evaluate(self, data=example[0], split=example[2], output=example[1]) test_pred.i_check_create_source(self) test_pred.i_check_create_dataset(self, suffix=None) test_pred.i_check_create_dataset(self, suffix='train ') test_pred.i_check_create_dataset(self, suffix='test ') test_pred.i_check_create_model(self) test_pred.i_check_create_evaluation(self) evaluation.i_check_evaluation_key(self, key=example[3], value=example[4])