def test_surround_config(self): path = os.path.dirname(__file__) config = Config() config.read_config_files([os.path.join(path, "config.yaml")]) data = AssemblerState() assembler = Assembler("Surround config", InputValidator(), HelloStage(), config) assembler.run(data) self.assertEqual(data.config_value, "Scott")
def test_pipeline_stop_on_error_estimator(self): data = AssemblerState() data.estimator_add_error = True assembler = Assembler("Fail test").set_stages( [InputValidator(), HelloStage(), PostFilter()]) # This should fail to execute PostFilter assembler.run(data) self.assertFalse(data.post_filter_ran)
def test_pipeline_stop_on_exception_validator(self): data = AssemblerState() data.stage1 = "Now it will fail in the validator" assembler = Assembler("Fail test").set_stages( [InputValidator(), HelloStage(), PostFilter()]) # This should fail to execute HelloStage & PostFilter assembler.run(data) self.assertIsNone(data.text) self.assertFalse(data.post_filter_ran)
def test_pipeline_stop_on_error_filter(self): # Use the errors list in state instead data = AssemblerState() data.use_errors_instead = True assembler = Assembler("Fail test").set_stages( [InputValidator(), BadFilter(), HelloStage(), PostFilter()]) # This should fail to execute HelloStage & PostFilter assembler.run(data) self.assertIsNone(data.text) self.assertFalse(data.post_filter_ran)
def execute_assembler(mode): assembler = Assembler("Default project", ValidateData(), Main()) if mode == "train": BatchRunner(assembler).run(True) elif mode == "batch": BatchRunner(assembler).run() else: WebRunner(assembler).run()
def test_finaliser_successful_pipeline(self): data = AssemblerState() assembler = Assembler("Finalizer test", InputValidator(), HelloStage(), Config()) assembler.set_finaliser(TestFinalStage()) assembler.init_assembler() # Run assembler which will succeed assembler.run(data) # Finalizer should be executed self.assertTrue(data.final_ran)
def test_rejecting_attributes(self): data = AssemblerState() assembler = Assembler("Reject attribute").set_stages( [InputValidator(), HelloStage()]).set_config(Config()) assembler.init_assembler() assembler.run(data) self.assertRaises(AttributeError, getattr, data, "no_text")
def test_rejecting_attributes(self): data = BasicData() assembler = Assembler("Reject attribute", ValidateData(), HelloStage(), Config()) assembler.init_assembler() assembler.run(data) self.assertRaises(AttributeError, getattr, data, "no_text")
def test_happy_path(self): data = BasicData() assembler = Assembler("Happy path", ValidateData(), HelloStage(), Config()) assembler.init_assembler() assembler.run(data) self.assertEqual(data.text, test_text)
def test_finaliser_fail_pipeline(self): # Ensure pipeline will crash data = AssemblerState() data.text = "" assembler = Assembler("Finalizer test", InputValidator(), HelloStage(), Config()) assembler.set_finaliser(TestFinalStage()) assembler.init_assembler() # Run assembler which will fail assembler.run(data) # Finalizer should still be executed self.assertTrue(data.final_ran)
def test_happy_path(self): data = AssemblerState() assembler = Assembler("Happy path").set_stages( [InputValidator(), HelloStage()]).set_config(Config()) assembler.init_assembler() assembler.run(data) self.assertEqual(data.text, test_text)
def test_pipeline_stop_on_exception_filter(self): data = AssemblerState() assembler = Assembler("Fail test").set_validator(InputValidator()) assembler.set_estimator(HelloStage(), [BadFilter()], [PostFilter()]) # This should fail to execute HelloStage assembler.run(data) self.assertIsNone(data.text) self.assertFalse(data.post_filter_ran)
from surround import Assembler from stages import AddWorld, AddHello, AddSurround, BasicData, ValidateData if __name__ == "__main__": data = BasicData() assembler = Assembler("Pre Post Example", ValidateData()) assembler.set_estimator(AddWorld(), [AddHello()], [AddSurround()]) assembler.run(data) print("Text is '%s'" % data.text)
import logging from surround import SurroundData, Validator, Estimator, Assembler class HelloWorld(Estimator): def estimate(self, surround_data, config): surround_data.text = "Hello world" def fit(self, surround_data, config): print("No training implemented") class ValidateData(Validator): def validate(self, surround_data, config): if surround_data.text: raise ValueError("'text' is not None") class BasicData(SurroundData): text = None if __name__ == "__main__": logging.basicConfig(level=logging.INFO) data = BasicData() assembler = Assembler("Hello world example", ValidateData(), HelloWorld()) assembler.run(data) print("Text is '%s'" % data.text)
state.company = row['Company'] state.outputs.append((state.word_count, state.company)) def fit(self, state, config): print("No training implemented") class AssemblerState(State): outputs = [] rows = [] row = None word_count = None company = None csv_file = None if __name__ == "__main__": logging.basicConfig(level=logging.INFO) dir_extension = os.path.dirname(__file__) if dir_extension not in os.getcwd(): prefix = dir_extension + "/" app_config = Config() app_config.read_config_files([prefix + "config.yaml"]) assembler = Assembler("Loader example").set_stages( [CSVValidator(), ProcessCSV()]).set_config(app_config) MainRunner(assembler).run()
print("No training implemented") class AssemblerState(State): text = None class InputValidator(Stage): def operate(self, state, config): if state.text: raise ValueError("'text' is not None") if __name__ == "__main__": logging.basicConfig(level=logging.INFO) dir_extension = os.path.dirname(__file__) if dir_extension not in os.getcwd(): prefix = dir_extension + "/" app_config = Config() app_config.read_config_files([prefix + "config.yaml"]) data = AssemblerState() assembler = Assembler("Init state example").set_stages( [InputValidator(), HelloWorld()]).set_config(app_config) assembler.init_assembler() assembler.run(data) print("Text is '%s'" % data.text)
import logging from surround import State, Validator, Estimator, Assembler class HelloWorld(Estimator): def estimate(self, state, config): state.text = "Hello world" def fit(self, state, config): print("No training implemented") class InputValidator(Validator): def validate(self, state, config): if state.text: raise ValueError("'text' is not None") class AssemblerState(State): text = None if __name__ == "__main__": logging.basicConfig(level=logging.INFO) data = AssemblerState() assembler = Assembler("Hello world example", InputValidator(), HelloWorld()) assembler.run(data) print("Text is '%s'" % data.text)
def test_assembler_init_pass(self): assembler = Assembler("Pass test").set_stages( [InputValidator(), HelloStage()]) self.assertTrue(assembler.init_assembler())
print("Not training implementation") class AssemblerState(State): text = None class InputValidator(Validator): def validate(self, state, config): if state.text: raise ValueError("'text' is not None") if __name__ == "__main__": logging.basicConfig(level=logging.INFO) path = os.path.dirname(os.path.realpath(__file__)) app_config = Config() app_config.read_config_files([path + "/config.yaml"]) assembler = Assembler("Dump output example") assembler.set_validator(InputValidator()) assembler.set_config(app_config) assembler.set_estimator(WriteWorld(path), [WriteHello(path)]) assembler.run(AssemblerState()) print("Hello output.txt contains '%s'" % open(path + hello_file_path, "r").read()) print("World output.txt contains '%s'" % open(path + world_file_path, "r").read())
import logging from surround import State, Validator, Estimator, Assembler class HelloWorld(Estimator): def estimate(self, state, config): state.text = "Hello world" def fit(self, state, config): print("No training implemented") class InputValidator(Validator): def validate(self, state, config): if state.text: raise ValueError("'text' is not None") class AssemblerState(State): text = None if __name__ == "__main__": logging.basicConfig(level=logging.INFO) data = AssemblerState() assembler = Assembler("Hello world example").set_validator( InputValidator()).set_estimator(HelloWorld()) assembler.run(data) print("Text is '%s'" % data.text)
def main(): assembler = Assembler("Default project").set_stages( [InputValidator(), HelloWorld()]) # Example for running batch processing BatchRunner(assembler).run()
def estimate(self, state, config): state.text = "World" def fit(self, state, config): print("Not training implementation") class AssemblerState(State): text = None class InputValidator(Stage): def operate(self, state, config): if state.text: raise ValueError("'text' is not None") if __name__ == "__main__": logging.basicConfig(level=logging.INFO) path = os.path.dirname(os.path.realpath(__file__)) app_config = Config() app_config.read_config_files([path + "/config.yaml"]) assembler = Assembler("Dump output example") assembler.set_stages([InputValidator(), WriteHello(path), WriteWorld(path)]) assembler.set_config(app_config) assembler.run(AssemblerState()) print("Hello output.txt contains '%s'" % open(path + hello_file_path, "r").read()) print("World output.txt contains '%s'" % open(path + world_file_path, "r").read())
class BasicData(SurroundData): training_message = None text = None class Formatter(Visualiser): def visualise(self, surround_data, config): print("Visualiser result: %s" % surround_data.training_message) if __name__ == "__main__": logging.basicConfig(level=logging.INFO) data = BasicData() assembler = Assembler("Training example", ValidateData(), HelloWorld()) assembler.init_assembler(True) assembler.set_visualiser(Formatter()) # Run assembler before training assembler.run(data) print("Text before training is '%s'" % data.text) data.text = None # Clear text to prevent validation raising error # Run training mode assembler.run(data, True) # Run assembler after training assembler.run(data) print("Text after training is '%s'" % data.text)
print("No training implemented") class BasicData(SurroundData): text = None class ValidateData(Validator): def validate(self, surround_data, config): if surround_data.text: raise ValueError("'text' is not None") if __name__ == "__main__": logging.basicConfig(level=logging.INFO) dir_extension = os.path.dirname(__file__) if dir_extension not in os.getcwd(): prefix = dir_extension + "/" app_config = Config() app_config.read_config_files([prefix + "config.yaml"]) data = BasicData() assembler = Assembler("Init state example", ValidateData(), HelloWorld(), app_config) assembler.init_assembler() assembler.run(data) print("Text is '%s'" % data.text)
class AssemblerState(State): training_message = None text = None class Formatter(Stage): def visualise(self, state, config): print("Visualiser result: %s" % state.training_message) if __name__ == "__main__": logging.basicConfig(level=logging.INFO) data = AssemblerState() assembler = Assembler("Training example") assembler.set_stages([HelloWorld(), Formatter()]) assembler.init_assembler() # Run assembler before training assembler.run(data, RunMode.TRAIN) print("Text before training is '%s'" % data.text) data.text = None # Clear text to prevent validation raising error # Run training mode assembler.run(data) # Run assembler after training assembler.run(data) print("Text after training is '%s'" % data.text)
def estimate(self, state, config): state.word_count = len(state.active_row['Consumer complaint narrative'].split()) if config and config.get_path("ProcessCSV.include_company"): state.company = state.active_row['Company'] state.outputs.append((state.word_count, state.company)) def fit(self, state, config): print("No training implemented") class AssemblerState(State): outputs = [] row = None word_count = None company = None if __name__ == "__main__": logging.basicConfig(level=logging.INFO) dir_extension = os.path.dirname(__file__) if dir_extension not in os.getcwd(): prefix = dir_extension + "/" app_config = Config() app_config.read_config_files([prefix + "config.yaml"]) assembler = Assembler("Loader example", CSVValidator(), ProcessCSV(), app_config) MainRunner(assembler).run()
def main(): assembler = Assembler("Default project", ValidateData(), HelloWorld()) # Example for running batch processing BatchRunner(assembler).run()
def fit(self, surround_data, config): print("Not training implementation") class BasicData(SurroundData): text = None class ValidateData(Validator): def validate(self, surround_data, config): if surround_data.text: raise ValueError("'text' is not None") if __name__ == "__main__": logging.basicConfig(level=logging.INFO) path = os.path.dirname(os.path.realpath(__file__)) app_config = Config() app_config.read_config_files([path + "/config.yaml"]) assembler = Assembler("Dump output example", ValidateData()) assembler.set_config(app_config) assembler.set_estimator(WriteWorld(path), [WriteHello(path)]) assembler.run(BasicData()) print("Hello output.txt contains '%s'" % open(path + hello_file_path, "r").read()) print("World output.txt contains '%s'" % open(path + world_file_path, "r").read())
import logging from surround import State, Stage, Estimator, Assembler class HelloWorld(Estimator): def estimate(self, state, config): state.text = "Hello world" def fit(self, state, config): print("No training implemented") class InputValidator(Stage): def operate(self, state, config): if state.text: raise ValueError("'text' is not None") class AssemblerState(State): text = None if __name__ == "__main__": logging.basicConfig(level=logging.INFO) data = AssemblerState() assembler = Assembler("Hello world example").set_stages( [InputValidator(), HelloWorld()]) assembler.run(data) print("Text is '%s'" % data.text)
def test_assembler_init_fail(self): assembler = Assembler("Fail test").set_stages( [InputValidator(), BadFilter(), HelloStage()]) self.assertFalse(assembler.init_assembler())