def test_surround_config(self):
path = os.path.dirname(__file__)
config = Config()
config.read_config_files([os.path.join(path, "config.yaml")])
data = AssemblerState()
assembler = Assembler("Surround config", InputValidator(), HelloStage(), config)
assembler.run(data)
self.assertEqual(data.config_value, "Scott")
def test_pipeline_stop_on_error_estimator(self):
data = AssemblerState()
data.estimator_add_error = True
assembler = Assembler("Fail test").set_stages(
[InputValidator(), HelloStage(),
PostFilter()])
# This should fail to execute PostFilter
assembler.run(data)
self.assertFalse(data.post_filter_ran)
def test_pipeline_stop_on_exception_validator(self):
data = AssemblerState()
data.stage1 = "Now it will fail in the validator"
assembler = Assembler("Fail test").set_stages(
[InputValidator(), HelloStage(),
PostFilter()])
# This should fail to execute HelloStage & PostFilter
assembler.run(data)
self.assertIsNone(data.text)
self.assertFalse(data.post_filter_ran)
def test_pipeline_stop_on_error_filter(self):
# Use the errors list in state instead
data = AssemblerState()
data.use_errors_instead = True
assembler = Assembler("Fail test").set_stages(
[InputValidator(),
BadFilter(),
HelloStage(),
PostFilter()])
# This should fail to execute HelloStage & PostFilter
assembler.run(data)
self.assertIsNone(data.text)
self.assertFalse(data.post_filter_ran)
def execute_assembler(mode):
assembler = Assembler("Default project", ValidateData(), Main())
if mode == "train":
BatchRunner(assembler).run(True)
elif mode == "batch":
BatchRunner(assembler).run()
else:
WebRunner(assembler).run()
def test_finaliser_successful_pipeline(self):
data = AssemblerState()
assembler = Assembler("Finalizer test", InputValidator(), HelloStage(), Config())
assembler.set_finaliser(TestFinalStage())
assembler.init_assembler()
# Run assembler which will succeed
assembler.run(data)
# Finalizer should be executed
self.assertTrue(data.final_ran)
def test_rejecting_attributes(self):
data = AssemblerState()
assembler = Assembler("Reject attribute").set_stages(
[InputValidator(), HelloStage()]).set_config(Config())
assembler.init_assembler()
assembler.run(data)
self.assertRaises(AttributeError, getattr, data, "no_text")
def test_rejecting_attributes(self):
data = BasicData()
assembler = Assembler("Reject attribute", ValidateData(), HelloStage(),
Config())
assembler.init_assembler()
assembler.run(data)
self.assertRaises(AttributeError, getattr, data, "no_text")
def test_happy_path(self):
data = BasicData()
assembler = Assembler("Happy path", ValidateData(), HelloStage(),
Config())
assembler.init_assembler()
assembler.run(data)
self.assertEqual(data.text, test_text)
def test_finaliser_fail_pipeline(self):
# Ensure pipeline will crash
data = AssemblerState()
data.text = ""
assembler = Assembler("Finalizer test", InputValidator(), HelloStage(), Config())
assembler.set_finaliser(TestFinalStage())
assembler.init_assembler()
# Run assembler which will fail
assembler.run(data)
# Finalizer should still be executed
self.assertTrue(data.final_ran)
def test_happy_path(self):
data = AssemblerState()
assembler = Assembler("Happy path").set_stages(
[InputValidator(), HelloStage()]).set_config(Config())
assembler.init_assembler()
assembler.run(data)
self.assertEqual(data.text, test_text)
def test_pipeline_stop_on_exception_filter(self):
data = AssemblerState()
assembler = Assembler("Fail test").set_validator(InputValidator())
assembler.set_estimator(HelloStage(), [BadFilter()], [PostFilter()])
# This should fail to execute HelloStage
assembler.run(data)
self.assertIsNone(data.text)
self.assertFalse(data.post_filter_ran)
from surround import Assembler
from stages import AddWorld, AddHello, AddSurround, BasicData, ValidateData
if __name__ == "__main__":
data = BasicData()
assembler = Assembler("Pre Post Example", ValidateData())
assembler.set_estimator(AddWorld(), [AddHello()], [AddSurround()])
assembler.run(data)
print("Text is '%s'" % data.text)
import logging
from surround import SurroundData, Validator, Estimator, Assembler
class HelloWorld(Estimator):
def estimate(self, surround_data, config):
surround_data.text = "Hello world"
def fit(self, surround_data, config):
print("No training implemented")
class ValidateData(Validator):
def validate(self, surround_data, config):
if surround_data.text:
raise ValueError("'text' is not None")
class BasicData(SurroundData):
text = None
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
data = BasicData()
assembler = Assembler("Hello world example", ValidateData(), HelloWorld())
assembler.run(data)
print("Text is '%s'" % data.text)
state.company = row['Company']
state.outputs.append((state.word_count, state.company))
def fit(self, state, config):
print("No training implemented")
class AssemblerState(State):
outputs = []
rows = []
row = None
word_count = None
company = None
csv_file = None
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
dir_extension = os.path.dirname(__file__)
if dir_extension not in os.getcwd():
prefix = dir_extension + "/"
app_config = Config()
app_config.read_config_files([prefix + "config.yaml"])
assembler = Assembler("Loader example").set_stages(
[CSVValidator(), ProcessCSV()]).set_config(app_config)
MainRunner(assembler).run()
print("No training implemented")
class AssemblerState(State):
text = None
class InputValidator(Stage):
def operate(self, state, config):
if state.text:
raise ValueError("'text' is not None")
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
dir_extension = os.path.dirname(__file__)
if dir_extension not in os.getcwd():
prefix = dir_extension + "/"
app_config = Config()
app_config.read_config_files([prefix + "config.yaml"])
data = AssemblerState()
assembler = Assembler("Init state example").set_stages(
[InputValidator(), HelloWorld()]).set_config(app_config)
assembler.init_assembler()
assembler.run(data)
print("Text is '%s'" % data.text)
import logging
from surround import State, Validator, Estimator, Assembler
class HelloWorld(Estimator):
def estimate(self, state, config):
state.text = "Hello world"
def fit(self, state, config):
print("No training implemented")
class InputValidator(Validator):
def validate(self, state, config):
if state.text:
raise ValueError("'text' is not None")
class AssemblerState(State):
text = None
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
data = AssemblerState()
assembler = Assembler("Hello world example", InputValidator(),
HelloWorld())
assembler.run(data)
print("Text is '%s'" % data.text)
def test_assembler_init_pass(self):
assembler = Assembler("Pass test").set_stages(
[InputValidator(), HelloStage()])
self.assertTrue(assembler.init_assembler())
print("Not training implementation")
class AssemblerState(State):
text = None
class InputValidator(Validator):
def validate(self, state, config):
if state.text:
raise ValueError("'text' is not None")
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
path = os.path.dirname(os.path.realpath(__file__))
app_config = Config()
app_config.read_config_files([path + "/config.yaml"])
assembler = Assembler("Dump output example")
assembler.set_validator(InputValidator())
assembler.set_config(app_config)
assembler.set_estimator(WriteWorld(path), [WriteHello(path)])
assembler.run(AssemblerState())
print("Hello output.txt contains '%s'" %
open(path + hello_file_path, "r").read())
print("World output.txt contains '%s'" %
open(path + world_file_path, "r").read())
import logging
from surround import State, Validator, Estimator, Assembler
class HelloWorld(Estimator):
def estimate(self, state, config):
state.text = "Hello world"
def fit(self, state, config):
print("No training implemented")
class InputValidator(Validator):
def validate(self, state, config):
if state.text:
raise ValueError("'text' is not None")
class AssemblerState(State):
text = None
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
data = AssemblerState()
assembler = Assembler("Hello world example").set_validator(
InputValidator()).set_estimator(HelloWorld())
assembler.run(data)
print("Text is '%s'" % data.text)
def main():
assembler = Assembler("Default project").set_stages(
[InputValidator(), HelloWorld()])
# Example for running batch processing
BatchRunner(assembler).run()
def estimate(self, state, config):
state.text = "World"
def fit(self, state, config):
print("Not training implementation")
class AssemblerState(State):
text = None
class InputValidator(Stage):
def operate(self, state, config):
if state.text:
raise ValueError("'text' is not None")
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
path = os.path.dirname(os.path.realpath(__file__))
app_config = Config()
app_config.read_config_files([path + "/config.yaml"])
assembler = Assembler("Dump output example")
assembler.set_stages([InputValidator(), WriteHello(path), WriteWorld(path)])
assembler.set_config(app_config)
assembler.run(AssemblerState())
print("Hello output.txt contains '%s'" % open(path + hello_file_path, "r").read())
print("World output.txt contains '%s'" % open(path + world_file_path, "r").read())
class BasicData(SurroundData):
training_message = None
text = None
class Formatter(Visualiser):
def visualise(self, surround_data, config):
print("Visualiser result: %s" % surround_data.training_message)
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
data = BasicData()
assembler = Assembler("Training example", ValidateData(), HelloWorld())
assembler.init_assembler(True)
assembler.set_visualiser(Formatter())
# Run assembler before training
assembler.run(data)
print("Text before training is '%s'" % data.text)
data.text = None # Clear text to prevent validation raising error
# Run training mode
assembler.run(data, True)
# Run assembler after training
assembler.run(data)
print("Text after training is '%s'" % data.text)
print("No training implemented")
class BasicData(SurroundData):
text = None
class ValidateData(Validator):
def validate(self, surround_data, config):
if surround_data.text:
raise ValueError("'text' is not None")
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
dir_extension = os.path.dirname(__file__)
if dir_extension not in os.getcwd():
prefix = dir_extension + "/"
app_config = Config()
app_config.read_config_files([prefix + "config.yaml"])
data = BasicData()
assembler = Assembler("Init state example", ValidateData(), HelloWorld(),
app_config)
assembler.init_assembler()
assembler.run(data)
print("Text is '%s'" % data.text)
class AssemblerState(State):
training_message = None
text = None
class Formatter(Stage):
def visualise(self, state, config):
print("Visualiser result: %s" % state.training_message)
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
data = AssemblerState()
assembler = Assembler("Training example")
assembler.set_stages([HelloWorld(), Formatter()])
assembler.init_assembler()
# Run assembler before training
assembler.run(data, RunMode.TRAIN)
print("Text before training is '%s'" % data.text)
data.text = None # Clear text to prevent validation raising error
# Run training mode
assembler.run(data)
# Run assembler after training
assembler.run(data)
print("Text after training is '%s'" % data.text)
def estimate(self, state, config):
state.word_count = len(state.active_row['Consumer complaint narrative'].split())
if config and config.get_path("ProcessCSV.include_company"):
state.company = state.active_row['Company']
state.outputs.append((state.word_count, state.company))
def fit(self, state, config):
print("No training implemented")
class AssemblerState(State):
outputs = []
row = None
word_count = None
company = None
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
dir_extension = os.path.dirname(__file__)
if dir_extension not in os.getcwd():
prefix = dir_extension + "/"
app_config = Config()
app_config.read_config_files([prefix + "config.yaml"])
assembler = Assembler("Loader example", CSVValidator(), ProcessCSV(), app_config)
MainRunner(assembler).run()
def main():
assembler = Assembler("Default project", ValidateData(), HelloWorld())
# Example for running batch processing
BatchRunner(assembler).run()
def fit(self, surround_data, config):
print("Not training implementation")
class BasicData(SurroundData):
text = None
class ValidateData(Validator):
def validate(self, surround_data, config):
if surround_data.text:
raise ValueError("'text' is not None")
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
path = os.path.dirname(os.path.realpath(__file__))
app_config = Config()
app_config.read_config_files([path + "/config.yaml"])
assembler = Assembler("Dump output example", ValidateData())
assembler.set_config(app_config)
assembler.set_estimator(WriteWorld(path), [WriteHello(path)])
assembler.run(BasicData())
print("Hello output.txt contains '%s'" %
open(path + hello_file_path, "r").read())
print("World output.txt contains '%s'" %
open(path + world_file_path, "r").read())
import logging
from surround import State, Stage, Estimator, Assembler
class HelloWorld(Estimator):
def estimate(self, state, config):
state.text = "Hello world"
def fit(self, state, config):
print("No training implemented")
class InputValidator(Stage):
def operate(self, state, config):
if state.text:
raise ValueError("'text' is not None")
class AssemblerState(State):
text = None
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
data = AssemblerState()
assembler = Assembler("Hello world example").set_stages(
[InputValidator(), HelloWorld()])
assembler.run(data)
print("Text is '%s'" % data.text)
def test_assembler_init_fail(self):
assembler = Assembler("Fail test").set_stages(
[InputValidator(), BadFilter(),
HelloStage()])
self.assertFalse(assembler.init_assembler())
