class Deployer:
def __init__(self, module_properties):
self._server_port = module_properties['server_port']
self._evaluator = EvaluatorFactory.new_evaluator(module_properties)
self._registry = RegistryClient(module_properties['registry'])
self._processor = StreamService.Processor(DeployerHandler(self._evaluator))
self._stream_server = Server(self._processor, self._server_port, module_properties['multi_threading'])
def run(self):
hostname = socket.gethostname()
address = Utils.get_address(hostname, self._server_port)
self._registry.reg(ComponentType.DEPLOYER, address)
self._stream_server.start()
class Learner:
def __init__(self, module_properties, dao, clf):
self._dao = dao
self._id = module_properties['id']
self._warmup_examples = module_properties['warmup_examples']
self._checkpoint = module_properties['checkpoint']
self._is_prequential = True if module_properties['eval_mode'] == 'prequential' else False
self._parser = ParserFactory.new_parser(module_properties)
self._evaluator = EvaluatorFactory.new_evaluator(module_properties)
self._offline_test = None
if self._is_prequential == False:
self._offline_test = self._parser.parse(module_properties['offline_test'])
self._classes = np.array(map(int, module_properties['classes'].split(',')))
self._server_port = module_properties['server_port']
##### Recovery and adding learners on the fly #########
self._clf, timestamp = self._dao.get_model(self._id)
self._checkpointed = None
# if there was a checkpoint, then see if there are some historical points beyond that and train
if self._clf:
self._checkpointed = True
examples = self._dao.get_examples_greater_than(timestamp)
if examples:
print 'catching up checkpointed model with historical points...'
X, y, timestamps = self._parser.parse_feature(examples)
self._clf.partial_fit(X, y)
else:
print 'no historical points to catch up the checkpointed model'
# will use the last metric saved
else:
self._checkpointed = False
self._clf = clf
examples = self._dao.get_examples()
if examples:
print 'catching up new model with historical points'
X, y, timestamps = self._parser.parse_feature(examples)
self._clf.partial_fit(X, y, self._classes)
else:
print 'no historical points to catch up the new model'
#######################################
self._registry = RegistryClient(module_properties['registry'])
hostname = socket.gethostname()
address = Utils.get_address(hostname, self._server_port)
self._stream_client_address = self._registry.reg(ComponentType.LEARNER, address)[0]
self._handler = LearnerHandler(self._stream_client_address, self._registry, self._parser, self._evaluator, self._dao, self._clf, self._classes,
self._warmup_examples, self._id, self._checkpoint, self._is_prequential,
self._checkpointed, self._offline_test)
self._processor = StreamService.Processor(self._handler)
self._stream_server = Server(self._processor, self._server_port, module_properties['multi_threading'])
def run(self):
self._stream_server.start()
def __init__(self, dao, module_properties):
self._dao = dao
self._parser = ParserFactory.new_parser(module_properties)
self._historical_batches = module_properties["historical_batches"]
self._registry = RegistryClient(module_properties["registry"])
self._server_port = module_properties["server_port"]
hostname = socket.gethostname()
address = Utils.get_address(hostname, self._server_port)
self._stream_clients_addresses = self._registry.reg(ComponentType.FEATGEN, address)
self._processor = StreamService.Processor(
FeatureGeneratorHandler(
self._registry, self._parser, self._dao, self._historical_batches, self._stream_clients_addresses
)
)
self._stream_server = Server(self._processor, self._server_port, module_properties["multi_threading"])
def __init__(self, module_properties): self._server_port = module_properties['server_port'] self._evaluator = EvaluatorFactory.new_evaluator(module_properties) self._registry = RegistryClient(module_properties['registry']) self._processor = StreamService.Processor(DeployerHandler(self._evaluator)) self._stream_server = Server(self._processor, self._server_port, module_properties['multi_threading'])