Exemplo n.º 1
0
  def __init__(self, module_properties, dao, clf):
    self._dao = dao
    self._id = module_properties['id']
    self._warmup_examples = module_properties['warmup_examples']
    self._checkpoint = module_properties['checkpoint']
    self._is_prequential = True if module_properties['eval_mode'] == 'prequential' else False
    self._parser = ParserFactory.new_parser(module_properties)
    self._evaluator = EvaluatorFactory.new_evaluator(module_properties)
    self._offline_test = None
    if self._is_prequential == False:
      self._offline_test = self._parser.parse(module_properties['offline_test'])
    self._classes = np.array(map(int, module_properties['classes'].split(',')))
    self._server_port = module_properties['server_port']

    ##### Recovery and adding learners on the fly #########
    self._clf, timestamp = self._dao.get_model(self._id)
    self._checkpointed = None
    # if there was a checkpoint, then see if there are some historical points beyond that and train
    if self._clf:
      self._checkpointed = True
      examples = self._dao.get_examples_greater_than(timestamp)
      if examples:
        print 'catching up checkpointed model with historical points...'
        X, y, timestamps = self._parser.parse_feature(examples)
        self._clf.partial_fit(X, y)
      else:
        print 'no historical points to catch up the checkpointed model'
        # will use the last metric saved
    else:
      self._checkpointed = False
      self._clf = clf
      examples = self._dao.get_examples()
      if examples:
        print 'catching up new model with historical points'
        X, y, timestamps = self._parser.parse_feature(examples)
        self._clf.partial_fit(X, y, self._classes)
      else:
        print 'no historical points to catch up the new model'

    #######################################        

    self._registry = RegistryClient(module_properties['registry'])
    hostname = socket.gethostname()
    address = Utils.get_address(hostname, self._server_port)
    self._stream_client_address = self._registry.reg(ComponentType.LEARNER, address)[0]    

    self._handler = LearnerHandler(self._stream_client_address, self._registry, self._parser, self._evaluator, self._dao, self._clf, self._classes, 
                              self._warmup_examples, self._id, self._checkpoint, self._is_prequential, 
                              self._checkpointed, self._offline_test)
    self._processor = StreamService.Processor(self._handler)
    self._stream_server = Server(self._processor, self._server_port, module_properties['multi_threading'])
Exemplo n.º 2
0
 def __init__(self, dao, module_properties):
     self._dao = dao
     self._parser = ParserFactory.new_parser(module_properties)
     self._historical_batches = module_properties["historical_batches"]
     self._registry = RegistryClient(module_properties["registry"])
     self._server_port = module_properties["server_port"]
     hostname = socket.gethostname()
     address = Utils.get_address(hostname, self._server_port)
     self._stream_clients_addresses = self._registry.reg(ComponentType.FEATGEN, address)
     self._processor = StreamService.Processor(
         FeatureGeneratorHandler(
             self._registry, self._parser, self._dao, self._historical_batches, self._stream_clients_addresses
         )
     )
     self._stream_server = Server(self._processor, self._server_port, module_properties["multi_threading"])