Beispiel #1
0
  def __init__(self, module_properties, dao, clf):
    self._dao = dao
    self._id = module_properties['id']
    self._warmup_examples = module_properties['warmup_examples']
    self._checkpoint = module_properties['checkpoint']
    self._is_prequential = True if module_properties['eval_mode'] == 'prequential' else False
    self._parser = ParserFactory.new_parser(module_properties)
    self._evaluator = EvaluatorFactory.new_evaluator(module_properties)
    self._offline_test = None
    if self._is_prequential == False:
      self._offline_test = self._parser.parse(module_properties['offline_test'])
    self._classes = np.array(map(int, module_properties['classes'].split(',')))
    self._server_port = module_properties['server_port']

    ##### Recovery and adding learners on the fly #########
    self._clf, timestamp = self._dao.get_model(self._id)
    self._checkpointed = None
    # if there was a checkpoint, then see if there are some historical points beyond that and train
    if self._clf:
      self._checkpointed = True
      examples = self._dao.get_examples_greater_than(timestamp)
      if examples:
        print 'catching up checkpointed model with historical points...'
        X, y, timestamps = self._parser.parse_feature(examples)
        self._clf.partial_fit(X, y)
      else:
        print 'no historical points to catch up the checkpointed model'
        # will use the last metric saved
    else:
      self._checkpointed = False
      self._clf = clf
      examples = self._dao.get_examples()
      if examples:
        print 'catching up new model with historical points'
        X, y, timestamps = self._parser.parse_feature(examples)
        self._clf.partial_fit(X, y, self._classes)
      else:
        print 'no historical points to catch up the new model'

    #######################################        

    self._registry = RegistryClient(module_properties['registry'])
    hostname = socket.gethostname()
    address = Utils.get_address(hostname, self._server_port)
    self._stream_client_address = self._registry.reg(ComponentType.LEARNER, address)[0]    

    self._handler = LearnerHandler(self._stream_client_address, self._registry, self._parser, self._evaluator, self._dao, self._clf, self._classes, 
                              self._warmup_examples, self._id, self._checkpoint, self._is_prequential, 
                              self._checkpointed, self._offline_test)
    self._processor = StreamService.Processor(self._handler)
    self._stream_server = Server(self._processor, self._server_port, module_properties['multi_threading'])
Beispiel #2
0
 def __init__(self, module_properties):
   self._server_port = module_properties['server_port']
   self._evaluator = EvaluatorFactory.new_evaluator(module_properties)
   self._registry = RegistryClient(module_properties['registry'])
   self._processor = StreamService.Processor(DeployerHandler(self._evaluator))
   self._stream_server = Server(self._processor, self._server_port, module_properties['multi_threading'])