def _make_stream(self, s: db.Stream): if s.learning_params and s.learning_threshold: learning_params = json.loads(s.learning_params) if isinstance( s.learning_params, str) else s.learning_params return StreamLearningController( s.name, s.predictor, learning_params, s.learning_threshold, stream_in=KafkaStream(s.stream_in, self.connection_info), stream_out=KafkaStream(s.stream_out, self.connection_info), in_thread=True) return StreamController( s.name, s.predictor, stream_in=KafkaStream(s.stream_in, self.connection_info), stream_out=KafkaStream(s.stream_out, self.connection_info), stream_anomaly=KafkaStream(s.anomaly_stream, self.connection_info) if s.anomaly_stream is not None else None, in_thread=True)
def test_length(self): print(f"\nExecuting {self._testMethodName}") from mindsdb_streams import KafkaStream stream = KafkaStream(f'test_stream_length_{STREAM_SUFFIX}', CONNECTION_PARAMS) self.assertEqual(len(list(stream.read())), 0) time.sleep(5) stream.write({'0': 0}) time.sleep(5) self.assertEqual(len(list(stream.read())), 1) stream.write({'0': 0}) stream.write({'0': 0}) time.sleep(5) self.assertEqual(len(list(stream.read())), 2) self.assertEqual(len(list(stream.read())), 0)
def test_3_making_stream_prediction(self): print(f"\nExecuting {self._testMethodName}") from mindsdb_streams import KafkaStream stream_in = KafkaStream(STREAM_IN, CONNECTION_PARAMS, mode='w') stream_out = KafkaStream(STREAM_OUT, CONNECTION_PARAMS, mode='r') # wait when the integration launches created stream time.sleep(10) for x in range(1, 3): stream_in.write({'x1': x, 'x2': 2*x}) time.sleep(5) time.sleep(10) self.assertEqual(len(list(stream_out.read())), 2)
def test_5_making_ts_stream_prediction(self): print(f"\nExecuting {self._testMethodName}") from mindsdb_streams import KafkaStream stream_in = KafkaStream(STREAM_IN_TS, CONNECTION_PARAMS) stream_out = KafkaStream(STREAM_OUT_TS, CONNECTION_PARAMS) # wait when the integration launches created stream time.sleep(10) for x in range(210, 221): stream_in.write({'x1': x, 'x2': 2*x, 'order': x, 'group': 'A', 'y': 3*x}) time.sleep(0.001) time.sleep(10) self.assertEqual(len(list(stream_out.read())), 2)
def __init__(self, config, name, db_info): self.connection_info = db_info['connection'] # Back compatibility with initial API version self.control_stream = db_info.get('control_stream') or db_info.get( 'topic') or None if 'advanced' in db_info: self.connection_info['advanced'] = db_info['advanced'] self.control_connection_info = deepcopy(self.connection_info) # don't need to read all records from the beginning of 'control stream' # since all active streams are saved in db. Use 'latest' auto_offset_reset for control stream if 'advanced' in self.control_connection_info: if 'consumer' in self.control_connection_info['advanced']: self.control_connection_info['advanced']['consumer'][ 'auto_offset_reset'] = 'latest' StreamIntegration.__init__( self, config, name, control_stream=KafkaStream(self.control_stream, self.control_connection_info) if self.control_stream else None)
def test_8_test_online_learning(self): print(f"\nExecuting {self._testMethodName}") from mindsdb_streams import KafkaStream control_stream = KafkaStream(CONTROL_STREAM, CONNECTION_PARAMS) stream_in = KafkaStream(STREAM_IN_OL, CONNECTION_PARAMS) stream_out = KafkaStream(STREAM_OUT_OL, CONNECTION_PARAMS) PREDICTOR_NAME = "ONLINE_LEARNING" control_stream.write({ 'action': 'create', 'name': f'{self._testMethodName}_{STREAM_SUFFIX}', 'predictor': PREDICTOR_NAME, 'learning_params': {"to_predict": "y", 'kwargs': { 'stop_training_in_x_seconds': 3} }, 'learning_threshold': 10, 'stream_in': STREAM_IN_OL, 'stream_out': STREAM_OUT_OL, }) for x in range(1, 101): stream_in.write({'x1': x, 'x2': 2 * x, 'y': 3 * x}) start_time = time.time() while (time.time() - start_time) < 30: time.sleep(5) res = list(stream_out.read()) if res and res[0]['status'] == 'success': break else: raise Exception('Create predictor timeout') url = f'{HTTP_API_ROOT}/predictors/{PREDICTOR_NAME}' res = requests.get(url) self.assertEqual(res.status_code, 200, f"expected to get {PREDICTOR_NAME} info, but have {res.text}")
def test_6_create_stream_kafka_native_api(self): print(f"\nExecuting {self._testMethodName}") from mindsdb_streams import KafkaStream control_stream = KafkaStream(CONTROL_STREAM, CONNECTION_PARAMS) control_stream.write({ 'action': 'create', 'name': f'{self._testMethodName}_{STREAM_SUFFIX}', 'predictor': DEFAULT_PREDICTOR, 'stream_in': STREAM_IN_NATIVE, 'stream_out': STREAM_OUT_NATIVE, }) time.sleep(5) stream_in = KafkaStream(STREAM_IN_NATIVE, CONNECTION_PARAMS) stream_out = KafkaStream(STREAM_OUT_NATIVE, CONNECTION_PARAMS) for x in range(1, 3): stream_in.write({'x1': x, 'x2': 2*x}) time.sleep(5) self.assertEqual(len(list(stream_out.read())), 2)