def _make_stream(self, s: db.Stream):
if s.learning_params and s.learning_threshold:
learning_params = json.loads(s.learning_params) if isinstance(
s.learning_params, str) else s.learning_params
return StreamLearningController(
s.name,
s.predictor,
learning_params,
s.learning_threshold,
stream_in=KafkaStream(s.stream_in, self.connection_info),
stream_out=KafkaStream(s.stream_out, self.connection_info),
in_thread=True)
return StreamController(
s.name,
s.predictor,
stream_in=KafkaStream(s.stream_in, self.connection_info),
stream_out=KafkaStream(s.stream_out, self.connection_info),
stream_anomaly=KafkaStream(s.anomaly_stream, self.connection_info)
if s.anomaly_stream is not None else None,
in_thread=True)
def test_length(self):
print(f"\nExecuting {self._testMethodName}")
from mindsdb_streams import KafkaStream
stream = KafkaStream(f'test_stream_length_{STREAM_SUFFIX}', CONNECTION_PARAMS)
self.assertEqual(len(list(stream.read())), 0)
time.sleep(5)
stream.write({'0': 0})
time.sleep(5)
self.assertEqual(len(list(stream.read())), 1)
stream.write({'0': 0})
stream.write({'0': 0})
time.sleep(5)
self.assertEqual(len(list(stream.read())), 2)
self.assertEqual(len(list(stream.read())), 0)
def test_3_making_stream_prediction(self):
print(f"\nExecuting {self._testMethodName}")
from mindsdb_streams import KafkaStream
stream_in = KafkaStream(STREAM_IN, CONNECTION_PARAMS, mode='w')
stream_out = KafkaStream(STREAM_OUT, CONNECTION_PARAMS, mode='r')
# wait when the integration launches created stream
time.sleep(10)
for x in range(1, 3):
stream_in.write({'x1': x, 'x2': 2*x})
time.sleep(5)
time.sleep(10)
self.assertEqual(len(list(stream_out.read())), 2)
def test_5_making_ts_stream_prediction(self):
print(f"\nExecuting {self._testMethodName}")
from mindsdb_streams import KafkaStream
stream_in = KafkaStream(STREAM_IN_TS, CONNECTION_PARAMS)
stream_out = KafkaStream(STREAM_OUT_TS, CONNECTION_PARAMS)
# wait when the integration launches created stream
time.sleep(10)
for x in range(210, 221):
stream_in.write({'x1': x, 'x2': 2*x, 'order': x, 'group': 'A', 'y': 3*x})
time.sleep(0.001)
time.sleep(10)
self.assertEqual(len(list(stream_out.read())), 2)
def __init__(self, config, name, db_info):
self.connection_info = db_info['connection']
# Back compatibility with initial API version
self.control_stream = db_info.get('control_stream') or db_info.get(
'topic') or None
if 'advanced' in db_info:
self.connection_info['advanced'] = db_info['advanced']
self.control_connection_info = deepcopy(self.connection_info)
# don't need to read all records from the beginning of 'control stream'
# since all active streams are saved in db. Use 'latest' auto_offset_reset for control stream
if 'advanced' in self.control_connection_info:
if 'consumer' in self.control_connection_info['advanced']:
self.control_connection_info['advanced']['consumer'][
'auto_offset_reset'] = 'latest'
StreamIntegration.__init__(
self,
config,
name,
control_stream=KafkaStream(self.control_stream,
self.control_connection_info)
if self.control_stream else None)
def test_8_test_online_learning(self):
print(f"\nExecuting {self._testMethodName}")
from mindsdb_streams import KafkaStream
control_stream = KafkaStream(CONTROL_STREAM, CONNECTION_PARAMS)
stream_in = KafkaStream(STREAM_IN_OL, CONNECTION_PARAMS)
stream_out = KafkaStream(STREAM_OUT_OL, CONNECTION_PARAMS)
PREDICTOR_NAME = "ONLINE_LEARNING"
control_stream.write({
'action': 'create',
'name': f'{self._testMethodName}_{STREAM_SUFFIX}',
'predictor': PREDICTOR_NAME,
'learning_params': {"to_predict": "y",
'kwargs': {
'stop_training_in_x_seconds': 3}
},
'learning_threshold': 10,
'stream_in': STREAM_IN_OL,
'stream_out': STREAM_OUT_OL,
})
for x in range(1, 101):
stream_in.write({'x1': x, 'x2': 2 * x, 'y': 3 * x})
start_time = time.time()
while (time.time() - start_time) < 30:
time.sleep(5)
res = list(stream_out.read())
if res and res[0]['status'] == 'success':
break
else:
raise Exception('Create predictor timeout')
url = f'{HTTP_API_ROOT}/predictors/{PREDICTOR_NAME}'
res = requests.get(url)
self.assertEqual(res.status_code, 200,
f"expected to get {PREDICTOR_NAME} info, but have {res.text}")
def test_6_create_stream_kafka_native_api(self):
print(f"\nExecuting {self._testMethodName}")
from mindsdb_streams import KafkaStream
control_stream = KafkaStream(CONTROL_STREAM, CONNECTION_PARAMS)
control_stream.write({
'action': 'create',
'name': f'{self._testMethodName}_{STREAM_SUFFIX}',
'predictor': DEFAULT_PREDICTOR,
'stream_in': STREAM_IN_NATIVE,
'stream_out': STREAM_OUT_NATIVE,
})
time.sleep(5)
stream_in = KafkaStream(STREAM_IN_NATIVE, CONNECTION_PARAMS)
stream_out = KafkaStream(STREAM_OUT_NATIVE, CONNECTION_PARAMS)
for x in range(1, 3):
stream_in.write({'x1': x, 'x2': 2*x})
time.sleep(5)
self.assertEqual(len(list(stream_out.read())), 2)