def test_length(self):
print(f"\nExecuting {self._testMethodName}")
from mindsdb_streams import KafkaStream
stream = KafkaStream(f'test_stream_length_{STREAM_SUFFIX}', CONNECTION_PARAMS)
self.assertEqual(len(list(stream.read())), 0)
time.sleep(5)
stream.write({'0': 0})
time.sleep(5)
self.assertEqual(len(list(stream.read())), 1)
stream.write({'0': 0})
stream.write({'0': 0})
time.sleep(5)
self.assertEqual(len(list(stream.read())), 2)
self.assertEqual(len(list(stream.read())), 0)
def test_3_making_stream_prediction(self):
print(f"\nExecuting {self._testMethodName}")
from mindsdb_streams import KafkaStream
stream_in = KafkaStream(STREAM_IN, CONNECTION_PARAMS, mode='w')
stream_out = KafkaStream(STREAM_OUT, CONNECTION_PARAMS, mode='r')
# wait when the integration launches created stream
time.sleep(10)
for x in range(1, 3):
stream_in.write({'x1': x, 'x2': 2*x})
time.sleep(5)
time.sleep(10)
self.assertEqual(len(list(stream_out.read())), 2)
def test_5_making_ts_stream_prediction(self):
print(f"\nExecuting {self._testMethodName}")
from mindsdb_streams import KafkaStream
stream_in = KafkaStream(STREAM_IN_TS, CONNECTION_PARAMS)
stream_out = KafkaStream(STREAM_OUT_TS, CONNECTION_PARAMS)
# wait when the integration launches created stream
time.sleep(10)
for x in range(210, 221):
stream_in.write({'x1': x, 'x2': 2*x, 'order': x, 'group': 'A', 'y': 3*x})
time.sleep(0.001)
time.sleep(10)
self.assertEqual(len(list(stream_out.read())), 2)
def test_6_create_stream_kafka_native_api(self):
print(f"\nExecuting {self._testMethodName}")
from mindsdb_streams import KafkaStream
control_stream = KafkaStream(CONTROL_STREAM, CONNECTION_PARAMS)
control_stream.write({
'action': 'create',
'name': f'{self._testMethodName}_{STREAM_SUFFIX}',
'predictor': DEFAULT_PREDICTOR,
'stream_in': STREAM_IN_NATIVE,
'stream_out': STREAM_OUT_NATIVE,
})
time.sleep(5)
stream_in = KafkaStream(STREAM_IN_NATIVE, CONNECTION_PARAMS)
stream_out = KafkaStream(STREAM_OUT_NATIVE, CONNECTION_PARAMS)
for x in range(1, 3):
stream_in.write({'x1': x, 'x2': 2*x})
time.sleep(5)
self.assertEqual(len(list(stream_out.read())), 2)
def test_8_test_online_learning(self):
print(f"\nExecuting {self._testMethodName}")
from mindsdb_streams import KafkaStream
control_stream = KafkaStream(CONTROL_STREAM, CONNECTION_PARAMS)
stream_in = KafkaStream(STREAM_IN_OL, CONNECTION_PARAMS)
stream_out = KafkaStream(STREAM_OUT_OL, CONNECTION_PARAMS)
PREDICTOR_NAME = "ONLINE_LEARNING"
control_stream.write({
'action': 'create',
'name': f'{self._testMethodName}_{STREAM_SUFFIX}',
'predictor': PREDICTOR_NAME,
'learning_params': {"to_predict": "y",
'kwargs': {
'stop_training_in_x_seconds': 3}
},
'learning_threshold': 10,
'stream_in': STREAM_IN_OL,
'stream_out': STREAM_OUT_OL,
})
for x in range(1, 101):
stream_in.write({'x1': x, 'x2': 2 * x, 'y': 3 * x})
start_time = time.time()
while (time.time() - start_time) < 30:
time.sleep(5)
res = list(stream_out.read())
if res and res[0]['status'] == 'success':
break
else:
raise Exception('Create predictor timeout')
url = f'{HTTP_API_ROOT}/predictors/{PREDICTOR_NAME}'
res = requests.get(url)
self.assertEqual(res.status_code, 200,
f"expected to get {PREDICTOR_NAME} info, but have {res.text}")