def handle(self):
log.debug('connect')
log.debug(str(self.server.socket))
self.session = Session(self.server.mindsdb_env)
first_byte = self.request.recv(1, socket.MSG_PEEK)
if first_byte == b'\x16':
# TLS 'client hello' starts from \x16
self._init_ssl()
while True:
header = self._read_bytes(16)
if header is False:
# connection closed by client
break
length, pos = unpack(INT, header)
request_id, pos = unpack(INT, header, pos)
response_to, pos = unpack(INT, header, pos)
opcode, pos = unpack(INT, header, pos)
log.debug(f'GET length={length} id={request_id} opcode={opcode}')
msg_bytes = self._read_bytes(length - pos)
answer = self.get_answer(request_id, opcode, msg_bytes)
if answer is not None:
self.request.send(answer)
db_session.close()
def make_prediction(self):
predict_record = session.query(DBPredictor).filter_by(
company_id=self.company_id, name=self.predictor).first()
if predict_record is None:
log.error(
f"Error creating stream: requested predictor {self.predictor} is not exist"
)
return
while not self.stop_event.wait(0.5):
try:
msg_str = next(self.consumer)
when_data = json.loads(msg_str.value)
result = self.native_interface.predict(self.predictor,
self.format_flag,
when_data=when_data)
log.error(f"STREAM: got {result}")
for res in result:
in_json = json.dumps({"prediction": res})
to_send = in_json.encode('utf-8')
log.error(f"sending {to_send}")
self.producer.send(self.stream_out_name, to_send)
except StopIteration:
pass
log.error("Stopping stream..")
self.producer.close()
self.consumer.close()
session.close()
def make_timeseries_predictions(self):
log.error("STREAM: running 'make_timeseries_predictions'")
predict_record = session.query(DBPredictor).filter_by(
company_id=self.company_id, name=self.predictor).first()
if predict_record is None:
log.error(
f"Error creating stream: requested predictor {self.predictor} is not exist"
)
return
self.target = self._get_target()
self.window = self._get_window_size()
self.gb = self._get_gb()
self.dt = self._get_dt()
while not self.stop_event.wait(0.5):
try:
msg_str = next(self.consumer)
when_data = json.loads(msg_str.value)
self.to_cache(when_data)
except StopIteration:
pass
log.error("Stopping stream..")
self.producer.close()
self.consumer.close()
session.close()
def analyzing_thread(name, default_store):
try:
from mindsdb.interfaces.storage.db import session
analysis = default_store.start_analysis(name)
session.close()
except Exception as e:
log.error(e)
def analyzing_thread(name, default_store):
global ds_analysis
ds_analysis[name] = None
ds = default_store.get_datasource(name)
analysis = default_store.get_analysis(ds['name'])
ds_analysis[name] = {
'created_at': datetime.datetime.utcnow(),
'data': analysis
}
session.close()
def work(self):
"""Creates a Streams by receiving initial information from control stream."""
self.log.error(
f"INTEGRATION HAS BEEN CREATED: {self.connection_params}")
self.log.error(
f"Integration {self.name}: start listening {self.control_stream_name} redis stream"
)
while not self.stop_event.wait(0.5):
try:
# break if no record about this integration has found in db
if not self.exist_in_db():
self.delete_all_streams()
break
# First, lets check that there are no new records in db, created via HTTP API for e.g.
self.start_stored_streams()
# stop streams which have been deleted from db.
self.stop_deleted_streams()
# Checking new incoming records(requests) for new stream.
# Could't use blocking reading here, because this loop must
# also check new records in db (created via HTTP api for e.g)
recs = self.control_stream.read()
for r in recs:
r_id = r[0]
binary_r = r[1]
stream_params = self._decode(binary_r)
if "input_stream" not in stream_params or "output_stream" not in stream_params:
self.delete_stream(stream_params['predictor'])
self.control_stream.delete(r_id)
continue
stream_params[
'name'] = f"{self.name}_{stream_params['predictor']}"
stream = self.get_stream_from_kwargs(**stream_params)
self.log.error(
f"Integration {self.name}: creating stream: {stream_params}"
)
stream.start()
# store created stream in database
self.store_stream(stream)
# need to delete record which steam is already created
self.control_stream.delete(r_id)
except Exception as e:
self.log.error(f"Integration {self.name}: {e}")
raise e
# received exit event
self.log.error(f"Integration {self.name}: exiting...")
self.client.close()
self.stop_streams()
session.close()
def _open_webbrowser(url: str, pid: int, port: int, init_static_thread, static_folder):
"""Open webbrowser with url when http service is started.
If some error then do nothing.
"""
init_static_thread.join()
inject_telemetry_to_static(static_folder)
logger = get_log('http')
try:
is_http_active = wait_func_is_true(func=is_pid_listen_port, timeout=10,
pid=pid, port=port)
if is_http_active:
webbrowser.open(url)
except Exception as e:
logger.error(f'Failed to open {url} in webbrowser with exception {e}')
logger.error(traceback.format_exc())
session.close()
def make_timeseries_predictions(self):
log.error("STREAM: running 'make_timeseries_predictions'")
predict_record = session.query(DBPredictor).filter_by(company_id=self.company_id, name=self.predictor).first()
if predict_record is None:
log.error(f"Error creating stream: requested predictor {self.predictor} is not exist")
return
while not self.stop_event.wait(0.5):
# block==0 is a blocking mode
predict_info = self.stream_in.read(block=0)
for record in predict_info:
record_id = record[0]
raw_when_data = record[1]
when_data = self.decode(raw_when_data)
log.error(f"STREAM: next record have read from {self.stream_in.key}: {when_data}")
self.to_cache(when_data)
self.stream_in.delete(record_id)
session.close()
def work(self):
if self.control_topic_name is not None:
#self.consumer = kafka.KafkaConsumer(bootstrap_servers=f"{self.host}:{self.port}",
# consumer_timeout_ms=1000)
self.consumer = kafka.KafkaConsumer(
**self.connection_params,
**self.advanced_info.get('consumer', {}))
self.consumer.subscribe([self.control_topic_name])
self.log.error(
f"Integration {self.name}: subscribed to {self.control_topic_name} kafka topic"
)
else:
self.consumer = None
self.log.error(f"Integration {self.name}: worked mode - DB only.")
while not self.stop_event.wait(0.5):
try:
# break if no record about this integration has found in db
if not self.exist_in_db():
self.delete_all_streams()
break
self.start_stored_streams()
self.stop_deleted_streams()
if self.consumer is not None:
try:
msg_str = next(self.consumer)
stream_params = json.loads(msg_str.value)
stream = self.get_stream_from_kwargs(**stream_params)
stream.start()
# store created stream in database
self.store_stream(stream)
except StopIteration:
pass
except Exception as e:
self.log.error(f"Integration {self.name}: {e}")
# received exit event
self.consumer.close()
self.stop_streams()
session.close()
self.log.error(f"Integration {self.name}: exiting...")
def make_predictions(self):
predict_record = session.query(DBPredictor).filter_by(company_id=self.company_id, name=self.predictor).first()
if predict_record is None:
log.error(f"Error creating stream: requested predictor {self.predictor} is not exist")
return
while not self.stop_event.wait(0.5):
predict_info = self.stream_in.read()
for record in predict_info:
record_id = record[0]
raw_when_data = record[1]
when_data = self.decode(raw_when_data)
result = self.native_interface.predict(self.predictor, self.format_flag, when_data=when_data)
log.error(f"STREAM: got {result}")
for res in result:
in_json = json.dumps(res)
self.stream_out.add({"prediction": in_json})
self.stream_in.delete(record_id)
session.close()
log.error("STREAM: stopping...")
def remove_session(*args, **kwargs):
session.close()
def initialize_static(config):
''' Update Scout files basing on compatible-config.json content.
Files will be downloaded and updated if new version of GUI > current.
Current GUI version stored in static/version.txt.
'''
log = get_log('http')
static_path = Path(config.paths['static'])
static_path.mkdir(parents=True, exist_ok=True)
try:
res = requests.get(
'https://mindsdb-web-builds.s3.amazonaws.com/compatible-config.json'
)
except (ConnectionError, requests.exceptions.ConnectionError) as e:
print(f'Is no connection. {e}')
return False
except Exception as e:
print(f'Is something wrong with getting compatible-config.json: {e}')
return False
if res.status_code != 200:
print(
f'Cant get compatible-config.json: returned status code = {res.status_code}'
)
return False
try:
versions = res.json()
except Exception as e:
print(f'Cant decode compatible-config.json: {e}')
return False
current_mindsdb_lv = LooseVersion(mindsdb_version)
try:
gui_versions = {}
max_mindsdb_lv = None
max_gui_lv = None
for el in versions['mindsdb']:
if el['mindsdb_version'] is None:
gui_lv = LooseVersion(el['gui_version'])
else:
mindsdb_lv = LooseVersion(el['mindsdb_version'])
gui_lv = LooseVersion(el['gui_version'])
if mindsdb_lv.vstring not in gui_versions or gui_lv > gui_versions[
mindsdb_lv.vstring]:
gui_versions[mindsdb_lv.vstring] = gui_lv
if max_mindsdb_lv is None or max_mindsdb_lv < mindsdb_lv:
max_mindsdb_lv = mindsdb_lv
if max_gui_lv is None or max_gui_lv < gui_lv:
max_gui_lv = gui_lv
all_mindsdb_lv = [LooseVersion(x) for x in gui_versions.keys()]
all_mindsdb_lv.sort()
if current_mindsdb_lv.vstring in gui_versions:
gui_version_lv = gui_versions[current_mindsdb_lv.vstring]
elif current_mindsdb_lv > all_mindsdb_lv[-1]:
gui_version_lv = max_gui_lv
else:
lower_versions = {
key: value
for key, value in gui_versions.items()
if LooseVersion(key) < current_mindsdb_lv
}
if len(lower_versions) == 0:
gui_version_lv = gui_versions[all_mindsdb_lv[0].vstring]
else:
all_lower_versions = [
LooseVersion(x) for x in lower_versions.keys()
]
gui_version_lv = gui_versions[all_lower_versions[-1].vstring]
except Exception as e:
log.error(f'Error in compatible-config.json structure: {e}')
return False
current_gui_version = None
version_txt_path = static_path.joinpath('version.txt')
if version_txt_path.is_file():
with open(version_txt_path, 'rt') as f:
current_gui_version = f.readline()
if current_gui_version is not None:
current_gui_lv = LooseVersion(current_gui_version)
if current_gui_lv >= gui_version_lv:
return True
log.info(
f'New version of GUI available ({gui_version_lv.vstring}). Downloading...'
)
shutil.rmtree(static_path)
static_path.mkdir(parents=True, exist_ok=True)
try:
css_zip_path = str(static_path.joinpath('css.zip'))
js_zip_path = str(static_path.joinpath('js.zip'))
media_zip_path = str(static_path.joinpath('media.zip'))
bucket = "https://mindsdb-web-builds.s3.amazonaws.com/"
gui_version = gui_version_lv.vstring
resources = [{
'url': bucket + 'css-V' + gui_version + '.zip',
'path': css_zip_path
}, {
'url': bucket + 'js-V' + gui_version + '.zip',
'path': js_zip_path
}, {
'url': bucket + 'indexV' + gui_version + '.html',
'path': str(static_path.joinpath('index.html'))
}, {
'url': bucket + 'favicon.ico',
'path': str(static_path.joinpath('favicon.ico'))
}, {
'url': bucket + 'media.zip',
'path': media_zip_path
}]
def get_resources(resource):
try:
response = requests.get(resource['url'])
if response.status_code != requests.status_codes.codes.ok:
return Exception(
f"Error {response.status_code} GET {resource['url']}")
open(resource['path'], 'wb').write(response.content)
except Exception as e:
return e
return None
for r in resources:
get_resources(r)
'''
# to make downloading faster download each resource in a separate thread
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
future_to_url = {executor.submit(get_resources, r): r for r in resources}
for future in concurrent.futures.as_completed(future_to_url):
res = future.result()
if res is not None:
raise res
'''
except Exception as e:
log.error(f'Error during downloading files from s3: {e}')
session.close()
return False
static_folder = static_path.joinpath('static')
static_folder.mkdir(parents=True, exist_ok=True)
# unzip process
for zip_path, dir_name in [[js_zip_path, 'js'], [css_zip_path, 'css']]:
temp_dir = static_path.joinpath(f'temp_{dir_name}')
temp_dir.mkdir(mode=0o777, exist_ok=True, parents=True)
ZipFile(zip_path).extractall(temp_dir)
files_path = static_path.joinpath('static', dir_name)
if temp_dir.joinpath('build', 'static', dir_name).is_dir():
shutil.move(temp_dir.joinpath('build', 'static', dir_name),
files_path)
shutil.rmtree(temp_dir)
else:
shutil.move(temp_dir, files_path)
ZipFile(media_zip_path).extractall(static_folder)
os.remove(js_zip_path)
os.remove(css_zip_path)
os.remove(media_zip_path)
with open(version_txt_path, 'wt') as f:
f.write(gui_version_lv.vstring)
log.info(f'GUI version updated to {gui_version_lv.vstring}')
session.close()
return True
def initialize_static():
success = update_static()
session.close()
return success
