def handle(self): log.debug('connect') log.debug(str(self.server.socket)) self.session = Session(self.server.mindsdb_env) first_byte = self.request.recv(1, socket.MSG_PEEK) if first_byte == b'\x16': # TLS 'client hello' starts from \x16 self._init_ssl() while True: header = self._read_bytes(16) if header is False: # connection closed by client break length, pos = unpack(INT, header) request_id, pos = unpack(INT, header, pos) response_to, pos = unpack(INT, header, pos) opcode, pos = unpack(INT, header, pos) log.debug(f'GET length={length} id={request_id} opcode={opcode}') msg_bytes = self._read_bytes(length - pos) answer = self.get_answer(request_id, opcode, msg_bytes) if answer is not None: self.request.send(answer) db_session.close()
def make_prediction(self): predict_record = session.query(DBPredictor).filter_by( company_id=self.company_id, name=self.predictor).first() if predict_record is None: log.error( f"Error creating stream: requested predictor {self.predictor} is not exist" ) return while not self.stop_event.wait(0.5): try: msg_str = next(self.consumer) when_data = json.loads(msg_str.value) result = self.native_interface.predict(self.predictor, self.format_flag, when_data=when_data) log.error(f"STREAM: got {result}") for res in result: in_json = json.dumps({"prediction": res}) to_send = in_json.encode('utf-8') log.error(f"sending {to_send}") self.producer.send(self.stream_out_name, to_send) except StopIteration: pass log.error("Stopping stream..") self.producer.close() self.consumer.close() session.close()
def make_timeseries_predictions(self): log.error("STREAM: running 'make_timeseries_predictions'") predict_record = session.query(DBPredictor).filter_by( company_id=self.company_id, name=self.predictor).first() if predict_record is None: log.error( f"Error creating stream: requested predictor {self.predictor} is not exist" ) return self.target = self._get_target() self.window = self._get_window_size() self.gb = self._get_gb() self.dt = self._get_dt() while not self.stop_event.wait(0.5): try: msg_str = next(self.consumer) when_data = json.loads(msg_str.value) self.to_cache(when_data) except StopIteration: pass log.error("Stopping stream..") self.producer.close() self.consumer.close() session.close()
def analyzing_thread(name, default_store): try: from mindsdb.interfaces.storage.db import session analysis = default_store.start_analysis(name) session.close() except Exception as e: log.error(e)
def analyzing_thread(name, default_store): global ds_analysis ds_analysis[name] = None ds = default_store.get_datasource(name) analysis = default_store.get_analysis(ds['name']) ds_analysis[name] = { 'created_at': datetime.datetime.utcnow(), 'data': analysis } session.close()
def work(self): """Creates a Streams by receiving initial information from control stream.""" self.log.error( f"INTEGRATION HAS BEEN CREATED: {self.connection_params}") self.log.error( f"Integration {self.name}: start listening {self.control_stream_name} redis stream" ) while not self.stop_event.wait(0.5): try: # break if no record about this integration has found in db if not self.exist_in_db(): self.delete_all_streams() break # First, lets check that there are no new records in db, created via HTTP API for e.g. self.start_stored_streams() # stop streams which have been deleted from db. self.stop_deleted_streams() # Checking new incoming records(requests) for new stream. # Could't use blocking reading here, because this loop must # also check new records in db (created via HTTP api for e.g) recs = self.control_stream.read() for r in recs: r_id = r[0] binary_r = r[1] stream_params = self._decode(binary_r) if "input_stream" not in stream_params or "output_stream" not in stream_params: self.delete_stream(stream_params['predictor']) self.control_stream.delete(r_id) continue stream_params[ 'name'] = f"{self.name}_{stream_params['predictor']}" stream = self.get_stream_from_kwargs(**stream_params) self.log.error( f"Integration {self.name}: creating stream: {stream_params}" ) stream.start() # store created stream in database self.store_stream(stream) # need to delete record which steam is already created self.control_stream.delete(r_id) except Exception as e: self.log.error(f"Integration {self.name}: {e}") raise e # received exit event self.log.error(f"Integration {self.name}: exiting...") self.client.close() self.stop_streams() session.close()
def _open_webbrowser(url: str, pid: int, port: int, init_static_thread, static_folder): """Open webbrowser with url when http service is started. If some error then do nothing. """ init_static_thread.join() inject_telemetry_to_static(static_folder) logger = get_log('http') try: is_http_active = wait_func_is_true(func=is_pid_listen_port, timeout=10, pid=pid, port=port) if is_http_active: webbrowser.open(url) except Exception as e: logger.error(f'Failed to open {url} in webbrowser with exception {e}') logger.error(traceback.format_exc()) session.close()
def make_timeseries_predictions(self): log.error("STREAM: running 'make_timeseries_predictions'") predict_record = session.query(DBPredictor).filter_by(company_id=self.company_id, name=self.predictor).first() if predict_record is None: log.error(f"Error creating stream: requested predictor {self.predictor} is not exist") return while not self.stop_event.wait(0.5): # block==0 is a blocking mode predict_info = self.stream_in.read(block=0) for record in predict_info: record_id = record[0] raw_when_data = record[1] when_data = self.decode(raw_when_data) log.error(f"STREAM: next record have read from {self.stream_in.key}: {when_data}") self.to_cache(when_data) self.stream_in.delete(record_id) session.close()
def work(self): if self.control_topic_name is not None: #self.consumer = kafka.KafkaConsumer(bootstrap_servers=f"{self.host}:{self.port}", # consumer_timeout_ms=1000) self.consumer = kafka.KafkaConsumer( **self.connection_params, **self.advanced_info.get('consumer', {})) self.consumer.subscribe([self.control_topic_name]) self.log.error( f"Integration {self.name}: subscribed to {self.control_topic_name} kafka topic" ) else: self.consumer = None self.log.error(f"Integration {self.name}: worked mode - DB only.") while not self.stop_event.wait(0.5): try: # break if no record about this integration has found in db if not self.exist_in_db(): self.delete_all_streams() break self.start_stored_streams() self.stop_deleted_streams() if self.consumer is not None: try: msg_str = next(self.consumer) stream_params = json.loads(msg_str.value) stream = self.get_stream_from_kwargs(**stream_params) stream.start() # store created stream in database self.store_stream(stream) except StopIteration: pass except Exception as e: self.log.error(f"Integration {self.name}: {e}") # received exit event self.consumer.close() self.stop_streams() session.close() self.log.error(f"Integration {self.name}: exiting...")
def make_predictions(self): predict_record = session.query(DBPredictor).filter_by(company_id=self.company_id, name=self.predictor).first() if predict_record is None: log.error(f"Error creating stream: requested predictor {self.predictor} is not exist") return while not self.stop_event.wait(0.5): predict_info = self.stream_in.read() for record in predict_info: record_id = record[0] raw_when_data = record[1] when_data = self.decode(raw_when_data) result = self.native_interface.predict(self.predictor, self.format_flag, when_data=when_data) log.error(f"STREAM: got {result}") for res in result: in_json = json.dumps(res) self.stream_out.add({"prediction": in_json}) self.stream_in.delete(record_id) session.close() log.error("STREAM: stopping...")
def remove_session(*args, **kwargs): session.close()
def initialize_static(config): ''' Update Scout files basing on compatible-config.json content. Files will be downloaded and updated if new version of GUI > current. Current GUI version stored in static/version.txt. ''' log = get_log('http') static_path = Path(config.paths['static']) static_path.mkdir(parents=True, exist_ok=True) try: res = requests.get( 'https://mindsdb-web-builds.s3.amazonaws.com/compatible-config.json' ) except (ConnectionError, requests.exceptions.ConnectionError) as e: print(f'Is no connection. {e}') return False except Exception as e: print(f'Is something wrong with getting compatible-config.json: {e}') return False if res.status_code != 200: print( f'Cant get compatible-config.json: returned status code = {res.status_code}' ) return False try: versions = res.json() except Exception as e: print(f'Cant decode compatible-config.json: {e}') return False current_mindsdb_lv = LooseVersion(mindsdb_version) try: gui_versions = {} max_mindsdb_lv = None max_gui_lv = None for el in versions['mindsdb']: if el['mindsdb_version'] is None: gui_lv = LooseVersion(el['gui_version']) else: mindsdb_lv = LooseVersion(el['mindsdb_version']) gui_lv = LooseVersion(el['gui_version']) if mindsdb_lv.vstring not in gui_versions or gui_lv > gui_versions[ mindsdb_lv.vstring]: gui_versions[mindsdb_lv.vstring] = gui_lv if max_mindsdb_lv is None or max_mindsdb_lv < mindsdb_lv: max_mindsdb_lv = mindsdb_lv if max_gui_lv is None or max_gui_lv < gui_lv: max_gui_lv = gui_lv all_mindsdb_lv = [LooseVersion(x) for x in gui_versions.keys()] all_mindsdb_lv.sort() if current_mindsdb_lv.vstring in gui_versions: gui_version_lv = gui_versions[current_mindsdb_lv.vstring] elif current_mindsdb_lv > all_mindsdb_lv[-1]: gui_version_lv = max_gui_lv else: lower_versions = { key: value for key, value in gui_versions.items() if LooseVersion(key) < current_mindsdb_lv } if len(lower_versions) == 0: gui_version_lv = gui_versions[all_mindsdb_lv[0].vstring] else: all_lower_versions = [ LooseVersion(x) for x in lower_versions.keys() ] gui_version_lv = gui_versions[all_lower_versions[-1].vstring] except Exception as e: log.error(f'Error in compatible-config.json structure: {e}') return False current_gui_version = None version_txt_path = static_path.joinpath('version.txt') if version_txt_path.is_file(): with open(version_txt_path, 'rt') as f: current_gui_version = f.readline() if current_gui_version is not None: current_gui_lv = LooseVersion(current_gui_version) if current_gui_lv >= gui_version_lv: return True log.info( f'New version of GUI available ({gui_version_lv.vstring}). Downloading...' ) shutil.rmtree(static_path) static_path.mkdir(parents=True, exist_ok=True) try: css_zip_path = str(static_path.joinpath('css.zip')) js_zip_path = str(static_path.joinpath('js.zip')) media_zip_path = str(static_path.joinpath('media.zip')) bucket = "https://mindsdb-web-builds.s3.amazonaws.com/" gui_version = gui_version_lv.vstring resources = [{ 'url': bucket + 'css-V' + gui_version + '.zip', 'path': css_zip_path }, { 'url': bucket + 'js-V' + gui_version + '.zip', 'path': js_zip_path }, { 'url': bucket + 'indexV' + gui_version + '.html', 'path': str(static_path.joinpath('index.html')) }, { 'url': bucket + 'favicon.ico', 'path': str(static_path.joinpath('favicon.ico')) }, { 'url': bucket + 'media.zip', 'path': media_zip_path }] def get_resources(resource): try: response = requests.get(resource['url']) if response.status_code != requests.status_codes.codes.ok: return Exception( f"Error {response.status_code} GET {resource['url']}") open(resource['path'], 'wb').write(response.content) except Exception as e: return e return None for r in resources: get_resources(r) ''' # to make downloading faster download each resource in a separate thread with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: future_to_url = {executor.submit(get_resources, r): r for r in resources} for future in concurrent.futures.as_completed(future_to_url): res = future.result() if res is not None: raise res ''' except Exception as e: log.error(f'Error during downloading files from s3: {e}') session.close() return False static_folder = static_path.joinpath('static') static_folder.mkdir(parents=True, exist_ok=True) # unzip process for zip_path, dir_name in [[js_zip_path, 'js'], [css_zip_path, 'css']]: temp_dir = static_path.joinpath(f'temp_{dir_name}') temp_dir.mkdir(mode=0o777, exist_ok=True, parents=True) ZipFile(zip_path).extractall(temp_dir) files_path = static_path.joinpath('static', dir_name) if temp_dir.joinpath('build', 'static', dir_name).is_dir(): shutil.move(temp_dir.joinpath('build', 'static', dir_name), files_path) shutil.rmtree(temp_dir) else: shutil.move(temp_dir, files_path) ZipFile(media_zip_path).extractall(static_folder) os.remove(js_zip_path) os.remove(css_zip_path) os.remove(media_zip_path) with open(version_txt_path, 'wt') as f: f.write(gui_version_lv.vstring) log.info(f'GUI version updated to {gui_version_lv.vstring}') session.close() return True
def initialize_static(): success = update_static() session.close() return success