def __init__(self, connection_info, advanced_info, topic_in, topic_out,
predictor, _type):
self.connection_info = connection_info
self.advanced_info = advanced_info
self.predictor = predictor
self.stream_in_name = topic_in
self.stream_out_name = topic_out
self.consumer = kafka.KafkaConsumer(
**self.connection_info, **self.advanced_info.get('consumer', {}))
self.consumer.subscribe(topics=[self.stream_in_name])
self.producer = kafka.KafkaProducer(
**self.connection_info, **self.advanced_info.get('producer', {}))
self.admin = kafka.KafkaAdminClient(**self.connection_info)
try:
self.topic = NewTopic(self.stream_out_name,
num_partitions=1,
replication_factor=1)
self.admin.create_topics([self.topic])
except kafka.errors.TopicAlreadyExistsError:
pass
self._type = _type
self.native_interface = NativeInterface()
self.format_flag = 'explain'
self.stop_event = Event()
self.company_id = os.environ.get('MINDSDB_COMPANY_ID', None)
self.caches = {}
if self._type == 'timeseries':
super().__init__(target=KafkaStream.make_timeseries_predictions,
args=(self, ))
else:
super().__init__(target=KafkaStream.make_prediction, args=(self, ))
def __init__(self):
self.config = Config()
self.fs_store = FsSotre()
self.company_id = os.environ.get('MINDSDB_COMPANY_ID', None)
self.dir = self.config.paths['datasources']
self.mindsdb_native = NativeInterface()
def run_fit(predictor_id: int, df: pd.DataFrame) -> None:
try:
predictor_record = session.query(db.Predictor).filter_by(id=predictor_id).first()
assert predictor_record is not None
fs_store = FsStore()
config = Config()
predictor_record.data = {'training_log': 'training'}
session.commit()
predictor: lightwood.PredictorInterface = lightwood.predictor_from_code(predictor_record.code)
predictor.learn(df)
session.refresh(predictor_record)
fs_name = f'predictor_{predictor_record.company_id}_{predictor_record.id}'
pickle_path = os.path.join(config['paths']['predictors'], fs_name)
predictor.save(pickle_path)
fs_store.put(fs_name, fs_name, config['paths']['predictors'])
predictor_record.data = predictor.model_analysis.to_dict()
predictor_record.dtype_dict = predictor.dtype_dict
session.commit()
dbw = DatabaseWrapper(predictor_record.company_id)
mi = ModelInterfaceWrapper(ModelInterface(), predictor_record.company_id)
dbw.register_predictors([mi.get_model_data(predictor_record.name)])
except Exception as e:
session.refresh(predictor_record)
predictor_record.data = {'error': f'{traceback.format_exc()}\nMain error: {e}'}
session.commit()
raise e
def __init__(self):
self.config = Config()
self.fs_store = FsSotre()
self.company_id = os.environ.get('MINDSDB_COMPANY_ID', None)
self.dbw = DatabaseWrapper()
self.storage_dir = self.config['paths']['custom_models']
os.makedirs(self.storage_dir, exist_ok=True)
self.model_cache = {}
self.mindsdb_native = NativeInterface()
self.dbw = DatabaseWrapper()
def run_fit(predictor_id: int, df: pd.DataFrame) -> None:
try:
predictor_record = Predictor.query.with_for_update().get(predictor_id)
assert predictor_record is not None
fs_store = FsStore()
config = Config()
predictor_record.data = {'training_log': 'training'}
session.commit()
predictor: lightwood.PredictorInterface = lightwood.predictor_from_code(
predictor_record.code)
predictor.learn(df)
session.refresh(predictor_record)
fs_name = f'predictor_{predictor_record.company_id}_{predictor_record.id}'
pickle_path = os.path.join(config['paths']['predictors'], fs_name)
predictor.save(pickle_path)
fs_store.put(fs_name, fs_name, config['paths']['predictors'])
predictor_record.data = predictor.model_analysis.to_dict()
# getting training time for each tried model. it is possible to do
# after training only
fit_mixers = list(predictor.runtime_log[x]
for x in predictor.runtime_log
if isinstance(x, tuple) and x[0] == "fit_mixer")
submodel_data = predictor_record.data.get("submodel_data", [])
# add training time to other mixers info
if submodel_data and fit_mixers and len(submodel_data) == len(
fit_mixers):
for i, tr_time in enumerate(fit_mixers):
submodel_data[i]["training_time"] = tr_time
predictor_record.data["submodel_data"] = submodel_data
predictor_record.dtype_dict = predictor.dtype_dict
session.commit()
dbw = DatabaseWrapper(predictor_record.company_id)
mi = WithKWArgsWrapper(ModelInterface(),
company_id=predictor_record.company_id)
except Exception as e:
session.refresh(predictor_record)
predictor_record.data = {
'error': f'{traceback.format_exc()}\nMain error: {e}'
}
session.commit()
raise e
try:
dbw.register_predictors([mi.get_model_data(predictor_record.name)])
except Exception as e:
log.warn(e)
class RedisStream(Thread):
def __init__(self, host, port, database, stream_in, stream_out, predictor, _type):
self.host = host
self.port = port
self.db = database
self.predictor = predictor
self.client = self._get_client()
self.stream_in_name = stream_in
self.stream_out_name = stream_out
self.stream_in = self.client.Stream(stream_in)
self.stream_out = self.client.Stream(stream_out)
self._type = _type
self.native_interface = NativeInterface()
self.format_flag = 'explain'
self.stop_event = Event()
self.company_id = os.environ.get('MINDSDB_COMPANY_ID', None)
super().__init__(target=RedisStream.make_prediction, args=(self,))
def _get_client(self):
return walrus.Database(host=self.host, port=self.port, db=self.db)
def make_prediction(self):
predict_record = session.query(DBPredictor).filter_by(company_id=self.company_id, name=self.predictor).first()
if predict_record is None:
log.error(f"Error creating stream: requested predictor {self.predictor} is not exist")
return
while not self.stop_event.wait(0.5):
# block==0 is a blocking mode
predict_info = self.stream_in.read(block=0)
for record in predict_info:
record_id = record[0]
raw_when_data = record[1]
when_data = self.decode(raw_when_data)
result = self.native_interface.predict(self.predictor, self.format_flag, when_data=when_data)
log.error(f"STREAM: got {result}")
for res in result:
in_json = json.dumps(res)
self.stream_out.add({"prediction": in_json})
self.stream_in.delete(record_id)
session.close()
def decode(self, redis_data):
decoded = {}
for k in redis_data:
decoded[k.decode('utf8')] = redis_data[k].decode('utf8')
return decoded
def __init__(self, config):
mongodb_config = config['api'].get('mongodb')
assert mongodb_config is not None, 'is no mongodb config!'
host = mongodb_config['host']
port = mongodb_config['port']
log.debug(f'start mongo server on {host}:{port}')
super().__init__((host, int(port)), MongoRequestHandler)
self.mindsdb_env = {
'config': config,
'origin_data_store': DataStore(),
'origin_model_interface': ModelInterface(),
'origin_datasource_controller': DatasourceController(),
}
self.mindsdb_env['mindsdb_native'] = WithKWArgsWrapper(
self.mindsdb_env['origin_model_interface'], company_id=None)
self.mindsdb_env['data_store'] = WithKWArgsWrapper(
self.mindsdb_env['origin_data_store'], company_id=None)
self.mindsdb_env['datasource_controller'] = WithKWArgsWrapper(
self.mindsdb_env['origin_datasource_controller'], company_id=None)
respondersCollection = RespondersCollection()
opQueryResponder = OpQueryResponder(respondersCollection)
opMsgResponder = OpMsgResponder(respondersCollection)
opInsertResponder = OpInsertResponder(respondersCollection)
self.operationsHandlersMap = {
OP_QUERY: opQueryResponder,
OP_MSG: opMsgResponder,
OP_INSERT: opInsertResponder
}
respondersCollection.add(when={'drop': 'system.sessions'},
result={'ok': 1})
respondersCollection.add(when={'update': 'system.version'},
result={'ok': 1})
respondersCollection.add(
when={'setFeatureCompatibilityVersion': helpers.is_true},
result={'ok': 1})
# OpMSG=OrderedDict([('features', 1), ('$clusterTime', OrderedDict([('clusterTime', Timestamp(1599748325, 1)), ('signature', OrderedDict([('hash', b'\xb8\xc3\x03\x18\xca\xe6bh\xf0\xcb47,\x924\x8a >\xfc\x91'), ('keyId', 6870854312365391875)]))])), ('$configServerState', OrderedDict([('opTime', OrderedDict([('ts', Timestamp(1599748325, 1)), ('t', 1)]))])), ('$db', 'admin')])
respondersCollection.add(when={'features': helpers.is_true},
result={'ok': 1})
# OpMSG=OrderedDict([('serverStatus', 1), ('$clusterTime', OrderedDict([('clusterTime', Timestamp(1599748366, 1)), ('signature', OrderedDict([('hash', b'\xa1E}\xbbIU\xc2D\x95++\x82\x88\xb5\x84\xf5\xda)+B'), ('keyId', 6870854312365391875)]))])), ('$configServerState', OrderedDict([('opTime', OrderedDict([('ts', Timestamp(1599748366, 1)), ('t', 1)]))])), ('$db', 'admin')])
respondersCollection.add(when={'serverStatus': helpers.is_true},
result={'ok': 1})
# OpMSG=OrderedDict([('ismaster', 1), ('$db', 'admin'), ('$clusterTime', OrderedDict([('clusterTime', Timestamp(1599749031, 1)), ('signature', OrderedDict([('hash', b'6\x87\xd5Y\xa7\xc7\xcf$\xab\x1e\xa2{\xe5B\xe5\x99\xdbl\x8d\xf4'), ('keyId', 6870854312365391875)]))])), ('$client', OrderedDict([('application', OrderedDict([('name', 'MongoDB Shell')])), ('driver', OrderedDict([('name', 'MongoDB Internal Client'), ('version', '3.6.3')])), ('os', OrderedDict([('type', 'Linux'), ('name', 'Ubuntu'), ('architecture', 'x86_64'), ('version', '18.04')])), ('mongos', OrderedDict([('host', 'maxs-comp:27103'), ('client', '127.0.0.1:52148'), ('version', '3.6.3')]))])), ('$configServerState', OrderedDict([('opTime', OrderedDict([('ts', Timestamp(1599749031, 1)), ('t', 1)]))]))])
respondersCollection.responders += responders
def __init__(self, host, port, database, stream_in, stream_out, predictor, _type):
self.host = host
self.port = port
self.db = database
self.predictor = predictor
self.client = self._get_client()
self.stream_in_name = stream_in
self.stream_out_name = stream_out
self.stream_in = self.client.Stream(stream_in)
self.stream_out = self.client.Stream(stream_out)
self._type = _type
self.native_interface = NativeInterface()
self.format_flag = 'explain'
self.stop_event = Event()
self.company_id = os.environ.get('MINDSDB_COMPANY_ID', None)
super().__init__(target=RedisStream.make_prediction, args=(self,))
def __init__(self,
name,
predictor,
stream_in,
stream_out,
anomaly_stream=None,
learning_stream=None,
learning_threshold=100):
self.name = name
self.predictor = predictor
self.stream_in = stream_in
self.stream_out = stream_out
self.anomaly_stream = anomaly_stream
self.learning_stream = learning_stream
self.learning_threshold = learning_threshold
self.learning_data = []
self.company_id = os.environ.get('MINDSDB_COMPANY_ID', None)
self.stop_event = Event()
self.model_interface = ModelInterfaceWrapper(ModelInterface())
self.data_store = DataStore()
self.config = Config()
p = db.session.query(db.Predictor).filter_by(
company_id=self.company_id, name=self.predictor).first()
if p is None:
raise Exception(f'Predictor {predictor} doesn\'t exist')
self.target = p.to_predict[0]
ts_settings = p.learn_args.get('timeseries_settings', None)
if not ts_settings['is_timeseries']:
ts_settings = None
if ts_settings is None:
self.thread = Thread(target=StreamController._make_predictions,
args=(self, ))
else:
self.ts_settings = ts_settings
self.thread = Thread(target=StreamController._make_ts_predictions,
args=(self, ))
self.thread.start()
def __init__(self, name, connection_info, advanced_info, stream_in, stream_out, predictor, _type):
self.stream_name = name
self.connection_info = connection_info
self.connection_info.update(advanced_info)
self.predictor = predictor
self.client = self._get_client()
self.stream_in_name = stream_in
self.stream_out_name = stream_out
self.stream_in = self.client.Stream(stream_in)
self.stream_out = self.client.Stream(stream_out)
self._type = _type
self.native_interface = NativeInterface()
self.format_flag = 'explain'
self.stop_event = Event()
self.company_id = os.environ.get('MINDSDB_COMPANY_ID', None)
if self._type == 'timeseries':
super().__init__(target=RedisStream.make_timeseries_predictions, args=(self,))
else:
super().__init__(target=RedisStream.make_predictions, args=(self,))
def __init__(self, name, connection_info, advanced_info, stream_in, stream_out, predictor, _type, **ts_params):
self.stream_name = name
self.connection_info = connection_info
self.connection_info.update(advanced_info)
self.predictor = predictor
self.client = self._get_client()
self.stream_in_name = stream_in
self.stream_out_name = stream_out
self.stream_in = self.client.Stream(stream_in)
self.stream_out = self.client.Stream(stream_out)
self._type = _type
self.native_interface = NativeInterface()
self.format_flag = 'explain'
self.stop_event = Event()
self.company_id = os.environ.get('MINDSDB_COMPANY_ID', None)
self.ts_params = ts_params
if self._type.lower() == StreamTypes.timeseries:
self.target = self.ts_params.get('target')
self.window = self.ts_params.get('window_size')
self.gb = self.ts_params.get('group_by')
self.dt = self.ts_params.get('order_by')
super().__init__(target=RedisStream.make_timeseries_predictions, args=(self,))
else:
super().__init__(target=RedisStream.make_predictions, args=(self,))
class KafkaStream(Thread):
def __init__(self, connection_info, advanced_info, topic_in, topic_out,
predictor, _type, **ts_params):
self.connection_info = connection_info
self.advanced_info = advanced_info
self.predictor = predictor
self.stream_in_name = topic_in
self.stream_out_name = topic_out
self.consumer = kafka.KafkaConsumer(
**self.connection_info, **self.advanced_info.get('consumer', {}))
self.consumer.subscribe(topics=[self.stream_in_name])
self.producer = kafka.KafkaProducer(
**self.connection_info, **self.advanced_info.get('producer', {}))
self.admin = kafka.KafkaAdminClient(**self.connection_info)
try:
self.topic = NewTopic(self.stream_out_name,
num_partitions=1,
replication_factor=1)
self.admin.create_topics([self.topic])
except kafka.errors.TopicAlreadyExistsError:
pass
self._type = _type
self.native_interface = NativeInterface()
self.format_flag = 'explain'
self.stop_event = Event()
self.company_id = os.environ.get('MINDSDB_COMPANY_ID', None)
self.caches = {}
self.ts_params = ts_params
if self._type.lower() == StreamTypes.timeseries:
self.target = self.ts_params.get('target')
self.window = self.ts_params.get('window_size')
self.gb = self.ts_params.get('group_by')
self.dt = self.ts_params.get('order_by')
super().__init__(target=KafkaStream.make_timeseries_predictions,
args=(self, ))
else:
super().__init__(target=KafkaStream.make_prediction, args=(self, ))
def predict_ts(self, cache_name):
when_list = [x for x in self.caches[cache_name]]
for x in when_list:
if self.target not in x:
x['make_predictions'] = False
else:
x['make_predictions'] = True
result = self.native_interface.predict(self.predictor,
self.format_flag,
when_data=when_list)
log.error(f"TIMESERIES STREAM: got {result}")
for res in result:
in_json = json.dumps(res)
to_send = in_json.encode('utf-8')
log.error(f"sending {to_send}")
self.producer.send(self.stream_out_name, to_send)
self.caches[cache_name] = self.caches[cache_name][1:]
def make_prediction_from_cache(self, cache_name):
cache = self.caches[cache_name]
log.error("STREAM: in make_prediction_from_cache")
if len(cache) >= self.window:
log.error(
f"STREAM: make_prediction_from_cache - len(cache) = {len(cache)}"
)
self.predict_ts(cache_name)
def to_cache(self, record):
gb_val = record[self.gb]
cache_name = f"cache.{gb_val}"
if cache_name not in self.caches:
cache = []
self.caches[cache_name] = cache
log.error(f"STREAM: cache {cache_name} has been created")
self.make_prediction_from_cache(cache_name)
self.handle_record(cache_name, record)
self.make_prediction_from_cache(cache_name)
log.error("STREAM in cache: current iteration has done.")
def handle_record(self, cache_name, record):
log.error(f"STREAM: handling cache {cache_name} and {record} record.")
cache = self.caches[cache_name]
cache.append(record)
cache = self.sort_cache(cache)
self.caches[cache_name] = cache
def sort_cache(self, cache):
return sorted(cache, key=lambda x: x[self.dt])
def make_timeseries_predictions(self):
log.error("STREAM: running 'make_timeseries_predictions'")
predict_record = session.query(DBPredictor).filter_by(
company_id=self.company_id, name=self.predictor).first()
if predict_record is None:
log.error(
f"Error creating stream: requested predictor {self.predictor} is not exist"
)
return
while not self.stop_event.wait(0.5):
try:
msg_str = next(self.consumer)
when_data = json.loads(msg_str.value)
self.to_cache(when_data)
except StopIteration:
pass
log.error("Stopping stream..")
self.producer.close()
self.consumer.close()
session.close()
def make_prediction(self):
predict_record = session.query(DBPredictor).filter_by(
company_id=self.company_id, name=self.predictor).first()
if predict_record is None:
log.error(
f"Error creating stream: requested predictor {self.predictor} is not exist"
)
return
while not self.stop_event.wait(0.5):
try:
msg_str = next(self.consumer)
when_data = json.loads(msg_str.value)
result = self.native_interface.predict(self.predictor,
self.format_flag,
when_data=when_data)
log.error(f"STREAM: got {result}")
for res in result:
in_json = json.dumps({"prediction": res})
to_send = in_json.encode('utf-8')
log.error(f"sending {to_send}")
self.producer.send(self.stream_out_name, to_send)
except StopIteration:
pass
log.error("Stopping stream..")
self.producer.close()
self.consumer.close()
session.close()
os.environ['DEFAULT_LOG_LEVEL'] = config['log']['level']['console']
os.environ['LIGHTWOOD_LOG_LEVEL'] = config['log']['level']['console']
# Switch to this once the native interface has it's own thread :/
ctx = mp.get_context('spawn')
from mindsdb.__about__ import __version__ as mindsdb_version
print(f'Version {mindsdb_version}')
print(f'Configuration file:\n {config.config_path}')
print(f"Storage path:\n {config['paths']['root']}")
# @TODO Backwards compatibiltiy for tests, remove later
from mindsdb.interfaces.database.integrations import DatasourceController
dbw = DatabaseWrapper(COMPANY_ID)
model_interface = WithKWArgsWrapper(ModelInterface(),
company_id=COMPANY_ID)
datasource_interface = WithKWArgsWrapper(DatasourceController(),
company_id=COMPANY_ID)
raw_model_data_arr = model_interface.get_models()
model_data_arr = []
for model in raw_model_data_arr:
if model['status'] == 'complete':
x = model_interface.get_model_data(model['name'])
try:
model_data_arr.append(
model_interface.get_model_data(model['name']))
except Exception:
pass
is_cloud = config.get('cloud', False)
class RedisStream(Thread):
def __init__(self, name, connection_info, advanced_info, stream_in, stream_out, predictor, _type):
self.stream_name = name
self.connection_info = connection_info
self.connection_info.update(advanced_info)
self.predictor = predictor
self.client = self._get_client()
self.stream_in_name = stream_in
self.stream_out_name = stream_out
self.stream_in = self.client.Stream(stream_in)
self.stream_out = self.client.Stream(stream_out)
self._type = _type
self.native_interface = NativeInterface()
self.format_flag = 'explain'
self.stop_event = Event()
self.company_id = os.environ.get('MINDSDB_COMPANY_ID', None)
if self._type == 'timeseries':
super().__init__(target=RedisStream.make_timeseries_predictions, args=(self,))
else:
super().__init__(target=RedisStream.make_predictions, args=(self,))
def _get_client(self):
return walrus.Database(**self.connection_info)
def _get_target(self):
return "pnew_case"
# pass
def _get_window_size(self):
return 10
# pass
def _get_gb(self):
return "state"
# pass
def _get_dt(self):
return "time"
# pass
def predict(self, stream_in, stream_out, timeseries_mode=False):
predict_info = stream_in.read(block=0)
when_list = []
for record in predict_info:
record_id = record[0]
raw_when_data = record[1]
when_data = self.decode(raw_when_data)
if timeseries_mode:
# if self.target not in when_data:
# when_data['make_predictions'] = False
# else:
# when_data['make_predictions'] = True
when_list.append(when_data)
else:
result = self.native_interface.predict(self.predictor, self.format_flag, when_data=when_data)
log.error(f"STREAM: got {result}")
for res in result:
in_json = json.dumps(res)
stream_out.add({"prediction": in_json})
stream_in.delete(record_id)
if timeseries_mode:
result = self.native_interface.predict(self.predictor, self.format_flag, when_data=when_list)
log.error(f"TIMESERIES STREAM: got {result}")
for res in result:
in_json = json.dumps(res)
stream_out.add({"prediction": in_json})
stream_in.trim(len(stream_in) - 1, approximate=False)
def make_prediction_from_cache(self, cache):
log.error("STREAM: in make_prediction_from_cache")
if len(cache) >= self.window:
log.error(f"STREAM: make_prediction_from_cache - len(cache) = {len(cache)}")
self.predict(cache, self.stream_out, timeseries_mode=True)
def make_timeseries_predictions(self):
log.error("STREAM: running 'make_timeseries_predictions'")
predict_record = session.query(DBPredictor).filter_by(company_id=self.company_id, name=self.predictor).first()
if predict_record is None:
log.error(f"Error creating stream: requested predictor {self.predictor} is not exist")
return
self.target = self._get_target()
self.window = self._get_window_size()
self.gb = self._get_gb()
self.dt = self._get_dt()
while not self.stop_event.wait(0.5):
# block==0 is a blocking mode
predict_info = self.stream_in.read(block=0)
for record in predict_info:
record_id = record[0]
raw_when_data = record[1]
when_data = self.decode(raw_when_data)
log.error(f"STREAM: next record have read from {self.stream_in.key}: {when_data}")
self.to_cache(when_data)
self.stream_in.delete(record_id)
session.close()
def to_cache(self, record):
gb_val = record[self.gb]
cache = self.client.Stream(f"{self.stream_name}.cache.{gb_val}")
log.error(f"STREAM: cache {cache.key} has been created")
self.make_prediction_from_cache(cache)
self.handle_record(cache, record)
self.make_prediction_from_cache(cache)
log.error("STREAM in cache: current iteration has done.")
def handle_record(self, cache, record):
log.error(f"STREAM: handling cache {cache.key} and {record} record.")
records = cache.read()
# log.error(f"STREAM: current {cache.key} state: {records}")
records = [self.decode(x[1]) for x in records]
log.error(f"STREAM: read {records} from cache.")
records.append(record)
records = self.sort_cache(records)
log.error(f"STREAM: after updating and sorting - {records}.")
cache.trim(0, approximate=False)
for rec in records:
cache.add(rec)
log.error(f"STREAM: finish updating {cache.key}")
def sort_cache(self, cache):
return sorted(cache, key=lambda x: x[self.dt])
def make_predictions(self):
predict_record = session.query(DBPredictor).filter_by(company_id=self.company_id, name=self.predictor).first()
if predict_record is None:
log.error(f"Error creating stream: requested predictor {self.predictor} is not exist")
return
while not self.stop_event.wait(0.5):
predict_info = self.stream_in.read()
for record in predict_info:
record_id = record[0]
raw_when_data = record[1]
when_data = self.decode(raw_when_data)
result = self.native_interface.predict(self.predictor, self.format_flag, when_data=when_data)
log.error(f"STREAM: got {result}")
for res in result:
in_json = json.dumps(res)
self.stream_out.add({"prediction": in_json})
self.stream_in.delete(record_id)
session.close()
log.error("STREAM: stopping...")
def decode(self, redis_data):
decoded = {}
for k in redis_data:
decoded[k.decode('utf8')] = redis_data[k].decode('utf8')
return decoded
def __init__(self, config):
self.config = Config()
self.mindsdb_native = NativeInterface()
self.custom_models = CustomModels()
self.ai_table = AITable_store()
self.default_store = DataStore()
class CustomModels():
def __init__(self):
self.config = Config()
self.fs_store = FsSotre()
self.company_id = os.environ.get('MINDSDB_COMPANY_ID', None)
self.dbw = DatabaseWrapper()
self.storage_dir = self.config['paths']['custom_models']
os.makedirs(self.storage_dir, exist_ok=True)
self.model_cache = {}
self.mindsdb_native = NativeInterface()
self.dbw = DatabaseWrapper()
def _dir(self, name):
return str(os.path.join(self.storage_dir, name))
def _internal_load(self, name):
self.fs_store.get(name, f'custom_model_{self.company_id}_{name}',
self.storage_dir)
sys.path.insert(0, self._dir(name))
module = __import__(name)
try:
model = module.Model.load(
os.path.join(self._dir(name), 'model.pickle'))
except Exception as e:
model = module.Model()
model.initialize_column_types()
if hasattr(model, 'setup'):
model.setup()
self.model_cache[name] = model
return model
def learn(self, name, from_data, to_predict, datasource_id, kwargs={}):
model_data = self.get_model_data(name)
model_data['status'] = 'training'
self.save_model_data(name, model_data)
to_predict = to_predict if isinstance(to_predict,
list) else [to_predict]
data_source = getattr(mindsdb_datasources,
from_data['class'])(*from_data['args'],
**from_data['kwargs'])
data_frame = data_source.df
model = self._internal_load(name)
model.to_predict = to_predict
model_data = self.get_model_data(name)
model_data['predict'] = model.to_predict
self.save_model_data(name, model_data)
data_analysis = self.mindsdb_native.analyse_dataset(
data_source)['data_analysis_v2']
model_data = self.get_model_data(name)
model_data['data_analysis_v2'] = data_analysis
self.save_model_data(name, model_data)
model.fit(data_frame, to_predict, data_analysis, kwargs)
model.save(os.path.join(self._dir(name), 'model.pickle'))
self.model_cache[name] = model
model_data = self.get_model_data(name)
model_data['status'] = 'completed'
model_data['columns'] = list(data_analysis.keys())
self.save_model_data(name, model_data)
self.fs_store.put(name, f'custom_model_{self.company_id}_{name}',
self.storage_dir)
self.dbw.unregister_predictor(name)
self.dbw.register_predictors([self.get_model_data(name)])
def predict(self, name, when_data=None, from_data=None, kwargs=None):
self.fs_store.get(name, f'custom_model_{self.company_id}_{name}',
self.storage_dir)
if kwargs is None:
kwargs = {}
if from_data is not None:
if isinstance(from_data, dict):
data_source = getattr(mindsdb_datasources, from_data['class'])(
*from_data['args'], **from_data['kwargs'])
# assume that particular instance of any DataSource class is provided
else:
data_source = from_data
data_frame = data_source.df
elif when_data is not None:
if isinstance(when_data, dict):
for k in when_data:
when_data[k] = [when_data[k]]
data_frame = pd.DataFrame(when_data)
else:
data_frame = pd.DataFrame(when_data)
model = self._internal_load(name)
predictions = model.predict(data_frame, kwargs)
pred_arr = []
for i in range(len(predictions)):
pred_arr.append({})
pred_arr[-1] = {}
for col in predictions.columns:
pred_arr[-1][col] = {}
pred_arr[-1][col]['predicted_value'] = predictions[col].iloc[i]
return pred_arr
def get_model_data(self, name):
predictor_record = Predictor.query.filter_by(
company_id=self.company_id, name=name, is_custom=True).first()
return predictor_record.data
def save_model_data(self, name, data):
predictor_record = Predictor.query.filter_by(
company_id=self.company_id, name=name, is_custom=True).first()
if predictor_record is None:
predictor_record = Predictor(company_id=self.company_id,
name=name,
is_custom=True,
data=data)
session.add(predictor_record)
else:
predictor_record.data = data
session.commit()
def get_models(self):
predictor_names = [
x.name
for x in Predictor.query.filter_by(company_id=self.company_id,
is_custom=True)
]
models = []
for name in predictor_names:
models.append(self.get_model_data(name))
return models
def delete_model(self, name):
Predictor.query.filter_by(company_id=self.company_id,
name=name,
is_custom=True).delete()
session.commit()
shutil.rmtree(self._dir(name))
self.dbw.unregister_predictor(name)
self.fs_store.delete(f'custom_model_{self.company_id}_{name}')
def rename_model(self, name, new_name):
self.fs_store.get(name, f'custom_model_{self.company_id}_{name}',
self.storage_dir)
self.dbw.unregister_predictor(name)
shutil.move(self._dir(name), self._dir(new_name))
shutil.move(os.path.join(self._dir(new_name) + f'{name}.py'),
os.path.join(self._dir(new_name), f'{new_name}.py'))
predictor_record = Predictor.query.filter_by(
company_id=self.company_id, name=name, is_custom=True).first()
predictor_record.name = new_name
session.commit()
self.dbw.register_predictors([self.get_model_data(new_name)])
self.fs_store.put(name, f'custom_model_{self.company_id}_{new_name}',
self.storage_dir)
self.fs_store.delete(f'custom_model_{self.company_id}_{name}')
def export_model(self, name):
shutil.make_archive(base_name=name,
format='zip',
root_dir=self._dir(name))
return str(self._dir(name)) + '.zip'
def load_model(self, fpath, name, trained_status):
shutil.unpack_archive(fpath, self._dir(name), 'zip')
shutil.move(os.path.join(self._dir(name), 'model.py'),
os.path.join(self._dir(name), f'{name}.py'))
model = self._internal_load(name)
model.to_predict = model.to_predict if isinstance(
model.to_predict, list) else [model.to_predict]
self.save_model_data(
name, {
'name': name,
'data_analysis_v2': model.column_type_map,
'predict': model.to_predict,
'status': trained_status,
'is_custom': True,
'columns': list(model.column_type_map.keys())
})
with open(os.path.join(self._dir(name), '__init__.py'), 'w') as fp:
fp.write('')
self.fs_store.put(name, f'custom_model_{self.company_id}_{name}',
self.storage_dir)
if trained_status == 'trained':
self.dbw.register_predictors([self.get_model_data(name)])
def __init__(self):
self.config = Config()
self.fs_store = FsStore()
self.dir = self.config['paths']['datasources']
self.model_interface = ModelInterface()
class DataStore():
def __init__(self):
self.config = Config()
self.fs_store = FsStore()
self.dir = self.config['paths']['datasources']
self.model_interface = ModelInterface()
def get_analysis(self, name, company_id=None):
datasource_record = session.query(Datasource).filter_by(
company_id=company_id, name=name).first()
if datasource_record.analysis is None:
return None
analysis = json.loads(datasource_record.analysis)
return analysis
def start_analysis(self, name, company_id=None):
datasource_record = session.query(Datasource).filter_by(
company_id=company_id, name=name).first()
if datasource_record.analysis is not None:
return None
semaphor_record = session.query(Semaphor).filter_by(
company_id=company_id,
entity_id=datasource_record.id,
entity_type='datasource').first()
if semaphor_record is None:
semaphor_record = Semaphor(company_id=company_id,
entity_id=datasource_record.id,
entity_type='datasource',
action='write')
session.add(semaphor_record)
session.commit()
else:
return
try:
analysis = self.model_interface.analyse_dataset(
ds=self.get_datasource_obj(name,
raw=True,
company_id=company_id),
company_id=company_id)
datasource_record = session.query(Datasource).filter_by(
company_id=company_id, name=name).first()
datasource_record.analysis = json.dumps(analysis,
cls=CustomJSONEncoder)
session.commit()
except Exception as e:
log.error(e)
finally:
semaphor_record = session.query(Semaphor).filter_by(
company_id=company_id,
entity_id=datasource_record.id,
entity_type='datasource').first()
session.delete(semaphor_record)
session.commit()
def get_datasources(self, name=None, company_id=None):
datasource_arr = []
if name is not None:
datasource_record_arr = session.query(Datasource).filter_by(
company_id=company_id, name=name)
else:
datasource_record_arr = session.query(Datasource).filter_by(
company_id=company_id)
for datasource_record in datasource_record_arr:
try:
if datasource_record.data is None:
continue
datasource = json.loads(datasource_record.data)
datasource['created_at'] = datasource_record.created_at
datasource['updated_at'] = datasource_record.updated_at
datasource['name'] = datasource_record.name
datasource['id'] = datasource_record.id
datasource_arr.append(datasource)
except Exception as e:
log.error(e)
return datasource_arr
def get_data(self,
name,
where=None,
limit=None,
offset=None,
company_id=None):
offset = 0 if offset is None else offset
ds = self.get_datasource_obj(name, company_id=company_id)
if limit is not None:
# @TODO Add `offset` to the `filter` method of the datasource and get rid of `offset`
filtered_ds = ds.filter(where=where,
limit=limit + offset).iloc[offset:]
else:
filtered_ds = ds.filter(where=where)
filtered_ds = filtered_ds.where(pd.notnull(filtered_ds), None)
data = filtered_ds.to_dict(orient='records')
return {
'data': data,
'rowcount': len(ds),
'columns_names': list(data[0].keys())
}
def get_datasource(self, name, company_id=None):
datasource_arr = self.get_datasources(name, company_id=company_id)
if len(datasource_arr) == 1:
return datasource_arr[0]
# @TODO: Remove when db swithc is more stable, this should never happen, but good santiy check while this is kinda buggy
elif len(datasource_arr) > 1:
log.error('Two or more datasource with the same name, (',
len(datasource_arr), ') | Full list: ', datasource_arr)
raise Exception('Two or more datasource with the same name')
return None
def delete_datasource(self, name, company_id=None):
datasource_record = Datasource.query.filter_by(company_id=company_id,
name=name).first()
if not Config()["force_datasource_removing"]:
linked_models = Predictor.query.filter_by(
company_id=company_id,
datasource_id=datasource_record.id).all()
if linked_models:
raise Exception(
"Can't delete {} datasource because there are next models linked to it: {}"
.format(name, [model.name for model in linked_models]))
session.query(Semaphor).filter_by(company_id=company_id,
entity_id=datasource_record.id,
entity_type='datasource').delete()
session.delete(datasource_record)
session.commit()
self.fs_store.delete(f'datasource_{company_id}_{datasource_record.id}')
try:
shutil.rmtree(os.path.join(self.dir, f'{company_id}@@@@@{name}'))
except Exception:
pass
def get_vacant_name(self, base=None, company_id=None):
''' returns name of datasource, which starts from 'base' and ds with that name is not exists yet
'''
if base is None:
base = 'datasource'
datasources = session.query(
Datasource.name).filter_by(company_id=company_id).all()
datasources_names = [x[0] for x in datasources]
if base not in datasources_names:
return base
for i in range(1, 1000):
candidate = f'{base}_{i}'
if candidate not in datasources_names:
return candidate
raise Exception(
f"Can not find appropriate name for datasource '{base}'")
def create_datasource(self,
source_type,
source,
file_path=None,
company_id=None,
ds_meta_dir=None):
datasource_controller = DatasourceController()
if source_type == 'file':
source = os.path.join(ds_meta_dir, source)
shutil.move(file_path, source)
ds = FileDS(source)
creation_info = {'class': 'FileDS', 'args': [source], 'kwargs': {}}
elif datasource_controller.get_db_integration(source_type,
company_id) is not None:
integration = datasource_controller.get_db_integration(
source_type, company_id)
ds_class_map = {
'clickhouse': ClickhouseDS,
'mariadb': MariaDS,
'mysql': MySqlDS,
'singlestore': MySqlDS,
'postgres': PostgresDS,
'cockroachdb': PostgresDS,
'mssql': MSSQLDS,
'mongodb': MongoDS,
'snowflake': SnowflakeDS,
'athena': AthenaDS,
'cassandra': CassandraDS,
'scylladb': ScyllaDS,
'trinodb': TrinoDS
}
try:
dsClass = ds_class_map[integration['type']]
except KeyError:
raise KeyError(
f"Unknown DS type: {source_type}, type is {integration['type']}"
)
if dsClass is None:
raise Exception(
f"Unsupported datasource: {source_type}, type is {integration['type']}, please install required dependencies!"
)
if integration['type'] in ['clickhouse']:
creation_info = {
'class': dsClass.__name__,
'args': [],
'kwargs': {
'query': source['query'],
'user': integration['user'],
'password': integration['password'],
'host': integration['host'],
'port': integration['port']
}
}
ds = dsClass(**creation_info['kwargs'])
elif integration['type'] in [
'mssql', 'postgres', 'cockroachdb', 'mariadb', 'mysql',
'singlestore', 'cassandra', 'scylladb'
]:
creation_info = {
'class': dsClass.__name__,
'args': [],
'kwargs': {
'query': source['query'],
'user': integration['user'],
'password': integration['password'],
'host': integration['host'],
'port': integration['port']
}
}
kwargs = creation_info['kwargs']
integration_folder_name = f'integration_files_{company_id}_{integration["id"]}'
if integration['type'] in ('mysql', 'mariadb'):
kwargs['ssl'] = integration.get('ssl')
kwargs['ssl_ca'] = integration.get('ssl_ca')
kwargs['ssl_cert'] = integration.get('ssl_cert')
kwargs['ssl_key'] = integration.get('ssl_key')
for key in ['ssl_ca', 'ssl_cert', 'ssl_key']:
if isinstance(kwargs[key],
str) and len(kwargs[key]) > 0:
kwargs[key] = os.path.join(
self.integrations_dir, integration_folder_name,
kwargs[key])
elif integration['type'] in ('cassandra', 'scylla'):
kwargs['secure_connect_bundle'] = integration.get(
'secure_connect_bundle')
if (isinstance(kwargs['secure_connect_bundle'], str)
and len(kwargs['secure_connect_bundle']) > 0):
kwargs['secure_connect_bundle'] = os.path.join(
self.integrations_dir, integration_folder_name,
kwargs['secure_connect_bundle'])
if 'database' in integration:
kwargs['database'] = integration['database']
if 'database' in source:
kwargs['database'] = source['database']
ds = dsClass(**kwargs)
elif integration['type'] == 'snowflake':
creation_info = {
'class': dsClass.__name__,
'args': [],
'kwargs': {
'query':
source['query'].replace('"', "'"),
'schema':
source.get('schema', integration['schema']),
'warehouse':
source.get('warehouse', integration['warehouse']),
'database':
source.get('database', integration['database']),
'host':
integration['host'],
'password':
integration['password'],
'user':
integration['user'],
'account':
integration['account']
}
}
ds = dsClass(**creation_info['kwargs'])
elif integration['type'] == 'mongodb':
if isinstance(source['find'], str):
source['find'] = json.loads(source['find'])
creation_info = {
'class': dsClass.__name__,
'args': [],
'kwargs': {
'database': source['database'],
'collection': source['collection'],
'query': source['find'],
'user': integration['user'],
'password': integration['password'],
'host': integration['host'],
'port': integration['port']
}
}
ds = dsClass(**creation_info['kwargs'])
elif integration['type'] == 'athena':
creation_info = {
'class': dsClass.__name__,
'args': [],
'kwargs': {
'query': source['query'],
'staging_dir': source['staging_dir'],
'database': source['database'],
'access_key': source['access_key'],
'secret_key': source['secret_key'],
'region_name': source['region_name']
}
}
ds = dsClass(**creation_info['kwargs'])
elif integration['type'] == 'trinodb':
creation_info = {
'class': dsClass.__name__,
'args': [],
'kwargs': {
'query': source['query'],
'user': integration['user'],
'password': integration['password'],
'host': integration['host'],
'port': integration['port'],
'schema': integration['schema'],
'catalog': integration['catalog']
}
}
ds = dsClass(**creation_info['kwargs'])
else:
# This probably only happens for urls
ds = FileDS(source)
creation_info = {'class': 'FileDS', 'args': [source], 'kwargs': {}}
return ds, creation_info
def save_datasource(self,
name,
source_type,
source,
file_path=None,
company_id=None):
if source_type == 'file' and (file_path is None):
raise Exception(
'`file_path` argument required when source_type == "file"')
datasource_record = session.query(Datasource).filter_by(
company_id=company_id, name=name).first()
while datasource_record is not None:
raise Exception(f'Datasource with name {name} already exists')
try:
datasource_record = Datasource(
company_id=company_id,
name=name,
datasources_version=mindsdb_datasources.__version__,
mindsdb_version=mindsdb_version)
session.add(datasource_record)
session.commit()
ds_meta_dir = os.path.join(self.dir, f'{company_id}@@@@@{name}')
os.mkdir(ds_meta_dir)
ds, creation_info = self.create_datasource(source_type, source,
file_path, company_id,
ds_meta_dir)
if hasattr(ds, 'get_columns') and hasattr(ds, 'get_row_count'):
try:
column_names = ds.get_columns()
row_count = ds.get_row_count()
except Exception:
df = ds.df
column_names = list(df.keys())
row_count = len(df)
else:
df = ds.df
column_names = list(df.keys())
row_count = len(df)
if '' in column_names or len(column_names) != len(
set(column_names)):
shutil.rmtree(ds_meta_dir)
raise Exception(
'Each column in datasource must have unique non-empty name'
)
datasource_record.creation_info = json.dumps(creation_info)
datasource_record.data = json.dumps({
'source_type':
source_type,
'source':
source,
'row_count':
row_count,
'columns': [dict(name=x) for x in column_names]
})
self.fs_store.put(
f'{company_id}@@@@@{name}',
f'datasource_{company_id}_{datasource_record.id}', self.dir)
session.commit()
except Exception as e:
log.error(f'Error creating datasource {name}, exception: {e}')
try:
self.delete_datasource(name, company_id=company_id)
except Exception:
pass
raise e
return self.get_datasource_obj(name, raw=True, company_id=company_id)
def get_datasource_obj(self,
name=None,
id=None,
raw=False,
company_id=None):
try:
if name is not None:
datasource_record = session.query(Datasource).filter_by(
company_id=company_id, name=name).first()
else:
datasource_record = session.query(Datasource).filter_by(
company_id=company_id, id=id).first()
self.fs_store.get(
f'{company_id}@@@@@{name}',
f'datasource_{company_id}_{datasource_record.id}', self.dir)
creation_info = json.loads(datasource_record.creation_info)
if raw:
return creation_info
else:
return eval(creation_info['class'])(*creation_info['args'],
**creation_info['kwargs'])
except Exception as e:
log.error(f'Error getting datasource {name}, exception: {e}')
return None
os.environ['DEFAULT_LOG_LEVEL'] = config['log']['level']['console']
os.environ['LIGHTWOOD_LOG_LEVEL'] = config['log']['level']['console']
# Switch to this once the native interface has it's own thread :/
ctx = mp.get_context('spawn')
from mindsdb.__about__ import __version__ as mindsdb_version
print(f'Version {mindsdb_version}')
print(f'Configuration file:\n {config.config_path}')
print(f"Storage path:\n {config['paths']['root']}")
# @TODO Backwards compatibiltiy for tests, remove later
from mindsdb.interfaces.database.integrations import add_db_integration, get_db_integration, remove_db_integration
dbw = DatabaseWrapper(COMPANY_ID)
model_interface = ModelInterfaceWrapper(ModelInterface(), COMPANY_ID)
raw_model_data_arr = model_interface.get_models()
model_data_arr = []
for model in raw_model_data_arr:
if model['status'] == 'complete':
x = model_interface.get_model_data(model['name'])
try:
model_data_arr.append(
model_interface.get_model_data(model['name']))
except Exception:
pass
is_cloud = config.get('cloud', False)
if not is_cloud:
for integration_name in get_db_integrations(COMPANY_ID,
sensitive_info=True):
class DataStore():
def __init__(self):
self.config = Config()
self.fs_store = FsSotre()
self.company_id = os.environ.get('MINDSDB_COMPANY_ID', None)
self.dir = self.config.paths['datasources']
self.mindsdb_native = NativeInterface()
def get_analysis(self, name):
datasource_record = session.query(Datasource).filter_by(
company_id=self.company_id, name=name).first()
if datasource_record.analysis is None:
return None
analysis = json.loads(datasource_record.analysis)
return analysis
def start_analysis(self, name):
datasource_record = session.query(Datasource).filter_by(
company_id=self.company_id, name=name).first()
if datasource_record.analysis is not None:
return None
semaphor_record = session.query(Semaphor).filter_by(
company_id=self.company_id,
entity_id=datasource_record.id,
entity_type='datasource').first()
if semaphor_record is None:
semaphor_record = Semaphor(company_id=self.company_id,
entity_id=datasource_record.id,
entity_type='datasource',
action='write')
session.add(semaphor_record)
session.commit()
else:
return
try:
analysis = self.mindsdb_native.analyse_dataset(
self.get_datasource_obj(name, raw=True))
datasource_record = session.query(Datasource).filter_by(
company_id=self.company_id, name=name).first()
datasource_record.analysis = json.dumps(analysis)
session.commit()
except Exception as e:
log.error(e)
finally:
semaphor_record = session.query(Semaphor).filter_by(
company_id=self.company_id,
entity_id=datasource_record.id,
entity_type='datasource').first()
session.delete(semaphor_record)
session.commit()
def get_datasources(self, name=None):
datasource_arr = []
if name is not None:
datasource_record_arr = session.query(Datasource).filter_by(
company_id=self.company_id, name=name)
else:
datasource_record_arr = session.query(Datasource).filter_by(
company_id=self.company_id)
for datasource_record in datasource_record_arr:
try:
if datasource_record.data is None:
continue
datasource = json.loads(datasource_record.data)
datasource['created_at'] = datasource_record.created_at
datasource['updated_at'] = datasource_record.updated_at
datasource['name'] = datasource_record.name
datasource['id'] = datasource_record.id
datasource_arr.append(datasource)
except Exception as e:
log.error(e)
return datasource_arr
def get_data(self, name, where=None, limit=None, offset=None):
offset = 0 if offset is None else offset
ds = self.get_datasource_obj(name)
if limit is not None:
# @TODO Add `offset` to the `filter` method of the datasource and get rid of `offset`
filtered_ds = ds.filter(where=where,
limit=limit + offset).iloc[offset:]
else:
filtered_ds = ds.filter(where=where)
filtered_ds = filtered_ds.where(pd.notnull(filtered_ds), None)
data = filtered_ds.to_dict(orient='records')
return {
'data': data,
'rowcount': len(ds),
'columns_names': filtered_ds.columns
}
def get_datasource(self, name):
datasource_arr = self.get_datasources(name)
if len(datasource_arr) == 1:
return datasource_arr[0]
# @TODO: Remove when db swithc is more stable, this should never happen, but good santiy check while this is kinda buggy
elif len(datasource_arr) > 1:
log.error('Two or more datasource with the same name, (',
len(datasource_arr), ') | Full list: ', datasource_arr)
raise Exception('Two or more datasource with the same name')
return None
def delete_datasource(self, name):
datasource_record = Datasource.query.filter_by(
company_id=self.company_id, name=name).first()
id = datasource_record.id
session.delete(datasource_record)
session.commit()
self.fs_store.delete(
f'datasource_{self.company_id}_{datasource_record.id}')
try:
shutil.rmtree(os.path.join(self.dir, name))
except Exception:
pass
def save_datasource(self, name, source_type, source, file_path=None):
if source_type == 'file' and (file_path is None):
raise Exception(
'`file_path` argument required when source_type == "file"')
datasource_record = session.query(Datasource).filter_by(
company_id=self.company_id, name=name).first()
while datasource_record is not None:
raise Exception(f'Datasource with name {name} already exists')
try:
datasource_record = Datasource(
company_id=self.company_id,
name=name,
datasources_version=mindsdb_datasources.__version__,
mindsdb_version=mindsdb_version)
session.add(datasource_record)
session.commit()
datasource_record = session.query(Datasource).filter_by(
company_id=self.company_id, name=name).first()
ds_meta_dir = os.path.join(self.dir, name)
os.mkdir(ds_meta_dir)
if source_type == 'file':
source = os.path.join(ds_meta_dir, source)
shutil.move(file_path, source)
ds = FileDS(source)
creation_info = {
'class': 'FileDS',
'args': [source],
'kwargs': {}
}
elif source_type in self.config['integrations']:
integration = self.config['integrations'][source_type]
ds_class_map = {
'clickhouse': ClickhouseDS,
'mariadb': MariaDS,
'mysql': MySqlDS,
'postgres': PostgresDS,
'mssql': MSSQLDS,
'mongodb': MongoDS,
'snowflake': SnowflakeDS,
'athena': AthenaDS
}
try:
dsClass = ds_class_map[integration['type']]
except KeyError:
raise KeyError(
f"Unknown DS type: {source_type}, type is {integration['type']}"
)
if dsClass is None:
raise Exception(
f'Unsupported datasource: {source_type}, please install required dependencies!'
)
if integration['type'] in ['clickhouse']:
creation_info = {
'class': dsClass.__name__,
'args': [],
'kwargs': {
'query': source['query'],
'user': integration['user'],
'password': integration['password'],
'host': integration['host'],
'port': integration['port']
}
}
ds = dsClass(**creation_info['kwargs'])
elif integration['type'] in [
'mssql', 'postgres', 'mariadb', 'mysql'
]:
creation_info = {
'class': dsClass.__name__,
'args': [],
'kwargs': {
'query': source['query'],
'user': integration['user'],
'password': integration['password'],
'host': integration['host'],
'port': integration['port']
}
}
if 'database' in integration:
creation_info['kwargs']['database'] = integration[
'database']
if 'database' in source:
creation_info['kwargs']['database'] = source[
'database']
ds = dsClass(**creation_info['kwargs'])
elif integration['type'] == 'snowflake':
creation_info = {
'class': dsClass.__name__,
'args': [],
'kwargs': {
'query': source['query'],
'schema': source['schema'],
'warehouse': source['warehouse'],
'database': source['database'],
'host': integration['host'],
'password': integration['password'],
'user': integration['user'],
'account': integration['account']
}
}
ds = dsClass(**creation_info['kwargs'])
elif integration['type'] == 'mongodb':
if isinstance(source['find'], str):
source['find'] = json.loads(source['find'])
creation_info = {
'class': dsClass.__name__,
'args': [],
'kwargs': {
'database': source['database'],
'collection': source['collection'],
'query': source['find'],
'user': integration['user'],
'password': integration['password'],
'host': integration['host'],
'port': integration['port']
}
}
ds = dsClass(**creation_info['kwargs'])
elif integration['type'] == 'athena':
creation_info = {
'class': dsClass.__name__,
'args': [],
'kwargs': {
'query': source['query'],
'staging_dir': source['staging_dir'],
'database': source['database'],
'access_key': source['access_key'],
'secret_key': source['secret_key'],
'region_name': source['region_name']
}
}
ds = dsClass(**creation_info['kwargs'])
else:
# This probably only happens for urls
ds = FileDS(source)
creation_info = {
'class': 'FileDS',
'args': [source],
'kwargs': {}
}
df = ds.df
if '' in df.columns or len(df.columns) != len(set(df.columns)):
shutil.rmtree(ds_meta_dir)
raise Exception(
'Each column in datasource must have unique non-empty name'
)
datasource_record.creation_info = json.dumps(creation_info)
datasource_record.data = json.dumps({
'source_type':
source_type,
'source':
source,
'row_count':
len(df),
'columns': [dict(name=x) for x in list(df.keys())]
})
self.fs_store.put(
name, f'datasource_{self.company_id}_{datasource_record.id}',
self.dir)
session.commit()
except Exception as e:
log.error(f'{e}')
try:
self.delete_datasource(name)
except Exception:
pass
raise e
return self.get_datasource_obj(name, raw=True), name
def get_datasource_obj(self, name, raw=False, id=None):
try:
if name is None:
datasource_record = session.query(Datasource).filter_by(
company_id=self.company_id, id=id).first()
else:
datasource_record = session.query(Datasource).filter_by(
company_id=self.company_id, name=name).first()
self.fs_store.get(
name, f'datasource_{self.company_id}_{datasource_record.id}',
self.dir)
creation_info = json.loads(datasource_record.creation_info)
if raw:
return creation_info
else:
return eval(creation_info['class'])(*creation_info['args'],
**creation_info['kwargs'])
except Exception as e:
log.error(f'\n{e}\n')
return None
class MindsDBDataNode(DataNode):
type = 'mindsdb'
def __init__(self, config):
self.config = Config()
self.mindsdb_native = NativeInterface()
self.custom_models = CustomModels()
self.ai_table = AITable_store()
self.default_store = DataStore()
def getTables(self):
models = self.mindsdb_native.get_models()
models = [x['name'] for x in models if x['status'] == 'complete']
models += ['predictors', 'commands']
models += [x['name'] for x in self.custom_models.get_models()]
ai_tables = self.ai_table.get_ai_tables()
models += [x['name'] for x in ai_tables]
return models
def hasTable(self, table):
return table in self.getTables()
def _get_ai_table_columns(self, table_name):
aitable_record = self.ai_table.get_ai_table(table_name)
columns = ([x['name'] for x in aitable_record.query_fields] +
[x['name'] for x in aitable_record.predictor_columns])
return columns
def _get_model_columns(self, table_name):
model = self.mindsdb_native.get_model_data(name=table_name)
columns = []
columns += model['columns']
columns += [f'{x}_original' for x in model['predict']]
for col in model['predict']:
if model['data_analysis_v2'][col]['typing'][
'data_type'] == 'Numeric':
columns += [f"{col}_min", f"{col}_max"]
columns += [f"{col}_confidence"]
columns += [f"{col}_explain"]
return columns
def getTableColumns(self, table):
try:
columns = self.custom_models.get_model_data(table)['columns']
columns += [
'external_datasource', 'select_data_query', 'when_data'
]
return columns
except Exception:
pass
if table == 'predictors':
return [
'name', 'status', 'accuracy', 'predict', 'select_data_query',
'external_datasource', 'training_options'
]
if table == 'commands':
return ['command']
columns = []
ai_tables = self.ai_table.get_ai_table(table)
if ai_tables is not None:
columns = self._get_ai_table_columns(table)
elif table in [x['name'] for x in self.mindsdb_native.get_models()]:
columns = self._get_model_columns(table)
columns += [
'when_data', 'select_data_query', 'external_datasource'
]
return columns
def _select_predictors(self):
models = self.mindsdb_native.get_models()
# TODO add custom models
return [
{
'name': x['name'],
'status': x['status'],
'accuracy':
str(x['accuracy']) if x['accuracy'] is not None else None,
'predict': ', '.join(x['predict']),
'select_data_query': '',
'external_datasource': '', # TODO
'training_options': '' # TODO ?
} for x in models
]
def delete_predictor(self, name):
self.mindsdb_native.delete_model(name)
def _select_from_ai_table(self, table, columns, where):
aitable_record = self.ai_table.get_ai_table(table)
integration = aitable_record.integration_name
query = aitable_record.integration_query
predictor_name = aitable_record.predictor_name
ds, ds_name = self.default_store.save_datasource(
'temp_ds', integration, {'query': query})
dso = self.default_store.get_datasource_obj(ds_name, raw=True)
res = self.mindsdb_native.predict(predictor_name,
'dict',
when_data=dso)
self.default_store.delete_datasource(ds_name)
keys_map = {}
for f in aitable_record.predictor_columns:
keys_map[f['value']] = f['name']
for f in aitable_record.query_fields:
keys_map[f['name']] = f['name']
keys = list(keys_map.keys())
data = []
for i, el in enumerate(res):
data.append({keys_map[key]: el[key] for key in keys})
return data
def select(self,
table,
columns=None,
where=None,
where_data=None,
order_by=None,
group_by=None,
came_from=None):
''' NOTE WHERE statements can be just $eq joined with 'and'
'''
if table == 'predictors':
return self._select_predictors()
if table == 'commands':
return []
if self.ai_table.get_ai_table(table):
return self._select_from_ai_table(table, columns, where)
original_when_data = None
if 'when_data' in where:
if len(where) > 1:
raise ValueError(
"Should not be used any other keys in 'where', if 'when_data' used"
)
try:
original_when_data = where['when_data']['$eq']
where_data = json.loads(where['when_data']['$eq'])
if isinstance(where_data, list) is False:
where_data = [where_data]
except Exception:
raise ValueError(
f'''Error while parse 'when_data'="{where_data}"''')
external_datasource = None
if 'external_datasource' in where:
external_datasource = where['external_datasource']['$eq']
del where['external_datasource']
select_data_query = None
if came_from is not None and 'select_data_query' in where:
select_data_query = where['select_data_query']['$eq']
del where['select_data_query']
dbtype = self.config['integrations'][came_from]['type']
if dbtype == 'clickhouse':
ch = Clickhouse(self.config, came_from)
res = ch._query(
select_data_query.strip(' ;\n') + ' FORMAT JSON')
data = res.json()['data']
elif dbtype == 'mariadb':
maria = Mariadb(self.config, came_from)
data = maria._query(select_data_query)
elif dbtype == 'mysql':
mysql = MySQL(self.config, came_from)
data = mysql._query(select_data_query)
elif dbtype == 'postgres':
mysql = PostgreSQL(self.config, came_from)
data = mysql._query(select_data_query)
elif dbtype == 'mssql':
mssql = MSSQL(self.config, came_from)
data = mssql._query(select_data_query, fetch=True)
else:
raise Exception(f'Unknown database type: {dbtype}')
if where_data is None:
where_data = data
else:
where_data += data
new_where = {}
if where_data is None:
for key, value in where.items():
if isinstance(value, dict) is False or len(
value.keys()) != 1 or list(value.keys())[0] != '$eq':
# TODO value should be just string or number
raise Exception()
new_where[key] = value['$eq']
if len(new_where) == 0:
return []
where_data = [new_where]
try:
model = self.custom_models.get_model_data(name=table)
except Exception:
model = self.mindsdb_native.get_model_data(name=table)
predicted_columns = model['predict']
original_target_values = {}
for col in predicted_columns:
if where_data is not None:
if col in where_data:
original_target_values[col + '_original'] = list(
where_data[col])
else:
original_target_values[col +
'_original'] = [None
] * len(where_data)
else:
original_target_values[col + '_original'] = [None]
if table in [x['name'] for x in self.custom_models.get_models()]:
res = self.custom_models.predict(name=table, when_data=where_data)
data = []
fields = model['columns']
for i, ele in enumerate(res):
row = {}
row['select_data_query'] = select_data_query
row['external_datasource'] = external_datasource
row['when_data'] = original_when_data
for key in ele:
row[key] = ele[key]['predicted_value']
# FIXME prefer get int from mindsdb_native in this case
if model['data_analysis_v2'][key]['typing'][
'data_subtype'] == 'Int':
row[key] = int(row[key])
for k in fields:
if k not in ele:
if isinstance(where_data, list):
if k in where_data[i]:
row[k] = where_data[i][k]
else:
row[k] = None
elif k in where_data.columns:
row[k] = where_data[k].iloc[i]
else:
row[k] = None
for k in original_target_values:
row[k] = original_target_values[k][i]
data.append(row)
field_types = {
f: model['data_analysis_v2'][f]['typing']['data_subtype']
for f in fields if 'typing' in model['data_analysis_v2'][f]
}
for row in data:
cast_row_types(row, field_types)
return data
else:
pred_dicts, explanations = self.mindsdb_native.predict(
table, 'dict&explain', when_data=where_data)
keys = [x for x in pred_dicts[0] if x in columns]
min_max_keys = []
for col in predicted_columns:
if model['data_analysis_v2'][col]['typing'][
'data_type'] == 'Numeric':
min_max_keys.append(col)
data = []
explains = []
for i, el in enumerate(pred_dicts):
data.append({key: el[key] for key in keys})
explains.append(explanations[i])
field_types = {
f: model['data_analysis_v2'][f]['typing']['data_subtype']
for f in model['columns']
if 'typing' in model['data_analysis_v2'][f]
}
for i, row in enumerate(data):
cast_row_types(row, field_types)
row['select_data_query'] = select_data_query
row['external_datasource'] = external_datasource
row['when_data'] = original_when_data
for k in original_target_values:
row[k] = original_target_values[k][i]
explanation = explains[i]
for key in predicted_columns:
row[key + '_confidence'] = explanation[key]['confidence']
row[key + '_explain'] = json.dumps(explanation[key],
cls=NumpyJSONEncoder,
ensure_ascii=False)
for key in min_max_keys:
row[key + '_min'] = min(
explanation[key]['confidence_interval'])
row[key + '_max'] = max(
explanation[key]['confidence_interval'])
return data
def initialize_interfaces(app):
app.original_data_store = DataStore()
app.original_model_interface = ModelInterface()
config = Config()
app.config_obj = config
