def initialize_interfaces(app):
app.default_store = DataStore()
app.naitve_interface = NativeInterface()
app.custom_models = CustomModels()
app.dbw = DatabaseWrapper()
config = Config()
app.config_obj = config
if api_name not in config['api']:
print(
f"Trying run '{api_name}' API, but is no config for this api.")
print(f"Please, fill config['api']['{api_name}']")
sys.exit(0)
start_functions = {
'http': start_http,
'mysql': start_mysql,
'mongodb': start_mongo
}
archive_obsolete_predictors(config, '2.11.0')
mdb = MindsdbNative(config)
cst = CustomModels(config)
# @TODO Maybe just use `get_model_data` directly here ? Seems like a useless abstraction
model_data_arr = [{
'name':
x['name'],
'predict':
x['predict'],
'data_analysis':
mdb.get_model_data(x['name'])['data_analysis_v2']
} for x in mdb.get_models()]
model_data_arr.extend(cst.get_models())
dbw = DatabaseWrapper(config)
dbw.register_predictors(model_data_arr)
def __init__(self, config):
self.config = config
self.mindsdb_native = NativeInterface(config)
self.custom_models = CustomModels(config)
class MindsDBDataNode(DataNode):
type = 'mindsdb'
def __init__(self, config):
self.config = config
self.mindsdb_native = NativeInterface(config)
self.custom_models = CustomModels(config)
def getTables(self):
models = self.mindsdb_native.get_models()
models = [x['name'] for x in models if x['status'] == 'complete']
models += ['predictors', 'commands']
models += [x['name'] for x in self.custom_models.get_models()]
return models
def hasTable(self, table):
return table in self.getTables()
def getTableColumns(self, table):
try:
columns = self.custom_models.get_model_data(
table)['data_analysis_v2']['columns']
columns += [
'external_datasource', 'select_data_query', 'when_data'
]
return columns
except Exception:
pass
if table == 'predictors':
return [
'name', 'status', 'accuracy', 'predict', 'select_data_query',
'external_datasource', 'training_options'
]
if table == 'commands':
return ['command']
model = self.mindsdb_native.get_model_data(name=table)
columns = []
columns += model['data_analysis_v2']['columns']
columns += [f'{x}_original' for x in model['predict']]
for col in model['predict']:
if model['data_analysis_v2'][col]['typing'][
'data_type'] == 'Numeric':
columns += [f"{col}_min", f"{col}_max"]
columns += [f"{col}_confidence"]
columns += [f"{col}_explain"]
# TODO this should be added just for clickhouse queries
columns += ['when_data', 'select_data_query', 'external_datasource']
return columns
def _select_predictors(self):
models = self.mindsdb_native.get_models()
# TODO add custom models
return [
{
'name': x['name'],
'status': x['status'],
'accuracy':
str(x['accuracy']) if x['accuracy'] is not None else None,
'predict': ', '.join(x['predict']),
'select_data_query': '',
'external_datasource': '', # TODO
'training_options': '' # TODO ?
} for x in models
]
def delete_predictor(self, name):
self.mindsdb_native.delete_model(name)
def select(self,
table,
columns=None,
where=None,
where_data=None,
order_by=None,
group_by=None,
came_from=None):
''' NOTE WHERE statements can be just $eq joined with 'and'
'''
if table == 'predictors':
return self._select_predictors()
if table == 'commands':
return []
original_when_data = None
if 'when_data' in where:
if len(where) > 1:
raise ValueError(
"Should not be used any other keys in 'where', if 'when_data' used"
)
try:
original_when_data = where['when_data']['$eq']
where_data = json.loads(where['when_data']['$eq'])
if isinstance(where_data, list) is False:
where_data = [where_data]
except Exception:
raise ValueError(
f'''Error while parse 'when_data'="{where_data}"''')
external_datasource = None
if 'external_datasource' in where:
external_datasource = where['external_datasource']['$eq']
del where['external_datasource']
select_data_query = None
if came_from is not None and 'select_data_query' in where:
select_data_query = where['select_data_query']['$eq']
del where['select_data_query']
dbtype = self.config['integrations'][came_from]['type']
if dbtype == 'clickhouse':
ch = Clickhouse(self.config, came_from)
res = ch._query(
select_data_query.strip(' ;\n') + ' FORMAT JSON')
data = res.json()['data']
elif dbtype == 'mariadb':
maria = Mariadb(self.config, came_from)
data = maria._query(select_data_query)
elif dbtype == 'mysql':
mysql = MySQL(self.config, came_from)
data = mysql._query(select_data_query)
elif dbtype == 'postgres':
mysql = PostgreSQL(self.config, came_from)
data = mysql._query(select_data_query)
elif dbtype == 'mssql':
mssql = MSSQL(self.config, came_from)
data = mssql._query(select_data_query, fetch=True)
else:
raise Exception(f'Unknown database type: {dbtype}')
if where_data is None:
where_data = data
else:
where_data += data
new_where = {}
if where_data is not None:
where_data = pandas.DataFrame(where_data)
else:
for key, value in where.items():
if isinstance(value, dict) is False or len(
value.keys()) != 1 or list(value.keys())[0] != '$eq':
# TODO value should be just string or number
raise Exception()
new_where[key] = value['$eq']
if len(new_where) == 0:
return []
where_data = [new_where]
try:
model = self.custom_models.get_model_data(name=table)
except Exception:
model = self.mindsdb_native.get_model_data(name=table)
predicted_columns = model['predict']
original_target_values = {}
for col in predicted_columns:
if where_data is not None:
if col in where_data:
original_target_values[col + '_original'] = list(
where_data[col])
else:
original_target_values[col +
'_original'] = [None
] * len(where_data)
else:
original_target_values[col + '_original'] = [None]
if table in [x['name'] for x in self.custom_models.get_models()]:
res = self.custom_models.predict(name=table, when_data=where_data)
data = []
fields = model['data_analysis_v2']['columns']
for i, ele in enumerate(res):
row = {}
row['select_data_query'] = select_data_query
row['external_datasource'] = external_datasource
row['when_data'] = original_when_data
for key in ele:
row[key] = ele[key]['predicted_value']
# FIXME prefer get int from mindsdb_native in this case
if model['data_analysis_v2'][key]['typing'][
'data_subtype'] == 'Int':
row[key] = int(row[key])
for k in fields:
if k not in ele:
if isinstance(where_data, list):
if k in where_data[i]:
row[k] = where_data[i][k]
else:
row[k] = None
elif k in where_data.columns:
row[k] = where_data[k].iloc[i]
else:
row[k] = None
for k in original_target_values:
row[k] = original_target_values[k][i]
data.append(row)
field_types = {
f: model['data_analysis_v2'][f]['typing']['data_subtype']
for f in fields if 'typing' in model['data_analysis_v2'][f]
}
for row in data:
cast_row_types(row, field_types)
return data
else:
res = self.mindsdb_native.predict(name=table, when_data=where_data)
keys = [x for x in list(res._data.keys()) if x in columns]
min_max_keys = []
for col in predicted_columns:
if model['data_analysis_v2'][col]['typing'][
'data_type'] == 'Numeric':
min_max_keys.append(col)
data = []
explains = []
for i, el in enumerate(res):
data.append({key: el[key] for key in keys})
explains.append(el.explain())
field_types = {
f: model['data_analysis_v2'][f]['typing']['data_subtype']
for f in model['data_analysis_v2']['columns']
if 'typing' in model['data_analysis_v2'][f]
}
for row in data:
cast_row_types(row, field_types)
row['select_data_query'] = select_data_query
row['external_datasource'] = external_datasource
row['when_data'] = original_when_data
for k in original_target_values:
row[k] = original_target_values[k][i]
explanation = explains[i]
for key in predicted_columns:
row[key + '_confidence'] = explanation[key]['confidence']
row[key + '_explain'] = json.dumps(explanation[key],
cls=NumpyJSONEncoder)
for key in min_max_keys:
row[key + '_min'] = min(
explanation[key]['confidence_interval'])
row[key + '_max'] = max(
explanation[key]['confidence_interval'])
return data
for api_name in apis.keys():
if api_name not in config['api']:
print(f"Trying run '{api_name}' API, but is no config for this api.")
print(f"Please, fill config['api']['{api_name}']")
sys.exit(0)
start_functions = {
'http': start_http,
'mysql': start_mysql,
'mongodb': start_mongo
}
archive_obsolete_predictors(config, '2.11.0')
mdb = MindsdbNative(config)
cst = CustomModels(config)
remove_corrupted_predictors(config, mdb)
model_data_arr = get_all_models_meta_data(mdb, cst)
dbw = DatabaseWrapper(config)
dbw.register_predictors(model_data_arr)
for broken_name in [name for name, connected in dbw.check_connections().items() if connected is False]:
log.error(f'Error failed to integrate with database aliased: {broken_name}')
ctx = mp.get_context('spawn')
for api_name, api_data in apis.items():
print(f'{api_name} API: starting...')
def __init__(self, config):
self.config = Config()
self.mindsdb_native = NativeInterface()
self.custom_models = CustomModels()
self.ai_table = AITable_store()
self.default_store = DataStore()
def initialize_interfaces(config, app):
app.default_store = DataStore(config)
app.mindsdb_native = MindsdbNative(config)
app.custom_models = CustomModels(config)
app.config_obj = config
def initialize_interfaces(config, app):
app.default_store = DataStore(config)
app.mindsdb_native = NativeInterface(config)
app.custom_models = CustomModels(config)
app.dbw = DatabaseWrapper(config)
app.config_obj = config
'port': config['api'][api]['port'],
'process': None,
'started': False
} for api in api_arr
}
if not ray_based:
apis['rcp'] = {'process': rpc_proc, 'started': True}
start_functions = {
'http': start_http,
'mysql': start_mysql,
'mongodb': start_mongo
}
mdb = NativeInterface()
cst = CustomModels()
model_data_arr = get_all_models_meta_data(mdb, cst)
dbw = DatabaseWrapper()
for db_alias in config['integrations']:
dbw.setup_integration(db_alias)
dbw.register_predictors(model_data_arr)
for broken_name in [name for name, connected in dbw.check_connections().items() if connected is False]:
log.error(f'Error failed to integrate with database aliased: {broken_name}')
for api_name, api_data in apis.items():
if api_data['started']:
continue
print(f'{api_name} API: starting...')
