def learn(self, name, from_data, to_predict, datasource_id, kwargs={}):
from mindsdb.interfaces.model.learn_process import LearnProcess, run_learn
create_process_mark('learn')
join_learn_process = kwargs.get('join_learn_process', False)
if 'join_learn_process' in kwargs:
del kwargs['join_learn_process']
self._setup_for_creation(name)
if self.ray_based:
run_learn(name, from_data, to_predict, kwargs, datasource_id)
else:
p = LearnProcess(name, from_data, to_predict, kwargs,
datasource_id)
p.start()
if join_learn_process is True:
p.join()
if p.exitcode != 0:
delete_process_mark('learn')
raise Exception('Learning process failed !')
delete_process_mark('learn')
return 0
def analyse_dataset(self, ds):
from mindsdb_datasources import FileDS, ClickhouseDS, MariaDS, MySqlDS, PostgresDS, MSSQLDS, MongoDS, SnowflakeDS, AthenaDS
from mindsdb_native import F
create_process_mark('analyse')
ds = eval(ds['class'])(*ds['args'], **ds['kwargs'])
analysis = F.analyse_dataset(ds)
delete_process_mark('analyse')
return self._pack(analysis)
def predict(self, name, pred_format, when_data=None, kwargs={}):
from mindsdb_datasources import FileDS, ClickhouseDS, MariaDS, MySqlDS, PostgresDS, MSSQLDS, MongoDS, SnowflakeDS, AthenaDS
import mindsdb_native
from mindsdb.interfaces.storage.db import session, Predictor
create_process_mark('predict')
if name not in self.predictor_cache:
# Clear the cache entirely if we have less than 1.2 GB left
if psutil.virtual_memory().available < 1.2 * pow(10, 9):
self.predictor_cache = {}
predictor_record = Predictor.query.filter_by(
company_id=self.company_id, name=name,
is_custom=False).first()
if predictor_record.data['status'] == 'complete':
self.fs_store.get(
name, f'predictor_{self.company_id}_{predictor_record.id}',
self.config['paths']['predictors'])
self.predictor_cache[name] = {
'predictor':
mindsdb_native.Predictor(name=name,
run_env={'trigger': 'mindsdb'}),
'created':
datetime.datetime.now()
}
if isinstance(when_data,
dict) and 'kwargs' in when_data and 'args' in when_data:
data_source = getattr(mindsdb_datasources,
when_data['class'])(*when_data['args'],
**when_data['kwargs'])
else:
# @TODO: Replace with Datasource
try:
data_source = pd.DataFrame(when_data)
except Exception:
data_source = when_data
predictions = self.predictor_cache[name]['predictor'].predict(
when_data=data_source, **kwargs)
if pred_format == 'explain' or pred_format == 'new_explain':
predictions = [p.explain() for p in predictions]
elif pred_format == 'dict':
predictions = [p.as_dict() for p in predictions]
elif pred_format == 'dict&explain':
predictions = [[p.as_dict() for p in predictions],
[p.explain() for p in predictions]]
else:
delete_process_mark('predict')
raise Exception(f'Unkown predictions format: {pred_format}')
delete_process_mark('predict')
return self._pack(predictions)
def run_learn(name, from_data, to_predict, kwargs, datasource_id):
import mindsdb_native
import mindsdb_datasources
import mindsdb
create_process_mark('learn')
config = Config()
fs_store = FsSotre()
company_id = os.environ.get('MINDSDB_COMPANY_ID', None)
mdb = mindsdb_native.Predictor(name=name, run_env={'trigger': 'mindsdb'})
predictor_record = Predictor.query.filter_by(company_id=company_id,
name=name).first()
predictor_record.datasource_id = datasource_id
predictor_record.to_predict = to_predict
predictor_record.native_version = mindsdb_native.__version__
predictor_record.mindsdb_version = mindsdb_version
predictor_record.learn_args = {'to_predict': to_predict, 'kwargs': kwargs}
predictor_record.data = {'name': name, 'status': 'training'}
session.commit()
to_predict = to_predict if isinstance(to_predict, list) else [to_predict]
data_source = getattr(mindsdb_datasources,
from_data['class'])(*from_data['args'],
**from_data['kwargs'])
try:
mdb.learn(from_data=data_source, to_predict=to_predict, **kwargs)
except Exception as e:
log = logging.getLogger('mindsdb.main')
log.error(f'Predictor learn error: {e}')
predictor_record.data = {'name': name, 'status': 'error'}
session.commit()
delete_process_mark('learn')
return
fs_store.put(name, f'predictor_{company_id}_{predictor_record.id}',
config['paths']['predictors'])
model_data = mindsdb_native.F.get_model_data(name)
predictor_record = Predictor.query.filter_by(company_id=company_id,
name=name).first()
predictor_record.data = model_data
session.commit()
DatabaseWrapper().register_predictors([model_data])
delete_process_mark('learn')
def wrapper(*args, **kwargs):
create_process_mark(name)
try:
return func(*args, **kwargs)
finally:
delete_process_mark(name)
