def learn(self, name, from_data, to_predict, datasource_id, kwargs={}): from mindsdb.interfaces.model.learn_process import LearnProcess, run_learn create_process_mark('learn') join_learn_process = kwargs.get('join_learn_process', False) if 'join_learn_process' in kwargs: del kwargs['join_learn_process'] self._setup_for_creation(name) if self.ray_based: run_learn(name, from_data, to_predict, kwargs, datasource_id) else: p = LearnProcess(name, from_data, to_predict, kwargs, datasource_id) p.start() if join_learn_process is True: p.join() if p.exitcode != 0: delete_process_mark('learn') raise Exception('Learning process failed !') delete_process_mark('learn') return 0
def analyse_dataset(self, ds): from mindsdb_datasources import FileDS, ClickhouseDS, MariaDS, MySqlDS, PostgresDS, MSSQLDS, MongoDS, SnowflakeDS, AthenaDS from mindsdb_native import F create_process_mark('analyse') ds = eval(ds['class'])(*ds['args'], **ds['kwargs']) analysis = F.analyse_dataset(ds) delete_process_mark('analyse') return self._pack(analysis)
def predict(self, name, pred_format, when_data=None, kwargs={}): from mindsdb_datasources import FileDS, ClickhouseDS, MariaDS, MySqlDS, PostgresDS, MSSQLDS, MongoDS, SnowflakeDS, AthenaDS import mindsdb_native from mindsdb.interfaces.storage.db import session, Predictor create_process_mark('predict') if name not in self.predictor_cache: # Clear the cache entirely if we have less than 1.2 GB left if psutil.virtual_memory().available < 1.2 * pow(10, 9): self.predictor_cache = {} predictor_record = Predictor.query.filter_by( company_id=self.company_id, name=name, is_custom=False).first() if predictor_record.data['status'] == 'complete': self.fs_store.get( name, f'predictor_{self.company_id}_{predictor_record.id}', self.config['paths']['predictors']) self.predictor_cache[name] = { 'predictor': mindsdb_native.Predictor(name=name, run_env={'trigger': 'mindsdb'}), 'created': datetime.datetime.now() } if isinstance(when_data, dict) and 'kwargs' in when_data and 'args' in when_data: data_source = getattr(mindsdb_datasources, when_data['class'])(*when_data['args'], **when_data['kwargs']) else: # @TODO: Replace with Datasource try: data_source = pd.DataFrame(when_data) except Exception: data_source = when_data predictions = self.predictor_cache[name]['predictor'].predict( when_data=data_source, **kwargs) if pred_format == 'explain' or pred_format == 'new_explain': predictions = [p.explain() for p in predictions] elif pred_format == 'dict': predictions = [p.as_dict() for p in predictions] elif pred_format == 'dict&explain': predictions = [[p.as_dict() for p in predictions], [p.explain() for p in predictions]] else: delete_process_mark('predict') raise Exception(f'Unkown predictions format: {pred_format}') delete_process_mark('predict') return self._pack(predictions)
def run_learn(name, from_data, to_predict, kwargs, datasource_id): import mindsdb_native import mindsdb_datasources import mindsdb create_process_mark('learn') config = Config() fs_store = FsSotre() company_id = os.environ.get('MINDSDB_COMPANY_ID', None) mdb = mindsdb_native.Predictor(name=name, run_env={'trigger': 'mindsdb'}) predictor_record = Predictor.query.filter_by(company_id=company_id, name=name).first() predictor_record.datasource_id = datasource_id predictor_record.to_predict = to_predict predictor_record.native_version = mindsdb_native.__version__ predictor_record.mindsdb_version = mindsdb_version predictor_record.learn_args = {'to_predict': to_predict, 'kwargs': kwargs} predictor_record.data = {'name': name, 'status': 'training'} session.commit() to_predict = to_predict if isinstance(to_predict, list) else [to_predict] data_source = getattr(mindsdb_datasources, from_data['class'])(*from_data['args'], **from_data['kwargs']) try: mdb.learn(from_data=data_source, to_predict=to_predict, **kwargs) except Exception as e: log = logging.getLogger('mindsdb.main') log.error(f'Predictor learn error: {e}') predictor_record.data = {'name': name, 'status': 'error'} session.commit() delete_process_mark('learn') return fs_store.put(name, f'predictor_{company_id}_{predictor_record.id}', config['paths']['predictors']) model_data = mindsdb_native.F.get_model_data(name) predictor_record = Predictor.query.filter_by(company_id=company_id, name=name).first() predictor_record.data = model_data session.commit() DatabaseWrapper().register_predictors([model_data]) delete_process_mark('learn')
def wrapper(*args, **kwargs): create_process_mark(name) try: return func(*args, **kwargs) finally: delete_process_mark(name)