def run_fit(predictor_id: int, df: pd.DataFrame) -> None: try: predictor_record = session.query(db.Predictor).filter_by(id=predictor_id).first() assert predictor_record is not None fs_store = FsStore() config = Config() predictor_record.data = {'training_log': 'training'} session.commit() predictor: lightwood.PredictorInterface = lightwood.predictor_from_code(predictor_record.code) predictor.learn(df) session.refresh(predictor_record) fs_name = f'predictor_{predictor_record.company_id}_{predictor_record.id}' pickle_path = os.path.join(config['paths']['predictors'], fs_name) predictor.save(pickle_path) fs_store.put(fs_name, fs_name, config['paths']['predictors']) predictor_record.data = predictor.model_analysis.to_dict() predictor_record.dtype_dict = predictor.dtype_dict session.commit() dbw = DatabaseWrapper(predictor_record.company_id) mi = ModelInterfaceWrapper(ModelInterface(), predictor_record.company_id) dbw.register_predictors([mi.get_model_data(predictor_record.name)]) except Exception as e: session.refresh(predictor_record) predictor_record.data = {'error': f'{traceback.format_exc()}\nMain error: {e}'} session.commit() raise e
def rename_model(self, old_name, new_name, company_id: int): db_p = db.session.query(db.Predictor).filter_by(company_id=company_id, name=old_name).first() db_p.name = new_name db.session.commit() dbw = DatabaseWrapper(company_id) dbw.unregister_predictor(old_name) dbw.register_predictors([self.get_model_data(new_name, company_id)])
def put(self, name): params = request.json.get('params') if not isinstance(params, dict): abort(400, "type of 'params' must be dict") is_test = params.get('test', False) if is_test: del params['test'] integration = get_integration(name) if integration is not None: abort(400, f"Integration with name '{name}' already exists") try: if 'enabled' in params: params['publish'] = params['enabled'] del params['enabled'] ca.config_obj.add_db_integration(name, params) mdb = ca.mindsdb_native cst = ca.custom_models model_data_arr = get_all_models_meta_data(mdb, cst) dbw = DatabaseWrapper(ca.config_obj) dbw.register_predictors(model_data_arr) except Exception as e: print(traceback.format_exc()) abort(500, f'Error during config update: {str(e)}') if is_test: cons = dbw.check_connections() ca.config_obj.remove_db_integration(name) return {'success': cons[name]}, 200 return '', 200
def run_fit(predictor_id: int, df: pd.DataFrame) -> None: try: predictor_record = Predictor.query.with_for_update().get(predictor_id) assert predictor_record is not None fs_store = FsStore() config = Config() predictor_record.data = {'training_log': 'training'} session.commit() predictor: lightwood.PredictorInterface = lightwood.predictor_from_code( predictor_record.code) predictor.learn(df) session.refresh(predictor_record) fs_name = f'predictor_{predictor_record.company_id}_{predictor_record.id}' pickle_path = os.path.join(config['paths']['predictors'], fs_name) predictor.save(pickle_path) fs_store.put(fs_name, fs_name, config['paths']['predictors']) predictor_record.data = predictor.model_analysis.to_dict() # getting training time for each tried model. it is possible to do # after training only fit_mixers = list(predictor.runtime_log[x] for x in predictor.runtime_log if isinstance(x, tuple) and x[0] == "fit_mixer") submodel_data = predictor_record.data.get("submodel_data", []) # add training time to other mixers info if submodel_data and fit_mixers and len(submodel_data) == len( fit_mixers): for i, tr_time in enumerate(fit_mixers): submodel_data[i]["training_time"] = tr_time predictor_record.data["submodel_data"] = submodel_data predictor_record.dtype_dict = predictor.dtype_dict session.commit() dbw = DatabaseWrapper(predictor_record.company_id) mi = WithKWArgsWrapper(ModelInterface(), company_id=predictor_record.company_id) except Exception as e: session.refresh(predictor_record) predictor_record.data = { 'error': f'{traceback.format_exc()}\nMain error: {e}' } session.commit() raise e try: dbw.register_predictors([mi.get_model_data(predictor_record.name)]) except Exception as e: log.warn(e)
version.parse(predictor_record.mindsdb_version) < last_compatible_version): predictor_record.update_status = 'available' is_modified = True if is_modified is True: db.session.commit() # endregion for integration_name in datasource_interface.get_db_integrations( sensitive_info=True): print(f"Setting up integration: {integration_name}") if datasource_interface.get_db_integration(integration_name).get( 'publish', False): # do setup and register only if it is 'publish' integration dbw.setup_integration(integration_name) dbw.register_predictors(model_data_arr, integration_name=integration_name) for integration_name in config.get('integrations', {}): try: it = datasource_interface.get_db_integration(integration_name) if it is not None: datasource_interface.remove_db_integration( integration_name) print(f'Adding: {integration_name}') datasource_interface.add_db_integration( integration_name, config['integrations'][integration_name] ) # Setup for user `None`, since we don't need this for cloud if config['integrations'][integration_name].get( 'publish', False) and not is_cloud: dbw.setup_integration(integration_name) dbw.register_predictors(model_data_arr,
mdb = MindsdbNative(config) cst = CustomModels(config) # @TODO Maybe just use `get_model_data` directly here ? Seems like a useless abstraction model_data_arr = [{ 'name': x['name'], 'predict': x['predict'], 'data_analysis': mdb.get_model_data(x['name'])['data_analysis_v2'] } for x in mdb.get_models()] model_data_arr.extend(cst.get_models()) dbw = DatabaseWrapper(config) dbw.register_predictors(model_data_arr) for broken_name in [ name for name, connected in dbw.check_connections().items() if connected is False ]: print( f'Error failed to integrate with database aliased: {broken_name}') ctx = mp.get_context('spawn') for api_name, api_data in apis.items(): print(f'{api_name} API: starting...') try: p = ctx.Process(target=start_functions[api_name], args=(config_path, args.verbose))
class CustomModels(): def __init__(self): self.config = Config() self.fs_store = FsSotre() self.company_id = os.environ.get('MINDSDB_COMPANY_ID', None) self.dbw = DatabaseWrapper() self.storage_dir = self.config['paths']['custom_models'] os.makedirs(self.storage_dir, exist_ok=True) self.model_cache = {} self.mindsdb_native = NativeInterface() self.dbw = DatabaseWrapper() def _dir(self, name): return str(os.path.join(self.storage_dir, name)) def _internal_load(self, name): self.fs_store.get(name, f'custom_model_{self.company_id}_{name}', self.storage_dir) sys.path.insert(0, self._dir(name)) module = __import__(name) try: model = module.Model.load( os.path.join(self._dir(name), 'model.pickle')) except Exception as e: model = module.Model() model.initialize_column_types() if hasattr(model, 'setup'): model.setup() self.model_cache[name] = model return model def learn(self, name, from_data, to_predict, datasource_id, kwargs={}): model_data = self.get_model_data(name) model_data['status'] = 'training' self.save_model_data(name, model_data) to_predict = to_predict if isinstance(to_predict, list) else [to_predict] data_source = getattr(mindsdb_datasources, from_data['class'])(*from_data['args'], **from_data['kwargs']) data_frame = data_source.df model = self._internal_load(name) model.to_predict = to_predict model_data = self.get_model_data(name) model_data['predict'] = model.to_predict self.save_model_data(name, model_data) data_analysis = self.mindsdb_native.analyse_dataset( data_source)['data_analysis_v2'] model_data = self.get_model_data(name) model_data['data_analysis_v2'] = data_analysis self.save_model_data(name, model_data) model.fit(data_frame, to_predict, data_analysis, kwargs) model.save(os.path.join(self._dir(name), 'model.pickle')) self.model_cache[name] = model model_data = self.get_model_data(name) model_data['status'] = 'completed' model_data['columns'] = list(data_analysis.keys()) self.save_model_data(name, model_data) self.fs_store.put(name, f'custom_model_{self.company_id}_{name}', self.storage_dir) self.dbw.unregister_predictor(name) self.dbw.register_predictors([self.get_model_data(name)]) def predict(self, name, when_data=None, from_data=None, kwargs=None): self.fs_store.get(name, f'custom_model_{self.company_id}_{name}', self.storage_dir) if kwargs is None: kwargs = {} if from_data is not None: if isinstance(from_data, dict): data_source = getattr(mindsdb_datasources, from_data['class'])( *from_data['args'], **from_data['kwargs']) # assume that particular instance of any DataSource class is provided else: data_source = from_data data_frame = data_source.df elif when_data is not None: if isinstance(when_data, dict): for k in when_data: when_data[k] = [when_data[k]] data_frame = pd.DataFrame(when_data) else: data_frame = pd.DataFrame(when_data) model = self._internal_load(name) predictions = model.predict(data_frame, kwargs) pred_arr = [] for i in range(len(predictions)): pred_arr.append({}) pred_arr[-1] = {} for col in predictions.columns: pred_arr[-1][col] = {} pred_arr[-1][col]['predicted_value'] = predictions[col].iloc[i] return pred_arr def get_model_data(self, name): predictor_record = Predictor.query.filter_by( company_id=self.company_id, name=name, is_custom=True).first() return predictor_record.data def save_model_data(self, name, data): predictor_record = Predictor.query.filter_by( company_id=self.company_id, name=name, is_custom=True).first() if predictor_record is None: predictor_record = Predictor(company_id=self.company_id, name=name, is_custom=True, data=data) session.add(predictor_record) else: predictor_record.data = data session.commit() def get_models(self): predictor_names = [ x.name for x in Predictor.query.filter_by(company_id=self.company_id, is_custom=True) ] models = [] for name in predictor_names: models.append(self.get_model_data(name)) return models def delete_model(self, name): Predictor.query.filter_by(company_id=self.company_id, name=name, is_custom=True).delete() session.commit() shutil.rmtree(self._dir(name)) self.dbw.unregister_predictor(name) self.fs_store.delete(f'custom_model_{self.company_id}_{name}') def rename_model(self, name, new_name): self.fs_store.get(name, f'custom_model_{self.company_id}_{name}', self.storage_dir) self.dbw.unregister_predictor(name) shutil.move(self._dir(name), self._dir(new_name)) shutil.move(os.path.join(self._dir(new_name) + f'{name}.py'), os.path.join(self._dir(new_name), f'{new_name}.py')) predictor_record = Predictor.query.filter_by( company_id=self.company_id, name=name, is_custom=True).first() predictor_record.name = new_name session.commit() self.dbw.register_predictors([self.get_model_data(new_name)]) self.fs_store.put(name, f'custom_model_{self.company_id}_{new_name}', self.storage_dir) self.fs_store.delete(f'custom_model_{self.company_id}_{name}') def export_model(self, name): shutil.make_archive(base_name=name, format='zip', root_dir=self._dir(name)) return str(self._dir(name)) + '.zip' def load_model(self, fpath, name, trained_status): shutil.unpack_archive(fpath, self._dir(name), 'zip') shutil.move(os.path.join(self._dir(name), 'model.py'), os.path.join(self._dir(name), f'{name}.py')) model = self._internal_load(name) model.to_predict = model.to_predict if isinstance( model.to_predict, list) else [model.to_predict] self.save_model_data( name, { 'name': name, 'data_analysis_v2': model.column_type_map, 'predict': model.to_predict, 'status': trained_status, 'is_custom': True, 'columns': list(model.column_type_map.keys()) }) with open(os.path.join(self._dir(name), '__init__.py'), 'w') as fp: fp.write('') self.fs_store.put(name, f'custom_model_{self.company_id}_{name}', self.storage_dir) if trained_status == 'trained': self.dbw.register_predictors([self.get_model_data(name)])
class NativeInterface(): def __init__(self, config): self.config = config self.dbw = DatabaseWrapper(self.config) self.predictor_cache = {} def _invalidate_cached_predictors(self): # @TODO: Cache will become stale if the respective NativeInterface is not invoked yet a bunch of predictors remained cached, no matter where we invoke it. In practice shouldn't be a big issue though for predictor_name in list(self.predictor_cache.keys()): if (datetime.datetime.now() - self.predictor_cache[predictor_name]['created'] ).total_seconds() > 1200: del self.predictor_cache[predictor_name] def _setup_for_creation(self, name): if name in self.predictor_cache: del self.predictor_cache[name] # Here for no particular reason, because we want to run this sometimes but not too often self._invalidate_cached_predictors() predictor_dir = Path(self.config.paths['predictors']).joinpath(name) create_directory(predictor_dir) versions_file_path = predictor_dir.joinpath('versions.json') with open(str(versions_file_path), 'wt') as f: json.dump(self.config.versions, f, indent=4, sort_keys=True) def create(self, name): self._setup_for_creation(name) predictor = mindsdb_native.Predictor(name=name, run_env={'trigger': 'mindsdb'}) return predictor def learn(self, name, from_data, to_predict, kwargs={}): join_learn_process = kwargs.get('join_learn_process', False) if 'join_learn_process' in kwargs: del kwargs['join_learn_process'] self._setup_for_creation(name) p = LearnProcess(name, from_data, to_predict, kwargs, self.config.get_all()) p.start() if join_learn_process is True: p.join() if p.exitcode != 0: raise Exception('Learning process failed !') def predict(self, name, when_data=None, kwargs={}): if name not in self.predictor_cache: # Clear the cache entirely if we have less than .12 GB left if psutil.virtual_memory().available < 1.2 * pow(10, 9): self.predictor_cache = {} if F.get_model_data(name)['status'] == 'complete': self.predictor_cache[name] = { 'predictor': mindsdb_native.Predictor(name=name, run_env={'trigger': 'mindsdb'}), 'created': datetime.datetime.now() } predictions = self.predictor_cache[name]['predictor'].predict( when_data=when_data, **kwargs) return predictions def analyse_dataset(self, ds): return F.analyse_dataset(ds) def get_model_data(self, name, db_fix=True): model = F.get_model_data(name) # Make some corrections for databases not to break when dealing with empty columns if db_fix: data_analysis = model['data_analysis_v2'] for column in data_analysis['columns']: analysis = data_analysis.get(column) if isinstance(analysis, dict) and (len(analysis) == 0 or analysis.get( 'empty', {}).get('is_empty', False)): data_analysis[column]['typing'] = { 'data_subtype': DATA_SUBTYPES.INT } return model def get_models(self): models = [] predictors = [ x for x in Path(self.config.paths['predictors']).iterdir() if x.is_dir() and x.joinpath('light_model_metadata.pickle'). is_file() and x.joinpath('heavy_model_metadata.pickle').is_file() ] for p in predictors: model_name = p.name try: model_data = self.get_model_data(model_name, db_fix=False) if model_data['status'] == 'training' and parse_datetime( model_data['created_at']) < parse_datetime( self.config['mindsdb_last_started_at']): continue reduced_model_data = {} for k in [ 'name', 'version', 'is_active', 'predict', 'status', 'current_phase', 'accuracy', 'data_source' ]: reduced_model_data[k] = model_data.get(k, None) for k in ['train_end_at', 'updated_at', 'created_at']: reduced_model_data[k] = model_data.get(k, None) if reduced_model_data[k] is not None: try: reduced_model_data[k] = parse_datetime( str(reduced_model_data[k]).split('.')[0]) except Exception as e: # @TODO Does this ever happen print( f'Date parsing exception while parsing: {k} in get_models: ', e) reduced_model_data[k] = parse_datetime( str(reduced_model_data[k])) models.append(reduced_model_data) except Exception as e: print( f"Can't list data for model: '{model_name}' when calling `get_models(), error: {e}`" ) return models def delete_model(self, name): F.delete_model(name) self.dbw.unregister_predictor(name) def rename_model(self, name, new_name): self.dbw.unregister_predictor(self.get_model_data(name)) F.rename_model(name, new_name) self.dbw.register_predictors(self.get_model_data(new_name)) def load_model(self, fpath): name = F.import_model(model_archive_path=fpath) self.dbw.register_predictors(self.get_model_data(name), setup=False) def export_model(self, name): F.export_predictor(model_name=name)
class MindsdbNative(): def __init__(self, config): self.config = config self.dbw = DatabaseWrapper(self.config) def _setup_for_creation(self, name): predictor_dir = Path(self.config.paths['predictors']).joinpath(name) create_directory(predictor_dir) versions_file_path = predictor_dir.joinpath('versions.json') with open(str(versions_file_path), 'wt') as f: json.dump(self.config.versions, f, indent=4, sort_keys=True) def create(self, name): self._setup_for_creation(name) predictor = mindsdb_native.Predictor(name=name, run_env={'trigger': 'mindsdb'}) return predictor def learn(self, name, from_data, to_predict, kwargs={}): join_learn_process = kwargs.get('join_learn_process', False) if 'join_learn_process' in kwargs: del kwargs['join_learn_process'] self._setup_for_creation(name) p = PredictorProcess(name, from_data, to_predict, kwargs, self.config.get_all(), 'learn') p.start() if join_learn_process is True: p.join() if p.exitcode != 0: raise Exception('Learning process failed !') def predict(self, name, when_data=None, kwargs={}): # @TODO Separate into two paths, one for "normal" predictions and one for "real time" predictions. Use the multiprocessing code commented out bellow for normal (once we figure out how to return the prediction object... else use the inline code but with the "real time" predict functionality of mindsdb_native taht will be implemented later) ''' from_data = when if when is not None else when_data p = PredictorProcess(name, from_data, to_predict=None, kwargs=kwargs, config=self.config.get_all(), 'predict') p.start() predictions = p.join() ''' mdb = mindsdb_native.Predictor(name=name, run_env={'trigger': 'mindsdb'}) predictions = mdb.predict(when_data=when_data, **kwargs) return predictions def analyse_dataset(self, ds): return F.analyse_dataset(ds) def get_model_data(self, name, native_view=False): model = F.get_model_data(name) if native_view: return model data_analysis = model['data_analysis_v2'] for column in data_analysis['columns']: if len(data_analysis[column]) == 0 or data_analysis[column].get( 'empty', {}).get('is_empty', False): data_analysis[column]['typing'] = { 'data_subtype': DATA_SUBTYPES.INT } return model def get_models(self, status='any'): models = F.get_models() if status != 'any': models = [x for x in models if x['status'] == status] models = [ x for x in models if x['status'] != 'training' or parse_datetime(x['created_at']) > parse_datetime(self.config['mindsdb_last_started_at']) ] for i in range(len(models)): for k in ['train_end_at', 'updated_at', 'created_at']: if k in models[i] and models[i][k] is not None: try: models[i][k] = parse_datetime( str(models[i][k]).split('.')[0]) except Exception: models[i][k] = parse_datetime(str(models[i][k])) return models def delete_model(self, name): F.delete_model(name) self.dbw.unregister_predictor(name) def rename_model(self, name, new_name): self.dbw.unregister_predictor(self.get_model_data(name)) F.rename_model(name, new_name) self.dbw.register_predictors(self.get_model_data(new_name), setup=False) def load_model(self, fpath): F.import_model(model_archive_path=fpath) # @TODO How do we figure out the name here ? # dbw.register_predictors(...) def export_model(self, name): F.export_predictor(model_name=name)
class MindsdbNative(): def __init__(self, config): self.config = config self.dbw = DatabaseWrapper(self.config) def learn(self, name, from_data, to_predict, kwargs={}): join_learn_process = kwargs.get('join_learn_process', False) if 'join_learn_process' in kwargs: del kwargs['join_learn_process'] p = PredictorProcess(name, from_data, to_predict, kwargs, self.config.get_all(), 'learn') p.start() if join_learn_process is True: p.join() if p.exitcode != 0: raise Exception('Learning process failed !') def predict(self, name, when_data=None, kwargs={}): # @TODO Separate into two paths, one for "normal" predictions and one for "real time" predictions. Use the multiprocessing code commented out bellow for normal (once we figure out how to return the prediction object... else use the inline code but with the "real time" predict functionality of mindsdb_native taht will be implemented later) ''' from_data = when if when is not None else when_data p = PredictorProcess(name, from_data, to_predict=None, kwargs=kwargs, config=self.config.get_all(), 'predict') p.start() predictions = p.join() ''' mdb = mindsdb_native.Predictor(name=name) predictions = mdb.predict(when_data=when_data, run_confidence_variation_analysis=isinstance( when_data, list) is False or len(when_data) == 1, **kwargs) return predictions def analyse_dataset(self, ds): return F.analyse_dataset(ds) def get_model_data(self, name): return F.get_model_data(name) def get_models(self, status='any'): models = F.get_models() if status != 'any': models = [x for x in models if x['status'] == status] for i in range(len(models)): for k in ['train_end_at', 'updated_at', 'created_at']: if k in models[i] and models[i][k] is not None: try: models[i][k] = parse_datetime( str(models[i][k]).split('.')[0]) except Exception as e: models[i][k] = parse_datetime(str(models[i][k])) return models def delete_model(self, name): F.delete_model(name) self.dbw.unregister_predictor(name) def rename_model(self, name, new_name): self.dbw.unregister_predictor(name) F.rename_model(name, new_name) self.dbw.register_predictors(new_name) def load_model(self, fpath): F.import_model(model_archive_path=fpath) # @TODO How do we figure out the name here ? #dbw.register_predictor(...) def export_model(self, name): F.export_predictor(model_name=name)
class CustomModels(): def __init__(self, config): self.config = config self.dbw = DatabaseWrapper(self.config) self.storage_dir = os.path.join(config['storage_dir'], 'misc') os.makedirs(self.storage_dir, exist_ok=True) self.model_cache = {} self.mindsdb_native = MindsdbNative(self.config) self.dbw = DatabaseWrapper(self.config) def _dir(self, name): return str(os.path.join(self.storage_dir, 'custom_model_' + name)) def _internal_load(self, name): # Caching (2 lines bellow), currently disabled due to multiprocessing cache invalidation issues #if name in self.model_cache: # return self.model_cache[name] # "Proper" model loading (3 lines bellow), currently disabled due to pickling issues #spec = importlib.util.spec_from_file_location(name, self._dir(name) + '/model.py') #module = importlib.util.module_from_spec(spec) #spec.loader.exec_module(module) sys.path.insert(0, self._dir(name)) module = __import__(name) try: model = module.Model.load( os.path.join(self._dir(name), 'model.pickle')) except Exception as e: model = module.Model() model.initialize_column_types() if hasattr(model, 'setup'): model.setup() self.model_cache[name] = model return model def learn(self, name, from_data, to_predict, kwargs={}): model_data = self.get_model_data(name) model_data['status'] = 'training' self.save_model_data(name, model_data) to_predict = to_predict if isinstance(to_predict, list) else [to_predict] data_source = getattr(mindsdb_native, from_data['class'])(*from_data['args'], **from_data['kwargs']) data_frame = data_source.df model = self._internal_load(name) model.to_predict = to_predict model_data = self.get_model_data(name) model_data['predict'] = model.to_predict self.save_model_data(name, model_data) data_analysis = self.mindsdb_native.analyse_dataset( data_source)['data_analysis_v2'] model_data = self.get_model_data(name) model_data['data_analysis'] = data_analysis self.save_model_data(name, model_data) model.fit(data_frame, to_predict, data_analysis, kwargs) model.save(os.path.join(self._dir(name), 'model.pickle')) self.model_cache[name] = model model_data = self.get_model_data(name) model_data['status'] = 'completed' self.save_model_data(name, model_data) self.dbw.unregister_predictor(name) self.dbw.register_predictors([self.get_model_data(name)], setup=False) def predict(self, name, when_data=None, from_data=None, kwargs={}): if from_data is not None: data_source = getattr(mindsdb_native, from_data['class'])(*from_data['args'], **from_data['kwargs']) data_frame = data_source.df elif when_data is not None: if isinstance(when_data, dict): for k in when_data: when_data[k] = [when_data[k]] data_frame = pd.DataFrame(when_data) else: data_frame = pd.DataFrame(when_data) model = self._internal_load(name) predictions = model.predict(data_frame, kwargs) pred_arr = [] for i in range(len(predictions)): pred_arr.append({}) pred_arr[-1] = {} for col in predictions.columns: pred_arr[-1][col] = {} pred_arr[-1][col]['predicted_value'] = predictions[col].iloc[i] return pred_arr def get_model_data(self, name): with open(os.path.join(self._dir(name), 'metadata.json'), 'r') as fp: return json.load(fp) def save_model_data(self, name, data): with open(os.path.join(self._dir(name), 'metadata.json'), 'w') as fp: json.dump(data, fp) def get_models(self, status='any'): models = [] for model_dir in os.listdir(self.storage_dir): if 'custom_model_' in model_dir: name = model_dir.replace('custom_model_', '') try: models.append(self.get_model_data(name)) except: print(f'Model {name} not found !') return models def delete_model(self, name): shutil.rmtree(self._dir(name)) self.dbw.unregister_predictor(name) def rename_model(self, name, new_name): self.dbw.unregister_predictor(name) shutil.move(self._dir(name), self._dir(new_name)) shutil.move(os.path.join(self._dir(new_name) + f'{name}.py'), os.path.join(self._dir(new_name), f'{new_name}.py')) self.dbw.register_predictors([self.get_model_data(new_name)], setup=False) def export_model(self, name): shutil.make_archive(base_name=name, format='zip', root_dir=self._dir(name)) return str(self._dir(name)) + '.zip' def load_model(self, fpath, name, trained_status): shutil.unpack_archive(fpath, self._dir(name), 'zip') shutil.move(os.path.join(self._dir(name), 'model.py'), os.path.join(self._dir(name), f'{name}.py')) model = self._internal_load(name) model.to_predict = model.to_predict if isinstance( model.to_predict, list) else [model.to_predict] self.save_model_data( name, { 'name': name, 'data_analysis': model.column_type_map, 'predict': model.to_predict, 'status': trained_status, 'is_custom': True }) with open(os.path.join(self._dir(name), '__init__.py'), 'w') as fp: fp.write('') if trained_status == 'trained': self.dbw.register_predictors([self.get_model_data(name)], setup=False)