def _setup_for_creation(self, name): if name in self.predictor_cache: del self.predictor_cache[name] # Here for no particular reason, because we want to run this sometimes but not too often self._invalidate_cached_predictors() predictor_dir = Path(self.config.paths['predictors']).joinpath(name) create_directory(predictor_dir) versions_file_path = predictor_dir.joinpath('versions.json') with open(str(versions_file_path), 'wt') as f: json.dump(self.config.versions, f, indent=4, sort_keys=True)
def _setup_for_creation(self, name): if name in self.predictor_cache: del self.predictor_cache[name] # Here for no particular reason, because we want to run this sometimes but not too often self._invalidate_cached_predictors() predictor_dir = Path(self.config.paths['predictors']).joinpath(name) create_directory(predictor_dir) predictor_record = Predictor(company_id=self.company_id, name=name, is_custom=False) session.add(predictor_record) session.commit()
def learn(self, name, from_data, to_predict, kwargs={}): join_learn_process = kwargs.get('join_learn_process', False) if 'join_learn_process' in kwargs: del kwargs['join_learn_process'] predictor_dir = Path(self.config.paths['predictors']).joinpath(name) create_directory(predictor_dir) versions_file_path = predictor_dir.joinpath('versions.json') with open(str(versions_file_path), 'wt') as f: json.dump(self.config.versions, f, indent=4, sort_keys=True) p = PredictorProcess(name, from_data, to_predict, kwargs, self.config.get_all(), 'learn') p.start() if join_learn_process is True: p.join() if p.exitcode != 0: raise Exception('Learning process failed !')
def _setup_for_creation(self, name): from mindsdb_datasources import FileDS, ClickhouseDS, MariaDS, MySqlDS, PostgresDS, MSSQLDS, MongoDS, SnowflakeDS, AthenaDS import mindsdb_native from mindsdb_native import F from mindsdb_native.libs.constants.mindsdb import DATA_SUBTYPES from mindsdb.interfaces.storage.db import session, Predictor if name in self.predictor_cache: del self.predictor_cache[name] # Here for no particular reason, because we want to run this sometimes but not too often self._invalidate_cached_predictors() predictor_dir = Path(self.config.paths['predictors']).joinpath(name) create_directory(predictor_dir) predictor_record = Predictor(company_id=self.company_id, name=name, is_custom=False) session.add(predictor_record) session.commit()
config_dir, storage_dir = get_or_create_dir_struct() config_path = os.path.join(config_dir, 'config.json') if not os.path.exists(config_path): _ = cli_config(None, None, storage_dir, config_dir, use_default=True) args = args_parse() if args.config is not None: config_path = args.config try: mindsdb_config = Config(config_path) except Exception as e: print(str(e)) sys.exit(1) paths = mindsdb_config.paths for path in paths.values(): create_directory(path) os.environ['MINDSDB_STORAGE_PATH'] = paths['predictors'] os.environ['DEFAULT_LOG_LEVEL'] = os.environ.get('DEFAULT_LOG_LEVEL', 'ERROR') os.environ['LIGHTWOOD_LOG_LEVEL'] = os.environ.get('LIGHTWOOD_LOG_LEVEL', 'ERROR') from mindsdb_native import * # Figure out how to add this as a module import lightwood #import dataskillet import mindsdb.utilities.wizards as wizards from mindsdb.interfaces.custom.model_interface import ModelInterface
def __init__(self): self.config_path = os.environ['MINDSDB_CONFIG_PATH'] if self.config_path == 'absent': self._override_config = {} else: with open(self.config_path, 'r') as fp: self._override_config = json.load(fp) self.company_id = os.environ.get('MINDSDB_COMPANY_ID', None) self._db_config = {} self.last_updated = datetime.datetime.now() - datetime.timedelta( days=3600) self._read() self.last_updated = datetime.datetime.now() - datetime.timedelta( days=3600) # Now comes the stuff that gets stored in the db if len(self._db_config) == 0: self._db_config = { 'permanent_storage': { 'location': 'local' }, 'paths': {}, "log": { "level": { "console": "ERROR", "file": "WARNING" } }, "debug": False, "integrations": {}, "api": { "http": { "host": "127.0.0.1", "port": "47334" }, "mysql": { "host": "127.0.0.1", "password": "", "port": "47335", "user": "******", "database": "mindsdb", "ssl": True }, "mongodb": { "host": "127.0.0.1", "port": "47336", "database": "mindsdb" } } } self._db_config['paths']['root'] = os.environ[ 'MINDSDB_STORAGE_DIR'] self._db_config['paths']['datasources'] = os.path.join( self._db_config['paths']['root'], 'datasources') self._db_config['paths']['predictors'] = os.path.join( self._db_config['paths']['root'], 'predictors') self._db_config['paths']['custom_models'] = os.path.join( self._db_config['paths']['root'], 'custom_models') self._db_config['paths']['static'] = os.path.join( self._db_config['paths']['root'], 'static') self._db_config['paths']['tmp'] = os.path.join( self._db_config['paths']['root'], 'tmp') self._db_config['paths']['log'] = os.path.join( self._db_config['paths']['root'], 'log') self._db_config['paths']['storage_dir'] = self._db_config['paths'][ 'root'] self._db_config['storage_dir'] = self._db_config['paths']['root'] for path in self._db_config['paths']: create_directory(path) self._save() self._read()
try: config = Config(config_path) except Exception as e: print(str(e)) sys.exit(1) try: datasource_dir = config['interface']['datastore']['storage_dir'] except KeyError: pass try: predictor_dir = config['interface']['mindsdb_native']['storage_dir'] except KeyError: pass create_directory(datasource_dir) create_directory(predictor_dir) temp_path = os.path.join(predictor_dir, 'tmp') create_directory(temp_path) os.environ['MINDSDB_STORAGE_PATH'] = predictor_dir os.environ['MINDSDB_TEMP_PATH'] = temp_path from mindsdb_native import * # Figure out how to add this as a module import lightwood #import dataskillet import mindsdb.utilities.wizards as wizards
def _setup_for_creation(self, name): predictor_dir = Path(self.config.paths['predictors']).joinpath(name) create_directory(predictor_dir) versions_file_path = predictor_dir.joinpath('versions.json') with open(str(versions_file_path), 'wt') as f: json.dump(self.config.versions, f, indent=4, sort_keys=True)
def add_db_integration(name, data, company_id): if 'database_name' not in data: data['database_name'] = name if 'publish' not in data: data['publish'] = True bundle_path = data.get('secure_connect_bundle') if data.get('type') in ('cassandra', 'scylla') and _is_not_empty_str(bundle_path): if os.path.isfile(bundle_path) is False: raise Exception(f'Can not get access to file: {bundle_path}') integrations_dir = Config()['paths']['integrations'] p = Path(bundle_path) data['secure_connect_bundle'] = p.name integration_record = Integration(name=name, data=data, company_id=company_id) session.add(integration_record) session.commit() integration_id = integration_record.id folder_name = f'integration_files_{company_id}_{integration_id}' integration_dir = os.path.join(integrations_dir, folder_name) create_directory(integration_dir) shutil.copyfile(bundle_path, os.path.join(integration_dir, p.name)) FsStore().put(folder_name, integration_dir, integrations_dir) elif data.get('type') in ('mysql', 'mariadb'): ssl = data.get('ssl') files = {} temp_dir = None if ssl is True: for key in ['ssl_ca', 'ssl_cert', 'ssl_key']: if key not in data: continue if os.path.isfile(data[key]) is False: if _is_not_empty_str(data[key]) is False: raise Exception( "'ssl_ca', 'ssl_cert' and 'ssl_key' must be paths or inline certs" ) if temp_dir is None: temp_dir = tempfile.mkdtemp( prefix='integration_files_') cert_file_name = data.get(f'{key}_name', f'{key}.pem') cert_file_path = os.path.join(temp_dir, cert_file_name) with open(cert_file_path, 'wt') as f: f.write(data[key]) data[key] = cert_file_path files[key] = data[key] p = Path(data[key]) data[key] = p.name integration_record = Integration(name=name, data=data, company_id=company_id) session.add(integration_record) session.commit() integration_id = integration_record.id if len(files) > 0: integrations_dir = Config()['paths']['integrations'] folder_name = f'integration_files_{company_id}_{integration_id}' integration_dir = os.path.join(integrations_dir, folder_name) create_directory(integration_dir) for file_path in files.values(): p = Path(file_path) shutil.copyfile(file_path, os.path.join(integration_dir, p.name)) FsStore().put(folder_name, integration_dir, integrations_dir) else: integration_record = Integration(name=name, data=data, company_id=company_id) session.add(integration_record) session.commit()
config_dir, storage_dir = get_or_create_dir_struct() config_path = os.path.join(config_dir, 'config.json') if not os.path.exists(config_path): _ = cli_config(None, None, storage_dir, config_dir, use_default=True) args = args_parse() if args.config is not None: config_path = args.config try: config = Config(config_path) except Exception as e: print(str(e)) sys.exit(1) paths = config.paths create_directory(paths['datasources']) create_directory(paths['predictors']) create_directory(paths['static']) create_directory(paths['tmp']) os.environ['MINDSDB_STORAGE_PATH'] = paths['predictors'] from mindsdb_native import * # Figure out how to add this as a module import lightwood #import dataskillet import mindsdb.utilities.wizards as wizards
def __init__(self): self.config_path = os.environ['MINDSDB_CONFIG_PATH'] if self.config_path == 'absent': self._override_config = {} else: with open(self.config_path, 'r') as fp: self._override_config = json.load(fp) self._default_config = { 'permanent_storage': { 'location': 'local' }, 'paths': {}, "log": { "level": { "console": "INFO", "file": "DEBUG", "db": "WARNING" } }, "debug": False, "integrations": {}, "api": { "http": { "host": "127.0.0.1", "port": "47334" }, "mysql": { "host": "127.0.0.1", "password": "", "port": "47335", "user": "******", "database": "mindsdb", "ssl": True }, "mongodb": { "host": "127.0.0.1", "port": "47336", "database": "mindsdb" } }, "cache": { "type": "local" }, "force_datasource_removing": False } self._default_config['paths']['root'] = os.environ['MINDSDB_STORAGE_DIR'] self._default_config['paths']['storage'] = os.path.join(self._default_config['paths']['root'], 'storage') self._default_config['paths']['datasources'] = os.path.join(self._default_config['paths']['root'], 'datasources') self._default_config['paths']['predictors'] = os.path.join(self._default_config['paths']['root'], 'predictors') self._default_config['paths']['static'] = os.path.join(self._default_config['paths']['root'], 'static') self._default_config['paths']['tmp'] = os.path.join(self._default_config['paths']['root'], 'tmp') self._default_config['paths']['log'] = os.path.join(self._default_config['paths']['root'], 'log') self._default_config['paths']['storage_dir'] = self._default_config['paths']['root'] self._default_config['paths']['cache'] = os.path.join(self._default_config['paths']['root'], 'cache') self._default_config['paths']['integrations'] = os.path.join(self._default_config['paths']['root'], 'integrations') self._default_config['storage_dir'] = self._default_config['paths']['root'] for path_name in self._default_config['paths']: create_directory(self._default_config['paths'][path_name]) self._config = _merge_configs(self._default_config, self._override_config)
args = args_parse() if args.config is not None: config_path = args.config try: config = Config(config_path) except Exception as e: print(str(e)) sys.exit(1) try: datasource_dir = config['interface']['datastore']['storage_dir'] except KeyError: pass try: predictor_dir = config['interface']['mindsdb_native']['storage_dir'] except KeyError: pass create_directory(datasource_dir) create_directory(predictor_dir) os.environ['MINDSDB_STORAGE_PATH'] = datasource_dir from mindsdb_native import * # Figure out how to add this as a module import lightwood #import dataskillet import mindsdb.utilities.wizards as wizards