def _setup_for_creation(self, name):
if name in self.predictor_cache:
del self.predictor_cache[name]
# Here for no particular reason, because we want to run this sometimes but not too often
self._invalidate_cached_predictors()
predictor_dir = Path(self.config.paths['predictors']).joinpath(name)
create_directory(predictor_dir)
versions_file_path = predictor_dir.joinpath('versions.json')
with open(str(versions_file_path), 'wt') as f:
json.dump(self.config.versions, f, indent=4, sort_keys=True)
def _setup_for_creation(self, name):
if name in self.predictor_cache:
del self.predictor_cache[name]
# Here for no particular reason, because we want to run this sometimes but not too often
self._invalidate_cached_predictors()
predictor_dir = Path(self.config.paths['predictors']).joinpath(name)
create_directory(predictor_dir)
predictor_record = Predictor(company_id=self.company_id,
name=name,
is_custom=False)
session.add(predictor_record)
session.commit()
def learn(self, name, from_data, to_predict, kwargs={}):
join_learn_process = kwargs.get('join_learn_process', False)
if 'join_learn_process' in kwargs:
del kwargs['join_learn_process']
predictor_dir = Path(self.config.paths['predictors']).joinpath(name)
create_directory(predictor_dir)
versions_file_path = predictor_dir.joinpath('versions.json')
with open(str(versions_file_path), 'wt') as f:
json.dump(self.config.versions, f, indent=4, sort_keys=True)
p = PredictorProcess(name, from_data, to_predict, kwargs,
self.config.get_all(), 'learn')
p.start()
if join_learn_process is True:
p.join()
if p.exitcode != 0:
raise Exception('Learning process failed !')
def _setup_for_creation(self, name):
from mindsdb_datasources import FileDS, ClickhouseDS, MariaDS, MySqlDS, PostgresDS, MSSQLDS, MongoDS, SnowflakeDS, AthenaDS
import mindsdb_native
from mindsdb_native import F
from mindsdb_native.libs.constants.mindsdb import DATA_SUBTYPES
from mindsdb.interfaces.storage.db import session, Predictor
if name in self.predictor_cache:
del self.predictor_cache[name]
# Here for no particular reason, because we want to run this sometimes but not too often
self._invalidate_cached_predictors()
predictor_dir = Path(self.config.paths['predictors']).joinpath(name)
create_directory(predictor_dir)
predictor_record = Predictor(company_id=self.company_id, name=name, is_custom=False)
session.add(predictor_record)
session.commit()
config_dir, storage_dir = get_or_create_dir_struct()
config_path = os.path.join(config_dir, 'config.json')
if not os.path.exists(config_path):
_ = cli_config(None, None, storage_dir, config_dir, use_default=True)
args = args_parse()
if args.config is not None:
config_path = args.config
try:
mindsdb_config = Config(config_path)
except Exception as e:
print(str(e))
sys.exit(1)
paths = mindsdb_config.paths
for path in paths.values():
create_directory(path)
os.environ['MINDSDB_STORAGE_PATH'] = paths['predictors']
os.environ['DEFAULT_LOG_LEVEL'] = os.environ.get('DEFAULT_LOG_LEVEL', 'ERROR')
os.environ['LIGHTWOOD_LOG_LEVEL'] = os.environ.get('LIGHTWOOD_LOG_LEVEL', 'ERROR')
from mindsdb_native import *
# Figure out how to add this as a module
import lightwood
#import dataskillet
import mindsdb.utilities.wizards as wizards
from mindsdb.interfaces.custom.model_interface import ModelInterface
def __init__(self):
self.config_path = os.environ['MINDSDB_CONFIG_PATH']
if self.config_path == 'absent':
self._override_config = {}
else:
with open(self.config_path, 'r') as fp:
self._override_config = json.load(fp)
self.company_id = os.environ.get('MINDSDB_COMPANY_ID', None)
self._db_config = {}
self.last_updated = datetime.datetime.now() - datetime.timedelta(
days=3600)
self._read()
self.last_updated = datetime.datetime.now() - datetime.timedelta(
days=3600)
# Now comes the stuff that gets stored in the db
if len(self._db_config) == 0:
self._db_config = {
'permanent_storage': {
'location': 'local'
},
'paths': {},
"log": {
"level": {
"console": "ERROR",
"file": "WARNING"
}
},
"debug": False,
"integrations": {},
"api": {
"http": {
"host": "127.0.0.1",
"port": "47334"
},
"mysql": {
"host": "127.0.0.1",
"password": "",
"port": "47335",
"user": "******",
"database": "mindsdb",
"ssl": True
},
"mongodb": {
"host": "127.0.0.1",
"port": "47336",
"database": "mindsdb"
}
}
}
self._db_config['paths']['root'] = os.environ[
'MINDSDB_STORAGE_DIR']
self._db_config['paths']['datasources'] = os.path.join(
self._db_config['paths']['root'], 'datasources')
self._db_config['paths']['predictors'] = os.path.join(
self._db_config['paths']['root'], 'predictors')
self._db_config['paths']['custom_models'] = os.path.join(
self._db_config['paths']['root'], 'custom_models')
self._db_config['paths']['static'] = os.path.join(
self._db_config['paths']['root'], 'static')
self._db_config['paths']['tmp'] = os.path.join(
self._db_config['paths']['root'], 'tmp')
self._db_config['paths']['log'] = os.path.join(
self._db_config['paths']['root'], 'log')
self._db_config['paths']['storage_dir'] = self._db_config['paths'][
'root']
self._db_config['storage_dir'] = self._db_config['paths']['root']
for path in self._db_config['paths']:
create_directory(path)
self._save()
self._read()
try:
config = Config(config_path)
except Exception as e:
print(str(e))
sys.exit(1)
try:
datasource_dir = config['interface']['datastore']['storage_dir']
except KeyError:
pass
try:
predictor_dir = config['interface']['mindsdb_native']['storage_dir']
except KeyError:
pass
create_directory(datasource_dir)
create_directory(predictor_dir)
temp_path = os.path.join(predictor_dir, 'tmp')
create_directory(temp_path)
os.environ['MINDSDB_STORAGE_PATH'] = predictor_dir
os.environ['MINDSDB_TEMP_PATH'] = temp_path
from mindsdb_native import *
# Figure out how to add this as a module
import lightwood
#import dataskillet
import mindsdb.utilities.wizards as wizards
def _setup_for_creation(self, name):
predictor_dir = Path(self.config.paths['predictors']).joinpath(name)
create_directory(predictor_dir)
versions_file_path = predictor_dir.joinpath('versions.json')
with open(str(versions_file_path), 'wt') as f:
json.dump(self.config.versions, f, indent=4, sort_keys=True)
def add_db_integration(name, data, company_id):
if 'database_name' not in data:
data['database_name'] = name
if 'publish' not in data:
data['publish'] = True
bundle_path = data.get('secure_connect_bundle')
if data.get('type') in ('cassandra',
'scylla') and _is_not_empty_str(bundle_path):
if os.path.isfile(bundle_path) is False:
raise Exception(f'Can not get access to file: {bundle_path}')
integrations_dir = Config()['paths']['integrations']
p = Path(bundle_path)
data['secure_connect_bundle'] = p.name
integration_record = Integration(name=name,
data=data,
company_id=company_id)
session.add(integration_record)
session.commit()
integration_id = integration_record.id
folder_name = f'integration_files_{company_id}_{integration_id}'
integration_dir = os.path.join(integrations_dir, folder_name)
create_directory(integration_dir)
shutil.copyfile(bundle_path, os.path.join(integration_dir, p.name))
FsStore().put(folder_name, integration_dir, integrations_dir)
elif data.get('type') in ('mysql', 'mariadb'):
ssl = data.get('ssl')
files = {}
temp_dir = None
if ssl is True:
for key in ['ssl_ca', 'ssl_cert', 'ssl_key']:
if key not in data:
continue
if os.path.isfile(data[key]) is False:
if _is_not_empty_str(data[key]) is False:
raise Exception(
"'ssl_ca', 'ssl_cert' and 'ssl_key' must be paths or inline certs"
)
if temp_dir is None:
temp_dir = tempfile.mkdtemp(
prefix='integration_files_')
cert_file_name = data.get(f'{key}_name', f'{key}.pem')
cert_file_path = os.path.join(temp_dir, cert_file_name)
with open(cert_file_path, 'wt') as f:
f.write(data[key])
data[key] = cert_file_path
files[key] = data[key]
p = Path(data[key])
data[key] = p.name
integration_record = Integration(name=name,
data=data,
company_id=company_id)
session.add(integration_record)
session.commit()
integration_id = integration_record.id
if len(files) > 0:
integrations_dir = Config()['paths']['integrations']
folder_name = f'integration_files_{company_id}_{integration_id}'
integration_dir = os.path.join(integrations_dir, folder_name)
create_directory(integration_dir)
for file_path in files.values():
p = Path(file_path)
shutil.copyfile(file_path,
os.path.join(integration_dir, p.name))
FsStore().put(folder_name, integration_dir, integrations_dir)
else:
integration_record = Integration(name=name,
data=data,
company_id=company_id)
session.add(integration_record)
session.commit()
config_dir, storage_dir = get_or_create_dir_struct()
config_path = os.path.join(config_dir, 'config.json')
if not os.path.exists(config_path):
_ = cli_config(None, None, storage_dir, config_dir, use_default=True)
args = args_parse()
if args.config is not None:
config_path = args.config
try:
config = Config(config_path)
except Exception as e:
print(str(e))
sys.exit(1)
paths = config.paths
create_directory(paths['datasources'])
create_directory(paths['predictors'])
create_directory(paths['static'])
create_directory(paths['tmp'])
os.environ['MINDSDB_STORAGE_PATH'] = paths['predictors']
from mindsdb_native import *
# Figure out how to add this as a module
import lightwood
#import dataskillet
import mindsdb.utilities.wizards as wizards
def __init__(self):
self.config_path = os.environ['MINDSDB_CONFIG_PATH']
if self.config_path == 'absent':
self._override_config = {}
else:
with open(self.config_path, 'r') as fp:
self._override_config = json.load(fp)
self._default_config = {
'permanent_storage': {
'location': 'local'
},
'paths': {},
"log": {
"level": {
"console": "INFO",
"file": "DEBUG",
"db": "WARNING"
}
},
"debug": False,
"integrations": {},
"api": {
"http": {
"host": "127.0.0.1",
"port": "47334"
},
"mysql": {
"host": "127.0.0.1",
"password": "",
"port": "47335",
"user": "******",
"database": "mindsdb",
"ssl": True
},
"mongodb": {
"host": "127.0.0.1",
"port": "47336",
"database": "mindsdb"
}
},
"cache": {
"type": "local"
},
"force_datasource_removing": False
}
self._default_config['paths']['root'] = os.environ['MINDSDB_STORAGE_DIR']
self._default_config['paths']['storage'] = os.path.join(self._default_config['paths']['root'], 'storage')
self._default_config['paths']['datasources'] = os.path.join(self._default_config['paths']['root'], 'datasources')
self._default_config['paths']['predictors'] = os.path.join(self._default_config['paths']['root'], 'predictors')
self._default_config['paths']['static'] = os.path.join(self._default_config['paths']['root'], 'static')
self._default_config['paths']['tmp'] = os.path.join(self._default_config['paths']['root'], 'tmp')
self._default_config['paths']['log'] = os.path.join(self._default_config['paths']['root'], 'log')
self._default_config['paths']['storage_dir'] = self._default_config['paths']['root']
self._default_config['paths']['cache'] = os.path.join(self._default_config['paths']['root'], 'cache')
self._default_config['paths']['integrations'] = os.path.join(self._default_config['paths']['root'], 'integrations')
self._default_config['storage_dir'] = self._default_config['paths']['root']
for path_name in self._default_config['paths']:
create_directory(self._default_config['paths'][path_name])
self._config = _merge_configs(self._default_config, self._override_config)
args = args_parse()
if args.config is not None:
config_path = args.config
try:
config = Config(config_path)
except Exception as e:
print(str(e))
sys.exit(1)
try:
datasource_dir = config['interface']['datastore']['storage_dir']
except KeyError:
pass
try:
predictor_dir = config['interface']['mindsdb_native']['storage_dir']
except KeyError:
pass
create_directory(datasource_dir)
create_directory(predictor_dir)
os.environ['MINDSDB_STORAGE_PATH'] = datasource_dir
from mindsdb_native import *
# Figure out how to add this as a module
import lightwood
#import dataskillet
import mindsdb.utilities.wizards as wizards