def __init__(self, config):
mongodb_config = config['api'].get('mongodb')
assert mongodb_config is not None, 'is no mongodb config!'
host = mongodb_config['host']
port = mongodb_config['port']
log.debug(f'start mongo server on {host}:{port}')
super().__init__((host, int(port)), MongoRequestHandler)
self.mindsdb_env = {
'config': config,
'origin_data_store': DataStore(),
'origin_model_interface': ModelInterface(),
'origin_datasource_controller': DatasourceController(),
}
self.mindsdb_env['mindsdb_native'] = WithKWArgsWrapper(
self.mindsdb_env['origin_model_interface'], company_id=None)
self.mindsdb_env['data_store'] = WithKWArgsWrapper(
self.mindsdb_env['origin_data_store'], company_id=None)
self.mindsdb_env['datasource_controller'] = WithKWArgsWrapper(
self.mindsdb_env['origin_datasource_controller'], company_id=None)
respondersCollection = RespondersCollection()
opQueryResponder = OpQueryResponder(respondersCollection)
opMsgResponder = OpMsgResponder(respondersCollection)
opInsertResponder = OpInsertResponder(respondersCollection)
self.operationsHandlersMap = {
OP_QUERY: opQueryResponder,
OP_MSG: opMsgResponder,
OP_INSERT: opInsertResponder
}
respondersCollection.add(when={'drop': 'system.sessions'},
result={'ok': 1})
respondersCollection.add(when={'update': 'system.version'},
result={'ok': 1})
respondersCollection.add(
when={'setFeatureCompatibilityVersion': helpers.is_true},
result={'ok': 1})
# OpMSG=OrderedDict([('features', 1), ('$clusterTime', OrderedDict([('clusterTime', Timestamp(1599748325, 1)), ('signature', OrderedDict([('hash', b'\xb8\xc3\x03\x18\xca\xe6bh\xf0\xcb47,\x924\x8a >\xfc\x91'), ('keyId', 6870854312365391875)]))])), ('$configServerState', OrderedDict([('opTime', OrderedDict([('ts', Timestamp(1599748325, 1)), ('t', 1)]))])), ('$db', 'admin')])
respondersCollection.add(when={'features': helpers.is_true},
result={'ok': 1})
# OpMSG=OrderedDict([('serverStatus', 1), ('$clusterTime', OrderedDict([('clusterTime', Timestamp(1599748366, 1)), ('signature', OrderedDict([('hash', b'\xa1E}\xbbIU\xc2D\x95++\x82\x88\xb5\x84\xf5\xda)+B'), ('keyId', 6870854312365391875)]))])), ('$configServerState', OrderedDict([('opTime', OrderedDict([('ts', Timestamp(1599748366, 1)), ('t', 1)]))])), ('$db', 'admin')])
respondersCollection.add(when={'serverStatus': helpers.is_true},
result={'ok': 1})
# OpMSG=OrderedDict([('ismaster', 1), ('$db', 'admin'), ('$clusterTime', OrderedDict([('clusterTime', Timestamp(1599749031, 1)), ('signature', OrderedDict([('hash', b'6\x87\xd5Y\xa7\xc7\xcf$\xab\x1e\xa2{\xe5B\xe5\x99\xdbl\x8d\xf4'), ('keyId', 6870854312365391875)]))])), ('$client', OrderedDict([('application', OrderedDict([('name', 'MongoDB Shell')])), ('driver', OrderedDict([('name', 'MongoDB Internal Client'), ('version', '3.6.3')])), ('os', OrderedDict([('type', 'Linux'), ('name', 'Ubuntu'), ('architecture', 'x86_64'), ('version', '18.04')])), ('mongos', OrderedDict([('host', 'maxs-comp:27103'), ('client', '127.0.0.1:52148'), ('version', '3.6.3')]))])), ('$configServerState', OrderedDict([('opTime', OrderedDict([('ts', Timestamp(1599749031, 1)), ('t', 1)]))]))])
respondersCollection.responders += responders
# Switch to this once the native interface has it's own thread :/
ctx = mp.get_context('spawn')
from mindsdb.__about__ import __version__ as mindsdb_version
print(f'Version {mindsdb_version}')
print(f'Configuration file:\n {config.config_path}')
print(f"Storage path:\n {config['paths']['root']}")
# @TODO Backwards compatibiltiy for tests, remove later
from mindsdb.interfaces.database.integrations import DatasourceController
dbw = DatabaseWrapper(COMPANY_ID)
model_interface = WithKWArgsWrapper(ModelInterface(),
company_id=COMPANY_ID)
datasource_interface = WithKWArgsWrapper(DatasourceController(),
company_id=COMPANY_ID)
raw_model_data_arr = model_interface.get_models()
model_data_arr = []
for model in raw_model_data_arr:
if model['status'] == 'complete':
x = model_interface.get_model_data(model['name'])
try:
model_data_arr.append(
model_interface.get_model_data(model['name']))
except Exception:
pass
is_cloud = config.get('cloud', False)
if not is_cloud:
# region Mark old predictors as outdated
def create_datasource(self,
source_type,
source,
file_path=None,
company_id=None,
ds_meta_dir=None):
datasource_controller = DatasourceController()
if source_type == 'file':
source = os.path.join(ds_meta_dir, source)
shutil.move(file_path, source)
ds = FileDS(source)
creation_info = {'class': 'FileDS', 'args': [source], 'kwargs': {}}
elif datasource_controller.get_db_integration(source_type,
company_id) is not None:
integration = datasource_controller.get_db_integration(
source_type, company_id)
ds_class_map = {
'clickhouse': ClickhouseDS,
'mariadb': MariaDS,
'mysql': MySqlDS,
'singlestore': MySqlDS,
'postgres': PostgresDS,
'cockroachdb': PostgresDS,
'mssql': MSSQLDS,
'mongodb': MongoDS,
'snowflake': SnowflakeDS,
'athena': AthenaDS,
'cassandra': CassandraDS,
'scylladb': ScyllaDS,
'trinodb': TrinoDS
}
try:
dsClass = ds_class_map[integration['type']]
except KeyError:
raise KeyError(
f"Unknown DS type: {source_type}, type is {integration['type']}"
)
if dsClass is None:
raise Exception(
f"Unsupported datasource: {source_type}, type is {integration['type']}, please install required dependencies!"
)
if integration['type'] in ['clickhouse']:
creation_info = {
'class': dsClass.__name__,
'args': [],
'kwargs': {
'query': source['query'],
'user': integration['user'],
'password': integration['password'],
'host': integration['host'],
'port': integration['port']
}
}
ds = dsClass(**creation_info['kwargs'])
elif integration['type'] in [
'mssql', 'postgres', 'cockroachdb', 'mariadb', 'mysql',
'singlestore', 'cassandra', 'scylladb'
]:
creation_info = {
'class': dsClass.__name__,
'args': [],
'kwargs': {
'query': source['query'],
'user': integration['user'],
'password': integration['password'],
'host': integration['host'],
'port': integration['port']
}
}
kwargs = creation_info['kwargs']
integration_folder_name = f'integration_files_{company_id}_{integration["id"]}'
if integration['type'] in ('mysql', 'mariadb'):
kwargs['ssl'] = integration.get('ssl')
kwargs['ssl_ca'] = integration.get('ssl_ca')
kwargs['ssl_cert'] = integration.get('ssl_cert')
kwargs['ssl_key'] = integration.get('ssl_key')
for key in ['ssl_ca', 'ssl_cert', 'ssl_key']:
if isinstance(kwargs[key],
str) and len(kwargs[key]) > 0:
kwargs[key] = os.path.join(
self.integrations_dir, integration_folder_name,
kwargs[key])
elif integration['type'] in ('cassandra', 'scylla'):
kwargs['secure_connect_bundle'] = integration.get(
'secure_connect_bundle')
if (isinstance(kwargs['secure_connect_bundle'], str)
and len(kwargs['secure_connect_bundle']) > 0):
kwargs['secure_connect_bundle'] = os.path.join(
self.integrations_dir, integration_folder_name,
kwargs['secure_connect_bundle'])
if 'database' in integration:
kwargs['database'] = integration['database']
if 'database' in source:
kwargs['database'] = source['database']
ds = dsClass(**kwargs)
elif integration['type'] == 'snowflake':
creation_info = {
'class': dsClass.__name__,
'args': [],
'kwargs': {
'query':
source['query'].replace('"', "'"),
'schema':
source.get('schema', integration['schema']),
'warehouse':
source.get('warehouse', integration['warehouse']),
'database':
source.get('database', integration['database']),
'host':
integration['host'],
'password':
integration['password'],
'user':
integration['user'],
'account':
integration['account']
}
}
ds = dsClass(**creation_info['kwargs'])
elif integration['type'] == 'mongodb':
if isinstance(source['find'], str):
source['find'] = json.loads(source['find'])
creation_info = {
'class': dsClass.__name__,
'args': [],
'kwargs': {
'database': source['database'],
'collection': source['collection'],
'query': source['find'],
'user': integration['user'],
'password': integration['password'],
'host': integration['host'],
'port': integration['port']
}
}
ds = dsClass(**creation_info['kwargs'])
elif integration['type'] == 'athena':
creation_info = {
'class': dsClass.__name__,
'args': [],
'kwargs': {
'query': source['query'],
'staging_dir': source['staging_dir'],
'database': source['database'],
'access_key': source['access_key'],
'secret_key': source['secret_key'],
'region_name': source['region_name']
}
}
ds = dsClass(**creation_info['kwargs'])
elif integration['type'] == 'trinodb':
creation_info = {
'class': dsClass.__name__,
'args': [],
'kwargs': {
'query': source['query'],
'user': integration['user'],
'password': integration['password'],
'host': integration['host'],
'port': integration['port'],
'schema': integration['schema'],
'catalog': integration['catalog']
}
}
ds = dsClass(**creation_info['kwargs'])
else:
# This probably only happens for urls
ds = FileDS(source)
creation_info = {'class': 'FileDS', 'args': [source], 'kwargs': {}}
return ds, creation_info
def initialize_interfaces(app):
app.original_data_store = DataStore()
app.original_model_interface = ModelInterface()
app.original_datasource_interface = DatasourceController()
config = Config()
app.config_obj = config
def __init__(self, company_id):
self.config = Config()
self.company_id = company_id
self.datasource_interface = WithKWArgsWrapper(DatasourceController(),
company_id=company_id)