def handle_exception(e):
get_log('http').error(f'http exception: {e}')
# pass through HTTP errors
if isinstance(e, HTTPException):
return {'message': str(e)}, e.code, e.get_response().headers
name = getattr(type(e), '__name__') or 'Unknown error'
return {'message': f'{name}: {str(e)}'}, 500
def get_last_compatible_gui_version() -> LooseVersion:
log = get_log('http')
try:
res = requests.get('https://mindsdb-web-builds.s3.amazonaws.com/compatible-config.json')
except (ConnectionError, requests.exceptions.ConnectionError) as e:
print(f'Is no connection. {e}')
return False
except Exception as e:
print(f'Is something wrong with getting compatible-config.json: {e}')
return False
if res.status_code != 200:
print(f'Cant get compatible-config.json: returned status code = {res.status_code}')
return False
try:
versions = res.json()
except Exception as e:
print(f'Cant decode compatible-config.json: {e}')
return False
current_mindsdb_lv = LooseVersion(mindsdb_version)
try:
gui_versions = {}
max_mindsdb_lv = None
max_gui_lv = None
for el in versions['mindsdb']:
if el['mindsdb_version'] is None:
gui_lv = LooseVersion(el['gui_version'])
else:
mindsdb_lv = LooseVersion(el['mindsdb_version'])
gui_lv = LooseVersion(el['gui_version'])
if mindsdb_lv.vstring not in gui_versions or gui_lv > gui_versions[mindsdb_lv.vstring]:
gui_versions[mindsdb_lv.vstring] = gui_lv
if max_mindsdb_lv is None or max_mindsdb_lv < mindsdb_lv:
max_mindsdb_lv = mindsdb_lv
if max_gui_lv is None or max_gui_lv < gui_lv:
max_gui_lv = gui_lv
all_mindsdb_lv = [LooseVersion(x) for x in gui_versions.keys()]
all_mindsdb_lv.sort()
if current_mindsdb_lv.vstring in gui_versions:
gui_version_lv = gui_versions[current_mindsdb_lv.vstring]
elif current_mindsdb_lv > all_mindsdb_lv[-1]:
gui_version_lv = max_gui_lv
else:
lower_versions = {key: value for key, value in gui_versions.items() if LooseVersion(key) < current_mindsdb_lv}
if len(lower_versions) == 0:
gui_version_lv = gui_versions[all_mindsdb_lv[0].vstring]
else:
all_lower_versions = [LooseVersion(x) for x in lower_versions.keys()]
gui_version_lv = gui_versions[all_lower_versions[-1].vstring]
except Exception as e:
log.error(f'Error in compatible-config.json structure: {e}')
return False
return gui_version_lv
def initialize_flask(config, init_static_thread, no_studio):
# Apparently there's a bug that causes the static path not to work if it's '/' -- https://github.com/pallets/flask/issues/3134, I think '' should achieve the same thing (???)
if no_studio:
app = Flask(
__name__
)
else:
static_path = os.path.join(config['paths']['static'], 'static/')
if os.path.isabs(static_path) is False:
static_path = os.path.join(os.getcwd(), static_path)
app = Flask(
__name__,
static_url_path='/static',
static_folder=static_path
)
app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 60
app.config['SWAGGER_HOST'] = 'http://localhost:8000/mindsdb'
app.json_encoder = CustomJSONEncoder
authorizations = {
'apikey': {
'type': 'session',
'in': 'query',
'name': 'session'
}
}
api = Swagger_Api(
app,
authorizations=authorizations,
security=['apikey'],
url_prefix=':8000',
prefix='/api',
doc='/doc/'
)
api.representations['application/json'] = custom_output_json
port = config['api']['http']['port']
host = config['api']['http']['host']
# NOTE rewrite it, that hotfix to see GUI link
if not no_studio:
log = get_log('http')
if host in ('', '0.0.0.0'):
url = f'http://127.0.0.1:{port}/'
else:
url = f'http://{host}:{port}/'
log.info(f' - GUI available at {url}')
pid = os.getpid()
x = threading.Thread(target=_open_webbrowser, args=(url, pid, port, init_static_thread, config['paths']['static']), daemon=True)
x.start()
return app, api
def before_request():
company_id = request.headers.get('company-id')
if company_id is not None:
try:
company_id = int(company_id)
except Exception as e:
get_log('http').error(
f'Cloud not parse company id: {company_id} | exception: {e}'
)
company_id = None
request.company_id = company_id
request.default_store = DataStoreWrapper(
data_store=current_app.original_data_store, company_id=company_id)
request.model_interface = ModelInterfaceWrapper(
model_interface=current_app.original_model_interface,
company_id=company_id)
def before_request():
company_id = request.headers.get('company-id')
if company_id is not None:
try:
company_id = int(company_id)
except Exception as e:
get_log('http').error(
f'Cloud not parse company id: {company_id} | exception: {e}'
)
company_id = None
request.company_id = company_id
request.default_store = WithKWArgsWrapper(
current_app.original_data_store, company_id=company_id)
request.model_interface = WithKWArgsWrapper(
current_app.original_model_interface, company_id=company_id)
request.integration_controller = WithKWArgsWrapper(
current_app.original_integration_controller, company_id=company_id)
def _open_webbrowser(url: str, pid: int, port: int, init_static_thread, static_folder):
"""Open webbrowser with url when http service is started.
If some error then do nothing.
"""
init_static_thread.join()
inject_telemetry_to_static(static_folder)
logger = get_log('http')
try:
is_http_active = wait_func_is_true(func=is_pid_listen_port, timeout=10,
pid=pid, port=port)
if is_http_active:
webbrowser.open(url)
except Exception as e:
logger.error(f'Failed to open {url} in webbrowser with exception {e}')
logger.error(traceback.format_exc())
session.close()
def update_static():
''' Update Scout files basing on compatible-config.json content.
Files will be downloaded and updated if new version of GUI > current.
Current GUI version stored in static/version.txt.
'''
config = Config()
log = get_log('http')
static_path = Path(config['paths']['static'])
last_gui_version_lv = get_last_compatible_gui_version()
current_gui_version_lv = get_current_gui_version()
if last_gui_version_lv is False:
return False
if current_gui_version_lv is not None:
if current_gui_version_lv >= last_gui_version_lv:
return True
log.info(
f'New version of GUI available ({last_gui_version_lv.vstring}). Downloading...'
)
temp_dir = tempfile.mkdtemp(prefix='mindsdb_gui_files_')
success = download_gui(temp_dir, last_gui_version_lv.vstring)
if success is False:
shutil.rmtree(temp_dir)
return False
temp_dir_for_rm = tempfile.mkdtemp(prefix='mindsdb_gui_files_')
shutil.rmtree(temp_dir_for_rm)
shutil.copytree(str(static_path), temp_dir_for_rm)
shutil.rmtree(str(static_path))
shutil.copytree(temp_dir, str(static_path))
shutil.rmtree(temp_dir_for_rm)
log.info(f'GUI version updated to {last_gui_version_lv.vstring}')
return True
def initialize_static(config):
''' Update Scout files basing on compatible-config.json content.
Files will be downloaded and updated if new version of GUI > current.
Current GUI version stored in static/version.txt.
'''
log = get_log('http')
static_path = Path(config.paths['static'])
static_path.mkdir(parents=True, exist_ok=True)
try:
res = requests.get(
'https://mindsdb-web-builds.s3.amazonaws.com/compatible-config.json'
)
except (ConnectionError, requests.exceptions.ConnectionError) as e:
print(f'Is no connection. {e}')
return False
except Exception as e:
print(f'Is something wrong with getting compatible-config.json: {e}')
return False
if res.status_code != 200:
print(
f'Cant get compatible-config.json: returned status code = {res.status_code}'
)
return False
try:
versions = res.json()
except Exception as e:
print(f'Cant decode compatible-config.json: {e}')
return False
current_mindsdb_lv = LooseVersion(mindsdb_version)
try:
gui_versions = {}
max_mindsdb_lv = None
max_gui_lv = None
for el in versions['mindsdb']:
if el['mindsdb_version'] is None:
gui_lv = LooseVersion(el['gui_version'])
else:
mindsdb_lv = LooseVersion(el['mindsdb_version'])
gui_lv = LooseVersion(el['gui_version'])
if mindsdb_lv.vstring not in gui_versions or gui_lv > gui_versions[
mindsdb_lv.vstring]:
gui_versions[mindsdb_lv.vstring] = gui_lv
if max_mindsdb_lv is None or max_mindsdb_lv < mindsdb_lv:
max_mindsdb_lv = mindsdb_lv
if max_gui_lv is None or max_gui_lv < gui_lv:
max_gui_lv = gui_lv
all_mindsdb_lv = [LooseVersion(x) for x in gui_versions.keys()]
all_mindsdb_lv.sort()
if current_mindsdb_lv.vstring in gui_versions:
gui_version_lv = gui_versions[current_mindsdb_lv.vstring]
elif current_mindsdb_lv > all_mindsdb_lv[-1]:
gui_version_lv = max_gui_lv
else:
lower_versions = {
key: value
for key, value in gui_versions.items()
if LooseVersion(key) < current_mindsdb_lv
}
if len(lower_versions) == 0:
gui_version_lv = gui_versions[all_mindsdb_lv[0].vstring]
else:
all_lower_versions = [
LooseVersion(x) for x in lower_versions.keys()
]
gui_version_lv = gui_versions[all_lower_versions[-1].vstring]
except Exception as e:
log.error(f'Error in compatible-config.json structure: {e}')
return False
current_gui_version = None
version_txt_path = static_path.joinpath('version.txt')
if version_txt_path.is_file():
with open(version_txt_path, 'rt') as f:
current_gui_version = f.readline()
if current_gui_version is not None:
current_gui_lv = LooseVersion(current_gui_version)
if current_gui_lv >= gui_version_lv:
return True
log.info(
f'New version of GUI available ({gui_version_lv.vstring}). Downloading...'
)
shutil.rmtree(static_path)
static_path.mkdir(parents=True, exist_ok=True)
try:
css_zip_path = str(static_path.joinpath('css.zip'))
js_zip_path = str(static_path.joinpath('js.zip'))
media_zip_path = str(static_path.joinpath('media.zip'))
bucket = "https://mindsdb-web-builds.s3.amazonaws.com/"
gui_version = gui_version_lv.vstring
resources = [{
'url': bucket + 'css-V' + gui_version + '.zip',
'path': css_zip_path
}, {
'url': bucket + 'js-V' + gui_version + '.zip',
'path': js_zip_path
}, {
'url': bucket + 'indexV' + gui_version + '.html',
'path': str(static_path.joinpath('index.html'))
}, {
'url': bucket + 'favicon.ico',
'path': str(static_path.joinpath('favicon.ico'))
}, {
'url': bucket + 'media.zip',
'path': media_zip_path
}]
def get_resources(resource):
try:
response = requests.get(resource['url'])
if response.status_code != requests.status_codes.codes.ok:
return Exception(
f"Error {response.status_code} GET {resource['url']}")
open(resource['path'], 'wb').write(response.content)
except Exception as e:
return e
return None
for r in resources:
get_resources(r)
'''
# to make downloading faster download each resource in a separate thread
with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor:
future_to_url = {executor.submit(get_resources, r): r for r in resources}
for future in concurrent.futures.as_completed(future_to_url):
res = future.result()
if res is not None:
raise res
'''
except Exception as e:
log.error(f'Error during downloading files from s3: {e}')
session.close()
return False
static_folder = static_path.joinpath('static')
static_folder.mkdir(parents=True, exist_ok=True)
# unzip process
for zip_path, dir_name in [[js_zip_path, 'js'], [css_zip_path, 'css']]:
temp_dir = static_path.joinpath(f'temp_{dir_name}')
temp_dir.mkdir(mode=0o777, exist_ok=True, parents=True)
ZipFile(zip_path).extractall(temp_dir)
files_path = static_path.joinpath('static', dir_name)
if temp_dir.joinpath('build', 'static', dir_name).is_dir():
shutil.move(temp_dir.joinpath('build', 'static', dir_name),
files_path)
shutil.rmtree(temp_dir)
else:
shutil.move(temp_dir, files_path)
ZipFile(media_zip_path).extractall(static_folder)
os.remove(js_zip_path)
os.remove(css_zip_path)
os.remove(media_zip_path)
with open(version_txt_path, 'wt') as f:
f.write(gui_version_lv.vstring)
log.info(f'GUI version updated to {gui_version_lv.vstring}')
session.close()
return True
from mindsdb.utilities.log import get_log
log = get_log('mysql')
def download_gui(destignation, version):
if isinstance(destignation, str):
destignation = Path(destignation)
log = get_log('http')
css_zip_path = str(destignation.joinpath('css.zip'))
js_zip_path = str(destignation.joinpath('js.zip'))
media_zip_path = str(destignation.joinpath('media.zip'))
bucket = "https://mindsdb-web-builds.s3.amazonaws.com/"
resources = [{
'url': bucket + 'css-V' + version + '.zip',
'path': css_zip_path
}, {
'url': bucket + 'js-V' + version + '.zip',
'path': js_zip_path
}, {
'url': bucket + 'indexV' + version + '.html',
'path': str(destignation.joinpath('index.html'))
}, {
'url': bucket + 'favicon.ico',
'path': str(destignation.joinpath('favicon.ico'))
}, {
'url': bucket + 'media.zip',
'path': media_zip_path
}]
def get_resources(resource):
response = requests.get(resource['url'])
if response.status_code != requests.status_codes.codes.ok:
raise Exception(
f"Error {response.status_code} GET {resource['url']}")
open(resource['path'], 'wb').write(response.content)
try:
for r in resources:
get_resources(r)
except Exception as e:
log.error(f'Error during downloading files from s3: {e}')
return False
for zip_path, dir_name in [[js_zip_path, 'js'], [css_zip_path, 'css']]:
temp_dir = destignation.joinpath(f'temp_{dir_name}')
temp_dir.mkdir(mode=0o777, exist_ok=True, parents=True)
ZipFile(zip_path).extractall(temp_dir)
files_path = destignation.joinpath('static', dir_name)
if temp_dir.joinpath('build', 'static', dir_name).is_dir():
shutil.move(temp_dir.joinpath('build', 'static', dir_name),
files_path)
shutil.rmtree(temp_dir)
else:
shutil.move(temp_dir, files_path)
static_folder = Path(destignation).joinpath('static')
static_folder.mkdir(parents=True, exist_ok=True)
ZipFile(media_zip_path).extractall(static_folder)
os.remove(js_zip_path)
os.remove(css_zip_path)
os.remove(media_zip_path)
version_txt_path = destignation.joinpath(
'version.txt') # os.path.join(destignation, 'version.txt')
with open(version_txt_path, 'wt') as f:
f.write(version)
return True
'''
from mindsdb.utilities.log import get_log
log = get_log('mongodb')
